fix: Revert "feat: Limit HTTP concurrency and frequency by default" (#27765)

lib/config/options/index.ts

@@ -2416,21 +2416,21 @@ const options: RenovateOptions[] = [
   },
   {
     name: 'concurrentRequestLimit',
-    description: 'Limit concurrent requests per host. Set to 0 for no limit.',
+    description: 'Limit concurrent requests per host.',
     type: 'integer',
-    default: 5,
     stage: 'repository',
     parents: ['hostRules'],
+    default: null,
     cli: false,
     env: false,
   },
   {
     name: 'maxRequestsPerSecond',
-    description: 'Limit requests rate per host. Set to 0 for no limit.',
+    description: 'Limit requests rate per host.',
     type: 'integer',
-    default: 5,
     stage: 'repository',
     parents: ['hostRules'],
+    default: 0,
     cli: false,
     env: false,
   },

lib/util/http/host-rules.spec.ts

@@ -3,12 +3,7 @@ import { bootstrap } from '../../proxy';
 import type { HostRule } from '../../types';
 import * as hostRules from '../host-rules';
 import { dnsLookup } from './dns';
-import {
+import { applyHostRule, findMatchingRule } from './host-rules';
-  applyHostRule,
-  findMatchingRule,
-  getConcurrentRequestsLimit,
-  getThrottleIntervalMs,
-} from './host-rules';
 import type { GotOptions } from './types';
 
 const url = 'https://github.com';
@@ -565,48 +560,4 @@ describe('util/http/host-rules', () => {
       },
     });
   });
-
-  describe('getConcurrentRequestsLimit', () => {
-    it('returns default value for undefined rule', () => {
-      expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(42);
-    });
-
-    it('returns null for 0', () => {
-      hostRules.add({
-        matchHost: 'https://example.com',
-        concurrentRequestLimit: 0,
-      });
-      expect(getConcurrentRequestsLimit('https://example.com', 42)).toBeNull();
-    });
-
-    it('returns positive limit', () => {
-      hostRules.add({
-        matchHost: 'https://example.com',
-        concurrentRequestLimit: 143,
-      });
-      expect(getConcurrentRequestsLimit('https://example.com', 42)).toBe(143);
-    });
-  });
-
-  describe('getThrottleIntervalMs', () => {
-    it('returns default value for undefined rule', () => {
-      expect(getThrottleIntervalMs('https://example.com', 42)).toBe(24); // 1000 / 42
-    });
-
-    it('returns null for 0', () => {
-      hostRules.add({
-        matchHost: 'https://example.com',
-        maxRequestsPerSecond: 0,
-      });
-      expect(getThrottleIntervalMs('https://example.com', 42)).toBeNull();
-    });
-
-    it('returns positive limit', () => {
-      hostRules.add({
-        matchHost: 'https://example.com',
-        maxRequestsPerSecond: 143,
-      });
-      expect(getThrottleIntervalMs('https://example.com', 42)).toBe(7); // 1000 / 143
-    });
-  });
 });

lib/util/http/host-rules.ts

@@ -217,36 +217,16 @@ export function applyHostRule<GotOptions extends HostRulesGotOptions>(
   return options;
 }
 
-export function getConcurrentRequestsLimit(
+export function getConcurrentRequestsLimit(url: string): number | null {
-  url: string,
-  defaultValue: number | null,
-): number | null {
   const { concurrentRequestLimit } = hostRules.find({ url });
-
+  return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0
-  if (!is.number(concurrentRequestLimit)) {
+    ? concurrentRequestLimit
-    return defaultValue;
+    : null;
-  }
-
-  if (concurrentRequestLimit > 0) {
-    return concurrentRequestLimit;
-  }
-
-  return null;
 }
 
-export function getThrottleIntervalMs(
+export function getThrottleIntervalMs(url: string): number | null {
-  url: string,
-  defaultValue: number | null,
-): number | null {
   const { maxRequestsPerSecond } = hostRules.find({ url });
-
+  return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0
-  if (!is.number(maxRequestsPerSecond)) {
+    ? Math.ceil(1000 / maxRequestsPerSecond)
-    return defaultValue ? Math.ceil(1000 / defaultValue) : defaultValue; // 5 requests per second
+    : null;
-  }
-
-  if (maxRequestsPerSecond > 0) {
-    return Math.ceil(1000 / maxRequestsPerSecond);
-  }
-
-  return null;
 }

lib/util/http/index.spec.ts

@@ -516,68 +516,34 @@ describe('util/http/index', () => {
   });
 
   describe('Throttling', () => {
-    const delta = 100;
-
-    beforeEach(() => {
-      jest.useFakeTimers({ advanceTimers: true });
-    });
-
     afterEach(() => {
       jest.useRealTimers();
     });
 
     it('works without throttling', async () => {
-      httpMock.scope(baseUrl).get('/foo').times(10).reply(200, 'bar');
+      jest.useFakeTimers({ advanceTimers: 1 });
+      httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar');
 
       const t1 = Date.now();
-      await Promise.all([
+      await http.get('http://renovate.com/foo');
-        http.get('http://renovate.com/foo'),
+      await http.get('http://renovate.com/foo');
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-      ]);
       const t2 = Date.now();
 
-      expect(t2 - t1).toBeLessThan(delta);
+      expect(t2 - t1).toBeLessThan(100);
     });
 
     it('limits request rate by host', async () => {
-      httpMock.scope(baseUrl).get('/foo').times(4).reply(200, 'bar');
+      jest.useFakeTimers({ advanceTimers: true });
-      hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 3 });
+      httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar');
+      hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 0.25 });
 
       const t1 = Date.now();
-      await Promise.all([
+      await http.get('http://renovate.com/foo');
-        http.get('http://renovate.com/foo'),
+      jest.advanceTimersByTime(4000);
-        http.get('http://renovate.com/foo'),
+      await http.get('http://renovate.com/foo');
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-      ]);
       const t2 = Date.now();
 
-      expect(t2 - t1).toBeWithin(1000, 1000 + delta);
+      expect(t2 - t1).toBeGreaterThanOrEqual(4000);
-    });
-
-    it('defaults to 5 requests per second', async () => {
-      Http.setDefaultLimits();
-      httpMock.scope(baseUrl).get('/foo').times(5).reply(200, 'bar');
-
-      const t1 = Date.now();
-      await Promise.all([
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-        http.get('http://renovate.com/foo'),
-      ]);
-      const t2 = Date.now();
-
-      expect(t2 - t1).toBeWithin(800, 800 + delta);
     });
   });
 

lib/util/http/index.ts

@@ -126,14 +126,6 @@ async function gotTask<T>(
 }
 
 export class Http<Opts extends HttpOptions = HttpOptions> {
-  private static defaultConcurrentRequestLimit: number | null = null;
-  private static defaultMaxRequestsPerSecond: number | null = null;
-
-  static setDefaultLimits(): void {
-    Http.defaultConcurrentRequestLimit = 5;
-    Http.defaultMaxRequestsPerSecond = 5;
-  }
-
   private options?: GotOptions;
 
   constructor(
@@ -151,7 +143,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
   }
 
   protected getThrottle(url: string): Throttle | null {
-    return getThrottle(url, Http.defaultMaxRequestsPerSecond);
+    return getThrottle(url);
   }
 
   protected async request<T>(
@@ -257,7 +249,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
         ? () => throttle.add<HttpResponse<T>>(httpTask)
         : httpTask;
 
-      const queue = getQueue(url, Http.defaultConcurrentRequestLimit);
+      const queue = getQueue(url);
       const queuedTask: GotTask<T> = queue
         ? () => queue.add<HttpResponse<T>>(throttledTask)
         : throttledTask;

lib/util/http/queue.spec.ts

@@ -12,15 +12,15 @@ describe('util/http/queue', () => {
   });
 
   it('returns null for invalid URL', () => {
-    expect(getQueue('$#@!', null)).toBeNull();
+    expect(getQueue('$#@!')).toBeNull();
   });
 
   it('returns queue for valid url', () => {
-    const q1a = getQueue('https://example.com', null);
+    const q1a = getQueue('https://example.com');
-    const q1b = getQueue('https://example.com', null);
+    const q1b = getQueue('https://example.com');
 
-    const q2a = getQueue('https://example.com:8080', null);
+    const q2a = getQueue('https://example.com:8080');
-    const q2b = getQueue('https://example.com:8080', null);
+    const q2b = getQueue('https://example.com:8080');
 
     expect(q1a).not.toBeNull();
     expect(q1a).toBe(q1b);

lib/util/http/queue.ts

@@ -5,10 +5,7 @@ import { getConcurrentRequestsLimit } from './host-rules';
 
 const hostQueues = new Map<string, PQueue | null>();
 
-export function getQueue(
+export function getQueue(url: string): PQueue | null {
-  url: string,
-  defaultConcurrentRequestLimit: number | null,
-): PQueue | null {
   const host = parseUrl(url)?.host;
   if (!host) {
     // should never happen
@@ -19,10 +16,7 @@ export function getQueue(
   let queue = hostQueues.get(host);
   if (queue === undefined) {
     queue = null; // null represents "no queue", as opposed to undefined
-    const concurrency = getConcurrentRequestsLimit(
+    const concurrency = getConcurrentRequestsLimit(url);
-      url,
-      defaultConcurrentRequestLimit,
-    );
     if (concurrency) {
       logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`);
       queue = new PQueue({ concurrency });

lib/util/http/throttle.spec.ts

@@ -12,15 +12,15 @@ describe('util/http/throttle', () => {
   });
 
   it('returns null for invalid URL', () => {
-    expect(getThrottle('$#@!', null)).toBeNull();
+    expect(getThrottle('$#@!')).toBeNull();
   });
 
   it('returns throttle for valid url', () => {
-    const t1a = getThrottle('https://example.com', null);
+    const t1a = getThrottle('https://example.com');
-    const t1b = getThrottle('https://example.com', null);
+    const t1b = getThrottle('https://example.com');
 
-    const t2a = getThrottle('https://example.com:8080', null);
+    const t2a = getThrottle('https://example.com:8080');
-    const t2b = getThrottle('https://example.com:8080', null);
+    const t2b = getThrottle('https://example.com:8080');
 
     expect(t1a).not.toBeNull();
     expect(t1a).toBe(t1b);

lib/util/http/throttle.ts

@@ -22,10 +22,7 @@ export class Throttle {
   }
 }
 
-export function getThrottle(
+export function getThrottle(url: string): Throttle | null {
-  url: string,
-  defaultMaxRequestsPerSecond: number | null,
-): Throttle | null {
   const host = parseUrl(url)?.host;
   if (!host) {
     // should never happen
@@ -36,10 +33,7 @@ export function getThrottle(
   let throttle = hostThrottles.get(host);
   if (throttle === undefined) {
     throttle = null; // null represents "no throttle", as opposed to undefined
-    const throttleOptions = getThrottleIntervalMs(
+    const throttleOptions = getThrottleIntervalMs(url);
-      url,
-      defaultMaxRequestsPerSecond,
-    );
     if (throttleOptions) {
       const intervalMs = throttleOptions;
       logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`);

lib/workers/global/initialize.ts

@@ -10,7 +10,6 @@ import * as packageCache from '../../util/cache/package';
 import { setEmojiConfig } from '../../util/emoji';
 import { validateGitVersion } from '../../util/git';
 import * as hostRules from '../../util/host-rules';
-import { Http } from '../../util/http';
 import { initMergeConfidence } from '../../util/merge-confidence';
 import { setMaxLimit } from './limits';
 
@@ -80,7 +79,6 @@ export async function globalInitialize(
   config_: AllConfig,
 ): Promise<RenovateConfig> {
   let config = config_;
-  Http.setDefaultLimits();
   await checkVersions();
   setGlobalHostRules(config);
   config = await initPlatform(config);