feat(host-rules): Support readOnly request matching (#28562)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

docs/usage/configuration-options.md

@@ -898,6 +898,27 @@ It will be compiled using Handlebars and the regex `groups` result.
 It will be compiled using Handlebars and the regex `groups` result.
 It will default to the value of `depName` if left unconfigured/undefined.
 
+### readOnly
+
+If the `readOnly` field is being set to `true` inside the host rule, it will match only against the requests that are known to be read operations.
+Examples are `GET` requests or `HEAD` requests, but also it could be certain types of GraphQL queries.
+
+This option could be used to avoid rate limits for certain platforms like GitHub or Bitbucket, by offloading the read operations to a different user.
+
+```json
+{
+  "hostRules": [
+    {
+      "matchHost": "api.github.com",
+      "readOnly": true,
+      "token": "********"
+    }
+  ]
+}
+```
+
+If more than one token matches for a read-only request then the `readOnly` token will be given preference.
+
 ### currentValueTemplate
 
 If the `currentValue` for a dependency is not captured with a named group then it can be defined in config using this field.

lib/config/options/index.ts

@@ -2476,6 +2476,16 @@ const options: RenovateOptions[] = [
     cli: false,
     env: false,
   },
+  {
+    name: 'readOnly',
+    description:
+      'Match against requests that only read data and do not mutate anything.',
+    type: 'boolean',
+    stage: 'repository',
+    parents: ['hostRules'],
+    cli: false,
+    env: false,
+  },
   {
     name: 'timeout',
     description: 'Timeout (in milliseconds) for queries to external endpoints.',

lib/modules/platform/github/index.ts

@@ -461,6 +461,7 @@ export async function initRepo({
   const opts = hostRules.find({
     hostType: 'github',
     url: platformConfig.endpoint,
+    readOnly: true,
   });
   config.renovateUsername = renovateUsername;
   [config.repositoryOwner, config.repositoryName] = repository.split('/');
@@ -499,6 +500,7 @@ export async function initRepo({
         name: config.repositoryName,
         user: renovateUsername,
       },
+      readOnly: true,
     });
 
     if (res?.errors) {
@@ -1214,6 +1216,7 @@ async function getIssues(): Promise<Issue[]> {
         name: config.repositoryName,
         user: config.renovateUsername,
       },
+      readOnly: true,
     },
   );
 
@@ -1975,6 +1978,7 @@ export async function getVulnerabilityAlerts(): Promise<VulnerabilityAlert[]> {
       variables: { owner: config.repositoryOwner, name: config.repositoryName },
       paginate: false,
       acceptHeader: 'application/vnd.github.vixen-preview+json',
+      readOnly: true,
     });
   } catch (err) {
     logger.debug({ err }, 'Error retrieving vulnerability alerts');

lib/types/host-rules.ts

@@ -25,9 +25,10 @@ export interface HostRule {
   hostType?: string;
   matchHost?: string;
   resolvedHost?: string;
+  readOnly?: boolean;
 }
 
 export type CombinedHostRule = Omit<
   HostRule,
-  'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost'
+  'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost' | 'readOnly'
 >;

lib/util/github/graphql/datasource-fetcher.ts

@@ -107,6 +107,7 @@ export class GithubGraphqlDatasourceFetcher<
     return {
       baseUrl,
       repository,
+      readOnly: true,
       body: { query, variables },
     };
   }

lib/util/host-rules.spec.ts

@@ -295,6 +295,25 @@ describe('util/host-rules', () => {
         }),
       ).toEqual({ token: 'longest' });
     });
+
+    it('matches readOnly requests', () => {
+      add({
+        matchHost: 'https://api.github.com/repos/',
+        token: 'aaa',
+        hostType: 'github',
+      });
+      add({
+        matchHost: 'https://api.github.com',
+        token: 'bbb',
+        readOnly: true,
+      });
+      expect(
+        find({
+          url: 'https://api.github.com/repos/foo/bar/tags',
+          readOnly: true,
+        }),
+      ).toEqual({ token: 'bbb' });
+    });
   });
 
   describe('hosts()', () => {

lib/util/host-rules.ts

@@ -73,6 +73,7 @@ export function add(params: HostRule): void {
 export interface HostRuleSearch {
   hostType?: string;
   url?: string;
+  readOnly?: boolean;
 }
 
 function matchesHost(url: string, matchHost: string): boolean {
@@ -107,8 +108,9 @@ function fromShorterToLongerMatchHost(a: HostRule, b: HostRule): number {
   return a.matchHost.length - b.matchHost.length;
 }
 
-function hostRuleRank({ hostType, matchHost }: HostRule): number {
+function hostRuleRank({ hostType, matchHost, readOnly }: HostRule): number {
-  if (hostType && matchHost) {
+  // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+  if ((hostType || readOnly) && matchHost) {
     return 3;
   }
 
@@ -142,6 +144,7 @@ export function find(search: HostRuleSearch): CombinedHostRule {
   for (const rule of sortedRules) {
     let hostTypeMatch = true;
     let hostMatch = true;
+    let readOnlyMatch = true;
 
     if (rule.hostType) {
       hostTypeMatch = false;
@@ -157,7 +160,15 @@ export function find(search: HostRuleSearch): CombinedHostRule {
       }
     }
 
-    if (hostTypeMatch && hostMatch) {
+    if (!is.undefined(rule.readOnly)) {
+      readOnlyMatch = false;
+      if (search.readOnly === rule.readOnly) {
+        readOnlyMatch = true;
+        hostTypeMatch = true; // When we match `readOnly`, we don't care about `hostType`
+      }
+    }
+
+    if (hostTypeMatch && readOnlyMatch && hostMatch) {
       matchedRules.push(clone(rule));
     }
   }
@@ -166,6 +177,7 @@ export function find(search: HostRuleSearch): CombinedHostRule {
   delete res.hostType;
   delete res.resolvedHost;
   delete res.matchHost;
+  delete res.readOnly;
   return res;
 }
 

lib/util/http/github.ts

@@ -276,7 +276,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
     options?: InternalHttpOptions & GithubHttpOptions,
     okToRetry = true,
   ): Promise<HttpResponse<T>> {
-    const opts: GithubHttpOptions = {
+    const opts: InternalHttpOptions & GithubHttpOptions = {
       baseUrl,
       ...options,
       throwHttpErrors: true,
@@ -296,8 +296,17 @@ export class GithubHttp extends Http<GithubHttpOptions> {
         );
       }
 
+      let readOnly = opts.readOnly;
+      const { method = 'get' } = opts;
+      if (
+        readOnly === undefined &&
+        ['get', 'head'].includes(method.toLowerCase())
+      ) {
+        readOnly = true;
+      }
       const { token } = findMatchingRule(authUrl.toString(), {
         hostType: this.hostType,
+        readOnly,
       });
       opts.token = token;
     }
@@ -393,6 +402,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
       baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path
       body,
       headers: { accept: options?.acceptHeader },
+      readOnly: options.readOnly,
     };
     if (options.token) {
       opts.token = options.token;

lib/util/http/host-rules.ts

@@ -14,10 +14,10 @@ import { matchRegexOrGlobList } from '../string-match';
 import { parseUrl } from '../url';
 import { dnsLookup } from './dns';
 import { keepAliveAgents } from './keep-alive';
-import type { GotOptions } from './types';
+import type { GotOptions, InternalHttpOptions } from './types';
 
 export type HostRulesGotOptions = Pick<
-  GotOptions,
+  GotOptions & InternalHttpOptions,
   | 'hostType'
   | 'url'
   | 'noAuth'
@@ -34,14 +34,15 @@ export type HostRulesGotOptions = Pick<
   | 'agent'
   | 'http2'
   | 'https'
+  | 'readOnly'
 >;
 
 export function findMatchingRule<GotOptions extends HostRulesGotOptions>(
   url: string,
   options: GotOptions,
 ): HostRule {
-  const { hostType } = options;
+  const { hostType, readOnly } = options;
-  let res = hostRules.find({ hostType, url });
+  let res = hostRules.find({ hostType, url, readOnly });
 
   if (
     is.nonEmptyString(res.token) ||

lib/util/http/index.ts

@@ -162,6 +162,13 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
 
     applyDefaultHeaders(options);
 
+    if (
+      is.undefined(options.readOnly) &&
+      ['head', 'get'].includes(options.method)
+    ) {
+      options.readOnly = true;
+    }
+
     const hostRule = findMatchingRule(url, options);
     options = applyHostRule(url, options, hostRule);
     if (options.enabled === false) {
@@ -457,6 +464,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     }
 
     applyDefaultHeaders(combinedOptions);
+
+    if (
+      is.undefined(combinedOptions.readOnly) &&
+      ['head', 'get'].includes(combinedOptions.method)
+    ) {
+      combinedOptions.readOnly = true;
+    }
+
     const hostRule = findMatchingRule(url, combinedOptions);
     combinedOptions = applyHostRule(resolvedUrl, combinedOptions, hostRule);
     if (combinedOptions.enabled === false) {

lib/util/http/types.ts

@@ -48,6 +48,7 @@ export interface GraphqlOptions {
   cursor?: string | null;
   acceptHeader?: string;
   token?: string;
+  readOnly?: boolean;
 }
 
 export interface HttpOptions {
@@ -67,6 +68,7 @@ export interface HttpOptions {
   token?: string;
   memCache?: boolean;
   cacheProvider?: HttpCacheProvider;
+  readOnly?: boolean;
 }
 
 export interface InternalHttpOptions extends HttpOptions {

lib/workers/repository/update/pr/changelog/github/source.ts

@@ -53,6 +53,7 @@ export class GitHubChangeLogSource extends ChangeLogSource {
     const { token } = hostRules.find({
       hostType: 'github',
       url,
+      readOnly: true,
     });
     // istanbul ignore if
     if (host && !token) {