feat(cache): etag caching for github GET (#26788)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/modules/platform/github/index.ts

@@ -35,7 +35,10 @@ import type {
 import * as hostRules from '../../../util/host-rules';
 import * as githubHttp from '../../../util/http/github';
 import type { GithubHttpOptions } from '../../../util/http/github';
-import type { HttpResponse } from '../../../util/http/types';
+import type {
+  HttpResponse,
+  InternalHttpOptions,
+} from '../../../util/http/types';
 import { coerceObject } from '../../../util/object';
 import { regEx } from '../../../util/regex';
 import { sanitize } from '../../../util/sanitize';
@@ -316,11 +319,15 @@ export async function getRawFile(
   branchOrTag?: string,
 ): Promise<string | null> {
   const repo = repoName ?? config.repository;
+  const httpOptions: InternalHttpOptions = {
+    // Only cache response if it's from the same repo
+    repoCache: repo === config.repository,
+  };
   let url = `repos/${repo}/contents/${fileName}`;
   if (branchOrTag) {
     url += `?ref=` + branchOrTag;
   }
-  const res = await githubApi.getJson<{ content: string }>(url);
+  const res = await githubApi.getJson<{ content: string }>(url, httpOptions);
   const buf = res.body.content;
   const str = fromBase64(buf);
   return str;
@@ -1220,7 +1227,7 @@ export async function getIssue(
     const issueBody = (
       await githubApi.getJson<{ body: string }>(
         `repos/${config.parentRepo ?? config.repository}/issues/${number}`,
-        { memCache: useCache },
+        { memCache: useCache, repoCache: true },
       )
     ).body.body;
     return {
@@ -1306,6 +1313,7 @@ export async function ensureIssue({
           `repos/${config.parentRepo ?? config.repository}/issues/${
             issue.number
           }`,
+          { repoCache: true },
         )
       ).body.body;
       if (

lib/modules/platform/github/pr.ts

@@ -67,6 +67,7 @@ export async function getPrCache(
       if (pageIdx === 1 && isInitial) {
         // Speed up initial fetch
         opts.paginate = true;
+        opts.repoCache = true;
       }
 
       const perPage = isInitial ? 100 : 20;

lib/util/cache/repository/types.ts

@@ -8,6 +8,7 @@ import type { BitbucketPrCacheData } from '../../../modules/platform/bitbucket/t
 import type { GiteaPrCacheData } from '../../../modules/platform/gitea/types';
 import type { RepoInitConfig } from '../../../workers/repository/init/types';
 import type { PrBlockedBy } from '../../../workers/types';
+import type { HttpResponse } from '../../http/types';
 
 export interface BaseBranchCache {
   sha: string; // branch commit sha
@@ -124,8 +125,15 @@ export interface BranchCache {
   result?: string;
 }
 
+export interface HttpCache {
+  etag: string;
+  httpResponse: HttpResponse<unknown>;
+  timeStamp: string;
+}
+
 export interface RepoCacheData {
   configFileName?: string;
+  httpCache?: Record<string, HttpCache>;
   semanticCommits?: 'enabled' | 'disabled';
   branches?: BranchCache[];
   init?: RepoInitConfig;

lib/util/http/index.spec.ts

@@ -76,13 +76,38 @@ describe('util/http/index', () => {
         },
       })
       .get('/')
-      .reply(200, '{ "test": true }');
+      .reply(200, '{ "test": true }', { etag: 'abc123' });
-    expect(await http.getJson('http://renovate.com')).toEqual({
+    expect(
+      await http.getJson('http://renovate.com', { repoCache: true }),
+    ).toEqual({
       authorization: false,
       body: {
         test: true,
       },
-      headers: {},
+      headers: {
+        etag: 'abc123',
+      },
+      statusCode: 200,
+    });
+
+    httpMock
+      .scope(baseUrl, {
+        reqheaders: {
+          accept: 'application/json',
+        },
+      })
+      .get('/')
+      .reply(304, '', { etag: 'abc123' });
+    expect(
+      await http.getJson('http://renovate.com', { repoCache: true }),
+    ).toEqual({
+      authorization: false,
+      body: {
+        test: true,
+      },
+      headers: {
+        etag: 'abc123',
+      },
       statusCode: 200,
     });
   });

lib/util/http/index.ts

@@ -7,6 +7,7 @@ import { pkg } from '../../expose.cjs';
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import * as memCache from '../cache/memory';
+import { getCache } from '../cache/repository';
 import { clone } from '../clone';
 import { hash } from '../hash';
 import { type AsyncResult, Result } from '../result';
@@ -163,6 +164,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       httpOptions,
     );
 
+    logger.trace(`HTTP request: ${options.method.toUpperCase()} ${url}`);
+
     const etagCache =
       httpOptions.etagCache && options.method === 'get'
         ? httpOptions.etagCache
@@ -210,6 +213,16 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
 
     // istanbul ignore else: no cache tests
     if (!resPromise) {
+      if (httpOptions.repoCache) {
+        const cachedEtag = getCache().httpCache?.[url]?.etag;
+        if (cachedEtag) {
+          logger.debug(`Using cached etag for ${url}`);
+          options.headers = {
+            ...options.headers,
+            'If-None-Match': cachedEtag,
+          };
+        }
+      }
       const startTime = Date.now();
       const httpTask: GotTask<T> = () => {
         const queueDuration = Date.now() - startTime;
@@ -243,6 +256,31 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304;
       const resCopy = copyResponse(res, deepCopyNeeded);
       resCopy.authorization = !!options?.headers?.authorization;
+      if (httpOptions.repoCache) {
+        const cache = getCache();
+        cache.httpCache ??= {};
+        if (resCopy.statusCode === 200 && resCopy.headers?.etag) {
+          logger.debug(
+            `Saving response to cache: ${url} with etag ${resCopy.headers.etag}`,
+          );
+          cache.httpCache[url] = {
+            etag: resCopy.headers.etag,
+            httpResponse: copyResponse(res, deepCopyNeeded),
+            timeStamp: new Date().toISOString(),
+          };
+        }
+        if (resCopy.statusCode === 304 && cache.httpCache[url]?.httpResponse) {
+          logger.debug(
+            `Using cached response: ${url} with etag ${resCopy.headers.etag} from ${cache.httpCache[url].timeStamp}`,
+          );
+          const cacheCopy = copyResponse(
+            cache.httpCache[url].httpResponse,
+            deepCopyNeeded,
+          );
+          cacheCopy.authorization = !!options?.headers?.authorization;
+          return cacheCopy as HttpResponse<T>;
+        }
+      }
       return resCopy;
     } catch (err) {
       const { abortOnError, abortIgnoreStatusCodes } = options;

lib/util/http/types.ts

@@ -65,6 +65,7 @@ export interface HttpOptions {
 
   token?: string;
   memCache?: boolean;
+  repoCache?: boolean;
 }
 
 export interface EtagCache<T = any> {
@@ -81,6 +82,7 @@ export interface InternalHttpOptions extends HttpOptions {
   responseType?: 'json' | 'buffer';
   method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
   parseJson?: ParseJsonFunction;
+  repoCache?: boolean;
 }
 
 export interface HttpHeaders extends IncomingHttpHeaders {