feat(http): Add pluggable HTTP cache implementation (#27998)

lib/util/cache/repository/types.ts

@@ -6,7 +6,6 @@ import type {
 import type { PackageFile } from '../../../modules/manager/types';
 import type { RepoInitConfig } from '../../../workers/repository/init/types';
 import type { PrBlockedBy } from '../../../workers/types';
-import type { HttpResponse } from '../../http/types';
 
 export interface BaseBranchCache {
   sha: string; // branch commit sha
@@ -123,16 +122,9 @@ export interface BranchCache {
   result?: string;
 }
 
-export interface HttpCache {
-  etag?: string;
-  httpResponse: HttpResponse<unknown>;
-  lastModified?: string;
-  timeStamp: string;
-}
-
 export interface RepoCacheData {
   configFileName?: string;
-  httpCache?: Record<string, HttpCache>;
+  httpCache?: Record<string, unknown>;
   semanticCommits?: 'enabled' | 'disabled';
   branches?: BranchCache[];
   init?: RepoInitConfig;

lib/util/http/cache/abstract-http-cache-provider.ts

@@ -0,0 +1,94 @@
+import { logger } from '../../../logger';
+import { HttpCacheStats } from '../../stats';
+import type { GotOptions, HttpResponse } from '../types';
+import { copyResponse } from '../util';
+import { HttpCacheSchema } from './schema';
+import type { HttpCache, HttpCacheProvider } from './types';
+
+export abstract class AbstractHttpCacheProvider implements HttpCacheProvider {
+  protected abstract load(url: string): Promise<unknown>;
+  protected abstract persist(url: string, data: HttpCache): Promise<void>;
+
+  async get(url: string): Promise<HttpCache | null> {
+    const cache = await this.load(url);
+    const httpCache = HttpCacheSchema.parse(cache);
+    if (!httpCache) {
+      return null;
+    }
+
+    return httpCache as HttpCache;
+  }
+
+  async setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+    url: string,
+    opts: T,
+  ): Promise<void> {
+    const httpCache = await this.get(url);
+    if (!httpCache) {
+      return;
+    }
+
+    opts.headers ??= {};
+
+    if (httpCache.etag) {
+      opts.headers['If-None-Match'] = httpCache.etag;
+    }
+
+    if (httpCache.lastModified) {
+      opts.headers['If-Modified-Since'] = httpCache.lastModified;
+    }
+  }
+
+  async wrapResponse<T>(
+    url: string,
+    resp: HttpResponse<T>,
+  ): Promise<HttpResponse<T>> {
+    if (resp.statusCode === 200) {
+      const etag = resp.headers?.['etag'];
+      const lastModified = resp.headers?.['last-modified'];
+
+      HttpCacheStats.incRemoteMisses(url);
+
+      const httpResponse = copyResponse(resp, true);
+      const timestamp = new Date().toISOString();
+
+      const newHttpCache = HttpCacheSchema.parse({
+        etag,
+        lastModified,
+        httpResponse,
+        timestamp,
+      });
+      if (newHttpCache) {
+        logger.debug(
+          `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`,
+        );
+        await this.persist(url, newHttpCache as HttpCache);
+      } else {
+        logger.debug(`http cache: failed to persist cache for ${url}`);
+      }
+
+      return resp;
+    }
+
+    if (resp.statusCode === 304) {
+      const httpCache = await this.get(url);
+      if (!httpCache) {
+        return resp;
+      }
+
+      const timestamp = httpCache.timestamp;
+      logger.debug(
+        `http cache: Using cached response: ${url} from ${timestamp}`,
+      );
+      HttpCacheStats.incRemoteHits(url);
+      const cachedResp = copyResponse(
+        httpCache.httpResponse as HttpResponse<T>,
+        true,
+      );
+      cachedResp.authorization = resp.authorization;
+      return cachedResp;
+    }
+
+    return resp;
+  }
+}

lib/util/http/cache/repository-http-cache-provider.spec.ts

@@ -0,0 +1,152 @@
+import { Http } from '..';
+import * as httpMock from '../../../../test/http-mock';
+import { logger } from '../../../../test/util';
+import { getCache, resetCache } from '../../cache/repository';
+import { repoCacheProvider } from './repository-http-cache-provider';
+
+describe('util/http/cache/repository-http-cache-provider', () => {
+  beforeEach(() => {
+    resetCache();
+  });
+
+  const http = new Http('test', {
+    cacheProvider: repoCacheProvider,
+  });
+
+  it('reuses data with etag', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('reuses data with last-modified', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope
+      .get('/foo/bar')
+      .reply(
+        200,
+        { msg: 'Hello, world!' },
+        { 'last-modified': 'Mon, 01 Jan 2000 00:00:00 GMT' },
+      );
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('uses older cache format', async () => {
+    const repoCache = getCache();
+    repoCache.httpCache = {
+      'https://example.com/foo/bar': {
+        etag: '123',
+        lastModified: 'Mon, 01 Jan 2000 00:00:00 GMT',
+        httpResponse: { statusCode: 200, body: { msg: 'Hello, world!' } },
+        timeStamp: new Date().toISOString(),
+      },
+    };
+    httpMock.scope('https://example.com').get('/foo/bar').reply(304);
+
+    const res = await http.getJson('https://example.com/foo/bar');
+
+    expect(res).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('reports if cache could not be persisted', async () => {
+    httpMock
+      .scope('https://example.com')
+      .get('/foo/bar')
+      .reply(200, { msg: 'Hello, world!' });
+
+    await http.getJson('https://example.com/foo/bar');
+
+    expect(logger.logger.debug).toHaveBeenCalledWith(
+      'http cache: failed to persist cache for https://example.com/foo/bar',
+    );
+  });
+
+  it('handles abrupt cache reset', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    resetCache();
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 304,
+      authorization: false,
+    });
+  });
+
+  it('bypasses for statuses other than 200 and 304', async () => {
+    const scope = httpMock.scope('https://example.com');
+    scope.get('/foo/bar').reply(203);
+
+    const res = await http.getJson('https://example.com/foo/bar');
+
+    expect(res).toMatchObject({
+      statusCode: 203,
+      authorization: false,
+    });
+  });
+
+  it('supports authorization', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar', {
+      headers: { authorization: 'Bearer 123' },
+    });
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: true,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar', {
+      headers: { authorization: 'Bearer 123' },
+    });
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: true,
+    });
+  });
+});

lib/util/http/cache/repository-http-cache-provider.ts

@@ -0,0 +1,20 @@
+import { getCache } from '../../cache/repository';
+import { AbstractHttpCacheProvider } from './abstract-http-cache-provider';
+import type { HttpCache } from './types';
+
+export class RepositoryHttpCacheProvider extends AbstractHttpCacheProvider {
+  override load(url: string): Promise<unknown> {
+    const cache = getCache();
+    cache.httpCache ??= {};
+    return Promise.resolve(cache.httpCache[url]);
+  }
+
+  override persist(url: string, data: HttpCache): Promise<void> {
+    const cache = getCache();
+    cache.httpCache ??= {};
+    cache.httpCache[url] = data;
+    return Promise.resolve();
+  }
+}
+
+export const repoCacheProvider = new RepositoryHttpCacheProvider();

lib/util/http/cache/schema.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+
+const invalidFieldsMsg =
+  'Cache object should have `etag` or `lastModified` fields';
+
+export const HttpCacheSchema = z
+  .object({
+    // TODO: remove this migration part during the Christmas eve 2024
+    timeStamp: z.string().optional(),
+    timestamp: z.string().optional(),
+  })
+  .passthrough()
+  .transform((data) => {
+    if (data.timeStamp) {
+      data.timestamp = data.timeStamp;
+      delete data.timeStamp;
+    }
+    return data;
+  })
+  .pipe(
+    z
+      .object({
+        etag: z.string().optional(),
+        lastModified: z.string().optional(),
+        httpResponse: z.unknown(),
+        timestamp: z.string(),
+      })
+      .refine(
+        ({ etag, lastModified }) => etag ?? lastModified,
+        invalidFieldsMsg,
+      ),
+  )
+  .nullable()
+  .catch(null);

lib/util/http/cache/types.ts

@@ -0,0 +1,17 @@
+import type { GotOptions, HttpResponse } from '../types';
+
+export interface HttpCache {
+  etag?: string;
+  lastModified?: string;
+  httpResponse: unknown;
+  timestamp: string;
+}
+
+export interface HttpCacheProvider {
+  setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+    url: string,
+    opts: T,
+  ): Promise<void>;
+
+  wrapResponse<T>(url: string, resp: HttpResponse<T>): Promise<HttpResponse<T>>;
+}

lib/util/http/index.ts

@@ -1,3 +1,4 @@
+import is from '@sindresorhus/is';
 import merge from 'deepmerge';
 import got, { Options, RequestError } from 'got';
 import type { SetRequired } from 'type-fest';
@@ -8,7 +9,6 @@ import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import * as memCache from '../cache/memory';
 import { getCache } from '../cache/repository';
-import { clone } from '../clone';
 import { hash } from '../hash';
 import { type AsyncResult, Result } from '../result';
 import {
@@ -27,12 +27,14 @@ import type {
   GotJSONOptions,
   GotOptions,
   GotTask,
+  HttpCache,
   HttpOptions,
   HttpResponse,
   InternalHttpOptions,
 } from './types';
 // TODO: refactor code to remove this (#9651)
 import './legacy';
+import { copyResponse } from './util';
 
 export { RequestError as HttpError };
 
@@ -49,26 +51,6 @@ type JsonArgs<
   schema?: Schema;
 };
 
-// Copying will help to avoid circular structure
-// and mutation of the cached response.
-function copyResponse<T>(
-  response: HttpResponse<T>,
-  deep: boolean,
-): HttpResponse<T> {
-  const { body, statusCode, headers } = response;
-  return deep
-    ? {
-        statusCode,
-        body: body instanceof Buffer ? (body.subarray() as T) : clone<T>(body),
-        headers: clone(headers),
-      }
-    : {
-        statusCode,
-        body,
-        headers,
-      };
-}
-
 function applyDefaultHeaders(options: Options): void {
   const renovateVersion = pkg.version;
   options.headers = {
@@ -142,13 +124,17 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     options: HttpOptions = {},
   ) {
     const retryLimit = process.env.NODE_ENV === 'test' ? 0 : 2;
-    this.options = merge<GotOptions>(options, {
+    this.options = merge<GotOptions>(
-      context: { hostType },
+      options,
-      retry: {
+      {
-        limit: retryLimit,
+        context: { hostType },
-        maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+        retry: {
+          limit: retryLimit,
+          maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+        },
       },
-    });
+      { isMergeableObject: is.plainObject },
+    );
   }
 
   protected getThrottle(url: string): Throttle | null {
@@ -164,13 +150,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       url = resolveBaseUrl(httpOptions.baseUrl, url);
     }
 
-    let options = merge<SetRequired<GotOptions, 'method'>, GotOptions>(
+    let options = merge<SetRequired<GotOptions, 'method'>, InternalHttpOptions>(
       {
         method: 'get',
         ...this.options,
         hostType: this.hostType,
       },
       httpOptions,
+      { isMergeableObject: is.plainObject },
     );
 
     logger.trace(`HTTP request: ${options.method.toUpperCase()} ${url}`);
@@ -212,7 +199,9 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     // istanbul ignore else: no cache tests
     if (!resPromise) {
       if (httpOptions.repoCache) {
-        const responseCache = getCache().httpCache?.[url];
+        const responseCache = getCache().httpCache?.[url] as
+          | HttpCache
+          | undefined;
         // Prefer If-Modified-Since over If-None-Match
         if (responseCache?.['lastModified']) {
           logger.debug(
@@ -232,6 +221,11 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
           };
         }
       }
+
+      if (options.cacheProvider) {
+        await options.cacheProvider.setCacheHeaders(url, options);
+      }
+
       const startTime = Date.now();
       const httpTask: GotTask<T> = () => {
         const queueMs = Date.now() - startTime;
@@ -261,6 +255,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304;
       const resCopy = copyResponse(res, deepCopyNeeded);
       resCopy.authorization = !!options?.headers?.authorization;
+
       if (httpOptions.repoCache) {
         const cache = getCache();
         cache.httpCache ??= {};
@@ -279,19 +274,25 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
             timeStamp: new Date().toISOString(),
           };
         }
-        if (resCopy.statusCode === 304 && cache.httpCache[url]?.httpResponse) {
+        const httpCache = cache.httpCache[url] as HttpCache | undefined;
+        if (resCopy.statusCode === 304 && httpCache) {
           logger.debug(
-            `http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`,
+            `http cache: Using cached response: ${url} from ${httpCache.timeStamp}`,
           );
           HttpCacheStats.incRemoteHits(url);
           const cacheCopy = copyResponse(
-            cache.httpCache[url].httpResponse,
+            httpCache.httpResponse,
             deepCopyNeeded,
           );
           cacheCopy.authorization = !!options?.headers?.authorization;
           return cacheCopy as HttpResponse<T>;
         }
       }
+
+      if (options.cacheProvider) {
+        return await options.cacheProvider.wrapResponse(url, resCopy);
+      }
+
       return resCopy;
     } catch (err) {
       const { abortOnError, abortIgnoreStatusCodes } = options;

lib/util/http/types.ts

@@ -4,6 +4,7 @@ import type {
   OptionsOfJSONResponseBody,
   ParseJsonFunction,
 } from 'got';
+import type { HttpCacheProvider } from './cache/types';
 
 export type GotContextOptions = {
   authType?: string;
@@ -65,7 +66,11 @@ export interface HttpOptions {
 
   token?: string;
   memCache?: boolean;
+  /**
+   * @deprecated
+   */
   repoCache?: boolean;
+  cacheProvider?: HttpCacheProvider;
 }
 
 export interface InternalHttpOptions extends HttpOptions {
@@ -73,6 +78,9 @@ export interface InternalHttpOptions extends HttpOptions {
   responseType?: 'json' | 'buffer';
   method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
   parseJson?: ParseJsonFunction;
+  /**
+   * @deprecated
+   */
   repoCache?: boolean;
 }
 
@@ -89,3 +97,13 @@ export interface HttpResponse<T = string> {
 
 export type Task<T> = () => Promise<T>;
 export type GotTask<T> = Task<HttpResponse<T>>;
+
+/**
+ * @deprecated
+ */
+export interface HttpCache {
+  etag?: string;
+  httpResponse: HttpResponse<unknown>;
+  lastModified?: string;
+  timeStamp: string;
+}

lib/util/http/util.ts

@@ -0,0 +1,22 @@
+import { clone } from '../clone';
+import type { HttpResponse } from './types';
+
+// Copying will help to avoid circular structure
+// and mutation of the cached response.
+export function copyResponse<T>(
+  response: HttpResponse<T>,
+  deep: boolean,
+): HttpResponse<T> {
+  const { body, statusCode, headers } = response;
+  return deep
+    ? {
+        statusCode,
+        body: body instanceof Buffer ? (body.subarray() as T) : clone<T>(body),
+        headers: clone(headers),
+      }
+    : {
+        statusCode,
+        body,
+        headers,
+      };
+}