feat: Http cache stats (#27956)

lib/modules/datasource/npm/get.ts

@@ -10,6 +10,7 @@ import * as packageCache from '../../../util/cache/package';
 import type { Http } from '../../../util/http';
 import type { HttpOptions } from '../../../util/http/types';
 import { regEx } from '../../../util/regex';
+import { HttpCacheStats } from '../../../util/stats';
 import { joinUrlParts } from '../../../util/url';
 import type { Release, ReleaseResult } from '../types';
 import type { CachedReleaseResult, NpmResponse } from './types';
@@ -91,10 +92,13 @@ export async function getDependency(
       );
       if (softExpireAt.isValid && softExpireAt > DateTime.local()) {
         logger.trace('Cached result is not expired - reusing');
+        HttpCacheStats.incLocalHits(packageUrl);
         delete cachedResult.cacheData;
         return cachedResult;
       }
+
       logger.trace('Cached result is soft expired');
+      HttpCacheStats.incLocalMisses(packageUrl);
     } else {
       logger.trace(
         `Package cache for npm package "${packageName}" is from an old revision - discarding`,
@@ -127,6 +131,7 @@ export async function getDependency(
     const raw = await http.getJson<NpmResponse>(packageUrl, options);
     if (cachedResult?.cacheData && raw.statusCode === 304) {
       logger.trace(`Cached npm result for ${packageName} is revalidated`);
+      HttpCacheStats.incRemoteHits(packageUrl);
       cachedResult.cacheData.softExpireAt = softExpireAt;
       await packageCache.set(
         cacheNamespace,
@@ -137,6 +142,7 @@ export async function getDependency(
       delete cachedResult.cacheData;
       return cachedResult;
     }
+    HttpCacheStats.incRemoteMisses(packageUrl);
     const etag = raw.headers.etag;
     const res = raw.body;
     if (!res.versions || !Object.keys(res.versions).length) {

lib/util/http/index.ts

@@ -11,7 +11,11 @@ import { getCache } from '../cache/repository';
 import { clone } from '../clone';
 import { hash } from '../hash';
 import { type AsyncResult, Result } from '../result';
-import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
+import {
+  HttpCacheStats,
+  type HttpRequestStatsDataPoint,
+  HttpStats,
+} from '../stats';
 import { resolveBaseUrl } from '../url';
 import { applyAuthorization, removeAuthorization } from './auth';
 import { hooks } from './hooks';
@@ -279,6 +283,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
           logger.debug(
             `http cache: saving ${url} (etag=${resCopy.headers.etag}, lastModified=${resCopy.headers['last-modified']})`,
           );
+          HttpCacheStats.incRemoteMisses(url);
           cache.httpCache[url] = {
             etag: resCopy.headers.etag,
             httpResponse: copyResponse(res, deepCopyNeeded),
@@ -290,6 +295,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
           logger.debug(
             `http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`,
           );
+          HttpCacheStats.incRemoteHits(url);
           const cacheCopy = copyResponse(
             cache.httpCache[url].httpResponse,
             deepCopyNeeded,

lib/util/stats.spec.ts

@@ -1,6 +1,7 @@
 import { logger } from '../../test/util';
 import * as memCache from './cache/memory';
 import {
+  HttpCacheStats,
   HttpStats,
   LookupStats,
   PackageCacheStats,
@@ -455,4 +456,79 @@ describe('util/stats', () => {
       });
     });
   });
+
+  describe('HttpCacheStats', () => {
+    it('returns empty data', () => {
+      const res = HttpCacheStats.getData();
+      expect(res).toEqual({});
+    });
+
+    it('ignores wrong url', () => {
+      HttpCacheStats.incLocalHits('<invalid>');
+      expect(HttpCacheStats.getData()).toEqual({});
+    });
+
+    it('writes data points', () => {
+      HttpCacheStats.incLocalHits('https://example.com/foo');
+      HttpCacheStats.incLocalHits('https://example.com/foo');
+      HttpCacheStats.incLocalMisses('https://example.com/foo');
+      HttpCacheStats.incLocalMisses('https://example.com/bar');
+      HttpCacheStats.incRemoteHits('https://example.com/bar');
+      HttpCacheStats.incRemoteMisses('https://example.com/bar');
+
+      const res = HttpCacheStats.getData();
+
+      expect(res).toEqual({
+        'https://example.com/bar': {
+          localHits: 0,
+          localMisses: 1,
+          localTotal: 1,
+          remoteHits: 1,
+          remoteMisses: 1,
+          remoteTotal: 2,
+        },
+        'https://example.com/foo': {
+          localHits: 2,
+          localMisses: 1,
+          localTotal: 3,
+          remoteHits: 0,
+          remoteMisses: 0,
+          remoteTotal: 0,
+        },
+      });
+    });
+
+    it('prints report', () => {
+      HttpCacheStats.incLocalHits('https://example.com/foo');
+      HttpCacheStats.incLocalHits('https://example.com/foo');
+      HttpCacheStats.incLocalMisses('https://example.com/foo');
+      HttpCacheStats.incLocalMisses('https://example.com/bar');
+      HttpCacheStats.incRemoteHits('https://example.com/bar');
+      HttpCacheStats.incRemoteMisses('https://example.com/bar');
+
+      HttpCacheStats.report();
+
+      expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+      const [data, msg] = logger.logger.debug.mock.calls[0];
+      expect(msg).toBe('HTTP cache statistics');
+      expect(data).toEqual({
+        'https://example.com/bar': {
+          localHits: 0,
+          localMisses: 1,
+          localTotal: 1,
+          remoteHits: 1,
+          remoteMisses: 1,
+          remoteTotal: 2,
+        },
+        'https://example.com/foo': {
+          localHits: 2,
+          localMisses: 1,
+          localTotal: 3,
+          remoteHits: 0,
+          remoteMisses: 0,
+          remoteTotal: 0,
+        },
+      });
+    });
+  });
 });

lib/util/stats.ts

@@ -235,3 +235,99 @@ export class HttpStats {
     logger.debug({ urls, hosts, requests }, 'HTTP statistics');
   }
 }
+
+interface HttpCacheHostStatsData {
+  localHits: number;
+  localMisses: number;
+  localTotal: number;
+  remoteHits: number;
+  remoteMisses: number;
+  remoteTotal: number;
+}
+
+type HttpCacheStatsData = Record<string, HttpCacheHostStatsData>;
+
+export class HttpCacheStats {
+  static getData(): HttpCacheStatsData {
+    return memCache.get<HttpCacheStatsData>('http-cache-stats') ?? {};
+  }
+
+  static read(key: string): HttpCacheHostStatsData {
+    return (
+      this.getData()?.[key] ?? {
+        localHits: 0,
+        localMisses: 0,
+        localTotal: 0,
+        remoteHits: 0,
+        remoteMisses: 0,
+        remoteTotal: 0,
+      }
+    );
+  }
+
+  static write(key: string, data: HttpCacheHostStatsData): void {
+    const stats = memCache.get<HttpCacheStatsData>('http-cache-stats') ?? {};
+    stats[key] = data;
+    memCache.set('http-cache-stats', stats);
+  }
+
+  static getBaseUrl(url: string): string | null {
+    const parsedUrl = parseUrl(url);
+    if (!parsedUrl) {
+      logger.debug({ url }, 'Failed to parse URL during cache stats');
+      return null;
+    }
+    const { origin, pathname } = parsedUrl;
+    const baseUrl = `${origin}${pathname}`;
+    return baseUrl;
+  }
+
+  static incLocalHits(url: string): void {
+    const baseUrl = HttpCacheStats.getBaseUrl(url);
+    if (baseUrl) {
+      const host = baseUrl;
+      const stats = HttpCacheStats.read(host);
+      stats.localHits += 1;
+      stats.localTotal += 1;
+      HttpCacheStats.write(host, stats);
+    }
+  }
+
+  static incLocalMisses(url: string): void {
+    const baseUrl = HttpCacheStats.getBaseUrl(url);
+    if (baseUrl) {
+      const host = baseUrl;
+      const stats = HttpCacheStats.read(host);
+      stats.localMisses += 1;
+      stats.localTotal += 1;
+      HttpCacheStats.write(host, stats);
+    }
+  }
+
+  static incRemoteHits(url: string): void {
+    const baseUrl = HttpCacheStats.getBaseUrl(url);
+    if (baseUrl) {
+      const host = baseUrl;
+      const stats = HttpCacheStats.read(host);
+      stats.remoteHits += 1;
+      stats.remoteTotal += 1;
+      HttpCacheStats.write(host, stats);
+    }
+  }
+
+  static incRemoteMisses(url: string): void {
+    const baseUrl = HttpCacheStats.getBaseUrl(url);
+    if (baseUrl) {
+      const host = baseUrl;
+      const stats = HttpCacheStats.read(host);
+      stats.remoteMisses += 1;
+      stats.remoteTotal += 1;
+      HttpCacheStats.write(host, stats);
+    }
+  }
+
+  static report(): void {
+    const stats = HttpCacheStats.getData();
+    logger.debug(stats, 'HTTP cache statistics');
+  }
+}

lib/workers/repository/index.ts

@@ -19,7 +19,12 @@ import { clearDnsCache, printDnsStats } from '../../util/http/dns';
 import * as queue from '../../util/http/queue';
 import * as throttle from '../../util/http/throttle';
 import { addSplit, getSplits, splitInit } from '../../util/split';
-import { HttpStats, LookupStats, PackageCacheStats } from '../../util/stats';
+import {
+  HttpCacheStats,
+  HttpStats,
+  LookupStats,
+  PackageCacheStats,
+} from '../../util/stats';
 import { setBranchCache } from './cache';
 import { extractRepoProblems } from './common';
 import { ensureDependencyDashboard } from './dependency-dashboard';
@@ -126,6 +131,7 @@ export async function renovateRepository(
   logger.debug(splits, 'Repository timing splits (milliseconds)');
   PackageCacheStats.report();
   HttpStats.report();
+  HttpCacheStats.report();
   LookupStats.report();
   printDnsStats();
   clearDnsCache();