diff --git a/lib/modules/datasource/index.ts b/lib/modules/datasource/index.ts index 006a0023c2..3815c69b6f 100644 --- a/lib/modules/datasource/index.ts +++ b/lib/modules/datasource/index.ts @@ -9,6 +9,7 @@ import * as memCache from '../../util/cache/memory'; import * as packageCache from '../../util/cache/package'; import { clone } from '../../util/clone'; import { AsyncResult, Result } from '../../util/result'; +import { DatasourceCacheStats } from '../../util/stats'; import { trimTrailingSlash } from '../../util/url'; import datasources from './api'; import { @@ -69,16 +70,22 @@ async function getRegistryReleases( cacheNamespace, cacheKey, ); + // istanbul ignore if if (cachedResult) { logger.trace({ cacheKey }, 'Returning cached datasource response'); + DatasourceCacheStats.hit(datasource.id, registryUrl, config.packageName); return cachedResult; } + + DatasourceCacheStats.miss(datasource.id, registryUrl, config.packageName); } + const res = await datasource.getReleases({ ...config, registryUrl }); if (res?.releases.length) { res.registryUrl ??= registryUrl; } + // cache non-null responses unless marked as private if (datasource.caching && res) { const cachePrivatePackages = GlobalConfig.get( @@ -89,8 +96,12 @@ async function getRegistryReleases( logger.trace({ cacheKey }, 'Caching datasource response'); const cacheMinutes = 15; await packageCache.set(cacheNamespace, cacheKey, res, cacheMinutes); + DatasourceCacheStats.set(datasource.id, registryUrl, config.packageName); + } else { + DatasourceCacheStats.skip(datasource.id, registryUrl, config.packageName); } } + return res; } diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts index cd8325f3a8..d479ea48e4 100644 --- a/lib/util/stats.spec.ts +++ b/lib/util/stats.spec.ts @@ -1,6 +1,7 @@ import { logger } from '../../test/util'; import * as memCache from './cache/memory'; import { + DatasourceCacheStats, HttpCacheStats, HttpStats, LookupStats, @@ -230,6 +231,60 @@ describe('util/stats', () => { }); }); + describe('DatasourceCacheStats', () => { + it('collects data points', () => { + DatasourceCacheStats.hit('crate', 'https://foo.example.com', 'foo'); + DatasourceCacheStats.miss('maven', 'https://bar.example.com', 'bar'); + DatasourceCacheStats.set('npm', 'https://baz.example.com', 'baz'); + DatasourceCacheStats.skip('rubygems', 'https://qux.example.com', 'qux'); + + const report = DatasourceCacheStats.getReport(); + + expect(report).toEqual({ + long: { + crate: { + 'https://foo.example.com': { foo: { read: 'hit' } }, + }, + maven: { + 'https://bar.example.com': { bar: { read: 'miss' } }, + }, + npm: { + 'https://baz.example.com': { baz: { write: 'set' } }, + }, + rubygems: { + 'https://qux.example.com': { qux: { write: 'skip' } }, + }, + }, + short: { + crate: { + 'https://foo.example.com': { hit: 1, miss: 0, set: 0, skip: 0 }, + }, + maven: { + 'https://bar.example.com': { hit: 0, miss: 1, set: 0, skip: 0 }, + }, + npm: { + 'https://baz.example.com': { hit: 0, miss: 0, set: 1, skip: 0 }, + }, + rubygems: { + 'https://qux.example.com': { hit: 0, miss: 0, set: 0, skip: 1 }, + }, + }, + }); + }); + + it('reports', () => { + DatasourceCacheStats.hit('crate', 'https://foo.example.com', 'foo'); + DatasourceCacheStats.miss('maven', 'https://bar.example.com', 'bar'); + DatasourceCacheStats.set('npm', 'https://baz.example.com', 'baz'); + DatasourceCacheStats.skip('rubygems', 'https://qux.example.com', 'qux'); + + DatasourceCacheStats.report(); + + expect(logger.logger.trace).toHaveBeenCalledTimes(1); + expect(logger.logger.debug).toHaveBeenCalledTimes(1); + }); + }); + describe('HttpStats', () => { it('returns empty report', () => { const res = HttpStats.getReport(); diff --git a/lib/util/stats.ts b/lib/util/stats.ts index c250020bd4..ecd933f924 100644 --- a/lib/util/stats.ts +++ b/lib/util/stats.ts @@ -105,6 +105,145 @@ export class PackageCacheStats { } } +interface DatasourceCacheDataPoint { + datasource: string; + registryUrl: string; + packageName: string; + action: 'hit' | 'miss' | 'set' | 'skip'; +} + +export interface DatasourceCacheReport { + long: { + [datasource in string]: { + [registryUrl in string]: { + [packageName in string]: { + read?: 'hit' | 'miss'; + write?: 'set' | 'skip'; + }; + }; + }; + }; + short: { + [datasource in string]: { + [registryUrl in string]: { + hit: number; + miss: number; + set: number; + skip: number; + }; + }; + }; +} + +export class DatasourceCacheStats { + private static getData(): DatasourceCacheDataPoint[] { + return ( + memCache.get('datasource-cache-stats') ?? [] + ); + } + + private static setData(data: DatasourceCacheDataPoint[]): void { + memCache.set('datasource-cache-stats', data); + } + + static hit( + datasource: string, + registryUrl: string, + packageName: string, + ): void { + const data = this.getData(); + data.push({ datasource, registryUrl, packageName, action: 'hit' }); + this.setData(data); + } + + static miss( + datasource: string, + registryUrl: string, + packageName: string, + ): void { + const data = this.getData(); + data.push({ datasource, registryUrl, packageName, action: 'miss' }); + this.setData(data); + } + + static set( + datasource: string, + registryUrl: string, + packageName: string, + ): void { + const data = this.getData(); + data.push({ datasource, registryUrl, packageName, action: 'set' }); + this.setData(data); + } + + static skip( + datasource: string, + registryUrl: string, + packageName: string, + ): void { + const data = this.getData(); + data.push({ datasource, registryUrl, packageName, action: 'skip' }); + this.setData(data); + } + + static getReport(): DatasourceCacheReport { + const data = this.getData(); + const result: DatasourceCacheReport = { long: {}, short: {} }; + for (const { datasource, registryUrl, packageName, action } of data) { + result.long[datasource] ??= {}; + result.long[datasource][registryUrl] ??= {}; + result.long[datasource][registryUrl] ??= {}; + result.long[datasource][registryUrl][packageName] ??= {}; + + result.short[datasource] ??= {}; + result.short[datasource][registryUrl] ??= { + hit: 0, + miss: 0, + set: 0, + skip: 0, + }; + + if (action === 'hit') { + result.long[datasource][registryUrl][packageName].read = 'hit'; + result.short[datasource][registryUrl].hit += 1; + continue; + } + + if (action === 'miss') { + result.long[datasource][registryUrl][packageName].read = 'miss'; + result.short[datasource][registryUrl].miss += 1; + continue; + } + + if (action === 'set') { + result.long[datasource][registryUrl][packageName].write = 'set'; + result.short[datasource][registryUrl].set += 1; + continue; + } + + if (action === 'skip') { + result.long[datasource][registryUrl][packageName].write = 'skip'; + result.short[datasource][registryUrl].skip += 1; + continue; + } + } + + return result; + } + + static report(): void { + const { long, short } = this.getReport(); + + if (Object.keys(short).length > 0) { + logger.debug(short, 'Datasource cache statistics'); + } + + if (Object.keys(long).length > 0) { + logger.trace(long, 'Datasource cache detailed statistics'); + } + } +} + export interface HttpRequestStatsDataPoint { method: string; url: string; diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index 9b05fd8420..6a5addcc08 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -20,6 +20,7 @@ import * as queue from '../../util/http/queue'; import * as throttle from '../../util/http/throttle'; import { addSplit, getSplits, splitInit } from '../../util/split'; import { + DatasourceCacheStats, HttpCacheStats, HttpStats, LookupStats, @@ -134,6 +135,7 @@ export async function renovateRepository( const splits = getSplits(); logger.debug(splits, 'Repository timing splits (milliseconds)'); PackageCacheStats.report(); + DatasourceCacheStats.report(); HttpStats.report(); HttpCacheStats.report(); LookupStats.report();