feat(cache): retain fingerprints for all matched managers (#20138)

lib/util/cache/repository/types.ts

@@ -8,6 +8,7 @@ import type { RepoInitConfig } from '../../../workers/repository/init/types';
 export interface BaseBranchCache {
   sha: string; // branch commit sha
   configHash: string; // object hash of config
+  extractionFingerprints: Record<string, string | undefined>; // matching manager fingerprints
   packageFiles: Record<string, PackageFile[]>; // extract result
 }
 

lib/workers/repository/extract/index.spec.ts

@@ -30,7 +30,7 @@ describe('workers/repository/extract/index', () => {
       config.enabledManagers = ['npm'];
       managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
       const res = await extractAllDependencies(config);
-      expect(res).toEqual({ packageFiles: { npm: [{}] } });
+      expect(res).toMatchObject({ packageFiles: { npm: [{}] } });
     });
 
     it('warns if no packages found for a enabled manager', async () => {

lib/workers/repository/extract/index.ts

@@ -2,7 +2,7 @@ import is from '@sindresorhus/is';
 import { getManagerConfig, mergeChildConfig } from '../../../config';
 import type { ManagerConfig, RenovateConfig } from '../../../config/types';
 import { logger } from '../../../logger';
-import { getManagerList } from '../../../modules/manager';
+import { getManagerList, hashMap } from '../../../modules/manager';
 import { getFileList } from '../../../util/git';
 import type { ExtractResult, WorkerExtractConfig } from '../../types';
 import { getMatchingFiles } from './file-match';
@@ -43,8 +43,15 @@ export async function extractAllDependencies(
 
   const extractResult: ExtractResult = {
     packageFiles: {},
+    extractionFingerprints: {},
   };
 
+  // Store the fingerprint of all managers which match any file (even if they do not find any dependencies)
+  // The cached result needs to be invalidated if the fingerprint of any matching manager changes
+  for (const { manager } of extractList) {
+    extractResult.extractionFingerprints[manager] = hashMap.get(manager);
+  }
+
   const extractResults = await Promise.all(
     extractList.map(async (managerConfig) => {
       const packageFiles = await getManagerPackageFiles(managerConfig);

lib/workers/repository/process/extract-update.spec.ts

@@ -87,6 +87,7 @@ describe('workers/repository/process/extract-update', () => {
           master: {
             sha: '123test',
             configHash: fingerprint(generateFingerprintConfig(config)),
+            extractionFingerprints: {},
             packageFiles,
           },
         },
@@ -99,19 +100,23 @@ describe('workers/repository/process/extract-update', () => {
   });
 
   describe('isCacheExtractValid()', () => {
-    let cachedExtract: BaseBranchCache = undefined as never;
+    let cachedExtract: BaseBranchCache;
+
+    beforeEach(() => {
+      cachedExtract = {
+        sha: 'sha',
+        configHash: undefined as never,
+        extractionFingerprints: {},
+        packageFiles: {},
+      };
+    });
 
     it('undefined cache', () => {
-      expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+      expect(isCacheExtractValid('sha', 'hash', undefined)).toBe(false);
       expect(logger.logger.debug).toHaveBeenCalledTimes(0);
     });
 
     it('partial cache', () => {
-      cachedExtract = {
-        sha: 'sha',
-        configHash: undefined as never,
-        packageFiles: {},
-      };
       expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
       expect(logger.logger.debug).toHaveBeenCalledTimes(0);
     });
@@ -126,7 +131,6 @@ describe('workers/repository/process/extract-update', () => {
     });
 
     it('config change', () => {
-      cachedExtract.sha = 'sha';
       cachedExtract.configHash = 'hash';
       expect(isCacheExtractValid('sha', 'new_hash', cachedExtract)).toBe(false);
       expect(logger.logger.debug).toHaveBeenCalledWith(
@@ -135,8 +139,30 @@ describe('workers/repository/process/extract-update', () => {
       expect(logger.logger.debug).toHaveBeenCalledTimes(1);
     });
 
+    it('invalid if no extractionFingerprints', () => {
+      cachedExtract.configHash = 'hash';
+      const { extractionFingerprints, ...restOfCache } = cachedExtract;
+      expect(
+        isCacheExtractValid(
+          'sha',
+          'hash',
+          restOfCache as never as BaseBranchCache
+        )
+      ).toBe(false);
+      expect(logger.logger.debug).toHaveBeenCalledWith(
+        'Cached extract is missing extractionFingerprints, so cannot be used'
+      );
+      expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+    });
+
+    it('invalid if changed fingerprints', () => {
+      cachedExtract.configHash = 'hash';
+      cachedExtract.extractionFingerprints = { npm: 'old-fingerprint' };
+      expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+      expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+    });
+
     it('valid cache and config', () => {
-      cachedExtract.sha = 'sha';
       cachedExtract.configHash = 'hash';
       expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(true);
       expect(logger.logger.debug).toHaveBeenCalledWith(

lib/workers/repository/process/extract-update.ts

@@ -1,6 +1,7 @@
 import is from '@sindresorhus/is';
 import type { RenovateConfig } from '../../../config/types';
 import { logger } from '../../../logger';
+import { hashMap } from '../../../modules/manager';
 import type { PackageFile } from '../../../modules/manager/types';
 import { getCache } from '../../../util/cache/repository';
 import type { BaseBranchCache } from '../../../util/cache/repository/types';
@@ -80,6 +81,27 @@ export function isCacheExtractValid(
     logger.debug('Cached extract result cannot be used due to config change');
     return false;
   }
+  if (!cachedExtract.extractionFingerprints) {
+    logger.debug(
+      'Cached extract is missing extractionFingerprints, so cannot be used'
+    );
+    return false;
+  }
+  const changedManagers = new Set();
+  for (const [manager, fingerprint] of Object.entries(
+    cachedExtract.extractionFingerprints
+  )) {
+    if (fingerprint !== hashMap.get(manager)) {
+      changedManagers.add(manager);
+    }
+  }
+  if (changedManagers.size > 0) {
+    logger.debug(
+      { changedManagers: [...changedManagers] },
+      'Manager fingerprint(s) have changed, extract cache cannot be reused'
+    );
+    return false;
+  }
   logger.debug(
     `Cached extract for sha=${baseBranchSha} is valid and can be used`
   );
@@ -114,12 +136,14 @@ export async function extract(
     }
   } else {
     await checkoutBranch(baseBranch!);
-    const extractResult = await extractAllDependencies(config);
+    const extractResult = (await extractAllDependencies(config)) || {};
-    packageFiles = extractResult?.packageFiles;
+    packageFiles = extractResult.packageFiles;
+    const { extractionFingerprints } = extractResult;
     // TODO: fix types (#7154)
     cache.scan[baseBranch!] = {
       sha: baseBranchSha!,
       configHash,
+      extractionFingerprints,
       packageFiles,
     };
     // Clean up cached branch extracts

lib/workers/types.ts

@@ -188,5 +188,6 @@ export interface UpgradeFingerprintConfig {
 }
 
 export interface ExtractResult {
+  extractionFingerprints: Record<string, string | undefined>;
   packageFiles: Record<string, PackageFile[]>;
 }