feat(datasource/github-releases)!: digest computation use git tag, not file digest (#20178)

The github-releases datasource has been copied into a new datasource called github-release-attachments.
The github-releases general datasource is updated to use the underlying Git tag of a GitHub release entry for digest computation.

Fixes #20160, Fixes #19552

BREAKING CHANGE: Regex Manager configurations relying on the github-release data-source with digests will have different digest semantics. The digest
will now always correspond to the underlying Git SHA of the release/version. The old behavior can be preserved by switching to the
github-release-attachments datasource.

lib/modules/datasource/api.ts

@@ -19,6 +19,7 @@ import { GalaxyDatasource } from './galaxy';
 import { GalaxyCollectionDatasource } from './galaxy-collection';
 import { GitRefsDatasource } from './git-refs';
 import { GitTagsDatasource } from './git-tags';
+import { GithubReleaseAttachmentsDatasource } from './github-release-attachments';
 import { GithubReleasesDatasource } from './github-releases';
 import { GithubTagsDatasource } from './github-tags';
 import { GitlabPackagesDatasource } from './gitlab-packages';
@@ -76,6 +77,10 @@ api.set(GalaxyDatasource.id, new GalaxyDatasource());
 api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource());
 api.set(GitRefsDatasource.id, new GitRefsDatasource());
 api.set(GitTagsDatasource.id, new GitTagsDatasource());
+api.set(
+  GithubReleaseAttachmentsDatasource.id,
+  new GithubReleaseAttachmentsDatasource()
+);
 api.set(GithubReleasesDatasource.id, new GithubReleasesDatasource());
 api.set(GithubTagsDatasource.id, new GithubTagsDatasource());
 api.set(GitlabPackagesDatasource.id, new GitlabPackagesDatasource());

lib/modules/datasource/github-release-attachments/digest.spec.ts

@@ -1,17 +1,17 @@
 import hasha from 'hasha';
 import * as httpMock from '../../../../test/http-mock';
 import type { GithubDigestFile } from '../../../util/github/types';
-import { GitHubReleaseMocker } from './test';
+import { GitHubReleaseAttachmentMocker } from './test';
 
-import { GithubReleasesDatasource } from '.';
+import { GithubReleaseAttachmentsDatasource } from '.';
 
-describe('modules/datasource/github-releases/digest', () => {
+describe('modules/datasource/github-release-attachments/digest', () => {
   const packageName = 'some/dep';
-  const releaseMock = new GitHubReleaseMocker(
+  const releaseMock = new GitHubReleaseAttachmentMocker(
     'https://api.github.com',
     packageName
   );
-  const githubReleases = new GithubReleasesDatasource();
+  const githubReleaseAttachments = new GithubReleaseAttachmentsDatasource();
 
   describe('findDigestAsset', () => {
     it('finds SHASUMS.txt file containing digest', async () => {
@@ -21,7 +21,7 @@ describe('modules/datasource/github-releases/digest', () => {
         'another-digest linux-arm64.tar.gz'
       );
 
-      const digestAsset = await githubReleases.findDigestAsset(
+      const digestAsset = await githubReleaseAttachments.findDigestAsset(
         release,
         'test-digest'
       );
@@ -40,7 +40,7 @@ describe('modules/datasource/github-releases/digest', () => {
         .get(`/repos/${packageName}/releases/download/v1.0.0/SHASUMS.txt`)
         .reply(200, '');
 
-      const digestAsset = await githubReleases.findDigestAsset(
+      const digestAsset = await githubReleaseAttachments.findDigestAsset(
         release,
         'test-digest'
       );
@@ -57,7 +57,7 @@ describe('modules/datasource/github-releases/digest', () => {
       });
       const contentDigest = await hasha.async(content, { algorithm: 'sha256' });
 
-      const digestAsset = await githubReleases.findDigestAsset(
+      const digestAsset = await githubReleaseAttachments.findDigestAsset(
         release,
         contentDigest
       );
@@ -67,7 +67,7 @@ describe('modules/datasource/github-releases/digest', () => {
 
     it('returns null when no assets available', async () => {
       const release = releaseMock.release('v1.0.0');
-      const digestAsset = await githubReleases.findDigestAsset(
+      const digestAsset = await githubReleaseAttachments.findDigestAsset(
         release,
         'test-digest'
       );
@@ -89,7 +89,7 @@ describe('modules/datasource/github-releases/digest', () => {
           'v1.0.1',
           'updated-digest  asset.zip'
         );
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAsset,
           release
         );
@@ -106,7 +106,7 @@ describe('modules/datasource/github-releases/digest', () => {
           'v1.0.1',
           'updated-digest  asset-1.0.1.zip'
         );
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAssetWithVersion,
           release
         );
@@ -118,7 +118,7 @@ describe('modules/datasource/github-releases/digest', () => {
           'v1.0.1',
           'moot-digest asset.tar.gz'
         );
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAsset,
           release
         );
@@ -127,7 +127,7 @@ describe('modules/datasource/github-releases/digest', () => {
 
       it('returns null when digest file not found', async () => {
         const release = releaseMock.release('v1.0.1');
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAsset,
           release
         );
@@ -151,7 +151,7 @@ describe('modules/datasource/github-releases/digest', () => {
           algorithm: 'sha256',
         });
 
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAsset,
           release
         );
@@ -160,7 +160,7 @@ describe('modules/datasource/github-releases/digest', () => {
 
       it('returns null when not found', async () => {
         const release = releaseMock.release('v1.0.1');
-        const digest = await githubReleases.mapDigestAssetToRelease(
+        const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
           digestAsset,
           release
         );

lib/modules/datasource/github-release-attachments/index.spec.ts

@@ -0,0 +1,154 @@
+import { getDigest, getPkgReleases } from '..';
+import { mocked } from '../../../../test/util';
+import * as githubGraphql from '../../../util/github/graphql';
+import * as _hostRules from '../../../util/host-rules';
+import { GitHubReleaseAttachmentMocker } from './test';
+import { GithubReleaseAttachmentsDatasource } from '.';
+
+jest.mock('../../../util/host-rules');
+const hostRules = mocked(_hostRules);
+
+const githubApiHost = 'https://api.github.com';
+
+describe('modules/datasource/github-release-attachments/index', () => {
+  beforeEach(() => {
+    hostRules.hosts.mockReturnValue([]);
+    hostRules.find.mockReturnValue({
+      token: 'some-token',
+    });
+  });
+
+  describe('getReleases', () => {
+    it('returns releases', async () => {
+      jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([
+        {
+          id: 1,
+          url: 'https://example.com',
+          name: 'some/dep2',
+          description: 'some description',
+          version: 'a',
+          releaseTimestamp: '2020-03-09T13:00:00Z',
+        },
+        {
+          id: 2,
+          url: 'https://example.com',
+          name: 'some/dep2',
+          description: 'some description',
+          version: 'v',
+          releaseTimestamp: '2020-03-09T12:00:00Z',
+        },
+        {
+          id: 3,
+          url: 'https://example.com',
+          name: 'some/dep2',
+          description: 'some description',
+          version: '1.0.0',
+          releaseTimestamp: '2020-03-09T11:00:00Z',
+        },
+        {
+          id: 4,
+          url: 'https://example.com',
+          name: 'some/dep2',
+          description: 'some description',
+          version: 'v1.1.0',
+          releaseTimestamp: '2020-03-09T10:00:00Z',
+        },
+        {
+          id: 5,
+          url: 'https://example.com',
+          name: 'some/dep2',
+          description: 'some description',
+          version: '2.0.0',
+          releaseTimestamp: '2020-04-09T10:00:00Z',
+          isStable: false,
+        },
+      ]);
+
+      const res = await getPkgReleases({
+        datasource: GithubReleaseAttachmentsDatasource.id,
+        packageName: 'some/dep',
+      });
+
+      expect(res).toMatchObject({
+        registryUrl: 'https://github.com',
+        releases: [
+          { releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' },
+          { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' },
+          {
+            version: '2.0.0',
+            releaseTimestamp: '2020-04-09T10:00:00.000Z',
+            isStable: false,
+          },
+        ],
+        sourceUrl: 'https://github.com/some/dep',
+      });
+    });
+  });
+
+  describe('getDigest', () => {
+    const packageName = 'some/dep';
+    const currentValue = 'v1.0.0';
+    const currentDigest = 'v1.0.0-digest';
+
+    const releaseMock = new GitHubReleaseAttachmentMocker(
+      githubApiHost,
+      packageName
+    );
+
+    it('requires currentDigest', async () => {
+      const digest = await getDigest(
+        { datasource: GithubReleaseAttachmentsDatasource.id, packageName },
+        currentValue
+      );
+      expect(digest).toBeNull();
+    });
+
+    it('defaults to currentDigest when currentVersion is missing', async () => {
+      const digest = await getDigest(
+        {
+          datasource: GithubReleaseAttachmentsDatasource.id,
+          packageName,
+          currentDigest,
+        },
+        currentValue
+      );
+      expect(digest).toEqual(currentDigest);
+    });
+
+    it('returns updated digest in new release', async () => {
+      releaseMock.withDigestFileAsset(
+        currentValue,
+        `${currentDigest} asset.zip`
+      );
+      const nextValue = 'v1.0.1';
+      const nextDigest = 'updated-digest';
+      releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
+      const digest = await getDigest(
+        {
+          datasource: GithubReleaseAttachmentsDatasource.id,
+          packageName,
+          currentValue,
+          currentDigest,
+        },
+        nextValue
+      );
+      expect(digest).toEqual(nextDigest);
+    });
+
+    // This is awkward, but I found returning `null` in this case to not produce an update
+    // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
+    it('ignores failures verifying currentDigest', async () => {
+      releaseMock.release(currentValue);
+      const digest = await getDigest(
+        {
+          datasource: GithubReleaseAttachmentsDatasource.id,
+          packageName,
+          currentValue,
+          currentDigest,
+        },
+        currentValue
+      );
+      expect(digest).toEqual(currentDigest);
+    });
+  });
+});

lib/modules/datasource/github-release-attachments/index.ts

@@ -0,0 +1,250 @@
+import is from '@sindresorhus/is';
+import hasha from 'hasha';
+import { logger } from '../../../logger';
+import { cache } from '../../../util/cache/package/decorator';
+import { queryReleases } from '../../../util/github/graphql';
+import type {
+  GithubDigestFile,
+  GithubRestAsset,
+  GithubRestRelease,
+} from '../../../util/github/types';
+import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
+import { GithubHttp } from '../../../util/http/github';
+import { newlineRegex, regEx } from '../../../util/regex';
+import { Datasource } from '../datasource';
+import type {
+  DigestConfig,
+  GetReleasesConfig,
+  Release,
+  ReleaseResult,
+} from '../types';
+
+export const cacheNamespace = 'datasource-github-releases';
+
+function inferHashAlg(digest: string): string {
+  switch (digest.length) {
+    case 64:
+      return 'sha256';
+    default:
+    case 96:
+      return 'sha512';
+  }
+}
+
+export class GithubReleaseAttachmentsDatasource extends Datasource {
+  static readonly id = 'github-release-attachments';
+
+  override readonly defaultRegistryUrls = ['https://github.com'];
+
+  override http: GithubHttp;
+
+  constructor() {
+    super(GithubReleaseAttachmentsDatasource.id);
+    this.http = new GithubHttp(GithubReleaseAttachmentsDatasource.id);
+  }
+
+  @cache({
+    ttlMinutes: 1440,
+    namespace: 'datasource-github-releases',
+    key: (release: GithubRestRelease, digest: string) =>
+      `${release.html_url}:${digest}`,
+  })
+  async findDigestFile(
+    release: GithubRestRelease,
+    digest: string
+  ): Promise<GithubDigestFile | null> {
+    const smallAssets = release.assets.filter(
+      (a: GithubRestAsset) => a.size < 5 * 1024
+    );
+    for (const asset of smallAssets) {
+      const res = await this.http.get(asset.browser_download_url);
+      for (const line of res.body.split(newlineRegex)) {
+        const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
+        if (lineDigest === digest) {
+          return {
+            assetName: asset.name,
+            digestedFileName: lineFilename,
+            currentVersion: release.tag_name,
+            currentDigest: lineDigest,
+          };
+        }
+      }
+    }
+    return null;
+  }
+
+  @cache({
+    ttlMinutes: 1440,
+    namespace: 'datasource-github-releases',
+    key: (asset: GithubRestAsset, algorithm: string) =>
+      `${asset.browser_download_url}:${algorithm}:assetDigest`,
+  })
+  async downloadAndDigest(
+    asset: GithubRestAsset,
+    algorithm: string
+  ): Promise<string> {
+    const res = this.http.stream(asset.browser_download_url);
+    const digest = await hasha.fromStream(res, { algorithm });
+    return digest;
+  }
+
+  async findAssetWithDigest(
+    release: GithubRestRelease,
+    digest: string
+  ): Promise<GithubDigestFile | null> {
+    const algorithm = inferHashAlg(digest);
+    const assetsBySize = release.assets.sort(
+      (a: GithubRestAsset, b: GithubRestAsset) => {
+        if (a.size < b.size) {
+          return -1;
+        }
+        if (a.size > b.size) {
+          return 1;
+        }
+        return 0;
+      }
+    );
+
+    for (const asset of assetsBySize) {
+      const assetDigest = await this.downloadAndDigest(asset, algorithm);
+      if (assetDigest === digest) {
+        return {
+          assetName: asset.name,
+          currentVersion: release.tag_name,
+          currentDigest: assetDigest,
+        };
+      }
+    }
+    return null;
+  }
+
+  /** Identify the asset associated with a known digest. */
+  async findDigestAsset(
+    release: GithubRestRelease,
+    digest: string
+  ): Promise<GithubDigestFile | null> {
+    const digestFile = await this.findDigestFile(release, digest);
+    if (digestFile) {
+      return digestFile;
+    }
+
+    const asset = await this.findAssetWithDigest(release, digest);
+    return asset;
+  }
+
+  /** Given a digest asset, find the equivalent digest in a different release. */
+  async mapDigestAssetToRelease(
+    digestAsset: GithubDigestFile,
+    release: GithubRestRelease
+  ): Promise<string | null> {
+    const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
+    const next = release.tag_name.replace(regEx(/^v/), '');
+    const releaseChecksumAssetName = digestAsset.assetName.replace(
+      current,
+      next
+    );
+    const releaseAsset = release.assets.find(
+      (a: GithubRestAsset) => a.name === releaseChecksumAssetName
+    );
+    if (!releaseAsset) {
+      return null;
+    }
+    if (digestAsset.digestedFileName) {
+      const releaseFilename = digestAsset.digestedFileName.replace(
+        current,
+        next
+      );
+      const res = await this.http.get(releaseAsset.browser_download_url);
+      for (const line of res.body.split(newlineRegex)) {
+        const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
+        if (lineFn === releaseFilename) {
+          return lineDigest;
+        }
+      }
+    } else {
+      const algorithm = inferHashAlg(digestAsset.currentDigest);
+      const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
+      return newDigest;
+    }
+    return null;
+  }
+
+  /**
+   * Attempts to resolve the digest for the specified package.
+   *
+   * The `newValue` supplied here should be a valid tag for the GitHub release.
+   * Requires `currentValue` and `currentDigest`.
+   *
+   * There may be many assets attached to the release. This function will:
+   *  - Identify the asset pinned by `currentDigest` in the `currentValue` release
+   *     - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
+   *     - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
+   *  - Map the hashed asset to `newValue` and return the updated digest as a string
+   */
+  override async getDigest(
+    {
+      packageName: repo,
+      currentValue,
+      currentDigest,
+      registryUrl,
+    }: DigestConfig,
+    newValue: string
+  ): Promise<string | null> {
+    logger.debug(
+      { repo, currentValue, currentDigest, registryUrl, newValue },
+      'getDigest'
+    );
+    if (!currentDigest) {
+      return null;
+    }
+    if (!currentValue) {
+      return currentDigest;
+    }
+
+    const apiBaseUrl = getApiBaseUrl(registryUrl);
+    const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
+      `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
+    );
+    const digestAsset = await this.findDigestAsset(
+      currentRelease,
+      currentDigest
+    );
+    let newDigest: string | null;
+    if (!digestAsset || newValue === currentValue) {
+      newDigest = currentDigest;
+    } else {
+      const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
+        `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
+      );
+      newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
+    }
+    return newDigest;
+  }
+
+  /**
+   * This function can be used to fetch releases with a customisable versioning
+   * (e.g. semver) and with releases.
+   *
+   * This function will:
+   *  - Fetch all releases
+   *  - Sanitize the versions if desired (e.g. strip out leading 'v')
+   *  - Return a dependency object containing sourceUrl string and releases array
+   */
+  async getReleases(config: GetReleasesConfig): Promise<ReleaseResult> {
+    const releasesResult = await queryReleases(config, this.http);
+    const releases = releasesResult.map((item) => {
+      const { version, releaseTimestamp, isStable } = item;
+      const result: Release = {
+        version,
+        gitRef: version,
+        releaseTimestamp,
+      };
+      if (is.boolean(isStable)) {
+        result.isStable = isStable;
+      }
+      return result;
+    });
+    const sourceUrl = getSourceUrl(config.packageName, config.registryUrl);
+    return { sourceUrl, releases };
+  }
+}

lib/modules/datasource/github-release-attachments/test/index.ts

@@ -2,7 +2,7 @@ import * as httpMock from '../../../../../test/http-mock';
 import { partial } from '../../../../../test/util';
 import type { GithubRestRelease } from '../../../../util/github/types';
 
-export class GitHubReleaseMocker {
+export class GitHubReleaseAttachmentMocker {
   constructor(
     private readonly githubApiHost: string,
     private readonly packageName: string

lib/modules/datasource/github-releases/index.spec.ts

@@ -1,17 +1,14 @@
 import { getDigest, getPkgReleases } from '..';
+import { mocked } from '../../../../test/util';
 import * as githubGraphql from '../../../util/github/graphql';
 import * as _hostRules from '../../../util/host-rules';
-import { GitHubReleaseMocker } from './test';
 import { GithubReleasesDatasource } from '.';
 
 jest.mock('../../../util/host-rules');
-const hostRules: any = _hostRules;
+const hostRules = mocked(_hostRules);
-
-const githubApiHost = 'https://api.github.com';
 
 describe('modules/datasource/github-releases/index', () => {
   beforeEach(() => {
-    jest.resetAllMocks();
     hostRules.hosts.mockReturnValue([]);
     hostRules.find.mockReturnValue({
       token: 'some-token',
@@ -88,38 +85,48 @@ describe('modules/datasource/github-releases/index', () => {
   describe('getDigest', () => {
     const packageName = 'some/dep';
     const currentValue = 'v1.0.0';
-    const currentDigest = 'v1.0.0-digest';
+    const currentDigest = 'sha-of-v1';
+    const newValue = 'v15.0.0';
+    const newDigest = 'sha-of-v15';
 
-    const releaseMock = new GitHubReleaseMocker(githubApiHost, packageName);
+    beforeEach(() => {
-
+      jest.spyOn(githubGraphql, 'queryTags').mockResolvedValueOnce([
-    it('requires currentDigest', async () => {
+        {
-      const digest = await getDigest(
+          version: 'v1.0.0',
-        { datasource: GithubReleasesDatasource.id, packageName },
+          gitRef: 'v1.0.0',
-        currentValue
+          releaseTimestamp: '2021-01-01',
-      );
+          hash: 'sha-of-v1',
-      expect(digest).toBeNull();
+        },
+        {
+          version: 'v15.0.0',
+          gitRef: 'v15.0.0',
+          releaseTimestamp: '2022-10-01',
+          hash: 'sha-of-v15',
+        },
+      ]);
     });
 
-    it('defaults to currentDigest when currentVersion is missing', async () => {
+    it('should be independent of the current digest', async () => {
       const digest = await getDigest(
         {
           datasource: GithubReleasesDatasource.id,
           packageName,
-          currentDigest,
+          currentValue,
         },
-        currentValue
+        newValue
       );
-      expect(digest).toEqual(currentDigest);
+      expect(digest).toBe(newDigest);
+    });
+
+    it('should be independent of the current value', async () => {
+      const digest = await getDigest(
+        { datasource: GithubReleasesDatasource.id, packageName },
+        newValue
+      );
+      expect(digest).toBe(newDigest);
     });
 
     it('returns updated digest in new release', async () => {
-      releaseMock.withDigestFileAsset(
-        currentValue,
-        `${currentDigest} asset.zip`
-      );
-      const nextValue = 'v1.0.1';
-      const nextDigest = 'updated-digest';
-      releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
       const digest = await getDigest(
         {
           datasource: GithubReleasesDatasource.id,
@@ -127,15 +134,12 @@ describe('modules/datasource/github-releases/index', () => {
           currentValue,
           currentDigest,
         },
-        nextValue
+        newValue
       );
-      expect(digest).toEqual(nextDigest);
+      expect(digest).toEqual(newDigest);
     });
 
-    // This is awkward, but I found returning `null` in this case to not produce an update
+    it('returns null if the new value/tag does not exist', async () => {
-    // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
-    it('ignores failures verifying currentDigest', async () => {
-      releaseMock.release(currentValue);
       const digest = await getDigest(
         {
           datasource: GithubReleasesDatasource.id,
@@ -143,9 +147,9 @@ describe('modules/datasource/github-releases/index', () => {
           currentValue,
           currentDigest,
         },
-        currentValue
+        'unknown-tag'
       );
-      expect(digest).toEqual(currentDigest);
+      expect(digest).toBeNull();
     });
   });
 });

lib/modules/datasource/github-releases/index.ts

@@ -1,17 +1,9 @@
-// TODO: types (#7154)
 import is from '@sindresorhus/is';
-import hasha from 'hasha';
 import { logger } from '../../../logger';
-import { cache } from '../../../util/cache/package/decorator';
 import { queryReleases } from '../../../util/github/graphql';
-import type {
+import { findCommitOfTag } from '../../../util/github/tags';
-  GithubDigestFile,
+import { getSourceUrl } from '../../../util/github/url';
-  GithubRestAsset,
-  GithubRestRelease,
-} from '../../../util/github/types';
-import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
 import { GithubHttp } from '../../../util/http/github';
-import { newlineRegex, regEx } from '../../../util/regex';
 import { Datasource } from '../datasource';
 import type {
   DigestConfig,
@@ -22,16 +14,6 @@ import type {
 
 export const cacheNamespace = 'datasource-github-releases';
 
-function inferHashAlg(digest: string): string {
-  switch (digest.length) {
-    case 64:
-      return 'sha256';
-    default:
-    case 96:
-      return 'sha512';
-  }
-}
-
 export class GithubReleasesDatasource extends Datasource {
   static readonly id = 'github-releases';
 
@@ -44,145 +26,17 @@ export class GithubReleasesDatasource extends Datasource {
     this.http = new GithubHttp(GithubReleasesDatasource.id);
   }
 
-  @cache({
-    ttlMinutes: 1440,
-    namespace: 'datasource-github-releases',
-    key: (release: GithubRestRelease, digest: string) =>
-      `${release.html_url}:${digest}`,
-  })
-  async findDigestFile(
-    release: GithubRestRelease,
-    digest: string
-  ): Promise<GithubDigestFile | null> {
-    const smallAssets = release.assets.filter(
-      (a: GithubRestAsset) => a.size < 5 * 1024
-    );
-    for (const asset of smallAssets) {
-      const res = await this.http.get(asset.browser_download_url);
-      for (const line of res.body.split(newlineRegex)) {
-        const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
-        if (lineDigest === digest) {
-          return {
-            assetName: asset.name,
-            digestedFileName: lineFilename,
-            currentVersion: release.tag_name,
-            currentDigest: lineDigest,
-          };
-        }
-      }
-    }
-    return null;
-  }
-
-  @cache({
-    ttlMinutes: 1440,
-    namespace: 'datasource-github-releases',
-    key: (asset: GithubRestAsset, algorithm: string) =>
-      `${asset.browser_download_url}:${algorithm}:assetDigest`,
-  })
-  async downloadAndDigest(
-    asset: GithubRestAsset,
-    algorithm: string
-  ): Promise<string> {
-    const res = this.http.stream(asset.browser_download_url);
-    const digest = await hasha.fromStream(res, { algorithm });
-    return digest;
-  }
-
-  async findAssetWithDigest(
-    release: GithubRestRelease,
-    digest: string
-  ): Promise<GithubDigestFile | null> {
-    const algorithm = inferHashAlg(digest);
-    const assetsBySize = release.assets.sort(
-      (a: GithubRestAsset, b: GithubRestAsset) => {
-        if (a.size < b.size) {
-          return -1;
-        }
-        if (a.size > b.size) {
-          return 1;
-        }
-        return 0;
-      }
-    );
-
-    for (const asset of assetsBySize) {
-      const assetDigest = await this.downloadAndDigest(asset, algorithm);
-      if (assetDigest === digest) {
-        return {
-          assetName: asset.name,
-          currentVersion: release.tag_name,
-          currentDigest: assetDigest,
-        };
-      }
-    }
-    return null;
-  }
-
-  /** Identify the asset associated with a known digest. */
-  async findDigestAsset(
-    release: GithubRestRelease,
-    digest: string
-  ): Promise<GithubDigestFile | null> {
-    const digestFile = await this.findDigestFile(release, digest);
-    if (digestFile) {
-      return digestFile;
-    }
-
-    const asset = await this.findAssetWithDigest(release, digest);
-    return asset;
-  }
-
-  /** Given a digest asset, find the equivalent digest in a different release. */
-  async mapDigestAssetToRelease(
-    digestAsset: GithubDigestFile,
-    release: GithubRestRelease
-  ): Promise<string | null> {
-    const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
-    const next = release.tag_name.replace(regEx(/^v/), '');
-    const releaseChecksumAssetName = digestAsset.assetName.replace(
-      current,
-      next
-    );
-    const releaseAsset = release.assets.find(
-      (a: GithubRestAsset) => a.name === releaseChecksumAssetName
-    );
-    if (!releaseAsset) {
-      return null;
-    }
-    if (digestAsset.digestedFileName) {
-      const releaseFilename = digestAsset.digestedFileName.replace(
-        current,
-        next
-      );
-      const res = await this.http.get(releaseAsset.browser_download_url);
-      for (const line of res.body.split(newlineRegex)) {
-        const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
-        if (lineFn === releaseFilename) {
-          return lineDigest;
-        }
-      }
-    } else {
-      const algorithm = inferHashAlg(digestAsset.currentDigest);
-      const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
-      return newDigest;
-    }
-    return null;
-  }
-
   /**
-   * github.getDigest
+   * Attempts to resolve the digest for the specified package.
    *
-   * The `newValue` supplied here should be a valid tag for the GitHub release.
+   * The `newValue` supplied here should be a valid tag for the GitHub release. The digest
-   * Requires `currentValue` and `currentDigest`.
+   * of a GitHub release will be the underlying SHA of the release tag.
    *
-   * There may be many assets attached to the release. This function will:
+   * Some managers like Bazel will deal with individual artifacts from releases and handle
-   *  - Identify the asset pinned by `currentDigest` in the `currentValue` release
+   * the artifact checksum computation separately. This data-source does not know about
-   *     - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
+   * specific artifacts being used, as that could vary per manager
-   *     - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
-   *  - Map the hashed asset to `newValue` and return the updated digest as a string
    */
-  override async getDigest(
+  override getDigest(
     {
       packageName: repo,
       currentValue,
@@ -195,37 +49,13 @@ export class GithubReleasesDatasource extends Datasource {
       { repo, currentValue, currentDigest, registryUrl, newValue },
       'getDigest'
     );
-    if (!currentDigest) {
-      return null;
-    }
-    if (!currentValue) {
-      return currentDigest;
-    }
 
-    const apiBaseUrl = getApiBaseUrl(registryUrl);
+    return findCommitOfTag(registryUrl, repo, newValue, this.http);
-    const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
-      `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
-    );
-    const digestAsset = await this.findDigestAsset(
-      currentRelease,
-      currentDigest
-    );
-    let newDigest: string | null;
-    if (!digestAsset || newValue === currentValue) {
-      newDigest = currentDigest;
-    } else {
-      const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
-        `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
-      );
-      newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
-    }
-    return newDigest;
   }
 
   /**
-   * github.getReleases
+   * This function can be used to fetch releases with a customizable versioning
-   *
+   * (e.g. semver) and with releases.
-   * This function can be used to fetch releases with a customisable versioning (e.g. semver) and with releases.
    *
    * This function will:
    *  - Fetch all releases

lib/modules/datasource/github-tags/index.ts

@@ -2,6 +2,7 @@ import is from '@sindresorhus/is';
 import { logger } from '../../../logger';
 import { queryReleases, queryTags } from '../../../util/github/graphql';
 import type { GithubReleaseItem } from '../../../util/github/graphql/types';
+import { findCommitOfTag } from '../../../util/github/tags';
 import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
 import { GithubHttp } from '../../../util/http/github';
 import { Datasource } from '../datasource';
@@ -24,42 +25,6 @@ export class GithubTagsDatasource extends Datasource {
     this.http = new GithubHttp(GithubTagsDatasource.id);
   }
 
-  async getTagCommit(
-    registryUrl: string | undefined,
-    packageName: string,
-    tag: string
-  ): Promise<string | null> {
-    logger.trace(`github-tags.getTagCommit(${packageName}, ${tag})`);
-    try {
-      const tags = await queryTags({ packageName, registryUrl }, this.http);
-      // istanbul ignore if
-      if (!tags.length) {
-        logger.debug(
-          `github-tags.getTagCommit(): No tags found for ${packageName}`
-        );
-      }
-      const tagItem = tags.find(({ version }) => version === tag);
-      if (tagItem) {
-        if (tagItem.hash) {
-          return tagItem.hash;
-        }
-        logger.debug(
-          `github-tags.getTagCommit(): Tag ${tag} has no hash for ${packageName}`
-        );
-      } else {
-        logger.debug(
-          `github-tags.getTagCommit(): Tag ${tag} not found for ${packageName}`
-        );
-      }
-    } catch (err) {
-      logger.debug(
-        { githubRepo: packageName, err },
-        'Error getting tag commit from GitHub repo'
-      );
-    }
-    return null;
-  }
-
   async getCommit(
     registryUrl: string | undefined,
     githubRepo: string
@@ -91,7 +56,7 @@ export class GithubTagsDatasource extends Datasource {
     newValue?: string
   ): Promise<string | null> {
     return newValue
-      ? this.getTagCommit(registryUrl, repo!, newValue)
+      ? findCommitOfTag(registryUrl, repo!, newValue, this.http)
       : this.getCommit(registryUrl, repo!);
   }
 

lib/util/github/tags.spec.ts

@@ -0,0 +1,78 @@
+import { GithubHttp } from '../http/github';
+import * as githubGraphql from './graphql';
+import { findCommitOfTag } from './tags';
+
+describe('util/github/tags', () => {
+  describe('findCommitOfTag', () => {
+    const http = new GithubHttp();
+    const queryTagsSpy = jest.spyOn(githubGraphql, 'queryTags');
+
+    it('should be able to find the hash of a Git tag', async () => {
+      queryTagsSpy.mockResolvedValueOnce([
+        {
+          version: 'v1.0.0',
+          gitRef: 'v1.0.0',
+          releaseTimestamp: '2021-01-01',
+          hash: '123',
+        },
+        {
+          version: 'v2.0.0',
+          gitRef: 'v2.0.0',
+          releaseTimestamp: '2022-01-01',
+          hash: 'abc',
+        },
+      ]);
+
+      const commit = await findCommitOfTag(
+        undefined,
+        'some-org/repo',
+        'v2.0.0',
+        http
+      );
+      expect(commit).toBe('abc');
+    });
+
+    it('should support passing a custom registry URL', async () => {
+      queryTagsSpy.mockResolvedValueOnce([]);
+
+      const commit = await findCommitOfTag(
+        'https://my-enterprise-github.dev',
+        'some-org/repo',
+        'v2.0.0',
+        http
+      );
+      expect(commit).toBeNull();
+      expect(githubGraphql.queryTags).toHaveBeenCalledWith(
+        {
+          packageName: 'some-org/repo',
+          registryUrl: 'https://my-enterprise-github.dev',
+        },
+        http
+      );
+    });
+
+    it('should return `null` if the tag does not exist', async () => {
+      queryTagsSpy.mockResolvedValueOnce([]);
+
+      const commit = await findCommitOfTag(
+        undefined,
+        'some-org/repo',
+        'v2.0.0',
+        http
+      );
+      expect(commit).toBeNull();
+    });
+
+    it('should gracefully return `null` if tags cannot be queried', async () => {
+      queryTagsSpy.mockRejectedValue(new Error('some error'));
+
+      const commit = await findCommitOfTag(
+        undefined,
+        'some-org/repo',
+        'v2.0.0',
+        http
+      );
+      expect(commit).toBeNull();
+    });
+  });
+});

lib/util/github/tags.ts

@@ -0,0 +1,39 @@
+import { logger } from '../../logger';
+import type { GithubHttp } from '../http/github';
+import { queryTags } from './graphql';
+
+export async function findCommitOfTag(
+  registryUrl: string | undefined,
+  packageName: string,
+  tag: string,
+  http: GithubHttp
+): Promise<string | null> {
+  logger.trace(`github/tags.findCommitOfTag(${packageName}, ${tag})`);
+  try {
+    const tags = await queryTags({ packageName, registryUrl }, http);
+    if (!tags.length) {
+      logger.debug(
+        `github/tags.findCommitOfTag(): No tags found for ${packageName}`
+      );
+    }
+    const tagItem = tags.find(({ version }) => version === tag);
+    if (tagItem) {
+      if (tagItem.hash) {
+        return tagItem.hash;
+      }
+      logger.debug(
+        `github/tags.findCommitOfTag: Tag ${tag} has no hash for ${packageName}`
+      );
+    } else {
+      logger.debug(
+        `github/tags.findCommitOfTag: Tag ${tag} not found for ${packageName}`
+      );
+    }
+  } catch (err) {
+    logger.debug(
+      { githubRepo: packageName, err },
+      'Error getting tag commit from GitHub repo'
+    );
+  }
+  return null;
+}