mirror of
https://github.com/renovatebot/renovate.git
synced 2025-01-13 15:36:25 +00:00
feat(datasource/github-releases)!: digest computation use git tag, not file digest (#20178)
The github-releases datasource has been copied into a new datasource called github-release-attachments. The github-releases general datasource is updated to use the underlying Git tag of a GitHub release entry for digest computation. Fixes #20160, Fixes #19552 BREAKING CHANGE: Regex Manager configurations relying on the github-release data-source with digests will have different digest semantics. The digest will now always correspond to the underlying Git SHA of the release/version. The old behavior can be preserved by switching to the github-release-attachments datasource.
This commit is contained in:
Paul Gschwendtner
Rhys Arkins
parent
b17bcf2789
commit
177ffedb85
10 changed files with 593 additions and 268 deletions
|
|
@ -19,6 +19,7 @@ import { GalaxyDatasource } from './galaxy';
|
|||
import { GalaxyCollectionDatasource } from './galaxy-collection';
|
||||
import { GitRefsDatasource } from './git-refs';
|
||||
import { GitTagsDatasource } from './git-tags';
|
||||
import { GithubReleaseAttachmentsDatasource } from './github-release-attachments';
|
||||
import { GithubReleasesDatasource } from './github-releases';
|
||||
import { GithubTagsDatasource } from './github-tags';
|
||||
import { GitlabPackagesDatasource } from './gitlab-packages';
|
||||
|
|
@ -76,6 +77,10 @@ api.set(GalaxyDatasource.id, new GalaxyDatasource());
|
|||
api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource());
|
||||
api.set(GitRefsDatasource.id, new GitRefsDatasource());
|
||||
api.set(GitTagsDatasource.id, new GitTagsDatasource());
|
||||
api.set(
|
||||
GithubReleaseAttachmentsDatasource.id,
|
||||
new GithubReleaseAttachmentsDatasource()
|
||||
);
|
||||
api.set(GithubReleasesDatasource.id, new GithubReleasesDatasource());
|
||||
api.set(GithubTagsDatasource.id, new GithubTagsDatasource());
|
||||
api.set(GitlabPackagesDatasource.id, new GitlabPackagesDatasource());
|
||||
|
|
|
|||
|
|
@ -1,17 +1,17 @@
|
|||
import hasha from 'hasha';
|
||||
import * as httpMock from '../../../../test/http-mock';
|
||||
import type { GithubDigestFile } from '../../../util/github/types';
|
||||
import { GitHubReleaseMocker } from './test';
|
||||
import { GitHubReleaseAttachmentMocker } from './test';
|
||||
|
||||
import { GithubReleasesDatasource } from '.';
|
||||
import { GithubReleaseAttachmentsDatasource } from '.';
|
||||
|
||||
describe('modules/datasource/github-releases/digest', () => {
|
||||
describe('modules/datasource/github-release-attachments/digest', () => {
|
||||
const packageName = 'some/dep';
|
||||
const releaseMock = new GitHubReleaseMocker(
|
||||
const releaseMock = new GitHubReleaseAttachmentMocker(
|
||||
'https://api.github.com',
|
||||
packageName
|
||||
);
|
||||
const githubReleases = new GithubReleasesDatasource();
|
||||
const githubReleaseAttachments = new GithubReleaseAttachmentsDatasource();
|
||||
|
||||
describe('findDigestAsset', () => {
|
||||
it('finds SHASUMS.txt file containing digest', async () => {
|
||||
|
|
@ -21,7 +21,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
'another-digest linux-arm64.tar.gz'
|
||||
);
|
||||
|
||||
const digestAsset = await githubReleases.findDigestAsset(
|
||||
const digestAsset = await githubReleaseAttachments.findDigestAsset(
|
||||
release,
|
||||
'test-digest'
|
||||
);
|
||||
|
|
@ -40,7 +40,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
.get(`/repos/${packageName}/releases/download/v1.0.0/SHASUMS.txt`)
|
||||
.reply(200, '');
|
||||
|
||||
const digestAsset = await githubReleases.findDigestAsset(
|
||||
const digestAsset = await githubReleaseAttachments.findDigestAsset(
|
||||
release,
|
||||
'test-digest'
|
||||
);
|
||||
|
|
@ -57,7 +57,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
});
|
||||
const contentDigest = await hasha.async(content, { algorithm: 'sha256' });
|
||||
|
||||
const digestAsset = await githubReleases.findDigestAsset(
|
||||
const digestAsset = await githubReleaseAttachments.findDigestAsset(
|
||||
release,
|
||||
contentDigest
|
||||
);
|
||||
|
|
@ -67,7 +67,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
|
||||
it('returns null when no assets available', async () => {
|
||||
const release = releaseMock.release('v1.0.0');
|
||||
const digestAsset = await githubReleases.findDigestAsset(
|
||||
const digestAsset = await githubReleaseAttachments.findDigestAsset(
|
||||
release,
|
||||
'test-digest'
|
||||
);
|
||||
|
|
@ -89,7 +89,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
'v1.0.1',
|
||||
'updated-digest asset.zip'
|
||||
);
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAsset,
|
||||
release
|
||||
);
|
||||
|
|
@ -106,7 +106,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
'v1.0.1',
|
||||
'updated-digest asset-1.0.1.zip'
|
||||
);
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAssetWithVersion,
|
||||
release
|
||||
);
|
||||
|
|
@ -118,7 +118,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
'v1.0.1',
|
||||
'moot-digest asset.tar.gz'
|
||||
);
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAsset,
|
||||
release
|
||||
);
|
||||
|
|
@ -127,7 +127,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
|
||||
it('returns null when digest file not found', async () => {
|
||||
const release = releaseMock.release('v1.0.1');
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAsset,
|
||||
release
|
||||
);
|
||||
|
|
@ -151,7 +151,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
algorithm: 'sha256',
|
||||
});
|
||||
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAsset,
|
||||
release
|
||||
);
|
||||
|
|
@ -160,7 +160,7 @@ describe('modules/datasource/github-releases/digest', () => {
|
|||
|
||||
it('returns null when not found', async () => {
|
||||
const release = releaseMock.release('v1.0.1');
|
||||
const digest = await githubReleases.mapDigestAssetToRelease(
|
||||
const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
|
||||
digestAsset,
|
||||
release
|
||||
);
|
||||
154
lib/modules/datasource/github-release-attachments/index.spec.ts
Normal file
154
lib/modules/datasource/github-release-attachments/index.spec.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
import { getDigest, getPkgReleases } from '..';
|
||||
import { mocked } from '../../../../test/util';
|
||||
import * as githubGraphql from '../../../util/github/graphql';
|
||||
import * as _hostRules from '../../../util/host-rules';
|
||||
import { GitHubReleaseAttachmentMocker } from './test';
|
||||
import { GithubReleaseAttachmentsDatasource } from '.';
|
||||
|
||||
jest.mock('../../../util/host-rules');
|
||||
const hostRules = mocked(_hostRules);
|
||||
|
||||
const githubApiHost = 'https://api.github.com';
|
||||
|
||||
describe('modules/datasource/github-release-attachments/index', () => {
|
||||
beforeEach(() => {
|
||||
hostRules.hosts.mockReturnValue([]);
|
||||
hostRules.find.mockReturnValue({
|
||||
token: 'some-token',
|
||||
});
|
||||
});
|
||||
|
||||
describe('getReleases', () => {
|
||||
it('returns releases', async () => {
|
||||
jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([
|
||||
{
|
||||
id: 1,
|
||||
url: 'https://example.com',
|
||||
name: 'some/dep2',
|
||||
description: 'some description',
|
||||
version: 'a',
|
||||
releaseTimestamp: '2020-03-09T13:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
url: 'https://example.com',
|
||||
name: 'some/dep2',
|
||||
description: 'some description',
|
||||
version: 'v',
|
||||
releaseTimestamp: '2020-03-09T12:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
url: 'https://example.com',
|
||||
name: 'some/dep2',
|
||||
description: 'some description',
|
||||
version: '1.0.0',
|
||||
releaseTimestamp: '2020-03-09T11:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
url: 'https://example.com',
|
||||
name: 'some/dep2',
|
||||
description: 'some description',
|
||||
version: 'v1.1.0',
|
||||
releaseTimestamp: '2020-03-09T10:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 5,
|
||||
url: 'https://example.com',
|
||||
name: 'some/dep2',
|
||||
description: 'some description',
|
||||
version: '2.0.0',
|
||||
releaseTimestamp: '2020-04-09T10:00:00Z',
|
||||
isStable: false,
|
||||
},
|
||||
]);
|
||||
|
||||
const res = await getPkgReleases({
|
||||
datasource: GithubReleaseAttachmentsDatasource.id,
|
||||
packageName: 'some/dep',
|
||||
});
|
||||
|
||||
expect(res).toMatchObject({
|
||||
registryUrl: 'https://github.com',
|
||||
releases: [
|
||||
{ releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' },
|
||||
{ version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' },
|
||||
{
|
||||
version: '2.0.0',
|
||||
releaseTimestamp: '2020-04-09T10:00:00.000Z',
|
||||
isStable: false,
|
||||
},
|
||||
],
|
||||
sourceUrl: 'https://github.com/some/dep',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDigest', () => {
|
||||
const packageName = 'some/dep';
|
||||
const currentValue = 'v1.0.0';
|
||||
const currentDigest = 'v1.0.0-digest';
|
||||
|
||||
const releaseMock = new GitHubReleaseAttachmentMocker(
|
||||
githubApiHost,
|
||||
packageName
|
||||
);
|
||||
|
||||
it('requires currentDigest', async () => {
|
||||
const digest = await getDigest(
|
||||
{ datasource: GithubReleaseAttachmentsDatasource.id, packageName },
|
||||
currentValue
|
||||
);
|
||||
expect(digest).toBeNull();
|
||||
});
|
||||
|
||||
it('defaults to currentDigest when currentVersion is missing', async () => {
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleaseAttachmentsDatasource.id,
|
||||
packageName,
|
||||
currentDigest,
|
||||
},
|
||||
currentValue
|
||||
);
|
||||
expect(digest).toEqual(currentDigest);
|
||||
});
|
||||
|
||||
it('returns updated digest in new release', async () => {
|
||||
releaseMock.withDigestFileAsset(
|
||||
currentValue,
|
||||
`${currentDigest} asset.zip`
|
||||
);
|
||||
const nextValue = 'v1.0.1';
|
||||
const nextDigest = 'updated-digest';
|
||||
releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleaseAttachmentsDatasource.id,
|
||||
packageName,
|
||||
currentValue,
|
||||
currentDigest,
|
||||
},
|
||||
nextValue
|
||||
);
|
||||
expect(digest).toEqual(nextDigest);
|
||||
});
|
||||
|
||||
// This is awkward, but I found returning `null` in this case to not produce an update
|
||||
// I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
|
||||
it('ignores failures verifying currentDigest', async () => {
|
||||
releaseMock.release(currentValue);
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleaseAttachmentsDatasource.id,
|
||||
packageName,
|
||||
currentValue,
|
||||
currentDigest,
|
||||
},
|
||||
currentValue
|
||||
);
|
||||
expect(digest).toEqual(currentDigest);
|
||||
});
|
||||
});
|
||||
});
|
||||
250
lib/modules/datasource/github-release-attachments/index.ts
Normal file
250
lib/modules/datasource/github-release-attachments/index.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import is from '@sindresorhus/is';
|
||||
import hasha from 'hasha';
|
||||
import { logger } from '../../../logger';
|
||||
import { cache } from '../../../util/cache/package/decorator';
|
||||
import { queryReleases } from '../../../util/github/graphql';
|
||||
import type {
|
||||
GithubDigestFile,
|
||||
GithubRestAsset,
|
||||
GithubRestRelease,
|
||||
} from '../../../util/github/types';
|
||||
import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
|
||||
import { GithubHttp } from '../../../util/http/github';
|
||||
import { newlineRegex, regEx } from '../../../util/regex';
|
||||
import { Datasource } from '../datasource';
|
||||
import type {
|
||||
DigestConfig,
|
||||
GetReleasesConfig,
|
||||
Release,
|
||||
ReleaseResult,
|
||||
} from '../types';
|
||||
|
||||
export const cacheNamespace = 'datasource-github-releases';
|
||||
|
||||
function inferHashAlg(digest: string): string {
|
||||
switch (digest.length) {
|
||||
case 64:
|
||||
return 'sha256';
|
||||
default:
|
||||
case 96:
|
||||
return 'sha512';
|
||||
}
|
||||
}
|
||||
|
||||
export class GithubReleaseAttachmentsDatasource extends Datasource {
|
||||
static readonly id = 'github-release-attachments';
|
||||
|
||||
override readonly defaultRegistryUrls = ['https://github.com'];
|
||||
|
||||
override http: GithubHttp;
|
||||
|
||||
constructor() {
|
||||
super(GithubReleaseAttachmentsDatasource.id);
|
||||
this.http = new GithubHttp(GithubReleaseAttachmentsDatasource.id);
|
||||
}
|
||||
|
||||
@cache({
|
||||
ttlMinutes: 1440,
|
||||
namespace: 'datasource-github-releases',
|
||||
key: (release: GithubRestRelease, digest: string) =>
|
||||
`${release.html_url}:${digest}`,
|
||||
})
|
||||
async findDigestFile(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const smallAssets = release.assets.filter(
|
||||
(a: GithubRestAsset) => a.size < 5 * 1024
|
||||
);
|
||||
for (const asset of smallAssets) {
|
||||
const res = await this.http.get(asset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
|
||||
if (lineDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
digestedFileName: lineFilename,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: lineDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@cache({
|
||||
ttlMinutes: 1440,
|
||||
namespace: 'datasource-github-releases',
|
||||
key: (asset: GithubRestAsset, algorithm: string) =>
|
||||
`${asset.browser_download_url}:${algorithm}:assetDigest`,
|
||||
})
|
||||
async downloadAndDigest(
|
||||
asset: GithubRestAsset,
|
||||
algorithm: string
|
||||
): Promise<string> {
|
||||
const res = this.http.stream(asset.browser_download_url);
|
||||
const digest = await hasha.fromStream(res, { algorithm });
|
||||
return digest;
|
||||
}
|
||||
|
||||
async findAssetWithDigest(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const algorithm = inferHashAlg(digest);
|
||||
const assetsBySize = release.assets.sort(
|
||||
(a: GithubRestAsset, b: GithubRestAsset) => {
|
||||
if (a.size < b.size) {
|
||||
return -1;
|
||||
}
|
||||
if (a.size > b.size) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
);
|
||||
|
||||
for (const asset of assetsBySize) {
|
||||
const assetDigest = await this.downloadAndDigest(asset, algorithm);
|
||||
if (assetDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: assetDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Identify the asset associated with a known digest. */
|
||||
async findDigestAsset(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const digestFile = await this.findDigestFile(release, digest);
|
||||
if (digestFile) {
|
||||
return digestFile;
|
||||
}
|
||||
|
||||
const asset = await this.findAssetWithDigest(release, digest);
|
||||
return asset;
|
||||
}
|
||||
|
||||
/** Given a digest asset, find the equivalent digest in a different release. */
|
||||
async mapDigestAssetToRelease(
|
||||
digestAsset: GithubDigestFile,
|
||||
release: GithubRestRelease
|
||||
): Promise<string | null> {
|
||||
const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
|
||||
const next = release.tag_name.replace(regEx(/^v/), '');
|
||||
const releaseChecksumAssetName = digestAsset.assetName.replace(
|
||||
current,
|
||||
next
|
||||
);
|
||||
const releaseAsset = release.assets.find(
|
||||
(a: GithubRestAsset) => a.name === releaseChecksumAssetName
|
||||
);
|
||||
if (!releaseAsset) {
|
||||
return null;
|
||||
}
|
||||
if (digestAsset.digestedFileName) {
|
||||
const releaseFilename = digestAsset.digestedFileName.replace(
|
||||
current,
|
||||
next
|
||||
);
|
||||
const res = await this.http.get(releaseAsset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineFn === releaseFilename) {
|
||||
return lineDigest;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const algorithm = inferHashAlg(digestAsset.currentDigest);
|
||||
const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
|
||||
return newDigest;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to resolve the digest for the specified package.
|
||||
*
|
||||
* The `newValue` supplied here should be a valid tag for the GitHub release.
|
||||
* Requires `currentValue` and `currentDigest`.
|
||||
*
|
||||
* There may be many assets attached to the release. This function will:
|
||||
* - Identify the asset pinned by `currentDigest` in the `currentValue` release
|
||||
* - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
|
||||
* - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
|
||||
* - Map the hashed asset to `newValue` and return the updated digest as a string
|
||||
*/
|
||||
override async getDigest(
|
||||
{
|
||||
packageName: repo,
|
||||
currentValue,
|
||||
currentDigest,
|
||||
registryUrl,
|
||||
}: DigestConfig,
|
||||
newValue: string
|
||||
): Promise<string | null> {
|
||||
logger.debug(
|
||||
{ repo, currentValue, currentDigest, registryUrl, newValue },
|
||||
'getDigest'
|
||||
);
|
||||
if (!currentDigest) {
|
||||
return null;
|
||||
}
|
||||
if (!currentValue) {
|
||||
return currentDigest;
|
||||
}
|
||||
|
||||
const apiBaseUrl = getApiBaseUrl(registryUrl);
|
||||
const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
|
||||
`${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
|
||||
);
|
||||
const digestAsset = await this.findDigestAsset(
|
||||
currentRelease,
|
||||
currentDigest
|
||||
);
|
||||
let newDigest: string | null;
|
||||
if (!digestAsset || newValue === currentValue) {
|
||||
newDigest = currentDigest;
|
||||
} else {
|
||||
const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
|
||||
`${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
|
||||
);
|
||||
newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
|
||||
}
|
||||
return newDigest;
|
||||
}
|
||||
|
||||
/**
|
||||
* This function can be used to fetch releases with a customisable versioning
|
||||
* (e.g. semver) and with releases.
|
||||
*
|
||||
* This function will:
|
||||
* - Fetch all releases
|
||||
* - Sanitize the versions if desired (e.g. strip out leading 'v')
|
||||
* - Return a dependency object containing sourceUrl string and releases array
|
||||
*/
|
||||
async getReleases(config: GetReleasesConfig): Promise<ReleaseResult> {
|
||||
const releasesResult = await queryReleases(config, this.http);
|
||||
const releases = releasesResult.map((item) => {
|
||||
const { version, releaseTimestamp, isStable } = item;
|
||||
const result: Release = {
|
||||
version,
|
||||
gitRef: version,
|
||||
releaseTimestamp,
|
||||
};
|
||||
if (is.boolean(isStable)) {
|
||||
result.isStable = isStable;
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const sourceUrl = getSourceUrl(config.packageName, config.registryUrl);
|
||||
return { sourceUrl, releases };
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import * as httpMock from '../../../../../test/http-mock';
|
|||
import { partial } from '../../../../../test/util';
|
||||
import type { GithubRestRelease } from '../../../../util/github/types';
|
||||
|
||||
export class GitHubReleaseMocker {
|
||||
export class GitHubReleaseAttachmentMocker {
|
||||
constructor(
|
||||
private readonly githubApiHost: string,
|
||||
private readonly packageName: string
|
||||
|
|
@ -1,17 +1,14 @@
|
|||
import { getDigest, getPkgReleases } from '..';
|
||||
import { mocked } from '../../../../test/util';
|
||||
import * as githubGraphql from '../../../util/github/graphql';
|
||||
import * as _hostRules from '../../../util/host-rules';
|
||||
import { GitHubReleaseMocker } from './test';
|
||||
import { GithubReleasesDatasource } from '.';
|
||||
|
||||
jest.mock('../../../util/host-rules');
|
||||
const hostRules: any = _hostRules;
|
||||
|
||||
const githubApiHost = 'https://api.github.com';
|
||||
const hostRules = mocked(_hostRules);
|
||||
|
||||
describe('modules/datasource/github-releases/index', () => {
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
hostRules.hosts.mockReturnValue([]);
|
||||
hostRules.find.mockReturnValue({
|
||||
token: 'some-token',
|
||||
|
|
@ -88,38 +85,48 @@ describe('modules/datasource/github-releases/index', () => {
|
|||
describe('getDigest', () => {
|
||||
const packageName = 'some/dep';
|
||||
const currentValue = 'v1.0.0';
|
||||
const currentDigest = 'v1.0.0-digest';
|
||||
const currentDigest = 'sha-of-v1';
|
||||
const newValue = 'v15.0.0';
|
||||
const newDigest = 'sha-of-v15';
|
||||
|
||||
const releaseMock = new GitHubReleaseMocker(githubApiHost, packageName);
|
||||
|
||||
it('requires currentDigest', async () => {
|
||||
const digest = await getDigest(
|
||||
{ datasource: GithubReleasesDatasource.id, packageName },
|
||||
currentValue
|
||||
);
|
||||
expect(digest).toBeNull();
|
||||
beforeEach(() => {
|
||||
jest.spyOn(githubGraphql, 'queryTags').mockResolvedValueOnce([
|
||||
{
|
||||
version: 'v1.0.0',
|
||||
gitRef: 'v1.0.0',
|
||||
releaseTimestamp: '2021-01-01',
|
||||
hash: 'sha-of-v1',
|
||||
},
|
||||
{
|
||||
version: 'v15.0.0',
|
||||
gitRef: 'v15.0.0',
|
||||
releaseTimestamp: '2022-10-01',
|
||||
hash: 'sha-of-v15',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('defaults to currentDigest when currentVersion is missing', async () => {
|
||||
it('should be independent of the current digest', async () => {
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleasesDatasource.id,
|
||||
packageName,
|
||||
currentDigest,
|
||||
currentValue,
|
||||
},
|
||||
currentValue
|
||||
newValue
|
||||
);
|
||||
expect(digest).toEqual(currentDigest);
|
||||
expect(digest).toBe(newDigest);
|
||||
});
|
||||
|
||||
it('should be independent of the current value', async () => {
|
||||
const digest = await getDigest(
|
||||
{ datasource: GithubReleasesDatasource.id, packageName },
|
||||
newValue
|
||||
);
|
||||
expect(digest).toBe(newDigest);
|
||||
});
|
||||
|
||||
it('returns updated digest in new release', async () => {
|
||||
releaseMock.withDigestFileAsset(
|
||||
currentValue,
|
||||
`${currentDigest} asset.zip`
|
||||
);
|
||||
const nextValue = 'v1.0.1';
|
||||
const nextDigest = 'updated-digest';
|
||||
releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleasesDatasource.id,
|
||||
|
|
@ -127,15 +134,12 @@ describe('modules/datasource/github-releases/index', () => {
|
|||
currentValue,
|
||||
currentDigest,
|
||||
},
|
||||
nextValue
|
||||
newValue
|
||||
);
|
||||
expect(digest).toEqual(nextDigest);
|
||||
expect(digest).toEqual(newDigest);
|
||||
});
|
||||
|
||||
// This is awkward, but I found returning `null` in this case to not produce an update
|
||||
// I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
|
||||
it('ignores failures verifying currentDigest', async () => {
|
||||
releaseMock.release(currentValue);
|
||||
it('returns null if the new value/tag does not exist', async () => {
|
||||
const digest = await getDigest(
|
||||
{
|
||||
datasource: GithubReleasesDatasource.id,
|
||||
|
|
@ -143,9 +147,9 @@ describe('modules/datasource/github-releases/index', () => {
|
|||
currentValue,
|
||||
currentDigest,
|
||||
},
|
||||
currentValue
|
||||
'unknown-tag'
|
||||
);
|
||||
expect(digest).toEqual(currentDigest);
|
||||
expect(digest).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,17 +1,9 @@
|
|||
// TODO: types (#7154)
|
||||
import is from '@sindresorhus/is';
|
||||
import hasha from 'hasha';
|
||||
import { logger } from '../../../logger';
|
||||
import { cache } from '../../../util/cache/package/decorator';
|
||||
import { queryReleases } from '../../../util/github/graphql';
|
||||
import type {
|
||||
GithubDigestFile,
|
||||
GithubRestAsset,
|
||||
GithubRestRelease,
|
||||
} from '../../../util/github/types';
|
||||
import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
|
||||
import { findCommitOfTag } from '../../../util/github/tags';
|
||||
import { getSourceUrl } from '../../../util/github/url';
|
||||
import { GithubHttp } from '../../../util/http/github';
|
||||
import { newlineRegex, regEx } from '../../../util/regex';
|
||||
import { Datasource } from '../datasource';
|
||||
import type {
|
||||
DigestConfig,
|
||||
|
|
@ -22,16 +14,6 @@ import type {
|
|||
|
||||
export const cacheNamespace = 'datasource-github-releases';
|
||||
|
||||
function inferHashAlg(digest: string): string {
|
||||
switch (digest.length) {
|
||||
case 64:
|
||||
return 'sha256';
|
||||
default:
|
||||
case 96:
|
||||
return 'sha512';
|
||||
}
|
||||
}
|
||||
|
||||
export class GithubReleasesDatasource extends Datasource {
|
||||
static readonly id = 'github-releases';
|
||||
|
||||
|
|
@ -44,145 +26,17 @@ export class GithubReleasesDatasource extends Datasource {
|
|||
this.http = new GithubHttp(GithubReleasesDatasource.id);
|
||||
}
|
||||
|
||||
@cache({
|
||||
ttlMinutes: 1440,
|
||||
namespace: 'datasource-github-releases',
|
||||
key: (release: GithubRestRelease, digest: string) =>
|
||||
`${release.html_url}:${digest}`,
|
||||
})
|
||||
async findDigestFile(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const smallAssets = release.assets.filter(
|
||||
(a: GithubRestAsset) => a.size < 5 * 1024
|
||||
);
|
||||
for (const asset of smallAssets) {
|
||||
const res = await this.http.get(asset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
|
||||
if (lineDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
digestedFileName: lineFilename,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: lineDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@cache({
|
||||
ttlMinutes: 1440,
|
||||
namespace: 'datasource-github-releases',
|
||||
key: (asset: GithubRestAsset, algorithm: string) =>
|
||||
`${asset.browser_download_url}:${algorithm}:assetDigest`,
|
||||
})
|
||||
async downloadAndDigest(
|
||||
asset: GithubRestAsset,
|
||||
algorithm: string
|
||||
): Promise<string> {
|
||||
const res = this.http.stream(asset.browser_download_url);
|
||||
const digest = await hasha.fromStream(res, { algorithm });
|
||||
return digest;
|
||||
}
|
||||
|
||||
async findAssetWithDigest(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const algorithm = inferHashAlg(digest);
|
||||
const assetsBySize = release.assets.sort(
|
||||
(a: GithubRestAsset, b: GithubRestAsset) => {
|
||||
if (a.size < b.size) {
|
||||
return -1;
|
||||
}
|
||||
if (a.size > b.size) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
);
|
||||
|
||||
for (const asset of assetsBySize) {
|
||||
const assetDigest = await this.downloadAndDigest(asset, algorithm);
|
||||
if (assetDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: assetDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Identify the asset associated with a known digest. */
|
||||
async findDigestAsset(
|
||||
release: GithubRestRelease,
|
||||
digest: string
|
||||
): Promise<GithubDigestFile | null> {
|
||||
const digestFile = await this.findDigestFile(release, digest);
|
||||
if (digestFile) {
|
||||
return digestFile;
|
||||
}
|
||||
|
||||
const asset = await this.findAssetWithDigest(release, digest);
|
||||
return asset;
|
||||
}
|
||||
|
||||
/** Given a digest asset, find the equivalent digest in a different release. */
|
||||
async mapDigestAssetToRelease(
|
||||
digestAsset: GithubDigestFile,
|
||||
release: GithubRestRelease
|
||||
): Promise<string | null> {
|
||||
const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
|
||||
const next = release.tag_name.replace(regEx(/^v/), '');
|
||||
const releaseChecksumAssetName = digestAsset.assetName.replace(
|
||||
current,
|
||||
next
|
||||
);
|
||||
const releaseAsset = release.assets.find(
|
||||
(a: GithubRestAsset) => a.name === releaseChecksumAssetName
|
||||
);
|
||||
if (!releaseAsset) {
|
||||
return null;
|
||||
}
|
||||
if (digestAsset.digestedFileName) {
|
||||
const releaseFilename = digestAsset.digestedFileName.replace(
|
||||
current,
|
||||
next
|
||||
);
|
||||
const res = await this.http.get(releaseAsset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineFn === releaseFilename) {
|
||||
return lineDigest;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const algorithm = inferHashAlg(digestAsset.currentDigest);
|
||||
const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
|
||||
return newDigest;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* github.getDigest
|
||||
* Attempts to resolve the digest for the specified package.
|
||||
*
|
||||
* The `newValue` supplied here should be a valid tag for the GitHub release.
|
||||
* Requires `currentValue` and `currentDigest`.
|
||||
* The `newValue` supplied here should be a valid tag for the GitHub release. The digest
|
||||
* of a GitHub release will be the underlying SHA of the release tag.
|
||||
*
|
||||
* There may be many assets attached to the release. This function will:
|
||||
* - Identify the asset pinned by `currentDigest` in the `currentValue` release
|
||||
* - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
|
||||
* - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
|
||||
* - Map the hashed asset to `newValue` and return the updated digest as a string
|
||||
* Some managers like Bazel will deal with individual artifacts from releases and handle
|
||||
* the artifact checksum computation separately. This data-source does not know about
|
||||
* specific artifacts being used, as that could vary per manager
|
||||
*/
|
||||
override async getDigest(
|
||||
override getDigest(
|
||||
{
|
||||
packageName: repo,
|
||||
currentValue,
|
||||
|
|
@ -195,37 +49,13 @@ export class GithubReleasesDatasource extends Datasource {
|
|||
{ repo, currentValue, currentDigest, registryUrl, newValue },
|
||||
'getDigest'
|
||||
);
|
||||
if (!currentDigest) {
|
||||
return null;
|
||||
}
|
||||
if (!currentValue) {
|
||||
return currentDigest;
|
||||
}
|
||||
|
||||
const apiBaseUrl = getApiBaseUrl(registryUrl);
|
||||
const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
|
||||
`${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
|
||||
);
|
||||
const digestAsset = await this.findDigestAsset(
|
||||
currentRelease,
|
||||
currentDigest
|
||||
);
|
||||
let newDigest: string | null;
|
||||
if (!digestAsset || newValue === currentValue) {
|
||||
newDigest = currentDigest;
|
||||
} else {
|
||||
const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
|
||||
`${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
|
||||
);
|
||||
newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
|
||||
}
|
||||
return newDigest;
|
||||
return findCommitOfTag(registryUrl, repo, newValue, this.http);
|
||||
}
|
||||
|
||||
/**
|
||||
* github.getReleases
|
||||
*
|
||||
* This function can be used to fetch releases with a customisable versioning (e.g. semver) and with releases.
|
||||
* This function can be used to fetch releases with a customizable versioning
|
||||
* (e.g. semver) and with releases.
|
||||
*
|
||||
* This function will:
|
||||
* - Fetch all releases
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import is from '@sindresorhus/is';
|
|||
import { logger } from '../../../logger';
|
||||
import { queryReleases, queryTags } from '../../../util/github/graphql';
|
||||
import type { GithubReleaseItem } from '../../../util/github/graphql/types';
|
||||
import { findCommitOfTag } from '../../../util/github/tags';
|
||||
import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
|
||||
import { GithubHttp } from '../../../util/http/github';
|
||||
import { Datasource } from '../datasource';
|
||||
|
|
@ -24,42 +25,6 @@ export class GithubTagsDatasource extends Datasource {
|
|||
this.http = new GithubHttp(GithubTagsDatasource.id);
|
||||
}
|
||||
|
||||
async getTagCommit(
|
||||
registryUrl: string | undefined,
|
||||
packageName: string,
|
||||
tag: string
|
||||
): Promise<string | null> {
|
||||
logger.trace(`github-tags.getTagCommit(${packageName}, ${tag})`);
|
||||
try {
|
||||
const tags = await queryTags({ packageName, registryUrl }, this.http);
|
||||
// istanbul ignore if
|
||||
if (!tags.length) {
|
||||
logger.debug(
|
||||
`github-tags.getTagCommit(): No tags found for ${packageName}`
|
||||
);
|
||||
}
|
||||
const tagItem = tags.find(({ version }) => version === tag);
|
||||
if (tagItem) {
|
||||
if (tagItem.hash) {
|
||||
return tagItem.hash;
|
||||
}
|
||||
logger.debug(
|
||||
`github-tags.getTagCommit(): Tag ${tag} has no hash for ${packageName}`
|
||||
);
|
||||
} else {
|
||||
logger.debug(
|
||||
`github-tags.getTagCommit(): Tag ${tag} not found for ${packageName}`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
{ githubRepo: packageName, err },
|
||||
'Error getting tag commit from GitHub repo'
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async getCommit(
|
||||
registryUrl: string | undefined,
|
||||
githubRepo: string
|
||||
|
|
@ -91,7 +56,7 @@ export class GithubTagsDatasource extends Datasource {
|
|||
newValue?: string
|
||||
): Promise<string | null> {
|
||||
return newValue
|
||||
? this.getTagCommit(registryUrl, repo!, newValue)
|
||||
? findCommitOfTag(registryUrl, repo!, newValue, this.http)
|
||||
: this.getCommit(registryUrl, repo!);
|
||||
}
|
||||
|
||||
|
|
|
|||
78
lib/util/github/tags.spec.ts
Normal file
78
lib/util/github/tags.spec.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { GithubHttp } from '../http/github';
|
||||
import * as githubGraphql from './graphql';
|
||||
import { findCommitOfTag } from './tags';
|
||||
|
||||
describe('util/github/tags', () => {
|
||||
describe('findCommitOfTag', () => {
|
||||
const http = new GithubHttp();
|
||||
const queryTagsSpy = jest.spyOn(githubGraphql, 'queryTags');
|
||||
|
||||
it('should be able to find the hash of a Git tag', async () => {
|
||||
queryTagsSpy.mockResolvedValueOnce([
|
||||
{
|
||||
version: 'v1.0.0',
|
||||
gitRef: 'v1.0.0',
|
||||
releaseTimestamp: '2021-01-01',
|
||||
hash: '123',
|
||||
},
|
||||
{
|
||||
version: 'v2.0.0',
|
||||
gitRef: 'v2.0.0',
|
||||
releaseTimestamp: '2022-01-01',
|
||||
hash: 'abc',
|
||||
},
|
||||
]);
|
||||
|
||||
const commit = await findCommitOfTag(
|
||||
undefined,
|
||||
'some-org/repo',
|
||||
'v2.0.0',
|
||||
http
|
||||
);
|
||||
expect(commit).toBe('abc');
|
||||
});
|
||||
|
||||
it('should support passing a custom registry URL', async () => {
|
||||
queryTagsSpy.mockResolvedValueOnce([]);
|
||||
|
||||
const commit = await findCommitOfTag(
|
||||
'https://my-enterprise-github.dev',
|
||||
'some-org/repo',
|
||||
'v2.0.0',
|
||||
http
|
||||
);
|
||||
expect(commit).toBeNull();
|
||||
expect(githubGraphql.queryTags).toHaveBeenCalledWith(
|
||||
{
|
||||
packageName: 'some-org/repo',
|
||||
registryUrl: 'https://my-enterprise-github.dev',
|
||||
},
|
||||
http
|
||||
);
|
||||
});
|
||||
|
||||
it('should return `null` if the tag does not exist', async () => {
|
||||
queryTagsSpy.mockResolvedValueOnce([]);
|
||||
|
||||
const commit = await findCommitOfTag(
|
||||
undefined,
|
||||
'some-org/repo',
|
||||
'v2.0.0',
|
||||
http
|
||||
);
|
||||
expect(commit).toBeNull();
|
||||
});
|
||||
|
||||
it('should gracefully return `null` if tags cannot be queried', async () => {
|
||||
queryTagsSpy.mockRejectedValue(new Error('some error'));
|
||||
|
||||
const commit = await findCommitOfTag(
|
||||
undefined,
|
||||
'some-org/repo',
|
||||
'v2.0.0',
|
||||
http
|
||||
);
|
||||
expect(commit).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
39
lib/util/github/tags.ts
Normal file
39
lib/util/github/tags.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { logger } from '../../logger';
|
||||
import type { GithubHttp } from '../http/github';
|
||||
import { queryTags } from './graphql';
|
||||
|
||||
export async function findCommitOfTag(
|
||||
registryUrl: string | undefined,
|
||||
packageName: string,
|
||||
tag: string,
|
||||
http: GithubHttp
|
||||
): Promise<string | null> {
|
||||
logger.trace(`github/tags.findCommitOfTag(${packageName}, ${tag})`);
|
||||
try {
|
||||
const tags = await queryTags({ packageName, registryUrl }, http);
|
||||
if (!tags.length) {
|
||||
logger.debug(
|
||||
`github/tags.findCommitOfTag(): No tags found for ${packageName}`
|
||||
);
|
||||
}
|
||||
const tagItem = tags.find(({ version }) => version === tag);
|
||||
if (tagItem) {
|
||||
if (tagItem.hash) {
|
||||
return tagItem.hash;
|
||||
}
|
||||
logger.debug(
|
||||
`github/tags.findCommitOfTag: Tag ${tag} has no hash for ${packageName}`
|
||||
);
|
||||
} else {
|
||||
logger.debug(
|
||||
`github/tags.findCommitOfTag: Tag ${tag} not found for ${packageName}`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
{ githubRepo: packageName, err },
|
||||
'Error getting tag commit from GitHub repo'
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
Loading…
Reference in a new issue