mirror of
https://github.com/renovatebot/renovate.git
synced 2025-01-11 14:36:25 +00:00
refactor(datasource/github-releases): Consolidate into single file (#14083)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
This commit is contained in:
Sergei Zharinov
GitHub
parent
96d777527f
commit
08acc9ad0e
7 changed files with 184 additions and 195 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from './common';
|
||||
import { GitHubReleaseMocker } from './test';
|
||||
import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from '.';
|
||||
|
||||
describe('datasource/github-releases/common', () => {
|
||||
describe('getSourceUrlBase', () => {
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
import { GithubHttp } from '../../util/http/github';
|
||||
import { ensureTrailingSlash } from '../../util/url';
|
||||
import type { GithubRelease } from './types';
|
||||
|
||||
const defaultSourceUrlBase = 'https://github.com/';
|
||||
export const id = 'github-releases';
|
||||
|
||||
export const cacheNamespace = 'datasource-github-releases';
|
||||
export const http = new GithubHttp(id);
|
||||
|
||||
export function getSourceUrlBase(registryUrl: string): string {
|
||||
// default to GitHub.com if no GHE host is specified.
|
||||
return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase);
|
||||
}
|
||||
|
||||
export function getApiBaseUrl(registryUrl: string): string {
|
||||
const sourceUrlBase = getSourceUrlBase(registryUrl);
|
||||
return sourceUrlBase === defaultSourceUrlBase
|
||||
? `https://api.github.com/`
|
||||
: `${sourceUrlBase}api/v3/`;
|
||||
}
|
||||
|
||||
export function getSourceUrl(lookupName: string, registryUrl?: string): string {
|
||||
const sourceUrlBase = getSourceUrlBase(registryUrl);
|
||||
return `${sourceUrlBase}${lookupName}`;
|
||||
}
|
||||
|
||||
export async function getGithubRelease(
|
||||
apiBaseUrl: string,
|
||||
repo: string,
|
||||
version: string
|
||||
): Promise<GithubRelease> {
|
||||
const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`;
|
||||
const res = await http.getJson<GithubRelease>(url);
|
||||
return res.body;
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import hasha from 'hasha';
|
||||
import * as httpMock from '../../../test/http-mock';
|
||||
import { findDigestAsset, mapDigestAssetToRelease } from './digest';
|
||||
import { GitHubReleaseMocker } from './test';
|
||||
import type { DigestAsset } from './types';
|
||||
import { findDigestAsset, mapDigestAssetToRelease } from '.';
|
||||
|
||||
describe('datasource/github-releases/digest', () => {
|
||||
const lookupName = 'some/dep';
|
||||
|
|
|
|||
|
|
@ -1,142 +0,0 @@
|
|||
import hasha from 'hasha';
|
||||
import * as packageCache from '../../util/cache/package';
|
||||
import { newlineRegex, regEx } from '../../util/regex';
|
||||
import { cacheNamespace, http } from './common';
|
||||
import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types';
|
||||
|
||||
async function findDigestFile(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset | null> {
|
||||
const smallAssets = release.assets.filter(
|
||||
(a: GithubReleaseAsset) => a.size < 5 * 1024
|
||||
);
|
||||
for (const asset of smallAssets) {
|
||||
const res = await http.get(asset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
digestedFileName: lineFn,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: lineDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inferHashAlg(digest: string): string {
|
||||
switch (digest.length) {
|
||||
case 64:
|
||||
return 'sha256';
|
||||
default:
|
||||
case 96:
|
||||
return 'sha512';
|
||||
}
|
||||
}
|
||||
|
||||
function getAssetDigestCacheKey(
|
||||
downloadUrl: string,
|
||||
algorithm: string
|
||||
): string {
|
||||
const type = 'assetDigest';
|
||||
return `${downloadUrl}:${algorithm}:${type}`;
|
||||
}
|
||||
|
||||
async function downloadAndDigest(
|
||||
asset: GithubReleaseAsset,
|
||||
algorithm: string
|
||||
): Promise<string> {
|
||||
const downloadUrl = asset.browser_download_url;
|
||||
const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm);
|
||||
const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey);
|
||||
// istanbul ignore if
|
||||
if (cachedResult) {
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
const res = http.stream(downloadUrl);
|
||||
const digest = await hasha.fromStream(res, { algorithm });
|
||||
|
||||
const cacheMinutes = 1440;
|
||||
await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
|
||||
return digest;
|
||||
}
|
||||
|
||||
async function findAssetWithDigest(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset | null> {
|
||||
const algorithm = inferHashAlg(digest);
|
||||
const assetsBySize = release.assets.sort(
|
||||
(a: GithubReleaseAsset, b: GithubReleaseAsset) => {
|
||||
if (a.size < b.size) {
|
||||
return -1;
|
||||
}
|
||||
if (a.size > b.size) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
);
|
||||
|
||||
for (const asset of assetsBySize) {
|
||||
const assetDigest = await downloadAndDigest(asset, algorithm);
|
||||
if (assetDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: assetDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Identify the asset associated with a known digest. */
|
||||
export async function findDigestAsset(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset> {
|
||||
const digestFile = await findDigestFile(release, digest);
|
||||
if (digestFile) {
|
||||
return digestFile;
|
||||
}
|
||||
|
||||
const asset = await findAssetWithDigest(release, digest);
|
||||
return asset;
|
||||
}
|
||||
|
||||
/** Given a digest asset, find the equivalent digest in a different release. */
|
||||
export async function mapDigestAssetToRelease(
|
||||
digestAsset: DigestAsset,
|
||||
release: GithubRelease
|
||||
): Promise<string | null> {
|
||||
const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
|
||||
const next = release.tag_name.replace(regEx(/^v/), '');
|
||||
const releaseChecksumAssetName = digestAsset.assetName.replace(current, next);
|
||||
const releaseAsset = release.assets.find(
|
||||
(a: GithubReleaseAsset) => a.name === releaseChecksumAssetName
|
||||
);
|
||||
if (!releaseAsset) {
|
||||
return null;
|
||||
}
|
||||
if (digestAsset.digestedFileName) {
|
||||
const releaseFilename = digestAsset.digestedFileName.replace(current, next);
|
||||
const res = await http.get(releaseAsset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineFn === releaseFilename) {
|
||||
return lineDigest;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const algorithm = inferHashAlg(digestAsset.currentDigest);
|
||||
const newDigest = await downloadAndDigest(releaseAsset, algorithm);
|
||||
return newDigest;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1,22 +1,186 @@
|
|||
import hasha from 'hasha';
|
||||
import { logger } from '../../logger';
|
||||
import * as packageCache from '../../util/cache/package';
|
||||
import { GithubHttp } from '../../util/http/github';
|
||||
import { newlineRegex, regEx } from '../../util/regex';
|
||||
import { ensureTrailingSlash } from '../../util/url';
|
||||
import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
|
||||
import {
|
||||
cacheNamespace,
|
||||
getApiBaseUrl,
|
||||
getGithubRelease,
|
||||
getSourceUrl,
|
||||
http,
|
||||
id,
|
||||
} from './common';
|
||||
import { findDigestAsset, mapDigestAssetToRelease } from './digest';
|
||||
import type { GithubRelease } from './types';
|
||||
import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types';
|
||||
|
||||
export { id };
|
||||
export const customRegistrySupport = true;
|
||||
export const defaultRegistryUrls = ['https://github.com'];
|
||||
export const registryStrategy = 'first';
|
||||
|
||||
const defaultSourceUrlBase = 'https://github.com/';
|
||||
export const id = 'github-releases';
|
||||
|
||||
export const cacheNamespace = 'datasource-github-releases';
|
||||
export const http = new GithubHttp(id);
|
||||
|
||||
async function findDigestFile(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset | null> {
|
||||
const smallAssets = release.assets.filter(
|
||||
(a: GithubReleaseAsset) => a.size < 5 * 1024
|
||||
);
|
||||
for (const asset of smallAssets) {
|
||||
const res = await http.get(asset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
digestedFileName: lineFn,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: lineDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function inferHashAlg(digest: string): string {
|
||||
switch (digest.length) {
|
||||
case 64:
|
||||
return 'sha256';
|
||||
default:
|
||||
case 96:
|
||||
return 'sha512';
|
||||
}
|
||||
}
|
||||
|
||||
function getAssetDigestCacheKey(
|
||||
downloadUrl: string,
|
||||
algorithm: string
|
||||
): string {
|
||||
const type = 'assetDigest';
|
||||
return `${downloadUrl}:${algorithm}:${type}`;
|
||||
}
|
||||
|
||||
async function downloadAndDigest(
|
||||
asset: GithubReleaseAsset,
|
||||
algorithm: string
|
||||
): Promise<string> {
|
||||
const downloadUrl = asset.browser_download_url;
|
||||
const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm);
|
||||
const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey);
|
||||
// istanbul ignore if
|
||||
if (cachedResult) {
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
const res = http.stream(downloadUrl);
|
||||
const digest = await hasha.fromStream(res, { algorithm });
|
||||
|
||||
const cacheMinutes = 1440;
|
||||
await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes);
|
||||
return digest;
|
||||
}
|
||||
|
||||
async function findAssetWithDigest(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset | null> {
|
||||
const algorithm = inferHashAlg(digest);
|
||||
const assetsBySize = release.assets.sort(
|
||||
(a: GithubReleaseAsset, b: GithubReleaseAsset) => {
|
||||
if (a.size < b.size) {
|
||||
return -1;
|
||||
}
|
||||
if (a.size > b.size) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
);
|
||||
|
||||
for (const asset of assetsBySize) {
|
||||
const assetDigest = await downloadAndDigest(asset, algorithm);
|
||||
if (assetDigest === digest) {
|
||||
return {
|
||||
assetName: asset.name,
|
||||
currentVersion: release.tag_name,
|
||||
currentDigest: assetDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Identify the asset associated with a known digest. */
|
||||
export async function findDigestAsset(
|
||||
release: GithubRelease,
|
||||
digest: string
|
||||
): Promise<DigestAsset> {
|
||||
const digestFile = await findDigestFile(release, digest);
|
||||
if (digestFile) {
|
||||
return digestFile;
|
||||
}
|
||||
|
||||
const asset = await findAssetWithDigest(release, digest);
|
||||
return asset;
|
||||
}
|
||||
|
||||
/** Given a digest asset, find the equivalent digest in a different release. */
|
||||
export async function mapDigestAssetToRelease(
|
||||
digestAsset: DigestAsset,
|
||||
release: GithubRelease
|
||||
): Promise<string | null> {
|
||||
const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
|
||||
const next = release.tag_name.replace(regEx(/^v/), '');
|
||||
const releaseChecksumAssetName = digestAsset.assetName.replace(current, next);
|
||||
const releaseAsset = release.assets.find(
|
||||
(a: GithubReleaseAsset) => a.name === releaseChecksumAssetName
|
||||
);
|
||||
if (!releaseAsset) {
|
||||
return null;
|
||||
}
|
||||
if (digestAsset.digestedFileName) {
|
||||
const releaseFilename = digestAsset.digestedFileName.replace(current, next);
|
||||
const res = await http.get(releaseAsset.browser_download_url);
|
||||
for (const line of res.body.split(newlineRegex)) {
|
||||
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
|
||||
if (lineFn === releaseFilename) {
|
||||
return lineDigest;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const algorithm = inferHashAlg(digestAsset.currentDigest);
|
||||
const newDigest = await downloadAndDigest(releaseAsset, algorithm);
|
||||
return newDigest;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getSourceUrlBase(registryUrl: string): string {
|
||||
// default to GitHub.com if no GHE host is specified.
|
||||
return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase);
|
||||
}
|
||||
|
||||
export function getApiBaseUrl(registryUrl: string): string {
|
||||
const sourceUrlBase = getSourceUrlBase(registryUrl);
|
||||
return sourceUrlBase === defaultSourceUrlBase
|
||||
? `https://api.github.com/`
|
||||
: `${sourceUrlBase}api/v3/`;
|
||||
}
|
||||
|
||||
export function getSourceUrl(lookupName: string, registryUrl?: string): string {
|
||||
const sourceUrlBase = getSourceUrlBase(registryUrl);
|
||||
return `${sourceUrlBase}${lookupName}`;
|
||||
}
|
||||
|
||||
export async function getGithubRelease(
|
||||
apiBaseUrl: string,
|
||||
repo: string,
|
||||
version: string
|
||||
): Promise<GithubRelease> {
|
||||
const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`;
|
||||
const res = await http.getJson<GithubRelease>(url);
|
||||
return res.body;
|
||||
}
|
||||
|
||||
function getReleasesCacheKey(registryUrl: string, repo: string): string {
|
||||
const type = 'tags';
|
||||
return `${registryUrl}:${repo}:${type}`;
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import { logger } from '../../logger';
|
||||
import * as packageCache from '../../util/cache/package';
|
||||
import { GithubHttp } from '../../util/http/github';
|
||||
import * as githubReleases from '../github-releases';
|
||||
import { getApiBaseUrl, getSourceUrl } from '../github-releases/common';
|
||||
import {
|
||||
getApiBaseUrl,
|
||||
getSourceUrl,
|
||||
getReleases as githubGetReleases,
|
||||
} from '../github-releases';
|
||||
import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
|
||||
import type { GitHubTag, TagResponse } from './types';
|
||||
|
||||
|
|
@ -158,7 +161,7 @@ export async function getReleases(
|
|||
|
||||
try {
|
||||
// Fetch additional data from releases endpoint when possible
|
||||
const releasesResult = await githubReleases.getReleases(config);
|
||||
const releasesResult = await githubGetReleases(config);
|
||||
const releaseByVersion = {};
|
||||
releasesResult?.releases?.forEach((release) => {
|
||||
const key = release.version;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { BitBucketTagsDatasource } from '../bitbucket-tags';
|
||||
import { getSourceUrl as githubSourceUrl } from '../github-releases/common';
|
||||
import { getSourceUrl as githubSourceUrl } from '../github-releases';
|
||||
import { id as githubDatasource } from '../github-tags';
|
||||
import { id as gitlabDatasource } from '../gitlab-tags';
|
||||
import { getSourceUrl as gitlabSourceUrl } from '../gitlab-tags/util';
|
||||
|
|
|
|||
Loading…
Reference in a new issue