renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-10 14:06:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

feat(datasource/azure-pipeline-tasks): read tasks from cdn (#22864)

Browse source
This commit is contained in:
Jamie Magee 2023-06-20 22:42:10 -07:00 committed by GitHub
parent d705796b3d
commit a51d6d9444
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 13649 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13559
data/azure-pipelines-tasks.json
View file

File diff suppressed because it is too large Load diff

19
lib/modules/datasource/azure-pipelines-tasks/index.spec.ts
View file

@ -3,12 +3,19 @@ import * as httpMock from '../../../../test/http-mock';
import { AzurePipelinesTasksDatasource } from '.';
const gitHubHost = 'https://raw.githubusercontent.com';
const builtinTasksPath =
'/renovatebot/azure-devops-marketplace/main/azure-pipelines-builtin-tasks.json';
const marketplaceTasksPath =
'/renovatebot/azure-devops-marketplace/main/azure-pipelines-marketplace-tasks.json';
describe('modules/datasource/azure-pipelines-tasks/index', () => {
it('returns null for unknown task', async () => {
httpMock.scope(gitHubHost).get(marketplaceTasksPath).reply(200, {});
httpMock
.scope(gitHubHost)
.get(builtinTasksPath)
.reply(200, {})
.get(marketplaceTasksPath)
.reply(200, {});
expect(
await getPkgReleases({
datasource: AzurePipelinesTasksDatasource.id,
@ -18,6 +25,10 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
});
it('supports built-in tasks', async () => {
httpMock
.scope(gitHubHost)
.get(builtinTasksPath)
.reply(200, { automatedanalysis: ['0.171.0', '0.198.0'] });
expect(
await getPkgReleases({
datasource: AzurePipelinesTasksDatasource.id,
@ -29,6 +40,8 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
it('supports marketplace tasks', async () => {
httpMock
.scope(gitHubHost)
.get(builtinTasksPath)
.reply(200, {})
.get(marketplaceTasksPath)
.reply(200, { 'automatedanalysis-marketplace': ['0.171.0', '0.198.0'] });
expect(
@ -40,6 +53,10 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
});
it('is case insensitive', async () => {
httpMock
.scope(gitHubHost)
.get(builtinTasksPath)
.reply(200, { automatedanalysis: ['0.171.0', '0.198.0'] });
expect(
await getPkgReleases({
datasource: AzurePipelinesTasksDatasource.id,

28
lib/modules/datasource/azure-pipelines-tasks/index.ts
View file

@ -1,22 +1,18 @@
import dataFiles from '../../../data-files.generated';
import { cache } from '../../../util/cache/package/decorator';
import { id as versioning } from '../../versioning/loose';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types';
const MARKETPLACE_TASKS_URL =
'https://raw.githubusercontent.com/renovatebot/azure-devops-marketplace/main/azure-pipelines-marketplace-tasks.json';
const TASKS_URL_BASE =
'https://raw.githubusercontent.com/renovatebot/azure-devops-marketplace/main';
const BUILT_IN_TASKS_URL = `${TASKS_URL_BASE}/azure-pipelines-builtin-tasks.json`;
const MARKETPLACE_TASKS_URL = `${TASKS_URL_BASE}/azure-pipelines-marketplace-tasks.json`;
export class AzurePipelinesTasksDatasource extends Datasource {
static readonly id = 'azure-pipelines-tasks';
private readonly builtInTasks: Record<string, string[]>;
constructor() {
super(AzurePipelinesTasksDatasource.id);
this.builtInTasks = JSON.parse(
dataFiles.get('data/azure-pipelines-tasks.json')!
);
}
override readonly customRegistrySupport = false;
@ -27,26 +23,24 @@ export class AzurePipelinesTasksDatasource extends Datasource {
packageName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
const versions =
this.builtInTasks[packageName.toLowerCase()] ??
(await this.getMarketplaceTasks())[packageName.toLowerCase()];
(await this.getTasks(BUILT_IN_TASKS_URL))[packageName.toLowerCase()] ??
(await this.getTasks(MARKETPLACE_TASKS_URL))[packageName.toLowerCase()];
if (versions) {
const releases = versions.map((version) => ({ version }));
return Promise.resolve({ releases });
return { releases };
}
return Promise.resolve(null);
return null;
}
@cache({
namespace: `datasource-${AzurePipelinesTasksDatasource.id}`,
key: 'azure-pipelines-marketplace-tasks',
key: (url: string) => url,
ttlMinutes: 24 * 60,
})
async getMarketplaceTasks(): Promise<Record<string, string[]>> {
const { body } = await this.http.getJson<Record<string, string[]>>(
MARKETPLACE_TASKS_URL
);
async getTasks(url: string): Promise<Record<string, string[]>> {
const { body } = await this.http.getJson<Record<string, string[]>>(url);
return body;
}
}

72
tools/static-data/generate-azure-pipelines-tasks.mjs
View file

@ -1,72 +0,0 @@
import os from 'node:os';
import fs from 'fs-extra';
import { glob } from 'glob';
import JSON5 from 'json5';
import Git from 'simple-git';
import path from 'upath';
import { updateJsonFile } from './utils.mjs';
const localPath = path.join(os.tmpdir(), 'azure-pipelines-tasks');
/**
* This script:
* 1. Clones the Azure Pipelines Tasks repo
* 2. Finds all `task.json` files
* 3. For each `task.json` it finds each commit that has that file
* 4. For each commit it gets the `task.json` content and extracts the task name, id and version
* 5. After all the `task.json` files have been processed it writes the results to `./data/azure-pipelines-tasks.json`
*/
await (async () => {
console.log('Generating azure pipelines tasks');
await fs.ensureDir(localPath);
const git = Git(localPath);
if (await git.checkIsRepo()) {
await git.pull();
} else {
await git.clone(
'https://github.com/microsoft/azure-pipelines-tasks.git',
'.'
);
}
// Find all `task.json` files
const files = (await glob(path.join(localPath, '**/task.json'))).map((file) =>
file.replace(`${localPath}/`, '')
);
/** @type {Record<string, Set<string>>} */
const tasks = {};
for (const file of files) {
// Find all commits that have the file
const revs = (await git.raw(['rev-list', 'HEAD', '--', file])).split('\n');
console.log(`Parsing ${file}`);
for (const rev of revs) {
try {
// Get the content of the file at the commit
const content = await git.show([`${rev}:${file}`]);
/** @type {{name: string, id: string, version: {Major: number, Minor: number, Patch: number}}} */
const parsedContent = JSON5.parse(content);
const version = `${parsedContent.version.Major}.${parsedContent.version.Minor}.${parsedContent.version.Patch}`;
tasks[parsedContent.name.toLowerCase()] =
tasks[parsedContent.name.toLowerCase()]?.add(version) ??
new Set([version]);
tasks[parsedContent.id.toLowerCase()] =
tasks[parsedContent.id.toLowerCase()]?.add(version) ??
new Set([version]);
} catch (e) {
console.error(`Failed to parse ${file} at ${rev}`);
console.error(e.toString());
}
}
}
const data = JSON.stringify(
tasks,
(_, value) => (value instanceof Set ? [...value] : value),
2
);
await updateJsonFile(`./data/azure-pipelines-tasks.json`, data);
})();