feat(datasource/azure-pipeline-tasks): read tasks from cdn (#22864)

lib/modules/datasource/azure-pipelines-tasks/index.spec.ts

@@ -3,12 +3,19 @@ import * as httpMock from '../../../../test/http-mock';
 import { AzurePipelinesTasksDatasource } from '.';
 
 const gitHubHost = 'https://raw.githubusercontent.com';
+const builtinTasksPath =
+  '/renovatebot/azure-devops-marketplace/main/azure-pipelines-builtin-tasks.json';
 const marketplaceTasksPath =
   '/renovatebot/azure-devops-marketplace/main/azure-pipelines-marketplace-tasks.json';
 
 describe('modules/datasource/azure-pipelines-tasks/index', () => {
   it('returns null for unknown task', async () => {
-    httpMock.scope(gitHubHost).get(marketplaceTasksPath).reply(200, {});
+    httpMock
+      .scope(gitHubHost)
+      .get(builtinTasksPath)
+      .reply(200, {})
+      .get(marketplaceTasksPath)
+      .reply(200, {});
     expect(
       await getPkgReleases({
         datasource: AzurePipelinesTasksDatasource.id,
@@ -18,6 +25,10 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
   });
 
   it('supports built-in tasks', async () => {
+    httpMock
+      .scope(gitHubHost)
+      .get(builtinTasksPath)
+      .reply(200, { automatedanalysis: ['0.171.0', '0.198.0'] });
     expect(
       await getPkgReleases({
         datasource: AzurePipelinesTasksDatasource.id,
@@ -29,6 +40,8 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
   it('supports marketplace tasks', async () => {
     httpMock
       .scope(gitHubHost)
+      .get(builtinTasksPath)
+      .reply(200, {})
       .get(marketplaceTasksPath)
       .reply(200, { 'automatedanalysis-marketplace': ['0.171.0', '0.198.0'] });
     expect(
@@ -40,6 +53,10 @@ describe('modules/datasource/azure-pipelines-tasks/index', () => {
   });
 
   it('is case insensitive', async () => {
+    httpMock
+      .scope(gitHubHost)
+      .get(builtinTasksPath)
+      .reply(200, { automatedanalysis: ['0.171.0', '0.198.0'] });
     expect(
       await getPkgReleases({
         datasource: AzurePipelinesTasksDatasource.id,

lib/modules/datasource/azure-pipelines-tasks/index.ts

@@ -1,22 +1,18 @@
-import dataFiles from '../../../data-files.generated';
 import { cache } from '../../../util/cache/package/decorator';
 import { id as versioning } from '../../versioning/loose';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
-const MARKETPLACE_TASKS_URL =
+const TASKS_URL_BASE =
-  'https://raw.githubusercontent.com/renovatebot/azure-devops-marketplace/main/azure-pipelines-marketplace-tasks.json';
+  'https://raw.githubusercontent.com/renovatebot/azure-devops-marketplace/main';
+const BUILT_IN_TASKS_URL = `${TASKS_URL_BASE}/azure-pipelines-builtin-tasks.json`;
+const MARKETPLACE_TASKS_URL = `${TASKS_URL_BASE}/azure-pipelines-marketplace-tasks.json`;
 
 export class AzurePipelinesTasksDatasource extends Datasource {
   static readonly id = 'azure-pipelines-tasks';
 
-  private readonly builtInTasks: Record<string, string[]>;
-
   constructor() {
     super(AzurePipelinesTasksDatasource.id);
-    this.builtInTasks = JSON.parse(
-      dataFiles.get('data/azure-pipelines-tasks.json')!
-    );
   }
 
   override readonly customRegistrySupport = false;
@@ -27,26 +23,24 @@ export class AzurePipelinesTasksDatasource extends Datasource {
     packageName,
   }: GetReleasesConfig): Promise<ReleaseResult | null> {
     const versions =
-      this.builtInTasks[packageName.toLowerCase()] ??
+      (await this.getTasks(BUILT_IN_TASKS_URL))[packageName.toLowerCase()] ??
-      (await this.getMarketplaceTasks())[packageName.toLowerCase()];
+      (await this.getTasks(MARKETPLACE_TASKS_URL))[packageName.toLowerCase()];
 
     if (versions) {
       const releases = versions.map((version) => ({ version }));
-      return Promise.resolve({ releases });
+      return { releases };
     }
 
-    return Promise.resolve(null);
+    return null;
   }
 
   @cache({
     namespace: `datasource-${AzurePipelinesTasksDatasource.id}`,
-    key: 'azure-pipelines-marketplace-tasks',
+    key: (url: string) => url,
     ttlMinutes: 24 * 60,
   })
-  async getMarketplaceTasks(): Promise<Record<string, string[]>> {
+  async getTasks(url: string): Promise<Record<string, string[]>> {
-    const { body } = await this.http.getJson<Record<string, string[]>>(
+    const { body } = await this.http.getJson<Record<string, string[]>>(url);
-      MARKETPLACE_TASKS_URL
-    );
     return body;
   }
 }

tools/static-data/generate-azure-pipelines-tasks.mjs

@@ -1,72 +0,0 @@
-import os from 'node:os';
-import fs from 'fs-extra';
-import { glob } from 'glob';
-import JSON5 from 'json5';
-import Git from 'simple-git';
-import path from 'upath';
-import { updateJsonFile } from './utils.mjs';
-
-const localPath = path.join(os.tmpdir(), 'azure-pipelines-tasks');
-
-/**
- * This script:
- *  1. Clones the Azure Pipelines Tasks repo
- *  2. Finds all `task.json` files
- *  3. For each `task.json` it finds each commit that has that file
- *  4. For each commit it gets the `task.json` content and extracts the task name, id and version
- *  5. After all the `task.json` files have been processed it writes the results to `./data/azure-pipelines-tasks.json`
- */
-await (async () => {
-  console.log('Generating azure pipelines tasks');
-  await fs.ensureDir(localPath);
-  const git = Git(localPath);
-
-  if (await git.checkIsRepo()) {
-    await git.pull();
-  } else {
-    await git.clone(
-      'https://github.com/microsoft/azure-pipelines-tasks.git',
-      '.'
-    );
-  }
-
-  // Find all `task.json` files
-  const files = (await glob(path.join(localPath, '**/task.json'))).map((file) =>
-    file.replace(`${localPath}/`, '')
-  );
-
-  /** @type {Record<string, Set<string>>} */
-  const tasks = {};
-
-  for (const file of files) {
-    // Find all commits that have the file
-    const revs = (await git.raw(['rev-list', 'HEAD', '--', file])).split('\n');
-    console.log(`Parsing ${file}`);
-    for (const rev of revs) {
-      try {
-        // Get the content of the file at the commit
-        const content = await git.show([`${rev}:${file}`]);
-        /** @type {{name: string, id: string, version: {Major: number, Minor: number, Patch: number}}} */
-        const parsedContent = JSON5.parse(content);
-        const version = `${parsedContent.version.Major}.${parsedContent.version.Minor}.${parsedContent.version.Patch}`;
-        tasks[parsedContent.name.toLowerCase()] =
-          tasks[parsedContent.name.toLowerCase()]?.add(version) ??
-          new Set([version]);
-        tasks[parsedContent.id.toLowerCase()] =
-          tasks[parsedContent.id.toLowerCase()]?.add(version) ??
-          new Set([version]);
-      } catch (e) {
-        console.error(`Failed to parse ${file} at ${rev}`);
-        console.error(e.toString());
-      }
-    }
-  }
-
-  const data = JSON.stringify(
-    tasks,
-    (_, value) => (value instanceof Set ? [...value] : value),
-    2
-  );
-
-  await updateJsonFile(`./data/azure-pipelines-tasks.json`, data);
-})();