feat: add datasource for Hermit package manager (#16257)

* feat: add datasource for Hermit package manager

* fix: make hermit datasource use http.stream instead of http.get

* chore: inline small function and return null on error in hermit datasource

* fix: use regExp to extract owner and repo from registryUrl

* fix: use http from super & use error instead of debug on datasource error

* fix: fix type check failure for github-release test

* Update lib/modules/datasource/hermit/index.spec.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.spec.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.spec.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* fix: wrap JSON.parse to avoid breaking process, plus more comment onto the search manifest streaming process

* fix: add test to cover the invalid json case

* fix: change some logger.errors to logger.warn in hermit datasource, error will exit the whole process

* Update lib/modules/datasource/hermit/index.spec.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.spec.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* Update lib/modules/datasource/hermit/index.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* fix: use parseUrl and use hermit as GithubHttp id

* fix: added test to include HermitDatasource type in GITHUB_API_USING_HOST_TYPES

* fix: handle invalid url & achieve code coverage

* fix: move url parsing to parent function

* Update lib/modules/datasource/hermit/index.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* chore: clean up HermitDataSource class doc

* fix: getHermitSearchManifest cache key function should expect undefined

* fix: pass GetReleasesConfig into getHermitSearchManifest to get proper caching working

* Update lib/modules/datasource/hermit/readme.md

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* Update lib/modules/datasource/hermit/readme.md

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* Update lib/modules/datasource/hermit/readme.md

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* Update lib/modules/datasource/hermit/readme.md

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* Update lib/modules/datasource/hermit/readme.md

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* chore: fix json block in datasource readme.md

* Put packageRules example in step 4

* fix: fix lint error

* fix: pass parsedUrl into getHermitSearchManifest, add warning and simplifies type

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

lib/constants/platform.spec.ts

@@ -4,6 +4,7 @@ import { GithubTagsDatasource } from '../modules/datasource/github-tags';
 import { GitlabPackagesDatasource } from '../modules/datasource/gitlab-packages';
 import { GitlabReleasesDatasource } from '../modules/datasource/gitlab-releases';
 import { GitlabTagsDatasource } from '../modules/datasource/gitlab-tags';
+import { HermitDatasource } from '../modules/datasource/hermit';
 import { PodDatasource } from '../modules/datasource/pod';
 import { id as GITHUB_CHANGELOG_ID } from '../workers/repository/update/pr/changelog/github';
 import { id as GITLAB_CHANGELOG_ID } from '../workers/repository/update/pr/changelog/gitlab';
@@ -43,6 +44,9 @@ describe('constants/platform', () => {
       GITHUB_API_USING_HOST_TYPES.includes(GithubReleasesDatasource.id)
     ).toBeTrue();
     expect(GITHUB_API_USING_HOST_TYPES.includes(PodDatasource.id)).toBeTrue();
+    expect(
+      GITHUB_API_USING_HOST_TYPES.includes(HermitDatasource.id)
+    ).toBeTrue();
     expect(
       GITHUB_API_USING_HOST_TYPES.includes(GITHUB_CHANGELOG_ID)
     ).toBeTrue();

lib/constants/platforms.ts

@@ -13,6 +13,7 @@ export const GITHUB_API_USING_HOST_TYPES = [
   'github-releases',
   'github-tags',
   'pod',
+  'hermit',
   'github-changelog',
 ];
 

lib/modules/datasource/api.ts

@@ -23,6 +23,7 @@ import { GoDatasource } from './go';
 import { GolangVersionDatasource } from './golang-version';
 import { GradleVersionDatasource } from './gradle-version';
 import { HelmDatasource } from './helm';
+import { HermitDatasource } from './hermit';
 import { HexDatasource } from './hex';
 import { JenkinsPluginsDatasource } from './jenkins-plugins';
 import { MavenDatasource } from './maven';
@@ -70,6 +71,7 @@ api.set(GoDatasource.id, new GoDatasource());
 api.set(GolangVersionDatasource.id, new GolangVersionDatasource());
 api.set(GradleVersionDatasource.id, new GradleVersionDatasource());
 api.set(HelmDatasource.id, new HelmDatasource());
+api.set(HermitDatasource.id, new HermitDatasource());
 api.set(HexDatasource.id, new HexDatasource());
 api.set(JenkinsPluginsDatasource.id, new JenkinsPluginsDatasource());
 api.set(MavenDatasource.id, new MavenDatasource());

lib/modules/datasource/github-releases/test/index.ts

@@ -24,10 +24,12 @@ export class GitHubReleaseMocker {
     });
     for (const assetFn of Object.keys(assets)) {
       const assetPath = `/repos/${this.packageName}/releases/download/${version}/${assetFn}`;
+      const urlPath = `/repos/${this.packageName}/releases/assets/${version}-${assetFn}`;
       const assetData = assets[assetFn];
       releaseData.assets.push({
         name: assetFn,
         size: assetData.length,
+        url: `${this.githubApiHost}${urlPath}`,
         browser_download_url: `${this.githubApiHost}${assetPath}`,
       });
       httpMock

lib/modules/datasource/github-releases/types.ts

@@ -13,6 +13,7 @@ export type GithubRelease = {
 
 export interface GithubReleaseAsset {
   name: string;
+  url: string;
   browser_download_url: string;
   size: number;
 }

lib/modules/datasource/hermit/index.spec.ts

@@ -0,0 +1,230 @@
+import * as httpMock from '../../../../test/http-mock';
+import type { HermitSearchResult } from './types';
+import { HermitDatasource } from './';
+
+const datasource = new HermitDatasource();
+const githubApiHost = 'https://api.github.com';
+const releaseUrl = '/repos/cashapp/hermit-packages/releases/tags/index';
+const indexAssetUrl = '/repos/cashapp/hermit-packages/releases/assets/38492';
+const sourceAssetUrl = '/repos/cashapp/hermit-packages/releases/assets/38492';
+const registryUrl = 'https://github.com/cashapp/hermit-packages';
+
+describe('modules/datasource/hermit/index', () => {
+  describe('getReleases', () => {
+    it('should return result from hermit list', async () => {
+      const resp: HermitSearchResult[] = [
+        {
+          Name: 'go',
+          Versions: ['1.17.9', '1.17.10', '1.18', '1.18.1'],
+          Channels: ['@1.17', '@1.18'],
+          CurrentVersion: '1.17.9',
+          Repository: 'https://github.com/golang/golang',
+          Description: 'golang',
+        },
+      ];
+      httpMock
+        .scope(githubApiHost)
+        .get(releaseUrl)
+        .reply(200, {
+          assets: [
+            {
+              name: 'source.tar.gz',
+              url: `${githubApiHost}${sourceAssetUrl}`,
+            },
+            {
+              name: 'index.json',
+              url: `${githubApiHost}${indexAssetUrl}`,
+            },
+          ],
+        });
+
+      httpMock.scope(githubApiHost).get(indexAssetUrl).reply(200, resp);
+
+      const res = await datasource.getReleases({
+        packageName: 'go',
+        registryUrl,
+      });
+
+      expect(res).toStrictEqual({
+        releases: [
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '1.17.9',
+          },
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '1.17.10',
+          },
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '1.18',
+          },
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '1.18.1',
+          },
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '@1.17',
+          },
+          {
+            sourceUrl: 'https://github.com/golang/golang',
+            version: '@1.18',
+          },
+        ],
+        sourceUrl: 'https://github.com/golang/golang',
+      });
+    });
+
+    it('should fail on no result found', async () => {
+      httpMock
+        .scope(githubApiHost)
+        .get(releaseUrl)
+        .reply(200, {
+          assets: [
+            {
+              name: 'source.tar.gz',
+              url: `${githubApiHost}${sourceAssetUrl}`,
+            },
+            {
+              name: 'index.json',
+              url: `${githubApiHost}${indexAssetUrl}`,
+            },
+          ],
+        });
+
+      httpMock.scope(githubApiHost).get(indexAssetUrl).reply(200, []);
+
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl,
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should fail on network error', async () => {
+      httpMock
+        .scope(githubApiHost)
+        .get(releaseUrl)
+        .reply(200, {
+          assets: [
+            {
+              name: 'source.tar.gz',
+              url: `${githubApiHost}${sourceAssetUrl}`,
+            },
+            {
+              name: 'index.json',
+              url: `${githubApiHost}${indexAssetUrl}`,
+            },
+          ],
+        });
+
+      httpMock.scope(githubApiHost).get(indexAssetUrl).reply(404);
+
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl,
+        })
+      ).rejects.toThrow();
+    });
+
+    it('should get null result on non github url given', async () => {
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl: 'https://gitlab.com/owner/project',
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should get null result on missing repo or owner', async () => {
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl: 'https://github.com/test',
+        })
+      ).resolves.toBeNull();
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl: 'https://github.com/',
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should get null for extra path provided in registry url', async () => {
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl: 'https://github.com/test/repo/extra-path',
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should get null result on empty registryUrl', async () => {
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should fail on missing index.json asset', async () => {
+      httpMock
+        .scope(githubApiHost)
+        .get(releaseUrl)
+        .reply(200, {
+          assets: [
+            {
+              name: 'source.tar.gz',
+              url: `${githubApiHost}${sourceAssetUrl}`,
+            },
+          ],
+        });
+
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl,
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should get null on invalid index.json asset', async () => {
+      httpMock
+        .scope(githubApiHost)
+        .get(releaseUrl)
+        .reply(200, {
+          assets: [
+            {
+              name: 'index.json',
+              url: `${githubApiHost}${indexAssetUrl}`,
+            },
+          ],
+        });
+
+      httpMock
+        .scope(githubApiHost)
+        .get(indexAssetUrl)
+        .reply(200, 'invalid content');
+
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl,
+        })
+      ).resolves.toBeNull();
+    });
+
+    it('should get null on invalid registry url', async () => {
+      await expect(
+        datasource.getReleases({
+          packageName: 'go',
+          registryUrl: 'invalid url',
+        })
+      ).resolves.toBeNull();
+    });
+  });
+});

lib/modules/datasource/hermit/index.ts

@@ -0,0 +1,160 @@
+import { logger } from '../../../logger';
+import { cache } from '../../../util/cache/package/decorator';
+import { GithubHttp } from '../../../util/http/github';
+import { regEx } from '../../../util/regex';
+import { streamToString } from '../../../util/streams';
+import { parseUrl } from '../../../util/url';
+import { id } from '../../versioning/hermit';
+import { Datasource } from '../datasource';
+import { getApiBaseUrl } from '../github-releases/common';
+import type { GithubRelease } from '../github-releases/types';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+import type { HermitSearchResult } from './types';
+
+/**
+ * Hermit Datasource searches a given package from the specified `hermit-packages`
+ * repository. It expects the search manifest to come from an asset `index.json` from
+ * a release named index.
+ */
+export class HermitDatasource extends Datasource {
+  static readonly id = 'hermit';
+
+  override readonly customRegistrySupport = true;
+
+  override readonly registryStrategy = 'first';
+
+  override readonly defaultVersioning = id;
+
+  override readonly defaultRegistryUrls = [
+    'https://github.com/cashapp/hermit-packages',
+  ];
+
+  pathRegex: RegExp;
+
+  constructor() {
+    super(HermitDatasource.id);
+    this.http = new GithubHttp(id);
+    this.pathRegex = regEx('^\\/(?<owner>[^/]+)\\/(?<repo>[^/]+)$');
+  }
+
+  @cache({
+    namespace: `datasource-hermit-package`,
+    key: ({ registryUrl, packageName }: GetReleasesConfig) =>
+      `${registryUrl ?? ''}-${packageName}`,
+  })
+  async getReleases({
+    packageName,
+    registryUrl,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    logger.trace(`HermitDataSource.getReleases()`);
+
+    if (!registryUrl) {
+      logger.error('registryUrl must be supplied');
+      return null;
+    }
+
+    const parsedUrl = parseUrl(registryUrl);
+    if (parsedUrl === null) {
+      logger.warn({ registryUrl }, 'invalid registryUrl given');
+      return null;
+    }
+
+    if (!registryUrl.startsWith('https://github.com/')) {
+      logger.warn({ registryUrl }, 'Only Github registryUrl is supported');
+      return null;
+    }
+
+    const items = await this.getHermitSearchManifest(parsedUrl);
+
+    if (items === null) {
+      return null;
+    }
+
+    const res = items.find((i) => i.Name === packageName);
+
+    if (!res) {
+      logger.debug({ packageName, registryUrl }, 'cannot find hermit package');
+      return null;
+    }
+
+    const sourceUrl = res.Repository;
+
+    return {
+      sourceUrl,
+      releases: [
+        ...res.Versions.map((v) => ({
+          version: v,
+          sourceUrl,
+        })),
+        ...res.Channels.map((v) => ({
+          version: v,
+          sourceUrl,
+        })),
+      ],
+    };
+  }
+
+  /**
+   * getHermitSearchManifest fetch the index.json from release
+   * named index, parses it and returned the parsed JSON result
+   */
+  @cache({
+    namespace: `datasource-hermit-search-manifest`,
+    key: (u) => u.toString(),
+  })
+  async getHermitSearchManifest(u: URL): Promise<HermitSearchResult[] | null> {
+    const registryUrl = u.toString();
+    const host = u.host ?? '';
+    const groups = this.pathRegex.exec(u.pathname ?? '')?.groups;
+    if (!groups) {
+      logger.warn(
+        { registryUrl },
+        'failed to get owner and repo from given url'
+      );
+      return null;
+    }
+
+    const { owner, repo } = groups;
+
+    const apiBaseUrl = getApiBaseUrl(`https://${host}`);
+
+    const indexRelease = await this.http.getJson<GithubRelease>(
+      `${apiBaseUrl}repos/${owner}/${repo}/releases/tags/index`
+    );
+
+    // finds asset with name index.json
+    const asset = indexRelease.body.assets.find(
+      (asset) => asset.name === 'index.json'
+    );
+
+    if (!asset) {
+      logger.warn(
+        { registryUrl },
+        `can't find asset index.json in the given registryUrl`
+      );
+      return null;
+    }
+
+    // stream down the content of index.json
+    // Note: need to use stream here with
+    // the accept header as octet-stream to
+    // download asset from private github repository
+    // see GithubDoc:
+    // https://docs.github.com/en/rest/releases/assets#get-a-release-asset
+    const indexContent = await streamToString(
+      this.http.stream(asset.url, {
+        headers: {
+          accept: 'application/octet-stream',
+        },
+      })
+    );
+
+    try {
+      return JSON.parse(indexContent) as HermitSearchResult[];
+    } catch (e) {
+      logger.warn('error parsing hermit search manifest from remote respond');
+    }
+
+    return null;
+  }
+}

lib/modules/datasource/hermit/readme.md

@@ -0,0 +1,22 @@
+By default [Hermit](https://cashapp.github.io/hermit/) looks up packages from the open source project [https://github.com/cashapp/hermit-packages](https://github.com/cashapp/hermit-packages).
+
+Hermit supports [private packages](https://cashapp.github.io/hermit/packaging/private/).
+To get Renovate to find your private packages, follow these steps:
+
+1. perform `hermit search --json` with your private Hermit distribution and save the file to `index.json`
+1. make a GitHub release in your private packages repository named `index` with the asset `index.json` generated in step 1.
+1. setup a CI pipeline to repeat step 1 & 2 on new commits to the private packages repository.
+1. Add a package rule for the Hermit manager, so that Renovate knows where to find your private packages:
+
+   ```json
+   {
+     "packageRules": [
+       {
+         "matchManagers": ["hermit"],
+         "defaultRegistryUrls": [
+           "https://github.com/your/private-hermit-packages"
+         ]
+       }
+     ]
+   }
+   ```

lib/modules/datasource/hermit/types.ts

@@ -0,0 +1,8 @@
+export interface HermitSearchResult {
+  Name: string;
+  Versions: string[];
+  Channels: string[];
+  CurrentVersion: string;
+  Description: string;
+  Repository: string;
+}