feat: add git-refs datasource (#5727)

lib/datasource/git-refs/index.spec.ts

@@ -0,0 +1,47 @@
+import _simpleGit from 'simple-git/promise';
+import { getPkgReleases } from '.';
+
+jest.mock('simple-git/promise');
+const simpleGit: any = _simpleGit;
+
+const lookupName = 'https://github.com/example/example.git';
+
+describe('datasource/git-refs', () => {
+  beforeEach(() => global.renovateCache.rmAll());
+  describe('getPkgReleases', () => {
+    it('returns nil if response is wrong', async () => {
+      simpleGit.mockReturnValue({
+        listRemote() {
+          return Promise.resolve(null);
+        },
+      });
+      const versions = await getPkgReleases({ lookupName });
+      expect(versions).toEqual(null);
+    });
+    it('returns nil if remote call throws exception', async () => {
+      simpleGit.mockReturnValue({
+        listRemote() {
+          throw new Error();
+        },
+      });
+      const versions = await getPkgReleases({ lookupName });
+      expect(versions).toEqual(null);
+    });
+    it('returns versions filtered from tags', async () => {
+      simpleGit.mockReturnValue({
+        listRemote() {
+          return Promise.resolve(
+            'commithash1\trefs/tags/0.0.1\ncommithash2\trefs/tags/v0.0.2\ncommithash3\trefs/tags/v0.0.2^{}\ncommithash4\trefs/heads/v0.0.3\ncommithash5\trefs/tags/v0.0.3'
+          );
+        },
+      });
+
+      const versions = await getPkgReleases({
+        lookupName,
+      });
+
+      const result = versions.releases.map(x => x.version).sort();
+      expect(result).toEqual(['0.0.1', 'v0.0.2', 'v0.0.3']);
+    });
+  });
+});

lib/datasource/git-refs/index.ts

@@ -0,0 +1,84 @@
+import simpleGit from 'simple-git/promise';
+import * as semver from '../../versioning/semver';
+import { logger } from '../../logger';
+import { ReleaseResult, GetReleasesConfig } from '../common';
+
+export const id = 'git-refs';
+
+const cacheMinutes = 10;
+
+// git will prompt for known hosts or passwords, unless we activate BatchMode
+process.env.GIT_SSH_COMMAND = 'ssh -o BatchMode=yes';
+
+export interface RawRefs {
+  type: string;
+  value: string;
+}
+
+export async function getRawRefs({
+  lookupName,
+}: GetReleasesConfig): Promise<RawRefs[] | null> {
+  const git = simpleGit();
+  try {
+    const cacheNamespace = 'git-raw-refs';
+
+    const cachedResult = await renovateCache.get<RawRefs[]>(
+      cacheNamespace,
+      lookupName
+    );
+    /* istanbul ignore next line */
+    if (cachedResult) {
+      return cachedResult;
+    }
+
+    // fetch remote tags
+    const lsRemote = await git.listRemote([lookupName, '--sort=-v:refname']);
+
+    if (!lsRemote) {
+      return null;
+    }
+
+    const refs = lsRemote.replace(/^.+?refs\//gm, '').split('\n');
+
+    const result = refs.map(ref => ({
+      type: /(.*?)\//.exec(ref)[1],
+      value: /\/(.*)/.exec(ref)[1],
+    }));
+
+    await renovateCache.set(cacheNamespace, lookupName, result, cacheMinutes);
+    return result;
+  } catch (err) {
+    logger.debug({ err }, `Git-Raw-Refs lookup error in ${lookupName}`);
+  }
+  return null;
+}
+
+export async function getPkgReleases({
+  lookupName,
+}: GetReleasesConfig): Promise<ReleaseResult | null> {
+  try {
+    const rawRefs: RawRefs[] = await getRawRefs({ lookupName });
+
+    const refs = rawRefs
+      .filter(ref => ref.type === 'tags' || ref.type === 'heads')
+      .map(ref => ref.value)
+      .filter(ref => semver.isVersion(ref));
+
+    const uniqueRefs = [...new Set(refs)];
+
+    const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, '');
+
+    const result: ReleaseResult = {
+      sourceUrl,
+      releases: uniqueRefs.map(ref => ({
+        version: ref,
+        gitRef: ref,
+      })),
+    };
+
+    return result;
+  } catch (err) {
+    logger.debug({ err }, `Git-Refs lookup error in ${lookupName}`);
+  }
+  return null;
+}

lib/datasource/git-submodules/index.ts

@@ -44,7 +44,7 @@ export async function getPkgReleases({
     await renovateCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
     return result;
   } catch (err) {
-    logger.debug(`Error looking up tags in ${lookupName}`);
+    logger.debug({ err }, `Git-SubModules lookup error in ${lookupName}`);
   }
   return null;
 }

lib/datasource/git-tags/index.spec.ts

@@ -4,7 +4,6 @@ import { getPkgReleases } from '.';
 jest.mock('simple-git/promise');
 const simpleGit: any = _simpleGit;
 
-// const lookupName = 'vapor';
 const lookupName = 'https://github.com/example/example.git';
 
 describe('datasource/git-tags', () => {

lib/datasource/git-tags/index.ts

@@ -1,42 +1,24 @@
-import simpleGit from 'simple-git/promise';
+import { ReleaseResult, GetReleasesConfig } from '../common';
 import * as semver from '../../versioning/semver';
 import { logger } from '../../logger';
-import { ReleaseResult, GetReleasesConfig } from '../common';
+import * as gitRefs from '../git-refs';
 
 export const id = 'git-tags';
 
-const cacheNamespace = 'git-tags';
-const cacheMinutes = 10;
-
-// git will prompt for known hosts or passwords, unless we activate BatchMode
-process.env.GIT_SSH_COMMAND = 'ssh -o BatchMode=yes';
-
 export async function getPkgReleases({
   lookupName,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  const git = simpleGit();
   try {
-    const cachedResult = await renovateCache.get<ReleaseResult>(
-      cacheNamespace,
-      lookupName
-    );
-    /* istanbul ignore next line */
-    if (cachedResult) {
-      return cachedResult;
-    }
-
     // fetch remote tags
-    const lsRemote = await git.listRemote([
+    const rawRefs: gitRefs.RawRefs[] = await gitRefs.getRawRefs({ lookupName });
-      '--sort=-v:refname',
+
-      '--tags',
+    const tags = rawRefs
-      lookupName,
+      .filter(ref => ref.type === 'tags')
-    ]);
+      .map(ref => ref.value)
-    // extract valid tags from git ls-remote which looks like 'commithash\trefs/tags/1.2.3
-    const tags = lsRemote
-      .replace(/^.+?refs\/tags\//gm, '')
-      .split('\n')
       .filter(tag => semver.isVersion(tag));
+
     const sourceUrl = lookupName.replace(/\.git$/, '').replace(/\/$/, '');
+
     const result: ReleaseResult = {
       sourceUrl,
       releases: tags.map(tag => ({
@@ -45,10 +27,9 @@ export async function getPkgReleases({
       })),
     };
 
-    await renovateCache.set(cacheNamespace, lookupName, result, cacheMinutes);
     return result;
-  } catch (e) {
+  } catch (err) {
-    logger.debug(`Error looking up tags in ${lookupName}`);
+    logger.debug({ err }, `Git-Tags lookup error in ${lookupName}`);
   }
   return null;
 }