feat(go): support self hosted go modules in gitlab (#8876)

2025-01-12 15:06:27 +00:00 · 2021-03-01 17:35:35 +00:00 · 2021-03-01 17:35:35 +00:00 · e3ccc26af7
commit e3ccc26af7
parent f4d5514281
7 changed files with 256 additions and 7 deletions
--- a/lib/datasource/go/snapshots/index.spec.ts.snap
+++ b/lib/datasource/go/snapshots/index.spec.ts.snap
@ -398,6 +398,48 @@ Array [
 ]
 `;

+exports[`datasource/go getReleases support self hosted gitlab private repositories 1`] = `
+Object {
+  "releases": Array [
+    Object {
+      "gitRef": "v1.0.0",
+      "version": "v1.0.0",
+    },
+    Object {
+      "gitRef": "v2.0.0",
+      "version": "v2.0.0",
+    },
+  ],
+  "sourceUrl": "https://my.custom.domain/golang/myrepo",
+}
+`;
+
+exports[`datasource/go getReleases support self hosted gitlab private repositories 2`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer some-token",
+      "host": "my.custom.domain",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://my.custom.domain/golang/myrepo?go-get=1",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/json",
+      "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer some-token",
+      "host": "my.custom.domain",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://my.custom.domain/api/v4/projects/golang%2Fmyrepo/repository/tags?per_page=100",
+  },
+]
+`;
+
 exports[`datasource/go getReleases unknown datasource returns null 1`] = `null`;

 exports[`datasource/go getReleases unknown datasource returns null 2`] = `
--- a/lib/datasource/go/index.spec.ts
+++ b/lib/datasource/go/index.spec.ts
@ -1,8 +1,13 @@
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../test/http-mock';
-import { logger } from '../../../test/util';
+import { logger, mocked } from '../../../test/util';
+import * as _hostRules from '../../util/host-rules';
 import { id as datasource, getDigest } from '.';

+jest.mock('../../util/host-rules');
+
+const hostRules = mocked(_hostRules);
+
 const res1 = `<!DOCTYPE html>
 <html>
 <head>
@ -16,6 +21,17 @@ Nothing to see here; <a href="https://godoc.org/golang.org/x/text">move along</a
 </body>
 </html>`;

+const resGitLabEE = `<html>
+<head>
+	<meta name="go-import" content="my.custom.domain/golang/myrepo git https://my.custom.domain/golang/myrepo.git" />
+	<meta name="go-source"
+		content="my.custom.domain/golang/myrepo https://my.custom.domain/golang/myrepo https://my.custom.domain/golang/myrepo/-/tree/master{/dir} https://my.custom.domain/golang/myrepo/-/blob/master{/dir}/{file}#L{line}" />
+</head>
+
+<body>go get https://my.custom.domain/golang/myrepo</body>
+
+</html>`;
+
 const resGitHubEnterprise = `<!DOCTYPE html>
 <html lang="en">
 <head>
@ -33,10 +49,13 @@ const resGitHubEnterprise = `<!DOCTYPE html>
 describe('datasource/go', () => {
  beforeEach(() => {
    httpMock.setup();
+    hostRules.find.mockReturnValue({});
+    hostRules.hosts.mockReturnValue([]);
  });

  afterEach(() => {
    httpMock.reset();
+    jest.resetAllMocks();
  });

  describe('getDigest', () => {
@ -194,6 +213,25 @@ describe('datasource/go', () => {
      expect(res).toBeDefined();
      expect(httpMock.getTrace()).toMatchSnapshot();
    });
+    it('support self hosted gitlab private repositories', async () => {
+      hostRules.find.mockReturnValue({ token: 'some-token' });
+      httpMock
+        .scope('https://my.custom.domain/')
+        .get('/golang/myrepo?go-get=1')
+        .reply(200, resGitLabEE);
+      httpMock
+        .scope('https://my.custom.domain/')
+        .get('/api/v4/projects/golang%2Fmyrepo/repository/tags?per_page=100')
+        .reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }]);
+      const res = await getPkgReleases({
+        datasource,
+        depName: 'my.custom.domain/golang/myrepo',
+      });
+      expect(res).toMatchSnapshot();
+      expect(res).not.toBeNull();
+      expect(res).toBeDefined();
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
    it('support bitbucket tags', async () => {
      httpMock
        .scope('https://api.bitbucket.org/')
@ -216,7 +254,7 @@ describe('datasource/go', () => {
      httpMock
        .scope('https://some.unknown.website/')
        .get('/example/module?go-get=1')
-        .reply(404);
+        .reply(200);
      const res = await getPkgReleases({
        datasource,
        depName: 'some.unknown.website/example/module',
@ -224,11 +262,11 @@ describe('datasource/go', () => {
      expect(res).toMatchSnapshot();
      expect(res).toBeNull();
      expect(httpMock.getTrace()).toMatchSnapshot();
-      expect(logger.logger.warn).toHaveBeenCalled();
-      expect(logger.logger.error).not.toHaveBeenCalledWith(
-        { lookupName: 'golang.org/foo/something' },
+      expect(logger.logger.warn).toHaveBeenCalledWith(
+        { lookupName: 'some.unknown.website/example/module' },
        'Unsupported dependency.'
      );
+      expect(logger.logger.error).not.toHaveBeenCalled();
      expect(logger.logger.fatal).not.toHaveBeenCalled();
    });
    it('support ghe', async () => {
--- a/lib/datasource/go/index.ts
+++ b/lib/datasource/go/index.ts
@ -1,5 +1,7 @@
 import URL from 'url';
+import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms';
 import { logger } from '../../logger';
+import * as hostRules from '../../util/host-rules';
 import { Http } from '../../util/http';
 import { regEx } from '../../util/regex';
 import * as bitbucket from '../bitbucket-tags';
@ -75,6 +77,27 @@ async function getDatasource(goModule: string): Promise<DataSource | null> {
        lookupName: gitlabRes[2].replace(/\/$/, ''),
      };
    }
+
+    const opts = hostRules.find({
+      hostType: PLATFORM_TYPE_GITLAB,
+      url: goSourceUrl,
+    });
+    if (opts.token) {
+      // get server base url from import url
+      const parsedUrl = URL.parse(goSourceUrl);
+
+      // split the go module from the URL: host/go/module -> go/module
+      const split = goModule.split('/');
+      const lookupName = split[1] + '/' + split[2];
+
+      const registryUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
+
+      return {
+        datasource: gitlab.id,
+        registryUrl,
+        lookupName,
+      };
+    }
  } else {
    // GitHub Enterprise only returns a go-import meta
    const importMatch = regEx(
--- a/lib/workers/pr/changelog/snapshots/release-notes.spec.ts.snap
+++ b/lib/workers/pr/changelog/snapshots/release-notes.spec.ts.snap
@ -104,6 +104,39 @@ Array [
 ]
 `;

+exports[`workers/pr/changelog/release-notes getReleaseList() should return release list for self hosted gitlab project 1`] = `
+Array [
+  Object {
+    "body": undefined,
+    "name": undefined,
+    "tag": "v1.0.0",
+    "url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases/v1.0.0",
+  },
+  Object {
+    "body": undefined,
+    "name": undefined,
+    "tag": "v1.0.1",
+    "url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases/v1.0.1",
+  },
+]
+`;
+
+exports[`workers/pr/changelog/release-notes getReleaseList() should return release list for self hosted gitlab project 2`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/json",
+      "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer some-token",
+      "host": "my.custom.domain",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases?per_page=100",
+  },
+]
+`;
+
 exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 1`] = `
 Object {
  "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124)
@ -923,3 +956,45 @@ Object {
  "url": "https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md#3100--2017-09-10",
 }
 `;
+
+exports[`workers/pr/changelog/release-notes getReleaseNotesMd() parses self hosted gitlab 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/json",
+      "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer some-token",
+      "host": "my.custom.domain",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://my.custom.domain/projects/gitlab-org%2fgitter%2fwebapp/repository/tree?per_page=100",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer some-token",
+      "host": "my.custom.domain",
+      "user-agent": "https://github.com/renovatebot/renovate",
+    },
+    "method": "GET",
+    "url": "https://my.custom.domain/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw",
+  },
+]
+`;
+
+exports[`workers/pr/changelog/release-notes getReleaseNotesMd() parses self hosted gitlab 2`] = `
+Object {
+  "body": "-   Removing markup from a part of the French translation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1878>
+-   Fix typo documentation -> documentation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1880>
+    -   Thanks to [@auua](https://gitlab.com/auua) for the contribution
+-   Fix \`/channel\` slash command name regex to accept hyphenated names, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1881>
+    -   Thanks to [@auua](https://gitlab.com/auua) for the contribution
+-   Add GitLab branding to the left-menu, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1865>
+-   Fix left-menu search state showing all rooms, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1883>
+-   Update Polish translation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1882>
+    -   Thanks to [@biesiad](https://gitlab.com/biesiad) for the contribution
+",
+  "url": "https://my.custom.domain/gitlab-org/gitter/webapp/blob/master/CHANGELOG.md#20260---2020-05-18",
+}
+`;
--- a/lib/workers/pr/changelog/release-notes.spec.ts
+++ b/lib/workers/pr/changelog/release-notes.spec.ts
@ -1,7 +1,8 @@
 import fs from 'fs-extra';
 import { DateTime } from 'luxon';
 import * as httpMock from '../../../../test/http-mock';
-import { getName } from '../../../../test/util';
+import { getName, mocked } from '../../../../test/util';
+import * as _hostRules from '../../../util/host-rules';
 import { ChangeLogNotes } from './common';
 import {
  addReleaseNotes,
@ -11,6 +12,10 @@ import {
  releaseNotesCacheMinutes,
 } from './release-notes';

+jest.mock('../../../util/host-rules');
+
+const hostRules = mocked(_hostRules);
+
 const angularJsChangelogMd = fs.readFileSync(
  'lib/workers/pr/__fixtures__/angular-js.md',
  'utf8'
@ -57,10 +62,13 @@ const gitlabTreeResponse = [
 describe(getName(__filename), () => {
  beforeEach(() => {
    httpMock.setup();
+    hostRules.find.mockReturnValue({});
+    hostRules.hosts.mockReturnValue([]);
  });

  afterEach(() => {
    httpMock.reset();
+    jest.resetAllMocks();
  });

  describe('releaseNotesCacheMinutes', () => {
@ -149,6 +157,28 @@ describe(getName(__filename), () => {
      expect(res).toMatchSnapshot();
      expect(httpMock.getTrace()).toMatchSnapshot();
    });
+    it('should return release list for self hosted gitlab project', async () => {
+      hostRules.find.mockReturnValue({ token: 'some-token' });
+      httpMock
+        .scope('https://my.custom.domain/')
+        .get(
+          '/api/v4/projects/some%2fyet-other-repository/releases?per_page=100'
+        )
+        .reply(200, [
+          { tag_name: `v1.0.0` },
+          {
+            tag_name: `v1.0.1`,
+            body:
+              'some body #123, [#124](https://my.custom.domain/some/yet-other-repository/issues/124)',
+          },
+        ]);
+      const res = await getReleaseList(
+        'https://my.custom.domain/api/v4/',
+        'some/yet-other-repository'
+      );
+      expect(res).toMatchSnapshot();
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
  });
  describe('getReleaseNotes()', () => {
    it('should return null for release notes without body', async () => {
@ -343,6 +373,27 @@ describe(getName(__filename), () => {
      expect(res).not.toBeNull();
      expect(res).toMatchSnapshot();
    });
+    it('parses self hosted gitlab', async () => {
+      hostRules.find.mockReturnValue({ token: 'some-token' });
+      jest.setTimeout(0);
+      httpMock
+        .scope('https://my.custom.domain/')
+        .get(
+          '/projects/gitlab-org%2fgitter%2fwebapp/repository/tree?per_page=100'
+        )
+        .reply(200, gitlabTreeResponse)
+        .get('/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw')
+        .reply(200, gitterWebappChangelogMd);
+      const res = await getReleaseNotesMd(
+        'gitlab-org/gitter/webapp',
+        '20.26.0',
+        'https://my.custom.domain/',
+        'https://my.custom.domain/'
+      );
+      expect(httpMock.getTrace()).toMatchSnapshot();
+      expect(res).not.toBeNull();
+      expect(res).toMatchSnapshot();
+    });
    it('parses jest', async () => {
      httpMock
        .scope('https://api.github.com')
--- a/lib/workers/pr/changelog/release-notes.ts
+++ b/lib/workers/pr/changelog/release-notes.ts
@ -4,9 +4,11 @@ import { linkify } from 'linkify-markdown';
 import { DateTime } from 'luxon';
 import MarkdownIt from 'markdown-it';

+import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms';
 import { logger } from '../../../logger';
 import * as memCache from '../../../util/cache/memory';
 import * as packageCache from '../../../util/cache/package';
+import * as hostRules from '../../../util/host-rules';
 import { ChangeLogFile, ChangeLogNotes, ChangeLogResult } from './common';
 import * as github from './github';
 import * as gitlab from './gitlab';
@ -28,6 +30,15 @@ export async function getReleaseList(
    if (apiBaseUrl.includes('gitlab')) {
      return await gitlab.getReleaseList(apiBaseUrl, repository);
    }
+
+    const opts = hostRules.find({
+      hostType: PLATFORM_TYPE_GITLAB,
+      url: apiBaseUrl,
+    });
+    if (opts.token) {
+      return await gitlab.getReleaseList(apiBaseUrl, repository);
+    }
+
    return await github.getReleaseList(apiBaseUrl, repository);
  } catch (err) /* istanbul ignore next */ {
    if (err.statusCode === 404) {
@ -169,6 +180,15 @@ export async function getReleaseNotesMdFileInner(
    if (apiBaseUrl.includes('gitlab')) {
      return await gitlab.getReleaseNotesMd(repository, apiBaseUrl);
    }
+
+    const opts = hostRules.find({
+      hostType: PLATFORM_TYPE_GITLAB,
+      url: apiBaseUrl,
+    });
+    if (opts.token) {
+      return await gitlab.getReleaseNotesMd(repository, apiBaseUrl);
+    }
+
    return await github.getReleaseNotesMd(repository, apiBaseUrl);
  } catch (err) /* istanbul ignore next */ {
    if (err.statusCode === 404) {
--- a/lib/workers/pr/changelog/source-github.ts
+++ b/lib/workers/pr/changelog/source-github.ts
@ -82,7 +82,7 @@ export async function getChangeLogJSON({
    .sort((a, b) => version.sortVersions(a.version, b.version));

  if (validReleases.length < 2) {
-    logger.debug('Not enough valid releases');
+    logger.debug(`Not enough valid releases for dep ${depName}`);
    return null;
  }