renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-12 23:16:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

feat(go): support self hosted go modules in gitlab (#8876)

Browse source
This commit is contained in:
davidgwcurve 2021-03-01 17:35:35 +00:00 committed by GitHub
parent f4d5514281
commit e3ccc26af7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 256 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
lib/datasource/go/__snapshots__/index.spec.ts.snap
View file

@ -398,6 +398,48 @@ Array [
] ]
`; `;
exports[`datasource/go getReleases support self hosted gitlab private repositories 1`] = `
Object {
"releases": Array [
Object {
"gitRef": "v1.0.0",
"version": "v1.0.0",
},
Object {
"gitRef": "v2.0.0",
"version": "v2.0.0",
},
],
"sourceUrl": "https://my.custom.domain/golang/myrepo",
}
`;
exports[`datasource/go getReleases support self hosted gitlab private repositories 2`] = `
Array [
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"authorization": "Bearer some-token",
"host": "my.custom.domain",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://my.custom.domain/golang/myrepo?go-get=1",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"authorization": "Bearer some-token",
"host": "my.custom.domain",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://my.custom.domain/api/v4/projects/golang%2Fmyrepo/repository/tags?per_page=100",
},
]
`;
exports[`datasource/go getReleases unknown datasource returns null 1`] = `null`; exports[`datasource/go getReleases unknown datasource returns null 1`] = `null`;
exports[`datasource/go getReleases unknown datasource returns null 2`] = ` exports[`datasource/go getReleases unknown datasource returns null 2`] = `

48
lib/datasource/go/index.spec.ts
View file

@ -1,8 +1,13 @@
import { getPkgReleases } from '..'; import { getPkgReleases } from '..';
import * as httpMock from '../../../test/http-mock'; import * as httpMock from '../../../test/http-mock';
import { logger } from '../../../test/util'; import { logger, mocked } from '../../../test/util';
import * as _hostRules from '../../util/host-rules';
import { id as datasource, getDigest } from '.'; import { id as datasource, getDigest } from '.';
jest.mock('../../util/host-rules');
const hostRules = mocked(_hostRules);
const res1 = `<!DOCTYPE html> const res1 = `<!DOCTYPE html>
<html> <html>
<head> <head>
@ -16,6 +21,17 @@ Nothing to see here; <a href="https://godoc.org/golang.org/x/text">move along</a
</body> </body>
</html>`; </html>`;
const resGitLabEE = `<html>
<head>
<meta name="go-import" content="my.custom.domain/golang/myrepo git https://my.custom.domain/golang/myrepo.git" />
<meta name="go-source"
content="my.custom.domain/golang/myrepo https://my.custom.domain/golang/myrepo https://my.custom.domain/golang/myrepo/-/tree/master{/dir} https://my.custom.domain/golang/myrepo/-/blob/master{/dir}/{file}#L{line}" />
</head>
<body>go get https://my.custom.domain/golang/myrepo</body>
</html>`;
const resGitHubEnterprise = `<!DOCTYPE html> const resGitHubEnterprise = `<!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
@ -33,10 +49,13 @@ const resGitHubEnterprise = `<!DOCTYPE html>
describe('datasource/go', () => { describe('datasource/go', () => {
beforeEach(() => { beforeEach(() => {
httpMock.setup(); httpMock.setup();
hostRules.find.mockReturnValue({});
hostRules.hosts.mockReturnValue([]);
}); });
afterEach(() => { afterEach(() => {
httpMock.reset(); httpMock.reset();
jest.resetAllMocks();
}); });
describe('getDigest', () => { describe('getDigest', () => {
@ -194,6 +213,25 @@ describe('datasource/go', () => {
expect(res).toBeDefined(); expect(res).toBeDefined();
expect(httpMock.getTrace()).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot();
}); });
it('support self hosted gitlab private repositories', async () => {
hostRules.find.mockReturnValue({ token: 'some-token' });
httpMock
.scope('https://my.custom.domain/')
.get('/golang/myrepo?go-get=1')
.reply(200, resGitLabEE);
httpMock
.scope('https://my.custom.domain/')
.get('/api/v4/projects/golang%2Fmyrepo/repository/tags?per_page=100')
.reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }]);
const res = await getPkgReleases({
datasource,
depName: 'my.custom.domain/golang/myrepo',
});
expect(res).toMatchSnapshot();
expect(res).not.toBeNull();
expect(res).toBeDefined();
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('support bitbucket tags', async () => { it('support bitbucket tags', async () => {
httpMock httpMock
.scope('https://api.bitbucket.org/') .scope('https://api.bitbucket.org/')
@ -216,7 +254,7 @@ describe('datasource/go', () => {
httpMock httpMock
.scope('https://some.unknown.website/') .scope('https://some.unknown.website/')
.get('/example/module?go-get=1') .get('/example/module?go-get=1')
.reply(404); .reply(200);
const res = await getPkgReleases({ const res = await getPkgReleases({
datasource, datasource,
depName: 'some.unknown.website/example/module', depName: 'some.unknown.website/example/module',
@ -224,11 +262,11 @@ describe('datasource/go', () => {
expect(res).toMatchSnapshot(); expect(res).toMatchSnapshot();
expect(res).toBeNull(); expect(res).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot();
expect(logger.logger.warn).toHaveBeenCalled(); expect(logger.logger.warn).toHaveBeenCalledWith(
expect(logger.logger.error).not.toHaveBeenCalledWith( { lookupName: 'some.unknown.website/example/module' },
{ lookupName: 'golang.org/foo/something' },
'Unsupported dependency.' 'Unsupported dependency.'
); );
expect(logger.logger.error).not.toHaveBeenCalled();
expect(logger.logger.fatal).not.toHaveBeenCalled(); expect(logger.logger.fatal).not.toHaveBeenCalled();
}); });
it('support ghe', async () => { it('support ghe', async () => {

23
lib/datasource/go/index.ts
View file

@ -1,5 +1,7 @@
import URL from 'url'; import URL from 'url';
import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms';
import { logger } from '../../logger'; import { logger } from '../../logger';
import * as hostRules from '../../util/host-rules';
import { Http } from '../../util/http'; import { Http } from '../../util/http';
import { regEx } from '../../util/regex'; import { regEx } from '../../util/regex';
import * as bitbucket from '../bitbucket-tags'; import * as bitbucket from '../bitbucket-tags';
@ -75,6 +77,27 @@ async function getDatasource(goModule: string): Promise<DataSource | null> {
lookupName: gitlabRes[2].replace(/\/$/, ''), lookupName: gitlabRes[2].replace(/\/$/, ''),
}; };
} }
const opts = hostRules.find({
hostType: PLATFORM_TYPE_GITLAB,
url: goSourceUrl,
});
if (opts.token) {
// get server base url from import url
const parsedUrl = URL.parse(goSourceUrl);
// split the go module from the URL: host/go/module -> go/module
const split = goModule.split('/');
const lookupName = split[1] + '/' + split[2];
const registryUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
return {
datasource: gitlab.id,
registryUrl,
lookupName,
};
}
} else { } else {
// GitHub Enterprise only returns a go-import meta // GitHub Enterprise only returns a go-import meta
const importMatch = regEx( const importMatch = regEx(

75
lib/workers/pr/changelog/__snapshots__/release-notes.spec.ts.snap
View file

@ -104,6 +104,39 @@ Array [
] ]
`; `;
exports[`workers/pr/changelog/release-notes getReleaseList() should return release list for self hosted gitlab project 1`] = `
Array [
Object {
"body": undefined,
"name": undefined,
"tag": "v1.0.0",
"url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases/v1.0.0",
},
Object {
"body": undefined,
"name": undefined,
"tag": "v1.0.1",
"url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases/v1.0.1",
},
]
`;
exports[`workers/pr/changelog/release-notes getReleaseList() should return release list for self hosted gitlab project 2`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"authorization": "Bearer some-token",
"host": "my.custom.domain",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://my.custom.domain/api/v4/projects/some%2fyet-other-repository/releases?per_page=100",
},
]
`;
exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 1`] = ` exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 1`] = `
Object { Object {
"body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124)
@ -923,3 +956,45 @@ Object {
"url": "https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md#3100--2017-09-10", "url": "https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md#3100--2017-09-10",
} }
`; `;
exports[`workers/pr/changelog/release-notes getReleaseNotesMd() parses self hosted gitlab 1`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"authorization": "Bearer some-token",
"host": "my.custom.domain",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://my.custom.domain/projects/gitlab-org%2fgitter%2fwebapp/repository/tree?per_page=100",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"authorization": "Bearer some-token",
"host": "my.custom.domain",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://my.custom.domain/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw",
},
]
`;
exports[`workers/pr/changelog/release-notes getReleaseNotesMd() parses self hosted gitlab 2`] = `
Object {
"body": "- Removing markup from a part of the French translation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1878>
- Fix typo documentation -> documentation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1880>
- Thanks to [@auua](https://gitlab.com/auua) for the contribution
- Fix \`/channel\` slash command name regex to accept hyphenated names, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1881>
- Thanks to [@auua](https://gitlab.com/auua) for the contribution
- Add GitLab branding to the left-menu, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1865>
- Fix left-menu search state showing all rooms, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1883>
- Update Polish translation, <https://gitlab.com/gitlab-org/gitter/webapp/-/merge_requests/1882>
- Thanks to [@biesiad](https://gitlab.com/biesiad) for the contribution
",
"url": "https://my.custom.domain/gitlab-org/gitter/webapp/blob/master/CHANGELOG.md#20260---2020-05-18",
}
`;

53
lib/workers/pr/changelog/release-notes.spec.ts
View file

@ -1,7 +1,8 @@
import fs from 'fs-extra'; import fs from 'fs-extra';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import * as httpMock from '../../../../test/http-mock'; import * as httpMock from '../../../../test/http-mock';
import { getName } from '../../../../test/util'; import { getName, mocked } from '../../../../test/util';
import * as _hostRules from '../../../util/host-rules';
import { ChangeLogNotes } from './common'; import { ChangeLogNotes } from './common';
import { import {
addReleaseNotes, addReleaseNotes,
@ -11,6 +12,10 @@ import {
releaseNotesCacheMinutes, releaseNotesCacheMinutes,
} from './release-notes'; } from './release-notes';
jest.mock('../../../util/host-rules');
const hostRules = mocked(_hostRules);
const angularJsChangelogMd = fs.readFileSync( const angularJsChangelogMd = fs.readFileSync(
'lib/workers/pr/__fixtures__/angular-js.md', 'lib/workers/pr/__fixtures__/angular-js.md',
'utf8' 'utf8'
@ -57,10 +62,13 @@ const gitlabTreeResponse = [
describe(getName(__filename), () => { describe(getName(__filename), () => {
beforeEach(() => { beforeEach(() => {
httpMock.setup(); httpMock.setup();
hostRules.find.mockReturnValue({});
hostRules.hosts.mockReturnValue([]);
}); });
afterEach(() => { afterEach(() => {
httpMock.reset(); httpMock.reset();
jest.resetAllMocks();
}); });
describe('releaseNotesCacheMinutes', () => { describe('releaseNotesCacheMinutes', () => {
@ -149,6 +157,28 @@ describe(getName(__filename), () => {
expect(res).toMatchSnapshot(); expect(res).toMatchSnapshot();
expect(httpMock.getTrace()).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot();
}); });
it('should return release list for self hosted gitlab project', async () => {
hostRules.find.mockReturnValue({ token: 'some-token' });
httpMock
.scope('https://my.custom.domain/')
.get(
'/api/v4/projects/some%2fyet-other-repository/releases?per_page=100'
)
.reply(200, [
{ tag_name: `v1.0.0` },
{
tag_name: `v1.0.1`,
body:
'some body #123, [#124](https://my.custom.domain/some/yet-other-repository/issues/124)',
},
]);
const res = await getReleaseList(
'https://my.custom.domain/api/v4/',
'some/yet-other-repository'
);
expect(res).toMatchSnapshot();
expect(httpMock.getTrace()).toMatchSnapshot();
});
}); });
describe('getReleaseNotes()', () => { describe('getReleaseNotes()', () => {
it('should return null for release notes without body', async () => { it('should return null for release notes without body', async () => {
@ -343,6 +373,27 @@ describe(getName(__filename), () => {
expect(res).not.toBeNull(); expect(res).not.toBeNull();
expect(res).toMatchSnapshot(); expect(res).toMatchSnapshot();
}); });
it('parses self hosted gitlab', async () => {
hostRules.find.mockReturnValue({ token: 'some-token' });
jest.setTimeout(0);
httpMock
.scope('https://my.custom.domain/')
.get(
'/projects/gitlab-org%2fgitter%2fwebapp/repository/tree?per_page=100'
)
.reply(200, gitlabTreeResponse)
.get('/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw')
.reply(200, gitterWebappChangelogMd);
const res = await getReleaseNotesMd(
'gitlab-org/gitter/webapp',
'20.26.0',
'https://my.custom.domain/',
'https://my.custom.domain/'
);
expect(httpMock.getTrace()).toMatchSnapshot();
expect(res).not.toBeNull();
expect(res).toMatchSnapshot();
});
it('parses jest', async () => { it('parses jest', async () => {
httpMock httpMock
.scope('https://api.github.com') .scope('https://api.github.com')

20
lib/workers/pr/changelog/release-notes.ts
View file

@ -4,9 +4,11 @@ import { linkify } from 'linkify-markdown';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import MarkdownIt from 'markdown-it'; import MarkdownIt from 'markdown-it';
import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms';
import { logger } from '../../../logger'; import { logger } from '../../../logger';
import * as memCache from '../../../util/cache/memory'; import * as memCache from '../../../util/cache/memory';
import * as packageCache from '../../../util/cache/package'; import * as packageCache from '../../../util/cache/package';
import * as hostRules from '../../../util/host-rules';
import { ChangeLogFile, ChangeLogNotes, ChangeLogResult } from './common'; import { ChangeLogFile, ChangeLogNotes, ChangeLogResult } from './common';
import * as github from './github'; import * as github from './github';
import * as gitlab from './gitlab'; import * as gitlab from './gitlab';
@ -28,6 +30,15 @@ export async function getReleaseList(
if (apiBaseUrl.includes('gitlab')) { if (apiBaseUrl.includes('gitlab')) {
return await gitlab.getReleaseList(apiBaseUrl, repository); return await gitlab.getReleaseList(apiBaseUrl, repository);
} }
const opts = hostRules.find({
hostType: PLATFORM_TYPE_GITLAB,
url: apiBaseUrl,
});
if (opts.token) {
return await gitlab.getReleaseList(apiBaseUrl, repository);
}
return await github.getReleaseList(apiBaseUrl, repository); return await github.getReleaseList(apiBaseUrl, repository);
} catch (err) /* istanbul ignore next */ { } catch (err) /* istanbul ignore next */ {
if (err.statusCode === 404) { if (err.statusCode === 404) {
@ -169,6 +180,15 @@ export async function getReleaseNotesMdFileInner(
if (apiBaseUrl.includes('gitlab')) { if (apiBaseUrl.includes('gitlab')) {
return await gitlab.getReleaseNotesMd(repository, apiBaseUrl); return await gitlab.getReleaseNotesMd(repository, apiBaseUrl);
} }
const opts = hostRules.find({
hostType: PLATFORM_TYPE_GITLAB,
url: apiBaseUrl,
});
if (opts.token) {
return await gitlab.getReleaseNotesMd(repository, apiBaseUrl);
}
return await github.getReleaseNotesMd(repository, apiBaseUrl); return await github.getReleaseNotesMd(repository, apiBaseUrl);
} catch (err) /* istanbul ignore next */ { } catch (err) /* istanbul ignore next */ {
if (err.statusCode === 404) { if (err.statusCode === 404) {

2
lib/workers/pr/changelog/source-github.ts
View file

@ -82,7 +82,7 @@ export async function getChangeLogJSON({
.sort((a, b) => version.sortVersions(a.version, b.version)); .sort((a, b) => version.sortVersions(a.version, b.version));
if (validReleases.length < 2) { if (validReleases.length < 2) {
logger.debug('Not enough valid releases'); logger.debug(`Not enough valid releases for dep ${depName}`);
return null; return null;
} }