fix(git): wrong git url handling (#17380)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/datasource/sbt-package/index.ts

@@ -1,9 +1,9 @@
 import { XmlDocument } from 'xmldoc';
 import { logger } from '../../../logger';
+import { detectPlatform } from '../../../util/common';
 import { Http } from '../../../util/http';
 import { regEx } from '../../../util/regex';
 import { ensureTrailingSlash } from '../../../util/url';
-import { detectPlatform } from '../../platform/util';
 import * as ivyVersioning from '../../versioning/ivy';
 import { compare } from '../../versioning/maven/compare';
 import { MavenDatasource } from '../maven';

lib/modules/platform/util.spec.ts

@@ -1,42 +1,19 @@
 import * as hostRules from '../../util/host-rules';
-import { detectPlatform } from './util';
+import { repoFingerprint } from './util';
 
 describe('modules/platform/util', () => {
   beforeEach(() => hostRules.clear());
 
-  describe('getHostType', () => {
+  describe('repoFingerprint', () => {
     it.each`
-      url                                                    | hostType
+      repoId       | endpoint                | fingerprint
-      ${'some-invalid@url:::'}                               | ${null}
+      ${'some-id'} | ${null}                 | ${'361b1bf27a0c0ef8fa5d270f588aa5747ba9497b16de64a44f186253295bc80a3891ecfee768f5c88734a6a738eacca69ccca7e50b16529cfc50dca77226a760'}
-      ${'https://enterprise.example.com/chalk/chalk'}        | ${null}
+      ${'some-id'} | ${'https://github.com'} | ${'423e527a4f88a1b6aae8b70e72a4ae80b44fe83f11b90851f5bc654f39a3272c76b57d7ad30cabd727c04c254a3e7ea16109d05e398a228701ac805460344815'}
-      ${'https://github.com/semantic-release/gitlab'}        | ${'github'}
+    `(
-      ${'https://github-enterprise.example.com/chalk/chalk'} | ${'github'}
+      '("$repoId", "$endpoint") === $fingerprint',
-      ${'https://gitlab.com/chalk/chalk'}                    | ${'gitlab'}
+      ({ repoId, endpoint, fingerprint }) => {
-      ${'https://gitlab-enterprise.example.com/chalk/chalk'} | ${'gitlab'}
+        expect(repoFingerprint(repoId, endpoint)).toBe(fingerprint);
-    `('("$url") === $hostType', ({ url, hostType }) => {
+      }
-      expect(detectPlatform(url)).toBe(hostType);
-    });
-
-    it('uses host rules', () => {
-      hostRules.add({
-        hostType: 'gitlab-changelog',
-        matchHost: 'gl.example.com',
-      });
-      hostRules.add({
-        hostType: 'github-changelog',
-        matchHost: 'gh.example.com',
-      });
-      hostRules.add({
-        hostType: 'gitea',
-        matchHost: 'gt.example.com',
-      });
-      expect(detectPlatform('https://gl.example.com/chalk/chalk')).toBe(
-        'gitlab'
     );
-      expect(detectPlatform('https://gh.example.com/chalk/chalk')).toBe(
-        'github'
-      );
-      expect(detectPlatform('https://gt.example.com/chalk/chalk')).toBeNull();
-    });
   });
 });

lib/modules/platform/util.ts

@@ -1,41 +1,4 @@
 import hasha from 'hasha';
-import {
-  GITHUB_API_USING_HOST_TYPES,
-  GITLAB_API_USING_HOST_TYPES,
-} from '../../constants';
-import * as hostRules from '../../util/host-rules';
-import { parseUrl } from '../../util/url';
-
-/**
- * Tries to detect the `platform from a url.
- *
- * @param url the url to detect platform from
- * @returns matched `platform` if found, otherwise `null`
- */
-export function detectPlatform(url: string): 'gitlab' | 'github' | null {
-  const { hostname } = parseUrl(url) ?? {};
-  if (hostname === 'github.com' || hostname?.includes('github')) {
-    return 'github';
-  }
-  if (hostname === 'gitlab.com' || hostname?.includes('gitlab')) {
-    return 'gitlab';
-  }
-
-  const hostType = hostRules.hostType({ url: url });
-
-  if (!hostType) {
-    return null;
-  }
-
-  if (GITLAB_API_USING_HOST_TYPES.includes(hostType)) {
-    return 'gitlab';
-  }
-  if (GITHUB_API_USING_HOST_TYPES.includes(hostType)) {
-    return 'github';
-  }
-
-  return null;
-}
 
 export function repoFingerprint(
   repoId: number | string,

lib/util/common.spec.ts

@@ -0,0 +1,42 @@
+import { detectPlatform } from './common';
+import * as hostRules from './host-rules';
+
+describe('util/common', () => {
+  beforeEach(() => hostRules.clear());
+
+  describe('detectPlatform', () => {
+    it.each`
+      url                                                    | hostType
+      ${'some-invalid@url:::'}                               | ${null}
+      ${'https://enterprise.example.com/chalk/chalk'}        | ${null}
+      ${'https://github.com/semantic-release/gitlab'}        | ${'github'}
+      ${'https://github-enterprise.example.com/chalk/chalk'} | ${'github'}
+      ${'https://gitlab.com/chalk/chalk'}                    | ${'gitlab'}
+      ${'https://gitlab-enterprise.example.com/chalk/chalk'} | ${'gitlab'}
+    `('("$url") === $hostType', ({ url, hostType }) => {
+      expect(detectPlatform(url)).toBe(hostType);
+    });
+
+    it('uses host rules', () => {
+      hostRules.add({
+        hostType: 'gitlab-changelog',
+        matchHost: 'gl.example.com',
+      });
+      hostRules.add({
+        hostType: 'github-changelog',
+        matchHost: 'gh.example.com',
+      });
+      hostRules.add({
+        hostType: 'gitea',
+        matchHost: 'gt.example.com',
+      });
+      expect(detectPlatform('https://gl.example.com/chalk/chalk')).toBe(
+        'gitlab'
+      );
+      expect(detectPlatform('https://gh.example.com/chalk/chalk')).toBe(
+        'github'
+      );
+      expect(detectPlatform('https://gt.example.com/chalk/chalk')).toBeNull();
+    });
+  });
+});

lib/util/common.ts

@@ -0,0 +1,37 @@
+import {
+  GITHUB_API_USING_HOST_TYPES,
+  GITLAB_API_USING_HOST_TYPES,
+} from '../constants';
+import * as hostRules from './host-rules';
+import { parseUrl } from './url';
+
+/**
+ * Tries to detect the `platform` from a url.
+ *
+ * @param url the url to detect `platform` from
+ * @returns matched `platform` if found, otherwise `null`
+ */
+export function detectPlatform(url: string): 'gitlab' | 'github' | null {
+  const { hostname } = parseUrl(url) ?? {};
+  if (hostname === 'github.com' || hostname?.includes('github')) {
+    return 'github';
+  }
+  if (hostname === 'gitlab.com' || hostname?.includes('gitlab')) {
+    return 'gitlab';
+  }
+
+  const hostType = hostRules.hostType({ url: url });
+
+  if (!hostType) {
+    return null;
+  }
+
+  if (GITLAB_API_USING_HOST_TYPES.includes(hostType)) {
+    return 'gitlab';
+  }
+  if (GITHUB_API_USING_HOST_TYPES.includes(hostType)) {
+    return 'github';
+  }
+
+  return null;
+}

lib/util/git/auth.ts

@@ -1,10 +1,10 @@
-import gitUrlParse from 'git-url-parse';
 import { PlatformId } from '../../constants';
 import { logger } from '../../logger';
-import { detectPlatform } from '../../modules/platform/util';
 import type { HostRule } from '../../types';
+import { detectPlatform } from '../common';
 import { regEx } from '../regex';
 import type { AuthenticationRule } from './types';
+import { parseGitUrl } from './url';
 
 /**
  * Add authorization to a Git Url and returns a new environment variables object
@@ -91,15 +91,7 @@ export function getAuthenticationRules(
 ): AuthenticationRule[] {
   const authenticationRules = [];
   const hasUser = token.split(':').length > 1;
-  const insteadUrl = gitUrlParse(gitUrl);
+  const insteadUrl = parseGitUrl(gitUrl);
-
-  // Workaround for https://github.com/IonicaBizau/parse-path/issues/38
-  if (insteadUrl.port && insteadUrl.resource.endsWith(`:${insteadUrl.port}`)) {
-    insteadUrl.resource = insteadUrl.resource.substring(
-      0,
-      insteadUrl.resource.length - `:${insteadUrl.port}`.length
-    );
-  }
 
   const url = { ...insteadUrl };
   const protocol = regEx(/^https?$/).test(url.protocol)

lib/util/git/url.spec.ts

@@ -1,9 +1,38 @@
 import { hostRules } from '../../../test/util';
-import { getHttpUrl, getRemoteUrlWithToken } from './url';
+import { getHttpUrl, getRemoteUrlWithToken, parseGitUrl } from './url';
 
 jest.mock('../host-rules');
 
 describe('util/git/url', () => {
+  describe('parseGitUrl', () => {
+    it('supports ports', () => {
+      expect(parseGitUrl('https://gitlab.com:8443/')).toEqual({
+        filepath: '',
+        filepathtype: '',
+        full_name: '',
+        git_suffix: false,
+        hash: '',
+        href: 'https://gitlab.com:8443',
+        name: '',
+        organization: '',
+        owner: '',
+        password: '',
+        pathname: '/',
+        port: '8443',
+        protocol: 'https',
+        protocols: ['https'],
+        query: {},
+        ref: '',
+        resource: 'gitlab.com',
+        search: '',
+        source: 'gitlab.com:8443',
+        toString: expect.toBeFunction(),
+        token: '',
+        user: '',
+      });
+    });
+  });
+
   describe('getHttpUrl()', () => {
     it('returns https url for git url', () => {
       expect(getHttpUrl('git://foo.bar/')).toBe('https://foo.bar/');
@@ -16,6 +45,18 @@ describe('util/git/url', () => {
     it('returns http url for http url', () => {
       expect(getHttpUrl('http://foo.bar/')).toBe('http://foo.bar/');
     });
+
+    it('returns gitlab url with token', () => {
+      expect(getHttpUrl('http://gitlab.com/', 'token')).toBe(
+        'http://gitlab-ci-token:token@gitlab.com/'
+      );
+      expect(getHttpUrl('http://gitlab.com/', 'gitlab-ci-token:token')).toBe(
+        'http://gitlab-ci-token:token@gitlab.com/'
+      );
+      expect(
+        getHttpUrl('http://gitlab.com:8443/', 'gitlab-ci-token:token')
+      ).toBe('http://gitlab-ci-token:token@gitlab.com:8443/');
+    });
   });
 
   describe('getRemoteUrlWithToken()', () => {
@@ -92,5 +133,14 @@ describe('util/git/url', () => {
         'https://u%24er:p%40ss@foo.bar/'
       );
     });
+
+    it('returns https url with encoded gitlab token', () => {
+      hostRules.find.mockReturnValueOnce({
+        token: 'token',
+      });
+      expect(getRemoteUrlWithToken('ssh://gitlab.com/some/repo.git')).toBe(
+        'https://gitlab-ci-token:token@gitlab.com/some/repo.git'
+      );
+    });
   });
 });

lib/util/git/url.ts

@@ -1,13 +1,37 @@
-import GitUrlParse from 'git-url-parse';
+import gitUrlParse from 'git-url-parse';
 import { logger } from '../../logger';
+import { detectPlatform } from '../common';
 import * as hostRules from '../host-rules';
 import { regEx } from '../regex';
 
+export function parseGitUrl(url: string): gitUrlParse.GitUrl {
+  const parsed = gitUrlParse(url);
+
+  // Workaround for https://github.com/IonicaBizau/parse-path/issues/38
+  if (parsed.port && parsed.resource.endsWith(`:${parsed.port}`)) {
+    parsed.resource = parsed.resource.substring(
+      0,
+      parsed.resource.length - `:${parsed.port}`.length
+    );
+  }
+
+  return parsed;
+}
+
 export function getHttpUrl(url: string, token?: string): string {
-  const parsedUrl = GitUrlParse(url);
+  const parsedUrl = parseGitUrl(url);
 
   parsedUrl.token = token ?? '';
 
+  if (token) {
+    switch (detectPlatform(url)) {
+      case 'gitlab':
+        parsedUrl.token = token.includes(':')
+          ? token
+          : `gitlab-ci-token:${token}`;
+    }
+  }
+
   const protocol = regEx(/^https?$/).exec(parsedUrl.protocol)
     ? parsedUrl.protocol
     : 'https';

lib/workers/repository/update/pr/changelog/index.ts

@@ -1,6 +1,6 @@
 import { logger } from '../../../../../logger';
-import { detectPlatform } from '../../../../../modules/platform/util';
 import * as allVersioning from '../../../../../modules/versioning';
+import { detectPlatform } from '../../../../../util/common';
 import type { BranchUpgradeConfig } from '../../../../types';
 import * as sourceGithub from './source-github';
 import * as sourceGitlab from './source-gitlab';