feat(core:changelogs): better platform detection (#14989)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/datasource/packagist/index.spec.ts

@@ -1,6 +1,7 @@
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../../test/http-mock';
 import { loadJsonFixture } from '../../../../test/util';
+import type { HostRule } from '../../../types';
 import * as _hostRules from '../../../util/host-rules';
 import * as composerVersioning from '../../versioning/composer';
 import { id as versioning } from '../../versioning/loose';
@@ -23,7 +24,7 @@ describe('modules/datasource/packagist/index', () => {
     let config: any;
     beforeEach(() => {
       jest.resetAllMocks();
-      hostRules.find = jest.fn((input) => input);
+      hostRules.find = jest.fn((input: HostRule) => input);
       hostRules.hosts = jest.fn(() => []);
       config = {
         versioning: composerVersioning.id,

lib/modules/platform/util.spec.ts

@@ -0,0 +1,41 @@
+import * as hostRules from '../../util/host-rules';
+import { detectPlatform } from './util';
+
+describe('modules/platform/util', () => {
+  beforeEach(() => hostRules.clear());
+
+  describe('getHostType', () => {
+    it.each`
+      url                                                    | hostType
+      ${'some-invalid@url:::'}                               | ${null}
+      ${'https://enterprise.example.com/chalk/chalk'}        | ${null}
+      ${'https://github.com/semantic-release/gitlab'}        | ${'github'}
+      ${'https://github-enterprise.example.com/chalk/chalk'} | ${'github'}
+      ${'https://gitlab.com/chalk/chalk'}                    | ${'gitlab'}
+      ${'https://gitlab-enterprise.example.com/chalk/chalk'} | ${'gitlab'}
+    `('("$url") === $hostType', ({ url, hostType }) => {
+      expect(detectPlatform(url)).toBe(hostType);
+    });
+    it('uses host rules', () => {
+      hostRules.add({
+        hostType: 'gitlab-changelog',
+        matchHost: 'gl.example.com',
+      });
+      hostRules.add({
+        hostType: 'github-changelog',
+        matchHost: 'gh.example.com',
+      });
+      hostRules.add({
+        hostType: 'gitea',
+        matchHost: 'gt.example.com',
+      });
+      expect(detectPlatform('https://gl.example.com/chalk/chalk')).toBe(
+        'gitlab'
+      );
+      expect(detectPlatform('https://gh.example.com/chalk/chalk')).toBe(
+        'github'
+      );
+      expect(detectPlatform('https://gt.example.com/chalk/chalk')).toBeNull();
+    });
+  });
+});

lib/modules/platform/util.ts

@@ -0,0 +1,37 @@
+import {
+  GITHUB_API_USING_HOST_TYPES,
+  GITLAB_API_USING_HOST_TYPES,
+} from '../../constants';
+import * as hostRules from '../../util/host-rules';
+import { parseUrl } from '../../util/url';
+
+/**
+ * Tries to detect the `platform from a url.
+ *
+ * @param url the url to detect platform from
+ * @returns matched `platform` if found, otherwise `null`
+ */
+export function detectPlatform(url: string): 'gitlab' | 'github' | null {
+  const { hostname } = parseUrl(url) ?? {};
+  if (hostname === 'github.com' || hostname?.includes('github')) {
+    return 'github';
+  }
+  if (hostname === 'gitlab.com' || hostname?.includes('gitlab')) {
+    return 'gitlab';
+  }
+
+  const hostType = hostRules.hostType({ url: url });
+
+  if (!hostType) {
+    return null;
+  }
+
+  if (GITLAB_API_USING_HOST_TYPES.includes(hostType)) {
+    return 'gitlab';
+  }
+  if (GITHUB_API_USING_HOST_TYPES.includes(hostType)) {
+    return 'github';
+  }
+
+  return null;
+}

lib/types/host-rules.ts

@@ -1,17 +1,20 @@
-export interface HostRule {
+export interface HostRuleSearchResult {
   authType?: string;
-  hostType?: string;
-  matchHost?: string;
   token?: string;
   username?: string;
   password?: string;
   insecureRegistry?: boolean;
   timeout?: number;
-  encrypted?: HostRule;
   abortOnError?: boolean;
   abortIgnoreStatusCodes?: number[];
   enabled?: boolean;
   enableHttp2?: boolean;
   concurrentRequestLimit?: number;
+}
+
+export interface HostRule extends HostRuleSearchResult {
+  encrypted?: HostRule;
+  hostType?: string;
+  matchHost?: string;
   resolvedHost?: string;
 }

lib/util/host-rules.spec.ts

@@ -1,6 +1,15 @@
 import { PlatformId } from '../constants';
 import { NugetDatasource } from '../modules/datasource/nuget';
-import { add, clear, find, findAll, getAll, hosts } from './host-rules';
+import type { HostRule } from '../types';
+import {
+  add,
+  clear,
+  find,
+  findAll,
+  getAll,
+  hostType,
+  hosts,
+} from './host-rules';
 
 describe('util/host-rules', () => {
   beforeEach(() => {
@@ -13,7 +22,7 @@ describe('util/host-rules', () => {
           hostType: PlatformId.Azure,
           domainName: 'github.com',
           hostName: 'api.github.com',
-        } as any)
+        } as HostRule)
       ).toThrow();
     });
     it('throws if both domainName and baseUrl', () => {
@@ -22,7 +31,7 @@ describe('util/host-rules', () => {
           hostType: PlatformId.Azure,
           domainName: 'github.com',
           matchHost: 'https://api.github.com',
-        } as any)
+        } as HostRule)
       ).toThrow();
     });
     it('throws if both hostName and baseUrl', () => {
@@ -31,7 +40,7 @@ describe('util/host-rules', () => {
           hostType: PlatformId.Azure,
           hostName: 'api.github.com',
           matchHost: 'https://api.github.com',
-        } as any)
+        } as HostRule)
       ).toThrow();
     });
     it('supports baseUrl-only', () => {
@@ -39,7 +48,7 @@ describe('util/host-rules', () => {
         matchHost: 'https://some.endpoint',
         username: 'user1',
         password: 'pass1',
-      } as any);
+      });
       expect(find({ url: 'https://some.endpoint/v3/' })).toEqual({
         password: 'pass1',
         username: 'user1',
@@ -60,7 +69,7 @@ describe('util/host-rules', () => {
         username: 'root',
         password: 'p4$$w0rd',
         token: undefined,
-      } as any);
+      } as HostRule);
       expect(find({ hostType: NugetDatasource.id })).toEqual({});
       expect(
         find({ hostType: NugetDatasource.id, url: 'https://nuget.org' })
@@ -93,7 +102,7 @@ describe('util/host-rules', () => {
       add({
         domainName: 'github.com',
         token: 'def',
-      } as any);
+      } as HostRule);
       expect(
         find({ hostType: NugetDatasource.id, url: 'https://api.github.com' })
           .token
@@ -176,7 +185,7 @@ describe('util/host-rules', () => {
       add({
         hostName: 'nuget.local',
         token: 'abc',
-      } as any);
+      } as HostRule);
       expect(
         find({ hostType: NugetDatasource.id, url: 'https://nuget.local/api' })
       ).toEqual({ token: 'abc' });
@@ -218,7 +227,7 @@ describe('util/host-rules', () => {
         hostType: NugetDatasource.id,
         matchHost: 'https://nuget.local/api',
         token: 'abc',
-      } as any);
+      });
       expect(
         find({ hostType: NugetDatasource.id, url: 'https://nuget.local/api' })
           .token
@@ -229,7 +238,7 @@ describe('util/host-rules', () => {
         hostType: NugetDatasource.id,
         matchHost: 'https://nuget.local/api',
         token: 'abc',
-      } as any);
+      });
       expect(
         find({
           hostType: NugetDatasource.id,
@@ -241,17 +250,20 @@ describe('util/host-rules', () => {
       add({
         matchHost: 'https://nuget.local/api',
         token: 'longest',
-      } as any);
+      });
       add({
         matchHost: 'https://nuget.local/',
         token: 'shortest',
-      } as any);
+      });
       expect(
         find({
           url: 'https://nuget.local/api/sub-resource',
         })
       ).toEqual({ token: 'longest' });
     });
+  });
+
+  describe('hosts()', () => {
     it('returns hosts', () => {
       add({
         hostType: NugetDatasource.id,
@@ -261,12 +273,12 @@ describe('util/host-rules', () => {
         hostType: NugetDatasource.id,
         matchHost: 'https://nuget.local/api',
         token: 'abc',
-      } as any);
+      });
       add({
         hostType: NugetDatasource.id,
         hostName: 'my.local.registry',
         token: 'def',
-      } as any);
+      } as HostRule);
       add({
         hostType: NugetDatasource.id,
         matchHost: 'another.local.registry',
@@ -288,6 +300,7 @@ describe('util/host-rules', () => {
       ]);
     });
   });
+
   describe('findAll()', () => {
     it('warns and returns empty for bad search', () => {
       expect(findAll({ abc: 'def' } as any)).toEqual([]);
@@ -329,4 +342,55 @@ describe('util/host-rules', () => {
       expect(getAll()).toMatchObject([hostRule1, hostRule2]);
     });
   });
+
+  describe('hostType()', () => {
+    it('return hostType', () => {
+      add({
+        hostType: PlatformId.Github,
+        token: 'aaaaaa',
+      });
+      add({
+        hostType: PlatformId.Github,
+        matchHost: 'github.example.com',
+        token: 'abc',
+      });
+      add({
+        hostType: 'github-changelog',
+        matchHost: 'https://github.example.com/chalk/chalk',
+        token: 'def',
+      });
+      expect(
+        hostType({
+          url: 'https://github.example.com/chalk/chalk',
+        })
+      ).toBe('github-changelog');
+    });
+
+    it('returns null', () => {
+      add({
+        hostType: PlatformId.Github,
+        token: 'aaaaaa',
+      });
+      add({
+        hostType: PlatformId.Github,
+        matchHost: 'github.example.com',
+        token: 'abc',
+      });
+      add({
+        hostType: 'github-changelog',
+        matchHost: 'https://github.example.com/chalk/chalk',
+        token: 'def',
+      });
+      expect(
+        hostType({
+          url: 'https://github.example.com/chalk/chalk',
+        })
+      ).toBe('github-changelog');
+      expect(
+        hostType({
+          url: 'https://gitlab.example.com/chalk/chalk',
+        })
+      ).toBeNull();
+    });
+  });
 });

lib/util/host-rules.ts

@@ -1,7 +1,7 @@
 import is from '@sindresorhus/is';
 import merge from 'deepmerge';
 import { logger } from '../logger';
-import type { HostRule } from '../types';
+import type { HostRule, HostRuleSearchResult } from '../types';
 import { clone } from './clone';
 import * as sanitize from './sanitize';
 import { toBase64 } from './string';
@@ -118,7 +118,7 @@ function prioritizeLongestMatchHost(rule1: HostRule, rule2: HostRule): number {
   return rule1.matchHost.length - rule2.matchHost.length;
 }
 
-export function find(search: HostRuleSearch): HostRule {
+export function find(search: HostRuleSearch): HostRuleSearchResult {
   if (!(search.hostType || search.url)) {
     logger.warn({ search }, 'Invalid hostRules search');
     return {};
@@ -167,6 +167,17 @@ export function hosts({ hostType }: { hostType: string }): string[] {
     .filter(is.truthy);
 }
 
+export function hostType({ url }: { url: string }): string | null {
+  return (
+    hostRules
+      .filter((rule) => matchesHost(rule, { url }))
+      .sort(prioritizeLongestMatchHost)
+      .map((rule) => rule.hostType)
+      .filter(is.truthy)
+      .pop() ?? null
+  );
+}
+
 export function findAll({ hostType }: { hostType: string }): HostRule[] {
   return hostRules.filter((rule) => rule.hostType === hostType);
 }

lib/workers/repository/update/pr/changelog/index.ts

@@ -1,4 +1,5 @@
 import { logger } from '../../../../../logger';
+import { detectPlatform } from '../../../../../modules/platform/util';
 import * as allVersioning from '../../../../../modules/versioning';
 import type { BranchUpgradeConfig } from '../../../../types';
 import { getInRangeReleases } from './releases';
@@ -27,15 +28,24 @@ export async function getChangeLogJSON(
 
     let res: ChangeLogResult | null = null;
 
-    if (
-      args.sourceUrl?.includes('gitlab') ||
-      (args.platform === 'gitlab' &&
-        new URL(args.sourceUrl).hostname === new URL(args.endpoint).hostname)
-    ) {
-      res = await sourceGitlab.getChangeLogJSON({ ...args, releases });
-    } else {
-      res = await sourceGithub.getChangeLogJSON({ ...args, releases });
+    const platform = detectPlatform(sourceUrl);
+
+    switch (platform) {
+      case 'gitlab':
+        res = await sourceGitlab.getChangeLogJSON({ ...args, releases });
+        break;
+      case 'github':
+        res = await sourceGithub.getChangeLogJSON({ ...args, releases });
+        break;
+
+      default:
+        logger.info(
+          { sourceUrl, hostType: platform },
+          'Unknown platform, skipping changelog fetching.'
+        );
+        break;
     }
+
     return res;
   } catch (err) /* istanbul ignore next */ {
     logger.error({ config: args, err }, 'getChangeLogJSON error');