feat(internal): datasource registryStrategy (#6549)

lib/datasource/__snapshots__/index.spec.ts.snap

@@ -22,3 +22,18 @@ Object {
   "sourceUrl": "https://github.com/nodejs/node",
 }
 `;
+
+exports[`datasource/index merges registries and returns success 1`] = `
+Object {
+  "releases": Array [
+    Object {
+      "version": "1.0.0",
+    },
+    Object {
+      "version": "1.1.0",
+    },
+  ],
+}
+`;
+
+exports[`datasource/index warns if multiple registryUrls for registryStrategy=first 1`] = `null`;

lib/datasource/common.ts

@@ -7,7 +7,9 @@ export interface Config {
   registryUrls?: string[];
 }
 
-export type DigestConfig = Config;
+export interface DigestConfig extends Config {
+  registryUrl?: string;
+}
 
 interface ReleasesConfigBase {
   compatibility?: Record<string, string>;
@@ -17,6 +19,7 @@ interface ReleasesConfigBase {
 
 export interface GetReleasesConfig extends ReleasesConfigBase {
   lookupName: string;
+  registryUrl?: string;
 }
 
 export interface GetPkgReleasesConfig extends ReleasesConfigBase {
@@ -65,6 +68,7 @@ export interface ReleaseResult {
   sourceUrl?: string;
   tags?: Record<string, string>;
   versions?: any;
+  registryUrl?: string;
 }
 
 export interface Datasource {
@@ -74,6 +78,7 @@ export interface Datasource {
   defaultRegistryUrls?: string[];
   appendRegistryUrls?: string[];
   defaultConfig?: object;
+  registryStrategy?: 'first' | 'hunt' | 'merge';
 }
 
 export class DatasourceError extends Error {

lib/datasource/docker/index.spec.ts

@@ -31,13 +31,16 @@ describe('api/docker', () => {
 
   describe('getRegistryRepository', () => {
     it('handles local registries', () => {
-      const res = docker.getRegistryRepository('registry:5000/org/package', []);
+      const res = docker.getRegistryRepository(
+        'registry:5000/org/package',
+        'https://index.docker.io'
+      );
       expect(res).toMatchSnapshot();
     });
     it('supports registryUrls', () => {
       const res = docker.getRegistryRepository(
         'my.local.registry/prefix/image',
-        ['https://my.local.registry/prefix']
+        'https://my.local.registry/prefix'
       );
       expect(res).toMatchSnapshot();
     });

lib/datasource/docker/index.ts

@@ -1,6 +1,5 @@
 import { OutgoingHttpHeaders } from 'http';
 import URL from 'url';
-import is from '@sindresorhus/is';
 import AWS from 'aws-sdk';
 import hasha from 'hasha';
 import parseLinkHeader from 'parse-link-header';
@@ -16,6 +15,8 @@ import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
 // TODO: replace www-authenticate with https://www.npmjs.com/package/auth-header ?
 
 export const id = 'docker';
+export const defaultRegistryUrls = ['https://index.docker.io'];
+export const registryStrategy = 'first';
 
 export const defaultConfig = {
   managerBranchPrefix: 'docker-',
@@ -57,10 +58,10 @@ export interface RegistryRepository {
 
 export function getRegistryRepository(
   lookupName: string,
-  registryUrls: string[]
+  registryUrl: string
 ): RegistryRepository {
-  if (is.nonEmptyArray(registryUrls)) {
+  if (registryUrl !== defaultRegistryUrls[0]) {
-    const dockerRegistry = registryUrls[0]
+    const dockerRegistry = registryUrl
       .replace('https://', '')
       .replace(/\/?$/, '/');
     if (lookupName.startsWith(dockerRegistry)) {
@@ -77,10 +78,10 @@ export function getRegistryRepository(
     split.shift();
   }
   let repository = split.join('/');
-  if (!registry && is.nonEmptyArray(registryUrls)) {
+  if (!registry) {
-    [registry] = registryUrls;
+    registry = registryUrl;
   }
-  if (!registry || registry === 'docker.io') {
+  if (registry === 'docker.io') {
     registry = 'index.docker.io';
   }
   if (!/^https?:\/\//.exec(registry)) {
@@ -327,12 +328,12 @@ async function getManifestResponse(
  *  - Return the digest as a string
  */
 export async function getDigest(
-  { registryUrls, lookupName }: GetReleasesConfig,
+  { registryUrl, lookupName }: GetReleasesConfig,
   newValue?: string
 ): Promise<string | null> {
   const { registry, repository } = getRegistryRepository(
     lookupName,
-    registryUrls
+    registryUrl
   );
   logger.debug(`getDigest(${registry}, ${repository}, ${newValue})`);
   const newTag = newValue || 'latest';
@@ -608,11 +609,11 @@ async function getLabels(
  */
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const { registry, repository } = getRegistryRepository(
     lookupName,
-    registryUrls
+    registryUrl
   );
   const tags = await getTags(registry, repository);
   if (!tags) {

lib/datasource/gitlab-tags/index.ts

@@ -1,5 +1,4 @@
 import URL from 'url';
-import is from '@sindresorhus/is';
 import { logger } from '../../logger';
 import * as globalCache from '../../util/cache/global';
 import { GitlabHttp } from '../../util/http/gitlab';
@@ -8,6 +7,8 @@ import { GetReleasesConfig, ReleaseResult } from '../common';
 const gitlabApi = new GitlabHttp();
 
 export const id = 'gitlab-tags';
+export const defaultRegistryUrls = ['https://gitlab.com'];
+export const registryStrategy = 'first';
 
 const cacheNamespace = 'datasource-gitlab';
 function getCacheKey(depHost: string, repo: string): string {
@@ -23,13 +24,9 @@ type GitlabTag = {
 };
 
 export async function getReleases({
-  registryUrls,
+  registryUrl: depHost,
   lookupName: repo,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  // Use registryUrls if present, otherwise default to publid gitlab.com
-  const depHost = is.nonEmptyArray(registryUrls)
-    ? registryUrls[0].replace(/\/$/, '')
-    : 'https://gitlab.com';
   let gitlabTags: GitlabTag[];
   const cachedResult = await globalCache.get<ReleaseResult>(
     cacheNamespace,
@@ -65,7 +62,7 @@ export async function getReleases({
   }
 
   const dependency: ReleaseResult = {
-    sourceUrl: `${depHost}/${repo}`,
+    sourceUrl: URL.resolve(depHost, repo),
     releases: null,
   };
   dependency.releases = gitlabTags.map(({ name, commit }) => ({

lib/datasource/gradle-version/__snapshots__/index.spec.ts.snap

@@ -4,10 +4,6 @@ exports[`datasource/gradle-version getReleases calls configured registryUrls 1`]
 Object {
   "homepage": "https://gradle.org",
   "releases": Array [
-    Object {
-      "releaseTimestamp": "2009-07-20T08:50:13+0200",
-      "version": "0.7",
-    },
     Object {
       "releaseTimestamp": "2009-07-20T08:50:13+0200",
       "version": "0.7",
@@ -16,14 +12,6 @@ Object {
       "releaseTimestamp": "2009-09-28T14:01:59+0200",
       "version": "0.8",
     },
-    Object {
-      "releaseTimestamp": "2009-09-28T14:01:59+0200",
-      "version": "0.8",
-    },
-    Object {
-      "releaseTimestamp": "2010-12-19T12:50:06+1100",
-      "version": "0.9",
-    },
     Object {
       "releaseTimestamp": "2010-12-19T12:50:06+1100",
       "version": "0.9",
@@ -32,22 +20,10 @@ Object {
       "releaseTimestamp": "2011-01-02T11:40:57+1100",
       "version": "0.9.1",
     },
-    Object {
-      "releaseTimestamp": "2011-01-02T11:40:57+1100",
-      "version": "0.9.1",
-    },
     Object {
       "releaseTimestamp": "2011-01-23T13:34:21+1100",
       "version": "0.9.2",
     },
-    Object {
-      "releaseTimestamp": "2011-01-23T13:34:21+1100",
-      "version": "0.9.2",
-    },
-    Object {
-      "releaseTimestamp": "2012-06-12T02:56:21+0200",
-      "version": "1.0",
-    },
     Object {
       "releaseTimestamp": "2012-06-12T02:56:21+0200",
       "version": "1.0",
@@ -56,22 +32,10 @@ Object {
       "releaseTimestamp": "2012-07-31T13:24:32+0000",
       "version": "1.1",
     },
-    Object {
-      "releaseTimestamp": "2012-07-31T13:24:32+0000",
-      "version": "1.1",
-    },
     Object {
       "releaseTimestamp": "2012-09-12T10:46:02+0000",
       "version": "1.2",
     },
-    Object {
-      "releaseTimestamp": "2012-09-12T10:46:02+0000",
-      "version": "1.2",
-    },
-    Object {
-      "releaseTimestamp": "2012-11-20T11:37:38+0000",
-      "version": "1.3",
-    },
     Object {
       "releaseTimestamp": "2012-11-20T11:37:38+0000",
       "version": "1.3",
@@ -80,22 +44,10 @@ Object {
       "releaseTimestamp": "2013-01-28T03:42:46+0000",
       "version": "1.4",
     },
-    Object {
-      "releaseTimestamp": "2013-01-28T03:42:46+0000",
-      "version": "1.4",
-    },
     Object {
       "releaseTimestamp": "2013-03-27T14:09:35+0000",
       "version": "1.5",
     },
-    Object {
-      "releaseTimestamp": "2013-03-27T14:09:35+0000",
-      "version": "1.5",
-    },
-    Object {
-      "releaseTimestamp": "2013-05-07T09:12:14+0000",
-      "version": "1.6",
-    },
     Object {
       "releaseTimestamp": "2013-05-07T09:12:14+0000",
       "version": "1.6",
@@ -104,22 +56,10 @@ Object {
       "releaseTimestamp": "2013-08-06T11:19:56+0000",
       "version": "1.7",
     },
-    Object {
-      "releaseTimestamp": "2013-08-06T11:19:56+0000",
-      "version": "1.7",
-    },
     Object {
       "releaseTimestamp": "2013-09-24T07:32:33+0000",
       "version": "1.8",
     },
-    Object {
-      "releaseTimestamp": "2013-09-24T07:32:33+0000",
-      "version": "1.8",
-    },
-    Object {
-      "releaseTimestamp": "2013-11-19T08:20:02+0000",
-      "version": "1.9",
-    },
     Object {
       "releaseTimestamp": "2013-11-19T08:20:02+0000",
       "version": "1.9",
@@ -128,22 +68,10 @@ Object {
       "releaseTimestamp": "2013-12-17T09:28:15+0000",
       "version": "1.10",
     },
-    Object {
-      "releaseTimestamp": "2013-12-17T09:28:15+0000",
-      "version": "1.10",
-    },
     Object {
       "releaseTimestamp": "2014-02-11T11:34:39+0000",
       "version": "1.11",
     },
-    Object {
-      "releaseTimestamp": "2014-02-11T11:34:39+0000",
-      "version": "1.11",
-    },
-    Object {
-      "releaseTimestamp": "2014-04-29T09:24:31+0000",
-      "version": "1.12",
-    },
     Object {
       "releaseTimestamp": "2014-04-29T09:24:31+0000",
       "version": "1.12",
@@ -152,22 +80,10 @@ Object {
       "releaseTimestamp": "2014-07-01T07:45:34+0000",
       "version": "2.0",
     },
-    Object {
-      "releaseTimestamp": "2014-07-01T07:45:34+0000",
-      "version": "2.0",
-    },
     Object {
       "releaseTimestamp": "2014-09-08T10:40:39+0000",
       "version": "2.1",
     },
-    Object {
-      "releaseTimestamp": "2014-09-08T10:40:39+0000",
-      "version": "2.1",
-    },
-    Object {
-      "releaseTimestamp": "2014-11-10T13:31:44+0000",
-      "version": "2.2",
-    },
     Object {
       "releaseTimestamp": "2014-11-10T13:31:44+0000",
       "version": "2.2",
@@ -176,22 +92,10 @@ Object {
       "releaseTimestamp": "2014-11-24T09:45:35+0000",
       "version": "2.2.1",
     },
-    Object {
-      "releaseTimestamp": "2014-11-24T09:45:35+0000",
-      "version": "2.2.1",
-    },
     Object {
       "releaseTimestamp": "2015-02-16T05:09:33+0000",
       "version": "2.3",
     },
-    Object {
-      "releaseTimestamp": "2015-02-16T05:09:33+0000",
-      "version": "2.3",
-    },
-    Object {
-      "releaseTimestamp": "2015-05-05T08:09:24+0000",
-      "version": "2.4",
-    },
     Object {
       "releaseTimestamp": "2015-05-05T08:09:24+0000",
       "version": "2.4",
@@ -200,22 +104,10 @@ Object {
       "releaseTimestamp": "2015-07-08T07:38:37+0000",
       "version": "2.5",
     },
-    Object {
-      "releaseTimestamp": "2015-07-08T07:38:37+0000",
-      "version": "2.5",
-    },
     Object {
       "releaseTimestamp": "2015-08-10T13:15:06+0000",
       "version": "2.6",
     },
-    Object {
-      "releaseTimestamp": "2015-08-10T13:15:06+0000",
-      "version": "2.6",
-    },
-    Object {
-      "releaseTimestamp": "2015-09-14T07:26:16+0000",
-      "version": "2.7",
-    },
     Object {
       "releaseTimestamp": "2015-09-14T07:26:16+0000",
       "version": "2.7",
@@ -224,22 +116,10 @@ Object {
       "releaseTimestamp": "2015-10-20T03:46:36+0000",
       "version": "2.8",
     },
-    Object {
-      "releaseTimestamp": "2015-10-20T03:46:36+0000",
-      "version": "2.8",
-    },
     Object {
       "releaseTimestamp": "2015-11-17T07:02:17+0000",
       "version": "2.9",
     },
-    Object {
-      "releaseTimestamp": "2015-11-17T07:02:17+0000",
-      "version": "2.9",
-    },
-    Object {
-      "releaseTimestamp": "2015-12-21T21:15:04+0000",
-      "version": "2.10",
-    },
     Object {
       "releaseTimestamp": "2015-12-21T21:15:04+0000",
       "version": "2.10",
@@ -248,22 +128,10 @@ Object {
       "releaseTimestamp": "2016-02-08T07:59:16+0000",
       "version": "2.11",
     },
-    Object {
-      "releaseTimestamp": "2016-02-08T07:59:16+0000",
-      "version": "2.11",
-    },
     Object {
       "releaseTimestamp": "2016-03-14T08:32:03+0000",
       "version": "2.12",
     },
-    Object {
-      "releaseTimestamp": "2016-03-14T08:32:03+0000",
-      "version": "2.12",
-    },
-    Object {
-      "releaseTimestamp": "2016-04-25T04:10:10+0000",
-      "version": "2.13",
-    },
     Object {
       "releaseTimestamp": "2016-04-25T04:10:10+0000",
       "version": "2.13",
@@ -272,22 +140,10 @@ Object {
       "releaseTimestamp": "2016-06-14T07:16:37+0000",
       "version": "2.14",
     },
-    Object {
-      "releaseTimestamp": "2016-06-14T07:16:37+0000",
-      "version": "2.14",
-    },
     Object {
       "releaseTimestamp": "2016-07-18T06:38:37+0000",
       "version": "2.14.1",
     },
-    Object {
-      "releaseTimestamp": "2016-07-18T06:38:37+0000",
-      "version": "2.14.1",
-    },
-    Object {
-      "releaseTimestamp": "2016-08-15T13:15:01+0000",
-      "version": "3.0",
-    },
     Object {
       "releaseTimestamp": "2016-08-15T13:15:01+0000",
       "version": "3.0",
@@ -296,22 +152,10 @@ Object {
       "releaseTimestamp": "2016-09-19T10:53:53+0000",
       "version": "3.1",
     },
-    Object {
-      "releaseTimestamp": "2016-09-19T10:53:53+0000",
-      "version": "3.1",
-    },
     Object {
       "releaseTimestamp": "2016-11-14T12:32:59+0000",
       "version": "3.2",
     },
-    Object {
-      "releaseTimestamp": "2016-11-14T12:32:59+0000",
-      "version": "3.2",
-    },
-    Object {
-      "releaseTimestamp": "2016-11-22T15:19:54+0000",
-      "version": "3.2.1",
-    },
     Object {
       "releaseTimestamp": "2016-11-22T15:19:54+0000",
       "version": "3.2.1",
@@ -320,22 +164,10 @@ Object {
       "releaseTimestamp": "2017-01-03T15:31:04+0000",
       "version": "3.3",
     },
-    Object {
-      "releaseTimestamp": "2017-01-03T15:31:04+0000",
-      "version": "3.3",
-    },
     Object {
       "releaseTimestamp": "2017-02-20T14:49:26+0000",
       "version": "3.4",
     },
-    Object {
-      "releaseTimestamp": "2017-02-20T14:49:26+0000",
-      "version": "3.4",
-    },
-    Object {
-      "releaseTimestamp": "2017-03-03T19:45:41+0000",
-      "version": "3.4.1",
-    },
     Object {
       "releaseTimestamp": "2017-03-03T19:45:41+0000",
       "version": "3.4.1",
@@ -344,22 +176,10 @@ Object {
       "releaseTimestamp": "2017-04-10T13:37:25+0000",
       "version": "3.5",
     },
-    Object {
-      "releaseTimestamp": "2017-04-10T13:37:25+0000",
-      "version": "3.5",
-    },
     Object {
       "releaseTimestamp": "2017-06-16T14:36:27+0000",
       "version": "3.5.1",
     },
-    Object {
-      "releaseTimestamp": "2017-06-16T14:36:27+0000",
-      "version": "3.5.1",
-    },
-    Object {
-      "releaseTimestamp": "2017-06-14T15:11:08+0000",
-      "version": "4.0",
-    },
     Object {
       "releaseTimestamp": "2017-06-14T15:11:08+0000",
       "version": "4.0",
@@ -368,22 +188,10 @@ Object {
       "releaseTimestamp": "2017-07-07T14:02:41+0000",
       "version": "4.0.1",
     },
-    Object {
-      "releaseTimestamp": "2017-07-07T14:02:41+0000",
-      "version": "4.0.1",
-    },
     Object {
       "releaseTimestamp": "2017-07-26T16:19:18+0000",
       "version": "4.0.2",
     },
-    Object {
-      "releaseTimestamp": "2017-07-26T16:19:18+0000",
-      "version": "4.0.2",
-    },
-    Object {
-      "releaseTimestamp": "2017-08-07T14:38:48+0000",
-      "version": "4.1",
-    },
     Object {
       "releaseTimestamp": "2017-08-07T14:38:48+0000",
       "version": "4.1",
@@ -392,22 +200,10 @@ Object {
       "releaseTimestamp": "2017-09-20T14:48:23+0000",
       "version": "4.2",
     },
-    Object {
-      "releaseTimestamp": "2017-09-20T14:48:23+0000",
-      "version": "4.2",
-    },
     Object {
       "releaseTimestamp": "2017-10-02T15:36:21+0000",
       "version": "4.2.1",
     },
-    Object {
-      "releaseTimestamp": "2017-10-02T15:36:21+0000",
-      "version": "4.2.1",
-    },
-    Object {
-      "releaseTimestamp": "2017-10-30T15:43:29+0000",
-      "version": "4.3",
-    },
     Object {
       "releaseTimestamp": "2017-10-30T15:43:29+0000",
       "version": "4.3",
@@ -416,22 +212,10 @@ Object {
       "releaseTimestamp": "2017-11-08T08:59:45+0000",
       "version": "4.3.1",
     },
-    Object {
-      "releaseTimestamp": "2017-11-08T08:59:45+0000",
-      "version": "4.3.1",
-    },
     Object {
       "releaseTimestamp": "2017-12-06T09:05:06+0000",
       "version": "4.4",
     },
-    Object {
-      "releaseTimestamp": "2017-12-06T09:05:06+0000",
-      "version": "4.4",
-    },
-    Object {
-      "releaseTimestamp": "2017-12-20T15:45:23+0000",
-      "version": "4.4.1",
-    },
     Object {
       "releaseTimestamp": "2017-12-20T15:45:23+0000",
       "version": "4.4.1",
@@ -440,22 +224,10 @@ Object {
       "releaseTimestamp": "2018-01-24T17:04:52+0000",
       "version": "4.5",
     },
-    Object {
-      "releaseTimestamp": "2018-01-24T17:04:52+0000",
-      "version": "4.5",
-    },
     Object {
       "releaseTimestamp": "2018-02-05T13:22:49+0000",
       "version": "4.5.1",
     },
-    Object {
-      "releaseTimestamp": "2018-02-05T13:22:49+0000",
-      "version": "4.5.1",
-    },
-    Object {
-      "releaseTimestamp": "2018-02-28T13:36:36+0000",
-      "version": "4.6",
-    },
     Object {
       "releaseTimestamp": "2018-02-28T13:36:36+0000",
       "version": "4.6",
@@ -464,22 +236,10 @@ Object {
       "releaseTimestamp": "2018-04-18T09:09:12+0000",
       "version": "4.7",
     },
-    Object {
-      "releaseTimestamp": "2018-04-18T09:09:12+0000",
-      "version": "4.7",
-    },
     Object {
       "releaseTimestamp": "2018-06-04T10:39:58+0000",
       "version": "4.8",
     },
-    Object {
-      "releaseTimestamp": "2018-06-04T10:39:58+0000",
-      "version": "4.8",
-    },
-    Object {
-      "releaseTimestamp": "2018-06-21T07:53:06+0000",
-      "version": "4.8.1",
-    },
     Object {
       "releaseTimestamp": "2018-06-21T07:53:06+0000",
       "version": "4.8.1",
@@ -488,22 +248,10 @@ Object {
       "releaseTimestamp": "2018-07-16T08:14:03+0000",
       "version": "4.9",
     },
-    Object {
-      "releaseTimestamp": "2018-07-16T08:14:03+0000",
-      "version": "4.9",
-    },
     Object {
       "releaseTimestamp": "2018-08-27T18:35:06+0000",
       "version": "4.10",
     },
-    Object {
-      "releaseTimestamp": "2018-08-27T18:35:06+0000",
-      "version": "4.10",
-    },
-    Object {
-      "releaseTimestamp": "2018-09-12T11:33:27+0000",
-      "version": "4.10.1",
-    },
     Object {
       "releaseTimestamp": "2018-09-12T11:33:27+0000",
       "version": "4.10.1",
@@ -512,22 +260,10 @@ Object {
       "releaseTimestamp": "2018-09-19T18:10:15+0000",
       "version": "4.10.2",
     },
-    Object {
-      "releaseTimestamp": "2018-09-19T18:10:15+0000",
-      "version": "4.10.2",
-    },
     Object {
       "releaseTimestamp": null,
       "version": "4.10.3",
     },
-    Object {
-      "releaseTimestamp": null,
-      "version": "4.10.3",
-    },
-    Object {
-      "releaseTimestamp": null,
-      "version": "5.0",
-    },
     Object {
       "releaseTimestamp": null,
       "version": "5.0",

lib/datasource/gradle-version/index.ts

@@ -1,20 +1,14 @@
-import is from '@sindresorhus/is';
 import { logger } from '../../logger';
 import { Http } from '../../util/http';
 import { regEx } from '../../util/regex';
-import {
+import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
-  DatasourceError,
-  GetReleasesConfig,
-  Release,
-  ReleaseResult,
-} from '../common';
 
 export const id = 'gradle-version';
+export const defaultRegistryUrls = ['https://services.gradle.org/versions/all'];
+export const registryStrategy = 'merge';
 
 const http = new Http(id);
 
-const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all';
-
 interface GradleRelease {
   snapshot?: boolean;
   nightly?: boolean;
@@ -38,41 +32,32 @@ function formatBuildTime(timeStr: string): string | null {
 }
 
 export async function getReleases({
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult> {
-  const versionsUrls = is.nonEmptyArray(registryUrls)
+  let releases;
-    ? registryUrls
+  try {
-    : [GradleVersionsServiceUrl];
+    const response = await http.getJson<GradleRelease[]>(registryUrl);
-
+    releases = response.body
-  const allReleases: Release[][] = await Promise.all(
+      .filter((release) => !release.snapshot && !release.nightly)
-    versionsUrls.map(async (url) => {
+      .filter(
-      try {
+        (release) =>
-        const response = await http.getJson<GradleRelease[]>(url);
+          // some milestone have wrong metadata and need to be filtered by version name content
-        const releases = response.body
+          release.rcFor === '' && !release.version.includes('milestone')
-          .filter((release) => !release.snapshot && !release.nightly)
+      )
-          .filter(
+      .map((release) => ({
-            (release) =>
+        version: release.version,
-              // some milestone have wrong metadata and need to be filtered by version name content
+        releaseTimestamp: formatBuildTime(release.buildTime),
-              release.rcFor === '' && !release.version.includes('milestone')
+      }));
-          )
+  } catch (err) /* istanbul ignore next */ {
-          .map((release) => ({
+    if (err.host === 'services.gradle.org') {
-            version: release.version,
+      throw new DatasourceError(err);
-            releaseTimestamp: formatBuildTime(release.buildTime),
+    }
-          }));
+    logger.debug({ err }, 'gradle-version err');
-        return releases;
+    return null;
-      } catch (err) /* istanbul ignore next */ {
+  }
-        // istanbul ignore if
-        if (err.host === 'services.gradle.org') {
-          throw new DatasourceError(err);
-        }
-        logger.debug({ err }, 'gradle-version err');
-        return null;
-      }
-    })
-  );
 
   const res: ReleaseResult = {
-    releases: Array.prototype.concat.apply([], allReleases).filter(Boolean),
+    releases,
     homepage: 'https://gradle.org',
     sourceUrl: 'https://github.com/gradle/gradle',
   };

lib/datasource/helm/index.ts

@@ -13,6 +13,7 @@ const http = new Http(id);
 export const defaultRegistryUrls = [
   'https://kubernetes-charts.storage.googleapis.com/',
 ];
+export const registryStrategy = 'first';
 
 export async function getRepositoryData(
   repository: string
@@ -101,9 +102,8 @@ export async function getRepositoryData(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl: helmRepository,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  const [helmRepository] = registryUrls;
   const repositoryData = await getRepositoryData(helmRepository);
   if (!repositoryData) {
     logger.debug(`Couldn't get index.yaml file from ${helmRepository}`);

lib/datasource/index.spec.ts

@@ -1,16 +1,28 @@
 import { mocked } from '../../test/util';
+import { DATASOURCE_FAILURE } from '../constants/error-messages';
 import { loadModules } from '../util/modules';
+import { DatasourceError } from './common';
 import * as datasourceDocker from './docker';
 import * as datasourceGithubTags from './github-tags';
+import * as datasourceMaven from './maven';
 import * as datasourceNpm from './npm';
+import * as datasourcePackagist from './packagist';
 import * as datasource from '.';
 
 jest.mock('./docker');
+jest.mock('./maven');
 jest.mock('./npm');
+jest.mock('./packagist');
 
+const dockerDatasource = mocked(datasourceDocker);
+const mavenDatasource = mocked(datasourceMaven);
 const npmDatasource = mocked(datasourceNpm);
+const packagistDatasource = mocked(datasourcePackagist);
 
 describe('datasource/index', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
   it('returns datasources', () => {
     expect(datasource.getDatasources()).toBeDefined();
     expect(datasource.getDatasourceList()).toBeDefined();
@@ -98,6 +110,96 @@ describe('datasource/index', () => {
     expect(res).toMatchSnapshot();
     expect(res.sourceUrl).toBeDefined();
   });
+  it('warns if multiple registryUrls for registryStrategy=first', async () => {
+    dockerDatasource.getReleases.mockResolvedValue(null);
+    const res = await datasource.getPkgReleases({
+      datasource: datasourceDocker.id,
+      depName: 'something',
+      registryUrls: ['https://docker.com', 'https://docker.io'],
+    });
+    expect(res).toMatchSnapshot();
+  });
+  it('hunts registries and returns success', async () => {
+    packagistDatasource.getReleases.mockResolvedValueOnce(null);
+    packagistDatasource.getReleases.mockResolvedValueOnce({
+      releases: [{ version: '1.0.0' }],
+    });
+    const res = await datasource.getPkgReleases({
+      datasource: datasourcePackagist.id,
+      depName: 'something',
+      registryUrls: ['https://reg1.com', 'https://reg2.io'],
+    });
+    expect(res).not.toBeNull();
+  });
+  it('hunts registries and aborts on DatasourceError', async () => {
+    packagistDatasource.getReleases.mockImplementationOnce(() => {
+      throw new DatasourceError(new Error());
+    });
+    await expect(
+      datasource.getPkgReleases({
+        datasource: datasourcePackagist.id,
+        depName: 'something',
+        registryUrls: ['https://reg1.com', 'https://reg2.io'],
+      })
+    ).rejects.toThrow(DATASOURCE_FAILURE);
+  });
+  it('hunts registries and passes on error', async () => {
+    packagistDatasource.getReleases.mockImplementationOnce(() => {
+      throw new Error('a');
+    });
+    packagistDatasource.getReleases.mockImplementationOnce(() => {
+      throw new Error('b');
+    });
+    await expect(
+      datasource.getPkgReleases({
+        datasource: datasourcePackagist.id,
+        depName: 'something',
+        registryUrls: ['https://reg1.com', 'https://reg2.io'],
+      })
+    ).rejects.toThrow('b');
+  });
+  it('merges registries and returns success', async () => {
+    mavenDatasource.getReleases.mockResolvedValueOnce({
+      releases: [{ version: '1.0.0' }, { version: '1.1.0' }],
+    });
+    mavenDatasource.getReleases.mockResolvedValueOnce({
+      releases: [{ version: '1.0.0' }],
+    });
+    const res = await datasource.getPkgReleases({
+      datasource: datasourceMaven.id,
+      depName: 'something',
+      registryUrls: ['https://reg1.com', 'https://reg2.io'],
+    });
+    expect(res).toMatchSnapshot();
+    expect(res.releases).toHaveLength(2);
+  });
+  it('merges registries and aborts on DatasourceError', async () => {
+    mavenDatasource.getReleases.mockImplementationOnce(() => {
+      throw new DatasourceError(new Error());
+    });
+    await expect(
+      datasource.getPkgReleases({
+        datasource: datasourceMaven.id,
+        depName: 'something',
+        registryUrls: ['https://reg1.com', 'https://reg2.io'],
+      })
+    ).rejects.toThrow(DATASOURCE_FAILURE);
+  });
+  it('merges registries and passes on error', async () => {
+    mavenDatasource.getReleases.mockImplementationOnce(() => {
+      throw new Error('a');
+    });
+    mavenDatasource.getReleases.mockImplementationOnce(() => {
+      throw new Error('b');
+    });
+    await expect(
+      datasource.getPkgReleases({
+        datasource: datasourceMaven.id,
+        depName: 'something',
+        registryUrls: ['https://reg1.com', 'https://reg2.io'],
+      })
+    ).rejects.toThrow('b');
+  });
   it('trims sourceUrl', async () => {
     npmDatasource.getReleases.mockResolvedValue({
       sourceUrl: ' https://abc.com',

lib/datasource/index.ts

@@ -29,6 +29,102 @@ function load(datasource: string): Promise<Datasource> {
 
 type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig;
 
+function firstRegistry(
+  config: GetReleasesInternalConfig,
+  datasource: Datasource,
+  registryUrls: string[]
+): Promise<ReleaseResult> {
+  if (registryUrls.length > 1) {
+    logger.warn(
+      { datasource: datasource.id, depName: config.depName, registryUrls },
+      'Excess registryUrls found for datasource lookup - using first configured only'
+    );
+  }
+  const registryUrl = registryUrls[0];
+  return datasource.getReleases({
+    ...config,
+    registryUrl,
+  });
+}
+
+async function huntRegistries(
+  config: GetReleasesInternalConfig,
+  datasource: Datasource,
+  registryUrls: string[]
+): Promise<ReleaseResult> {
+  let res: ReleaseResult;
+  let datasourceError;
+  for (const registryUrl of registryUrls) {
+    try {
+      res = await datasource.getReleases({
+        ...config,
+        registryUrl,
+      });
+      if (res) {
+        break;
+      }
+    } catch (err) {
+      if (err instanceof DatasourceError) {
+        throw err;
+      }
+      // We'll always save the last-thrown error
+      datasourceError = err;
+      logger.trace({ err }, 'datasource hunt failure');
+    }
+  }
+  if (res === undefined && datasourceError) {
+    // if we failed to get a result and also got an error then throw it
+    throw datasourceError;
+  }
+  return res;
+}
+
+async function mergeRegistries(
+  config: GetReleasesInternalConfig,
+  datasource: Datasource,
+  registryUrls: string[]
+): Promise<ReleaseResult> {
+  let combinedRes: ReleaseResult;
+  let datasourceError;
+  for (const registryUrl of registryUrls) {
+    try {
+      const res = await datasource.getReleases({
+        ...config,
+        registryUrl,
+      });
+      if (combinedRes) {
+        combinedRes = { ...res, ...combinedRes };
+        combinedRes.releases = [...combinedRes.releases, ...res.releases];
+      } else {
+        combinedRes = res;
+      }
+    } catch (err) {
+      if (err instanceof DatasourceError) {
+        throw err;
+      }
+      // We'll always save the last-thrown error
+      datasourceError = err;
+      logger.trace({ err }, 'datasource merge failure');
+    }
+  }
+  if (combinedRes === undefined && datasourceError) {
+    // if we failed to get a result and also got an error then throw it
+    throw datasourceError;
+  }
+  // De-duplicate releases
+  if (combinedRes?.releases?.length) {
+    const seenVersions = new Set<string>();
+    combinedRes.releases = combinedRes.releases.filter((release) => {
+      if (seenVersions.has(release.version)) {
+        return false;
+      }
+      seenVersions.add(release.version);
+      return true;
+    });
+  }
+  return combinedRes;
+}
+
 function resolveRegistryUrls(
   datasource: Datasource,
   extractedUrls: string[]
@@ -49,12 +145,31 @@ async function fetchReleases(
   }
   const datasource = await load(datasourceName);
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
-  let dep = await datasource.getReleases({
+  let dep: ReleaseResult;
-    ...config,
+  if (datasource.registryStrategy) {
-    registryUrls,
+    // istanbul ignore if
-  });
+    if (!registryUrls.length) {
-  if (!(dep && dep.releases.length)) {
+      logger.warn(
-    dep = null;
+        { datasource: datasourceName, depName: config.depName },
+        'Missing registryUrls for registryStrategy'
+      );
+      return null;
+    }
+    if (datasource.registryStrategy === 'first') {
+      dep = await firstRegistry(config, datasource, registryUrls);
+    } else if (datasource.registryStrategy === 'hunt') {
+      dep = await huntRegistries(config, datasource, registryUrls);
+    } else if (datasource.registryStrategy === 'merge') {
+      dep = await mergeRegistries(config, datasource, registryUrls);
+    }
+  } else {
+    dep = await datasource.getReleases({
+      ...config,
+      registryUrls,
+    });
+  }
+  if (!dep?.releases?.length) {
+    return null;
   }
   addMetaData(dep, datasourceName, config.lookupName);
   return dep;
@@ -131,10 +246,11 @@ export async function getDigest(
   config: DigestConfig,
   value?: string
 ): Promise<string | null> {
+  const datasource = await load(config.datasource);
   const lookupName = config.lookupName || config.depName;
-  const { registryUrls } = config;
+  const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
-  return (await load(config.datasource)).getDigest(
+  return datasource.getDigest(
-    { lookupName, registryUrls },
+    { lookupName, registryUrl: registryUrls[0] },
     value
   );
 }

lib/datasource/maven/index.ts

@@ -13,6 +13,7 @@ import { downloadHttpProtocol, isHttpResourceExists } from './util';
 export { id } from './common';
 
 export const defaultRegistryUrls = [MAVEN_REPO];
+export const registryStrategy = 'merge';
 
 function containsPlaceholder(str: string): boolean {
   return /\${.*?}/g.test(str);
@@ -250,53 +251,39 @@ async function filterMissingArtifacts(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  const repositories = registryUrls.map((repository) =>
-    repository.replace(/\/?$/, '/')
-  );
   const dependency = getDependencyParts(lookupName);
   const versions: string[] = [];
   const repoForVersions = {};
-  for (let i = 0; i < repositories.length; i += 1) {
+  const repoUrl = registryUrl.replace(/\/?$/, '/');
-    const repoUrl = repositories[i];
+  logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`);
-    logger.debug(
+  const metadataVersions = await getVersionsFromMetadata(dependency, repoUrl);
-      `Looking up ${dependency.display} in repository #${i} - ${repoUrl}`
+  if (metadataVersions) {
+    const availableVersions = await filterMissingArtifacts(
+      dependency,
+      repoUrl,
+      metadataVersions
     );
-    const metadataVersions = await getVersionsFromMetadata(dependency, repoUrl);
+    const filteredVersions = availableVersions.filter(
-    if (metadataVersions) {
+      (version) => !versions.includes(version)
-      const availableVersions = await filterMissingArtifacts(
+    );
-        dependency,
+    versions.push(...filteredVersions);
-        repoUrl,
-        metadataVersions
-      );
-      const filteredVersions = availableVersions.filter(
-        (version) => !versions.includes(version)
-      );
-      versions.push(...filteredVersions);
 
-      const latestVersion = getLatestStableVersion(filteredVersions);
+    const latestVersion = getLatestStableVersion(filteredVersions);
-      if (latestVersion) {
+    if (latestVersion) {
-        repoForVersions[latestVersion] = repoUrl;
+      repoForVersions[latestVersion] = repoUrl;
-      }
-
-      logger.debug(`Found ${availableVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
     }
-  }
 
-  if (versions.length === 0) {
+    logger.debug(`Found ${availableVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
-    logger.debug(`No versions found for ${dependency.display} in ${repositories.length} repositories`); // prettier-ignore
-    return null;
   }
-  logger.debug(`Found ${versions.length} versions for ${dependency.display}`);
 
   let dependencyInfo = {};
   const latestVersion = getLatestStableVersion(versions);
   if (latestVersion) {
-    const repoUrl = repoForVersions[latestVersion];
     dependencyInfo = await getDependencyInfo(
       dependency,
-      repoUrl,
+      repoForVersions[latestVersion],
       latestVersion
     );
   }

lib/datasource/nuget/index.ts

@@ -1,6 +1,5 @@
 import urlApi from 'url';
 import { logger } from '../../logger';
-import { clone } from '../../util/clone';
 import { GetReleasesConfig, ReleaseResult } from '../common';
 import * as v2 from './v2';
 import * as v3 from './v3';
@@ -8,6 +7,7 @@ import * as v3 from './v3';
 export { id } from './common';
 
 export const defaultRegistryUrls = [v3.getDefaultFeed()];
+export const registryStrategy = 'merge';
 
 function parseRegistryUrl(
   registryUrl: string
@@ -32,43 +32,18 @@ function parseRegistryUrl(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult> {
   logger.trace(`nuget.getReleases(${lookupName})`);
-  let dep: ReleaseResult = null;
+  const { feedUrl, protocolVersion } = parseRegistryUrl(registryUrl);
-  for (const feed of registryUrls) {
+  if (protocolVersion === 2) {
-    const { feedUrl, protocolVersion } = parseRegistryUrl(feed);
+    return v2.getReleases(feedUrl, lookupName);
-    let res: ReleaseResult = null;
+  }
-    if (protocolVersion === 2) {
+  if (protocolVersion === 3) {
-      res = await v2.getReleases(feedUrl, lookupName);
+    const queryUrl = await v3.getQueryUrl(feedUrl);
-    } else if (protocolVersion === 3) {
+    if (queryUrl !== null) {
-      const queryUrl = await v3.getQueryUrl(feedUrl);
+      return v3.getReleases(feedUrl, queryUrl, lookupName);
-      if (queryUrl !== null) {
-        res = await v3.getReleases(feedUrl, queryUrl, lookupName);
-      }
-    }
-    if (res !== null) {
-      res = clone(res);
-      if (dep !== null) {
-        for (const resRelease of res.releases) {
-          if (
-            !dep.releases.find(
-              (depRelease) => depRelease.version === resRelease.version
-            )
-          ) {
-            dep.releases.push(resRelease);
-          }
-        }
-      } else {
-        dep = res;
-      }
     }
   }
-  if (dep === null) {
+  return null;
-    logger.debug(
-      { lookupName },
-      `Dependency lookup failure: not found in all feeds`
-    );
-  }
-  return dep;
 }

lib/datasource/packagist/index.ts

@@ -1,5 +1,4 @@
 import URL from 'url';
-import is from '@sindresorhus/is';
 
 import pAll from 'p-all';
 import { logger } from '../../logger';
@@ -10,6 +9,8 @@ import { Http, HttpOptions } from '../../util/http';
 import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'packagist';
+export const defaultRegistryUrls = ['https://packagist.org'];
+export const registryStrategy = 'hunt';
 
 const http = new Http(id);
 
@@ -325,19 +326,8 @@ async function packageLookup(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult> {
   logger.trace(`getReleases(${lookupName})`);
-
+  return packageLookup(registryUrl, lookupName);
-  let res: ReleaseResult;
-  const registries = is.nonEmptyArray(registryUrls)
-    ? registryUrls
-    : ['https://packagist.org'];
-  for (const regUrl of registries) {
-    res = await packageLookup(regUrl, lookupName);
-    if (res) {
-      break;
-    }
-  }
-  return res;
 }

lib/datasource/pod/index.ts

@@ -8,6 +8,7 @@ import { GetReleasesConfig, ReleaseResult } from '../common';
 export const id = 'pod';
 
 export const defaultRegistryUrls = ['https://cdn.cocoapods.org'];
+export const registryStrategy = 'hunt';
 
 const cacheNamespace = `datasource-${id}`;
 const cacheMinutes = 30;
@@ -149,39 +150,36 @@ function isDefaultRepo(url: string): boolean {
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const podName = lookupName.replace(/\/.*$/, '');
 
   const cachedResult = await globalCache.get<ReleaseResult>(
     cacheNamespace,
-    podName
+    registryUrl + podName
   );
-  /* istanbul ignore next line */
+
-  if (cachedResult) {
+  // istanbul ignore if
-    logger.debug(`CocoaPods: Return cached result for ${podName}`);
+  if (cachedResult !== undefined) {
+    logger.trace(`CocoaPods: Return cached result for ${podName}`);
     return cachedResult;
   }
 
+  let baseUrl = registryUrl.replace(/\/+$/, '');
+
+  // In order to not abuse github API limits, query CDN instead
+  if (isDefaultRepo(baseUrl)) {
+    [baseUrl] = defaultRegistryUrls;
+  }
+
   let result: ReleaseResult | null = null;
-  for (let idx = 0; !result && idx < registryUrls.length; idx += 1) {
+  if (githubRegex.exec(baseUrl)) {
-    let registryUrl = registryUrls[idx].replace(/\/+$/, '');
+    result = await getReleasesFromGithub(podName, baseUrl);
-
+  } else {
-    // In order to not abuse github API limits, query CDN instead
+    result = await getReleasesFromCDN(podName, baseUrl);
-    if (isDefaultRepo(registryUrl)) {
-      [registryUrl] = defaultRegistryUrls;
-    }
-
-    if (githubRegex.exec(registryUrl)) {
-      result = await getReleasesFromGithub(podName, registryUrl);
-    } else {
-      result = await getReleasesFromCDN(podName, registryUrl);
-    }
   }
 
-  if (result) {
+  await globalCache.set(cacheNamespace, podName, result, cacheMinutes);
-    await globalCache.set(cacheNamespace, podName, result, cacheMinutes);
-  }
 
   return result;
 }

lib/datasource/pypi/__snapshots__/index.spec.ts.snap

@@ -394,21 +394,6 @@ Array [
 ]
 `;
 
-exports[`datasource/pypi getReleases supports custom datasource url from environmental variable 1`] = `
-Array [
-  Object {
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "host": "my.pypi.python",
-      "user-agent": "https://github.com/renovatebot/renovate",
-    },
-    "method": "GET",
-    "url": "https://my.pypi.python/pypi/azure-cli-monitor/json",
-  },
-]
-`;
-
 exports[`datasource/pypi getReleases supports multiple custom datasource urls 1`] = `
 Array [
   Object {

lib/datasource/pypi/index.spec.ts

@@ -82,20 +82,6 @@ describe('datasource/pypi', () => {
       });
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
-    it('supports custom datasource url from environmental variable', async () => {
-      httpMock
-        .scope('https://my.pypi.python/pypi/')
-        .get('/azure-cli-monitor/json')
-        .reply(200, JSON.parse(res1));
-      const pipIndexUrl = process.env.PIP_INDEX_URL;
-      process.env.PIP_INDEX_URL = 'https://my.pypi.python/pypi/';
-      await getPkgReleases({
-        datasource,
-        depName: 'azure-cli-monitor',
-      });
-      expect(httpMock.getTrace()).toMatchSnapshot();
-      process.env.PIP_INDEX_URL = pipIndexUrl;
-    });
     it('supports multiple custom datasource urls', async () => {
       httpMock
         .scope('https://custom.pypi.net/foo')

lib/datasource/pypi/index.ts

@@ -1,14 +1,19 @@
 import url from 'url';
-import is from '@sindresorhus/is';
 import changelogFilenameRegex from 'changelog-filename-regex';
 import { parse } from 'node-html-parser';
 import { logger } from '../../logger';
 import { Http } from '../../util/http';
+import { ensureTrailingSlash } from '../../util/url';
 import { matches } from '../../versioning/pep440';
 import * as pep440 from '../../versioning/pep440';
 import { GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'pypi';
+export const defaultRegistryUrls = [
+  process.env.PIP_INDEX_URL || 'https://pypi.org/pypi/',
+];
+export const registryStrategy = 'hunt';
+
 const github_repo_pattern = /^https?:\/\/github\.com\/[^\\/]+\/[^\\/]+$/;
 const http = new Http(id);
 
@@ -208,36 +213,13 @@ async function getSimpleDependency(
 export async function getReleases({
   compatibility,
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  let hostUrls = ['https://pypi.org/pypi/'];
+  const hostUrl = ensureTrailingSlash(registryUrl);
-  if (is.nonEmptyArray(registryUrls)) {
+  if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
-    hostUrls = registryUrls;
+    logger.trace({ lookupName, hostUrl }, 'Looking up pypi simple dependency');
+    return getSimpleDependency(lookupName, hostUrl);
   }
-  if (process.env.PIP_INDEX_URL) {
+  logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency');
-    hostUrls = [process.env.PIP_INDEX_URL];
+  return getDependency(lookupName, hostUrl, compatibility);
-  }
-  let dep: ReleaseResult;
-  for (let index = 0; index < hostUrls.length && !dep; index += 1) {
-    let hostUrl = hostUrls[index];
-    hostUrl += hostUrl.endsWith('/') ? '' : '/';
-    if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
-      logger.trace(
-        { lookupName, hostUrl },
-        'Looking up pypi simple dependency'
-      );
-      dep = await getSimpleDependency(lookupName, hostUrl);
-    } else {
-      logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency');
-      dep = await getDependency(lookupName, hostUrl, compatibility);
-    }
-    if (dep !== null) {
-      logger.trace({ lookupName, hostUrl }, 'Found pypi result');
-    }
-  }
-  if (dep) {
-    return dep;
-  }
-  logger.debug({ lookupName, registryUrls }, 'No pypi result - returning null');
-  return null;
 }

lib/datasource/rubygems/index.ts

@@ -1,3 +1,4 @@
 export { getReleases } from './releases';
 export { id } from './common';
 export const defaultRegistryUrls = ['https://rubygems.org'];
+export const registryStrategy = 'hunt';

lib/datasource/rubygems/releases.ts

@@ -4,20 +4,11 @@ import { getRubygemsOrgDependency } from './get-rubygems-org';
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  for (const registry of registryUrls) {
+  // prettier-ignore
-    let pkg: ReleaseResult;
+  if (registryUrl.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization]
-    // prettier-ignore
+      return getRubygemsOrgDependency(lookupName);
-    if (registry.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization]
-      pkg = await getRubygemsOrgDependency(lookupName);
-    } else {
-      pkg = await getDependency({ dependency: lookupName, registry });
     }
-    if (pkg) {
+  return getDependency({ dependency: lookupName, registry: registryUrl });
-      return pkg;
-    }
-  }
-
-  return null;
 }

lib/datasource/sbt-package/index.ts

@@ -8,6 +8,7 @@ import { parseIndexDir } from '../sbt-plugin/util';
 export const id = 'sbt-package';
 
 export const defaultRegistryUrls = [MAVEN_REPO];
+export const registryStrategy = 'hunt';
 
 const ensureTrailingSlash = (str: string): string => str.replace(/\/?$/, '/');
 
@@ -65,20 +66,18 @@ export async function resolvePackageReleases(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const [groupId, artifactId] = lookupName.split(':');
   const groupIdSplit = groupId.split('.');
   const artifactIdSplit = artifactId.split('_');
   const [artifact, scalaVersion] = artifactIdSplit;
 
-  const repoRoots = registryUrls.map((x) => x.replace(/\/?$/, ''));
+  const repoRoot = ensureTrailingSlash(registryUrl);
   const searchRoots: string[] = [];
-  repoRoots.forEach((repoRoot) => {
+  // Optimize lookup order
-    // Optimize lookup order
+  searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`);
-    searchRoots.push(`${repoRoot}/${groupIdSplit.join('/')}`);
+  searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`);
-    searchRoots.push(`${repoRoot}/${groupIdSplit.join('.')}`);
-  });
 
   for (let idx = 0; idx < searchRoots.length; idx += 1) {
     const searchRoot = searchRoots[idx];

lib/datasource/sbt-plugin/index.ts

@@ -8,6 +8,7 @@ import { SBT_PLUGINS_REPO, parseIndexDir } from './util';
 export const id = 'sbt-plugin';
 
 export const defaultRegistryUrls = [SBT_PLUGINS_REPO];
+export const registryStrategy = 'hunt';
 
 const ensureTrailingSlash = (str: string): string => str.replace(/\/?$/, '/');
 
@@ -62,20 +63,18 @@ async function resolvePluginReleases(
 
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const [groupId, artifactId] = lookupName.split(':');
   const groupIdSplit = groupId.split('.');
   const artifactIdSplit = artifactId.split('_');
   const [artifact, scalaVersion] = artifactIdSplit;
 
-  const repoRoots = registryUrls.map((x) => x.replace(/\/?$/, ''));
+  const repoRoot = ensureTrailingSlash(registryUrl);
   const searchRoots: string[] = [];
-  repoRoots.forEach((repoRoot) => {
+  // Optimize lookup order
-    // Optimize lookup order
+  searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`);
-    searchRoots.push(`${repoRoot}/${groupIdSplit.join('.')}`);
+  searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`);
-    searchRoots.push(`${repoRoot}/${groupIdSplit.join('/')}`);
-  });
 
   for (let idx = 0; idx < searchRoots.length; idx += 1) {
     const searchRoot = searchRoots[idx];

lib/datasource/terraform-module/index.ts

@@ -1,10 +1,11 @@
-import is from '@sindresorhus/is';
 import { logger } from '../../logger';
 import * as globalCache from '../../util/cache/global';
 import { Http } from '../../util/http';
 import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
 
 export const id = 'terraform-module';
+export const defaultRegistryUrls = ['https://registry.terraform.io'];
+export const registryStrategy = 'first';
 
 const http = new Http(id);
 
@@ -15,17 +16,15 @@ interface RegistryRepository {
 
 function getRegistryRepository(
   lookupName: string,
-  registryUrls: string[]
+  registryUrl: string
 ): RegistryRepository {
   let registry: string;
   const split = lookupName.split('/');
   if (split.length > 3 && split[0].includes('.')) {
     [registry] = split;
     split.shift();
-  } else if (is.nonEmptyArray(registryUrls)) {
-    [registry] = registryUrls;
   } else {
-    registry = 'registry.terraform.io';
+    registry = registryUrl;
   }
   if (!/^https?:\/\//.test(registry)) {
     registry = `https://${registry}`;
@@ -54,11 +53,11 @@ interface TerraformRelease {
  */
 export async function getReleases({
   lookupName,
-  registryUrls,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const { registry, repository } = getRegistryRepository(
     lookupName,
-    registryUrls
+    registryUrl
   );
   logger.debug(
     { registry, terraformRepository: repository },

lib/datasource/terraform-provider/index.ts

@@ -22,7 +22,6 @@ interface TerraformProvider {
  */
 export async function getReleases({
   lookupName,
-  registryUrls,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const repository = `hashicorp/${lookupName}`;