fix(go)!: Don't fallback if GOPROXY used (#12407)

Current implementation tries to use GOPROXY and falls back to Renovate fetching mechanism if no releases found.

The new one is switches to GOPROXY implementaiton when environment variable is set and doesn't fallback.
However, when direct keyword is used, it will use Renovate-native mechanism that fetches directly from GitHub, etc.
When off keyword is encountered or no URLs left, we're done with no releases (i.e. no fallback to Renovate-native mechanism).

BREAKING CHANGE: Go modules lookups will now no longer fallback to Renovate native lookups if GOPROXY is configured and without "direct" explicitly configured.

docs/usage/configuration-options.md

@@ -763,7 +763,8 @@ Configuration added here applies for all Go-related updates, however currently t
 
 For self-hosted users, `GOPROXY`, `GONOPROXY` and `GOPRIVATE` environment variables are supported ([reference](https://golang.org/ref/mod#module-proxy)).
 
-But when you use the `direct` or `off` keywords Renovate will fallback to its own fetching strategy (i.e. directly from GitHub, etc).
+Usage of `direct` will fallback to Renovate-native release fetching mechanism.
+Also we support `off` keyword which immediately will stop any fetching.
 
 ## group
 

lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`datasource/go/releases-goproxy GOPROXY fetches release data from goproxy 1`] = `
+exports[`datasource/go/releases-goproxy getReleases fetches release data from goproxy 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -34,7 +34,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles comma fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles comma fallback 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -86,7 +86,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles pipe fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles pipe fallback 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -129,7 +129,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles timestamp fetch errors 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles timestamp fetch errors 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -163,34 +163,125 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY short-circuits with comma fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases short-circuits for errors other than 404 or 410 1`] = `
 Array [
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "foo.example.com",
+      "host": "foo.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://foo.example.com/github.com/google/btree/@v/list",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
   },
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "bar.example.com",
+      "host": "bar.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://bar.example.com/github.com/google/btree/@v/list",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
   },
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "baz.example.com",
+      "host": "baz.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://baz.example.com/github.com/google/btree/@v/list",
+    "url": "https://baz.com/github.com/foo/bar/@v/list",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases skips GONOPROXY and GOPRIVATE packages 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/google/btree/tags?per_page=100",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/google/btree/releases?per_page=100",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases supports "direct" keyword 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "foo.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "bar.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/foo/bar/tags?per_page=100",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/foo/bar/releases?per_page=100",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases supports "off" keyword 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "foo.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "bar.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
   },
 ]
 `;

lib/datasource/go/index.spec.ts

@@ -19,7 +19,6 @@ describe('datasource/go/index', () => {
       jest.resetAllMocks();
       hostRules.find.mockReturnValue({});
       hostRules.hosts.mockReturnValue([]);
-      process.env.GOPROXY = 'https://proxy.golang.org,direct';
     });
 
     afterEach(() => {
@@ -32,10 +31,10 @@ describe('datasource/go/index', () => {
       goproxy.getReleases.mockResolvedValue(null);
       direct.getReleases.mockResolvedValue(expected);
 
-      const res = await getReleases({ lookupName: 'golang.org/foo/something' });
+      const res = await getReleases({ lookupName: 'golang.org/foo/bar' });
 
       expect(res).toBe(expected);
-      expect(goproxy.getReleases).toHaveBeenCalled();
+      expect(goproxy.getReleases).not.toHaveBeenCalled();
       expect(direct.getReleases).toHaveBeenCalled();
     });
 
@@ -43,8 +42,9 @@ describe('datasource/go/index', () => {
       const expected = { releases: [{ version: '0.0.1' }] };
       goproxy.getReleases.mockResolvedValue(expected);
       direct.getReleases.mockResolvedValue(null);
+      process.env.GOPROXY = 'https://proxy.golang.org,direct';
 
-      const res = await getReleases({ lookupName: 'golang.org/foo/something' });
+      const res = await getReleases({ lookupName: 'golang.org/foo/bar' });
 
       expect(res).toBe(expected);
       expect(goproxy.getReleases).toHaveBeenCalled();

lib/datasource/go/index.ts

@@ -1,18 +1,15 @@
 import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { getReleases as directReleases } from './releases-direct';
+import * as direct from './releases-direct';
 import * as goproxy from './releases-goproxy';
 
 export { id } from './common';
 
 export const customRegistrySupport = false;
 
-export async function getReleases(
+export function getReleases(
   config: GetReleasesConfig
 ): Promise<ReleaseResult | null> {
-  const res = await goproxy.getReleases(config);
+  return process.env.GOPROXY
-  if (res) {
+    ? goproxy.getReleases(config)
-    return res;
+    : direct.getReleases(config);
-  }
-
-  return directReleases(config);
 }

lib/datasource/go/releases-goproxy.spec.ts

@@ -78,10 +78,17 @@ describe('datasource/go/releases-goproxy', () => {
       expect(parseGoproxy(undefined)).toBeEmpty();
       expect(parseGoproxy(null)).toBeEmpty();
       expect(parseGoproxy('')).toBeEmpty();
-      expect(parseGoproxy('off')).toBeEmpty();
+      expect(parseGoproxy('off')).toMatchObject([
-      expect(parseGoproxy('direct')).toBeEmpty();
+        { url: 'off', fallback: '|' },
+      ]);
+      expect(parseGoproxy('direct')).toMatchObject([
+        { url: 'direct', fallback: '|' },
+      ]);
       expect(parseGoproxy('foo,off|direct,qux')).toMatchObject([
         { url: 'foo', fallback: ',' },
+        { url: 'off', fallback: '|' },
+        { url: 'direct', fallback: ',' },
+        { url: 'qux', fallback: '|' },
       ]);
     });
   });
@@ -127,7 +134,7 @@ describe('datasource/go/releases-goproxy', () => {
     });
   });
 
-  describe('GOPROXY', () => {
+  describe('getReleases', () => {
     const baseUrl = 'https://proxy.golang.org';
 
     afterEach(() => {
@@ -140,11 +147,22 @@ describe('datasource/go/releases-goproxy', () => {
       process.env.GOPROXY = baseUrl;
       process.env.GOPRIVATE = 'github.com/google/*';
 
-      httpMock.scope('https://api.github.com/');
+      httpMock
+        .scope('https://api.github.com/')
+        .get('/repos/google/btree/tags?per_page=100')
+        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+        .get('/repos/google/btree/releases?per_page=100')
+        .reply(200, []);
 
       const res = await getReleases({ lookupName: 'github.com/google/btree' });
-      expect(httpMock.getTrace()).toBeEmpty();
+      expect(httpMock.getTrace()).toMatchSnapshot();
-      expect(res).toBeNull();
+      expect(res).toEqual({
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
+        ],
+        sourceUrl: 'https://github.com/google/btree',
+      });
     });
 
     it('fetches release data from goproxy', async () => {
@@ -246,31 +264,96 @@ describe('datasource/go/releases-goproxy', () => {
       ]);
     });
 
-    it('short-circuits with comma fallback', async () => {
+    it('short-circuits for errors other than 404 or 410', async () => {
       process.env.GOPROXY = [
-        'https://foo.example.com',
+        'https://foo.com',
-        'https://bar.example.com',
+        'https://bar.com',
-        'https://baz.example.com',
+        'https://baz.com',
-        baseUrl,
+        'direct',
       ].join(',');
 
       httpMock
-        .scope('https://foo.example.com/github.com/google/btree')
+        .scope('https://foo.com/github.com/foo/bar')
         .get('/@v/list')
         .reply(404);
 
       httpMock
-        .scope('https://bar.example.com/github.com/google/btree')
+        .scope('https://bar.com/github.com/foo/bar')
         .get('/@v/list')
         .reply(410);
 
       httpMock
-        .scope('https://baz.example.com/github.com/google/btree')
+        .scope('https://baz.com/github.com/foo/bar')
         .get('/@v/list')
         .replyWithError('unknown');
 
-      const res = await getReleases({ lookupName: 'github.com/google/btree' });
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
+      expect(httpMock.getTrace()).toMatchSnapshot([
+        { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://baz.com/github.com/foo/bar/@v/list' },
+      ]);
+      expect(res).toBeNull();
+    });
+
+    it('supports "direct" keyword', async () => {
+      process.env.GOPROXY = [
+        'https://foo.com',
+        'https://bar.com',
+        'direct',
+      ].join(',');
+
+      httpMock
+        .scope('https://foo.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(404);
+
+      httpMock
+        .scope('https://bar.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(410);
+
+      httpMock
+        .scope('https://api.github.com/')
+        .get('/repos/foo/bar/tags?per_page=100')
+        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+        .get('/repos/foo/bar/releases?per_page=100')
+        .reply(200, []);
+
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
       expect(httpMock.getTrace()).toMatchSnapshot();
+      expect(res).toEqual({
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
+        ],
+        sourceUrl: 'https://github.com/foo/bar',
+      });
+    });
+
+    it('supports "off" keyword', async () => {
+      process.env.GOPROXY = ['https://foo.com', 'https://bar.com', 'off'].join(
+        ','
+      );
+
+      httpMock
+        .scope('https://foo.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(404);
+
+      httpMock
+        .scope('https://bar.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(410);
+
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
+      expect(httpMock.getTrace()).toMatchSnapshot([
+        { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' },
+      ]);
       expect(res).toBeNull();
     });
   });

lib/datasource/go/releases-goproxy.ts

@@ -6,6 +6,7 @@ import * as packageCache from '../../util/cache/package';
 import { regEx } from '../../util/regex';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
 import { GoproxyFallback, http } from './common';
+import * as direct from './releases-direct';
 import type { GoproxyItem, VersionInfo } from './types';
 
 const parsedGoproxy: Record<string, GoproxyItem[]> = {};
@@ -34,7 +35,7 @@ export function parseGoproxy(
     return parsedGoproxy[input];
   }
 
-  let result: GoproxyItem[] = input
+  const result: GoproxyItem[] = input
     .split(/([^,|]*(?:,|\|))/) // TODO: #12070
     .filter(Boolean)
     .map((s) => s.split(/(?=,|\|)/)) // TODO: #12070
@@ -46,15 +47,6 @@ export function parseGoproxy(
           : GoproxyFallback.Always,
     }));
 
-  // Ignore hosts after any keyword
-  for (let idx = 0; idx < result.length; idx += 1) {
-    const { url } = result[idx];
-    if (['off', 'direct'].includes(url)) {
-      result = result.slice(0, idx);
-      break;
-    }
-  }
-
   parsedGoproxy[input] = result;
   return result;
 }
@@ -159,22 +151,18 @@ export async function versionInfo(
   return result;
 }
 
-export async function getReleases({
+export async function getReleases(
-  lookupName,
+  config: GetReleasesConfig
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
+): Promise<ReleaseResult | null> {
+  const { lookupName } = config;
   logger.trace(`goproxy.getReleases(${lookupName})`);
 
-  const noproxy = parseNoproxy();
-  if (noproxy?.test(lookupName)) {
-    logger.debug(`Skipping ${lookupName} via GONOPROXY match`);
-    return null;
-  }
-
   const goproxy = process.env.GOPROXY;
   const proxyList = parseGoproxy(goproxy);
+  const noproxy = parseNoproxy();
 
   const cacheNamespaces = 'datasource-go-proxy';
-  const cacheKey = `${lookupName}@@${goproxy}`;
+  const cacheKey = `${lookupName}@@${goproxy}@@${noproxy?.toString()}`;
   const cacheMinutes = 60;
   const cachedResult = await packageCache.get<ReleaseResult | null>(
     cacheNamespaces,
@@ -187,8 +175,22 @@ export async function getReleases({
 
   let result: ReleaseResult | null = null;
 
+  if (noproxy?.test(lookupName)) {
+    logger.debug(`Fetching ${lookupName} via GONOPROXY match`);
+    result = await direct.getReleases(config);
+    await packageCache.set(cacheNamespaces, cacheKey, result, cacheMinutes);
+    return result;
+  }
+
   for (const { url, fallback } of proxyList) {
     try {
+      if (url === 'off') {
+        break;
+      } else if (url === 'direct') {
+        result = await direct.getReleases(config);
+        break;
+      }
+
       const versions = await listVersions(url, lookupName);
       const queue = versions.map((version) => async (): Promise<Release> => {
         try {