fix(go)!: Don't fallback if GOPROXY used (#12407)

Current implementation tries to use GOPROXY and falls back to Renovate fetching mechanism if no releases found. The new one is switches to GOPROXY implementaiton when environment variable is set and doesn't fallback. However, when direct keyword is used, it will use Renovate-native mechanism that fetches directly from GitHub, etc. When off keyword is encountered or no URLs left, we're done with no releases (i.e. no fallback to Renovate-native mechanism). BREAKING CHANGE: Go modules lookups will now no longer fallback to Renovate native lookups if GOPROXY is configured and without "direct" explicitly configured.
renovatebot · Nov 5, 2021 · f759f16 · f759f16
1 parent 1b84c52
commit f759f16
Show file tree

Hide file tree

Showing 6 changed files with 233 additions and 59 deletions.
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
@@ -763,7 +763,8 @@ Configuration added here applies for all Go-related updates, however currently t
 
 For self-hosted users, `GOPROXY`, `GONOPROXY` and `GOPRIVATE` environment variables are supported ([reference](https://golang.org/ref/mod#module-proxy)).
 
-But when you use the `direct` or `off` keywords Renovate will fallback to its own fetching strategy (i.e. directly from GitHub, etc).
+Usage of `direct` will fallback to Renovate-native release fetching mechanism.
+Also we support `off` keyword which immediately will stop any fetching.
 
 ## group
 

diff --git a/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap b/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap
@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`datasource/go/releases-goproxy GOPROXY fetches release data from goproxy 1`] = `
+exports[`datasource/go/releases-goproxy getReleases fetches release data from goproxy 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -34,7 +34,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles comma fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles comma fallback 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -86,7 +86,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles pipe fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles pipe fallback 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -129,7 +129,7 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY handles timestamp fetch errors 1`] = `
+exports[`datasource/go/releases-goproxy getReleases handles timestamp fetch errors 1`] = `
 Array [
   Object {
     "headers": Object {
@@ -163,34 +163,125 @@ Array [
 ]
 `;
 
-exports[`datasource/go/releases-goproxy GOPROXY short-circuits with comma fallback 1`] = `
+exports[`datasource/go/releases-goproxy getReleases short-circuits for errors other than 404 or 410 1`] = `
 Array [
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "foo.example.com",
+      "host": "foo.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://foo.example.com/github.com/google/btree/@v/list",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
   },
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "bar.example.com",
+      "host": "bar.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://bar.example.com/github.com/google/btree/@v/list",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "baz.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://baz.com/github.com/foo/bar/@v/list",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases skips GONOPROXY and GOPRIVATE packages 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/google/btree/tags?per_page=100",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/google/btree/releases?per_page=100",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases supports "direct" keyword 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "foo.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "bar.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/foo/bar/tags?per_page=100",
+  },
+  Object {
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://api.github.com/repos/foo/bar/releases?per_page=100",
+  },
+]
+`;
+
+exports[`datasource/go/releases-goproxy getReleases supports "off" keyword 1`] = `
+Array [
+  Object {
+    "headers": Object {
+      "accept-encoding": "gzip, deflate, br",
+      "host": "foo.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "GET",
+    "url": "https://foo.com/github.com/foo/bar/@v/list",
   },
   Object {
     "headers": Object {
       "accept-encoding": "gzip, deflate, br",
-      "host": "baz.example.com",
+      "host": "bar.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
     },
     "method": "GET",
-    "url": "https://baz.example.com/github.com/google/btree/@v/list",
+    "url": "https://bar.com/github.com/foo/bar/@v/list",
   },
 ]
 `;

diff --git a/lib/datasource/go/index.spec.ts b/lib/datasource/go/index.spec.ts
@@ -19,7 +19,6 @@ describe('datasource/go/index', () => {
       jest.resetAllMocks();
       hostRules.find.mockReturnValue({});
       hostRules.hosts.mockReturnValue([]);
-      process.env.GOPROXY = 'https://proxy.golang.org,direct';
     });
 
     afterEach(() => {
@@ -32,19 +31,20 @@ describe('datasource/go/index', () => {
       goproxy.getReleases.mockResolvedValue(null);
       direct.getReleases.mockResolvedValue(expected);
 
-      const res = await getReleases({ lookupName: 'golang.org/foo/something' });
+      const res = await getReleases({ lookupName: 'golang.org/foo/bar' });
 
       expect(res).toBe(expected);
-      expect(goproxy.getReleases).toHaveBeenCalled();
+      expect(goproxy.getReleases).not.toHaveBeenCalled();
       expect(direct.getReleases).toHaveBeenCalled();
     });
 
     it('supports GOPROXY', async () => {
       const expected = { releases: [{ version: '0.0.1' }] };
       goproxy.getReleases.mockResolvedValue(expected);
       direct.getReleases.mockResolvedValue(null);
+      process.env.GOPROXY = 'https://proxy.golang.org,direct';
 
-      const res = await getReleases({ lookupName: 'golang.org/foo/something' });
+      const res = await getReleases({ lookupName: 'golang.org/foo/bar' });
 
       expect(res).toBe(expected);
       expect(goproxy.getReleases).toHaveBeenCalled();

diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts
@@ -1,18 +1,15 @@
 import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { getReleases as directReleases } from './releases-direct';
+import * as direct from './releases-direct';
 import * as goproxy from './releases-goproxy';
 
 export { id } from './common';
 
 export const customRegistrySupport = false;
 
-export async function getReleases(
+export function getReleases(
   config: GetReleasesConfig
 ): Promise<ReleaseResult | null> {
-  const res = await goproxy.getReleases(config);
-  if (res) {
-    return res;
-  }
-
-  return directReleases(config);
+  return process.env.GOPROXY
+    ? goproxy.getReleases(config)
+    : direct.getReleases(config);
 }
diff --git a/lib/datasource/go/releases-goproxy.spec.ts b/lib/datasource/go/releases-goproxy.spec.ts
@@ -78,10 +78,17 @@ describe('datasource/go/releases-goproxy', () => {
       expect(parseGoproxy(undefined)).toBeEmpty();
       expect(parseGoproxy(null)).toBeEmpty();
       expect(parseGoproxy('')).toBeEmpty();
-      expect(parseGoproxy('off')).toBeEmpty();
-      expect(parseGoproxy('direct')).toBeEmpty();
+      expect(parseGoproxy('off')).toMatchObject([
+        { url: 'off', fallback: '|' },
+      ]);
+      expect(parseGoproxy('direct')).toMatchObject([
+        { url: 'direct', fallback: '|' },
+      ]);
       expect(parseGoproxy('foo,off|direct,qux')).toMatchObject([
         { url: 'foo', fallback: ',' },
+        { url: 'off', fallback: '|' },
+        { url: 'direct', fallback: ',' },
+        { url: 'qux', fallback: '|' },
       ]);
     });
   });
@@ -127,7 +134,7 @@ describe('datasource/go/releases-goproxy', () => {
     });
   });
 
-  describe('GOPROXY', () => {
+  describe('getReleases', () => {
     const baseUrl = 'https://proxy.golang.org';
 
     afterEach(() => {
@@ -140,11 +147,22 @@ describe('datasource/go/releases-goproxy', () => {
       process.env.GOPROXY = baseUrl;
       process.env.GOPRIVATE = 'github.com/google/*';
 
-      httpMock.scope('https://api.github.com/');
+      httpMock
+        .scope('https://api.github.com/')
+        .get('/repos/google/btree/tags?per_page=100')
+        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+        .get('/repos/google/btree/releases?per_page=100')
+        .reply(200, []);
 
       const res = await getReleases({ lookupName: 'github.com/google/btree' });
-      expect(httpMock.getTrace()).toBeEmpty();
-      expect(res).toBeNull();
+      expect(httpMock.getTrace()).toMatchSnapshot();
+      expect(res).toEqual({
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
+        ],
+        sourceUrl: 'https://github.com/google/btree',
+      });
     });
 
     it('fetches release data from goproxy', async () => {
@@ -246,31 +264,96 @@ describe('datasource/go/releases-goproxy', () => {
       ]);
     });
 
-    it('short-circuits with comma fallback', async () => {
+    it('short-circuits for errors other than 404 or 410', async () => {
       process.env.GOPROXY = [
-        'https://foo.example.com',
-        'https://bar.example.com',
-        'https://baz.example.com',
-        baseUrl,
+        'https://foo.com',
+        'https://bar.com',
+        'https://baz.com',
+        'direct',
       ].join(',');
 
       httpMock
-        .scope('https://foo.example.com/github.com/google/btree')
+        .scope('https://foo.com/github.com/foo/bar')
         .get('/@v/list')
         .reply(404);
 
       httpMock
-        .scope('https://bar.example.com/github.com/google/btree')
+        .scope('https://bar.com/github.com/foo/bar')
         .get('/@v/list')
         .reply(410);
 
       httpMock
-        .scope('https://baz.example.com/github.com/google/btree')
+        .scope('https://baz.com/github.com/foo/bar')
         .get('/@v/list')
         .replyWithError('unknown');
 
-      const res = await getReleases({ lookupName: 'github.com/google/btree' });
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
+      expect(httpMock.getTrace()).toMatchSnapshot([
+        { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://baz.com/github.com/foo/bar/@v/list' },
+      ]);
+      expect(res).toBeNull();
+    });
+
+    it('supports "direct" keyword', async () => {
+      process.env.GOPROXY = [
+        'https://foo.com',
+        'https://bar.com',
+        'direct',
+      ].join(',');
+
+      httpMock
+        .scope('https://foo.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(404);
+
+      httpMock
+        .scope('https://bar.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(410);
+
+      httpMock
+        .scope('https://api.github.com/')
+        .get('/repos/foo/bar/tags?per_page=100')
+        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+        .get('/repos/foo/bar/releases?per_page=100')
+        .reply(200, []);
+
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
       expect(httpMock.getTrace()).toMatchSnapshot();
+      expect(res).toEqual({
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
+        ],
+        sourceUrl: 'https://github.com/foo/bar',
+      });
+    });
+
+    it('supports "off" keyword', async () => {
+      process.env.GOPROXY = ['https://foo.com', 'https://bar.com', 'off'].join(
+        ','
+      );
+
+      httpMock
+        .scope('https://foo.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(404);
+
+      httpMock
+        .scope('https://bar.com/github.com/foo/bar')
+        .get('/@v/list')
+        .reply(410);
+
+      const res = await getReleases({ lookupName: 'github.com/foo/bar' });
+
+      expect(httpMock.getTrace()).toMatchSnapshot([
+        { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' },
+        { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' },
+      ]);
       expect(res).toBeNull();
     });
   });