From f759f165204cee62666693d75d474345c696dc85 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 4 Nov 2021 11:46:08 +0300 Subject: [PATCH] fix(go)!: Don't fallback if GOPROXY used (#12407) Current implementation tries to use GOPROXY and falls back to Renovate fetching mechanism if no releases found. The new one is switches to GOPROXY implementaiton when environment variable is set and doesn't fallback. However, when direct keyword is used, it will use Renovate-native mechanism that fetches directly from GitHub, etc. When off keyword is encountered or no URLs left, we're done with no releases (i.e. no fallback to Renovate-native mechanism). BREAKING CHANGE: Go modules lookups will now no longer fallback to Renovate native lookups if GOPROXY is configured and without "direct" explicitly configured. --- docs/usage/configuration-options.md | 3 +- .../releases-goproxy.spec.ts.snap | 113 ++++++++++++++++-- lib/datasource/go/index.spec.ts | 8 +- lib/datasource/go/index.ts | 13 +- lib/datasource/go/releases-goproxy.spec.ts | 113 +++++++++++++++--- lib/datasource/go/releases-goproxy.ts | 42 +++---- 6 files changed, 233 insertions(+), 59 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index d7744bcc3549cc..d7c7f2dcd10a75 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -763,7 +763,8 @@ Configuration added here applies for all Go-related updates, however currently t For self-hosted users, `GOPROXY`, `GONOPROXY` and `GOPRIVATE` environment variables are supported ([reference](https://golang.org/ref/mod#module-proxy)). -But when you use the `direct` or `off` keywords Renovate will fallback to its own fetching strategy (i.e. directly from GitHub, etc). +Usage of `direct` will fallback to Renovate-native release fetching mechanism. +Also we support `off` keyword which immediately will stop any fetching. ## group diff --git a/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap b/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap index 4f5aea2112414d..bddd8aa02a4b08 100644 --- a/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap +++ b/lib/datasource/go/__snapshots__/releases-goproxy.spec.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`datasource/go/releases-goproxy GOPROXY fetches release data from goproxy 1`] = ` +exports[`datasource/go/releases-goproxy getReleases fetches release data from goproxy 1`] = ` Array [ Object { "headers": Object { @@ -34,7 +34,7 @@ Array [ ] `; -exports[`datasource/go/releases-goproxy GOPROXY handles comma fallback 1`] = ` +exports[`datasource/go/releases-goproxy getReleases handles comma fallback 1`] = ` Array [ Object { "headers": Object { @@ -86,7 +86,7 @@ Array [ ] `; -exports[`datasource/go/releases-goproxy GOPROXY handles pipe fallback 1`] = ` +exports[`datasource/go/releases-goproxy getReleases handles pipe fallback 1`] = ` Array [ Object { "headers": Object { @@ -129,7 +129,7 @@ Array [ ] `; -exports[`datasource/go/releases-goproxy GOPROXY handles timestamp fetch errors 1`] = ` +exports[`datasource/go/releases-goproxy getReleases handles timestamp fetch errors 1`] = ` Array [ Object { "headers": Object { @@ -163,34 +163,125 @@ Array [ ] `; -exports[`datasource/go/releases-goproxy GOPROXY short-circuits with comma fallback 1`] = ` +exports[`datasource/go/releases-goproxy getReleases short-circuits for errors other than 404 or 410 1`] = ` Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", - "host": "foo.example.com", + "host": "foo.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, "method": "GET", - "url": "https://foo.example.com/github.com/google/btree/@v/list", + "url": "https://foo.com/github.com/foo/bar/@v/list", }, Object { "headers": Object { "accept-encoding": "gzip, deflate, br", - "host": "bar.example.com", + "host": "bar.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, "method": "GET", - "url": "https://bar.example.com/github.com/google/btree/@v/list", + "url": "https://bar.com/github.com/foo/bar/@v/list", + }, + Object { + "headers": Object { + "accept-encoding": "gzip, deflate, br", + "host": "baz.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://baz.com/github.com/foo/bar/@v/list", + }, +] +`; + +exports[`datasource/go/releases-goproxy getReleases skips GONOPROXY and GOPRIVATE packages 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/vnd.github.v3+json", + "accept-encoding": "gzip, deflate, br", + "host": "api.github.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://api.github.com/repos/google/btree/tags?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/vnd.github.v3+json", + "accept-encoding": "gzip, deflate, br", + "host": "api.github.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://api.github.com/repos/google/btree/releases?per_page=100", + }, +] +`; + +exports[`datasource/go/releases-goproxy getReleases supports "direct" keyword 1`] = ` +Array [ + Object { + "headers": Object { + "accept-encoding": "gzip, deflate, br", + "host": "foo.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://foo.com/github.com/foo/bar/@v/list", + }, + Object { + "headers": Object { + "accept-encoding": "gzip, deflate, br", + "host": "bar.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://bar.com/github.com/foo/bar/@v/list", + }, + Object { + "headers": Object { + "accept": "application/vnd.github.v3+json", + "accept-encoding": "gzip, deflate, br", + "host": "api.github.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://api.github.com/repos/foo/bar/tags?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/vnd.github.v3+json", + "accept-encoding": "gzip, deflate, br", + "host": "api.github.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://api.github.com/repos/foo/bar/releases?per_page=100", + }, +] +`; + +exports[`datasource/go/releases-goproxy getReleases supports "off" keyword 1`] = ` +Array [ + Object { + "headers": Object { + "accept-encoding": "gzip, deflate, br", + "host": "foo.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://foo.com/github.com/foo/bar/@v/list", }, Object { "headers": Object { "accept-encoding": "gzip, deflate, br", - "host": "baz.example.com", + "host": "bar.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, "method": "GET", - "url": "https://baz.example.com/github.com/google/btree/@v/list", + "url": "https://bar.com/github.com/foo/bar/@v/list", }, ] `; diff --git a/lib/datasource/go/index.spec.ts b/lib/datasource/go/index.spec.ts index 67fb7750d3d223..2ae259dae7112d 100644 --- a/lib/datasource/go/index.spec.ts +++ b/lib/datasource/go/index.spec.ts @@ -19,7 +19,6 @@ describe('datasource/go/index', () => { jest.resetAllMocks(); hostRules.find.mockReturnValue({}); hostRules.hosts.mockReturnValue([]); - process.env.GOPROXY = 'https://proxy.golang.org,direct'; }); afterEach(() => { @@ -32,10 +31,10 @@ describe('datasource/go/index', () => { goproxy.getReleases.mockResolvedValue(null); direct.getReleases.mockResolvedValue(expected); - const res = await getReleases({ lookupName: 'golang.org/foo/something' }); + const res = await getReleases({ lookupName: 'golang.org/foo/bar' }); expect(res).toBe(expected); - expect(goproxy.getReleases).toHaveBeenCalled(); + expect(goproxy.getReleases).not.toHaveBeenCalled(); expect(direct.getReleases).toHaveBeenCalled(); }); @@ -43,8 +42,9 @@ describe('datasource/go/index', () => { const expected = { releases: [{ version: '0.0.1' }] }; goproxy.getReleases.mockResolvedValue(expected); direct.getReleases.mockResolvedValue(null); + process.env.GOPROXY = 'https://proxy.golang.org,direct'; - const res = await getReleases({ lookupName: 'golang.org/foo/something' }); + const res = await getReleases({ lookupName: 'golang.org/foo/bar' }); expect(res).toBe(expected); expect(goproxy.getReleases).toHaveBeenCalled(); diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts index 271fdc42823205..dcc924c25ff9ca 100644 --- a/lib/datasource/go/index.ts +++ b/lib/datasource/go/index.ts @@ -1,18 +1,15 @@ import type { GetReleasesConfig, ReleaseResult } from '../types'; -import { getReleases as directReleases } from './releases-direct'; +import * as direct from './releases-direct'; import * as goproxy from './releases-goproxy'; export { id } from './common'; export const customRegistrySupport = false; -export async function getReleases( +export function getReleases( config: GetReleasesConfig ): Promise { - const res = await goproxy.getReleases(config); - if (res) { - return res; - } - - return directReleases(config); + return process.env.GOPROXY + ? goproxy.getReleases(config) + : direct.getReleases(config); } diff --git a/lib/datasource/go/releases-goproxy.spec.ts b/lib/datasource/go/releases-goproxy.spec.ts index 1e17e7a946cc44..9361be5a0955ed 100644 --- a/lib/datasource/go/releases-goproxy.spec.ts +++ b/lib/datasource/go/releases-goproxy.spec.ts @@ -78,10 +78,17 @@ describe('datasource/go/releases-goproxy', () => { expect(parseGoproxy(undefined)).toBeEmpty(); expect(parseGoproxy(null)).toBeEmpty(); expect(parseGoproxy('')).toBeEmpty(); - expect(parseGoproxy('off')).toBeEmpty(); - expect(parseGoproxy('direct')).toBeEmpty(); + expect(parseGoproxy('off')).toMatchObject([ + { url: 'off', fallback: '|' }, + ]); + expect(parseGoproxy('direct')).toMatchObject([ + { url: 'direct', fallback: '|' }, + ]); expect(parseGoproxy('foo,off|direct,qux')).toMatchObject([ { url: 'foo', fallback: ',' }, + { url: 'off', fallback: '|' }, + { url: 'direct', fallback: ',' }, + { url: 'qux', fallback: '|' }, ]); }); }); @@ -127,7 +134,7 @@ describe('datasource/go/releases-goproxy', () => { }); }); - describe('GOPROXY', () => { + describe('getReleases', () => { const baseUrl = 'https://proxy.golang.org'; afterEach(() => { @@ -140,11 +147,22 @@ describe('datasource/go/releases-goproxy', () => { process.env.GOPROXY = baseUrl; process.env.GOPRIVATE = 'github.com/google/*'; - httpMock.scope('https://api.github.com/'); + httpMock + .scope('https://api.github.com/') + .get('/repos/google/btree/tags?per_page=100') + .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }]) + .get('/repos/google/btree/releases?per_page=100') + .reply(200, []); const res = await getReleases({ lookupName: 'github.com/google/btree' }); - expect(httpMock.getTrace()).toBeEmpty(); - expect(res).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + expect(res).toEqual({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v1.0.1', version: 'v1.0.1' }, + ], + sourceUrl: 'https://github.com/google/btree', + }); }); it('fetches release data from goproxy', async () => { @@ -246,31 +264,96 @@ describe('datasource/go/releases-goproxy', () => { ]); }); - it('short-circuits with comma fallback', async () => { + it('short-circuits for errors other than 404 or 410', async () => { process.env.GOPROXY = [ - 'https://foo.example.com', - 'https://bar.example.com', - 'https://baz.example.com', - baseUrl, + 'https://foo.com', + 'https://bar.com', + 'https://baz.com', + 'direct', ].join(','); httpMock - .scope('https://foo.example.com/github.com/google/btree') + .scope('https://foo.com/github.com/foo/bar') .get('/@v/list') .reply(404); httpMock - .scope('https://bar.example.com/github.com/google/btree') + .scope('https://bar.com/github.com/foo/bar') .get('/@v/list') .reply(410); httpMock - .scope('https://baz.example.com/github.com/google/btree') + .scope('https://baz.com/github.com/foo/bar') .get('/@v/list') .replyWithError('unknown'); - const res = await getReleases({ lookupName: 'github.com/google/btree' }); + const res = await getReleases({ lookupName: 'github.com/foo/bar' }); + + expect(httpMock.getTrace()).toMatchSnapshot([ + { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' }, + { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' }, + { method: 'GET', url: 'https://baz.com/github.com/foo/bar/@v/list' }, + ]); + expect(res).toBeNull(); + }); + + it('supports "direct" keyword', async () => { + process.env.GOPROXY = [ + 'https://foo.com', + 'https://bar.com', + 'direct', + ].join(','); + + httpMock + .scope('https://foo.com/github.com/foo/bar') + .get('/@v/list') + .reply(404); + + httpMock + .scope('https://bar.com/github.com/foo/bar') + .get('/@v/list') + .reply(410); + + httpMock + .scope('https://api.github.com/') + .get('/repos/foo/bar/tags?per_page=100') + .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }]) + .get('/repos/foo/bar/releases?per_page=100') + .reply(200, []); + + const res = await getReleases({ lookupName: 'github.com/foo/bar' }); + expect(httpMock.getTrace()).toMatchSnapshot(); + expect(res).toEqual({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v1.0.1', version: 'v1.0.1' }, + ], + sourceUrl: 'https://github.com/foo/bar', + }); + }); + + it('supports "off" keyword', async () => { + process.env.GOPROXY = ['https://foo.com', 'https://bar.com', 'off'].join( + ',' + ); + + httpMock + .scope('https://foo.com/github.com/foo/bar') + .get('/@v/list') + .reply(404); + + httpMock + .scope('https://bar.com/github.com/foo/bar') + .get('/@v/list') + .reply(410); + + const res = await getReleases({ lookupName: 'github.com/foo/bar' }); + + expect(httpMock.getTrace()).toMatchSnapshot([ + { method: 'GET', url: 'https://foo.com/github.com/foo/bar/@v/list' }, + { method: 'GET', url: 'https://bar.com/github.com/foo/bar/@v/list' }, + ]); expect(res).toBeNull(); }); }); diff --git a/lib/datasource/go/releases-goproxy.ts b/lib/datasource/go/releases-goproxy.ts index 6eab2abde0f1c8..d1baae1996a275 100644 --- a/lib/datasource/go/releases-goproxy.ts +++ b/lib/datasource/go/releases-goproxy.ts @@ -6,6 +6,7 @@ import * as packageCache from '../../util/cache/package'; import { regEx } from '../../util/regex'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import { GoproxyFallback, http } from './common'; +import * as direct from './releases-direct'; import type { GoproxyItem, VersionInfo } from './types'; const parsedGoproxy: Record = {}; @@ -34,7 +35,7 @@ export function parseGoproxy( return parsedGoproxy[input]; } - let result: GoproxyItem[] = input + const result: GoproxyItem[] = input .split(/([^,|]*(?:,|\|))/) // TODO: #12070 .filter(Boolean) .map((s) => s.split(/(?=,|\|)/)) // TODO: #12070 @@ -46,15 +47,6 @@ export function parseGoproxy( : GoproxyFallback.Always, })); - // Ignore hosts after any keyword - for (let idx = 0; idx < result.length; idx += 1) { - const { url } = result[idx]; - if (['off', 'direct'].includes(url)) { - result = result.slice(0, idx); - break; - } - } - parsedGoproxy[input] = result; return result; } @@ -159,22 +151,18 @@ export async function versionInfo( return result; } -export async function getReleases({ - lookupName, -}: GetReleasesConfig): Promise { +export async function getReleases( + config: GetReleasesConfig +): Promise { + const { lookupName } = config; logger.trace(`goproxy.getReleases(${lookupName})`); - const noproxy = parseNoproxy(); - if (noproxy?.test(lookupName)) { - logger.debug(`Skipping ${lookupName} via GONOPROXY match`); - return null; - } - const goproxy = process.env.GOPROXY; const proxyList = parseGoproxy(goproxy); + const noproxy = parseNoproxy(); const cacheNamespaces = 'datasource-go-proxy'; - const cacheKey = `${lookupName}@@${goproxy}`; + const cacheKey = `${lookupName}@@${goproxy}@@${noproxy?.toString()}`; const cacheMinutes = 60; const cachedResult = await packageCache.get( cacheNamespaces, @@ -187,8 +175,22 @@ export async function getReleases({ let result: ReleaseResult | null = null; + if (noproxy?.test(lookupName)) { + logger.debug(`Fetching ${lookupName} via GONOPROXY match`); + result = await direct.getReleases(config); + await packageCache.set(cacheNamespaces, cacheKey, result, cacheMinutes); + return result; + } + for (const { url, fallback } of proxyList) { try { + if (url === 'off') { + break; + } else if (url === 'direct') { + result = await direct.getReleases(config); + break; + } + const versions = await listVersions(url, lookupName); const queue = versions.map((version) => async (): Promise => { try {