feat(gomod): use git host rules as authentication for gosum updates

docs/usage/golang.md

@@ -53,3 +53,24 @@ As an example, say you want Renovate to use the latest patch version of the `1.1
 
 We do not support patch level versions for the minimum `go` version.
 This means you cannot use `go 1.16.6`, but you can use `go 1.16` as a constraint.
+
+### Custom registry support, and authentication
+
+This example shows how you can use a `hostRules` configuration to configure Renovate for use with a custom private Go module source using Git to pull the modules when updating `go.sum` and vendored modules.
+All token `hostRules` with a `hostType` (e.g. `github`, `gitlab`, `bitbucket`, ... ) and host rules without a `hostType` are setup for authentication.
+
+```js
+module.exports = {
+  hostRules: [
+    {
+      matchHost: 'github.enterprise.com',
+      token: process.env.GO_GITHUB_TOKEN,
+      hostType: 'github',
+    },
+    {
+      matchHost: 'someGitHost.enterprise.com',
+      token: process.env.GO_GIT_TOKEN,
+    },
+  ],
+};
+```

lib/manager/gomod/artifacts.spec.ts

@@ -210,6 +210,279 @@ describe('manager/gomod/artifacts', () => {
     ).not.toBeNull();
     expect(execSnapshots).toMatchSnapshot();
   });
+
+  it('supports docker mode with 2 credentials', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.find.mockReturnValueOnce({
+      token: 'some-token',
+    });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-enterprise-token',
+        matchHost: 'github.enterprise.com',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '2',
+              GIT_CONFIG_KEY_0: 'url.https://some-token@github.com/.insteadOf',
+              GIT_CONFIG_KEY_1:
+                'url.https://some-enterprise-token@github.enterprise.com/.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://github.com/',
+              GIT_CONFIG_VALUE_1: 'https://github.enterprise.com/',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
+  it('supports docker mode with single credential', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-enterprise-token',
+        matchHost: 'gitlab.enterprise.com',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '1',
+              GIT_CONFIG_KEY_0:
+                'url.https://some-enterprise-token@gitlab.enterprise.com/.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://gitlab.enterprise.com/',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
+  it('supports docker mode with multiple credentials for different paths', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-enterprise-token-repo1',
+        matchHost: 'https://gitlab.enterprise.com/repo1',
+      },
+      {
+        token: 'some-enterprise-token-repo2',
+        matchHost: 'https://gitlab.enterprise.com/repo2',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '2',
+              GIT_CONFIG_KEY_0:
+                'url.https://some-enterprise-token-repo1@gitlab.enterprise.com/repo1.insteadOf',
+              GIT_CONFIG_KEY_1:
+                'url.https://some-enterprise-token-repo2@gitlab.enterprise.com/repo2.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://gitlab.enterprise.com/repo1',
+              GIT_CONFIG_VALUE_1: 'https://gitlab.enterprise.com/repo2',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
+  it('supports docker mode and ignores non http credentials', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-token',
+        matchHost: 'ssh://github.enterprise.com',
+      },
+      {
+        token: 'some-gitlab-token',
+        matchHost: 'gitlab.enterprise.com',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '1',
+              GIT_CONFIG_KEY_0:
+                'url.https://some-gitlab-token@gitlab.enterprise.com/.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://gitlab.enterprise.com/',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
+  it('supports docker mode with many credentials', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.find.mockReturnValueOnce({
+      token: 'some-token',
+    });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-token',
+        matchHost: 'api.github.com',
+      },
+      {
+        token: 'some-enterprise-token',
+        matchHost: 'github.enterprise.com',
+      },
+      {
+        token: 'some-gitlab-token',
+        matchHost: 'gitlab.enterprise.com',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '4',
+              GIT_CONFIG_KEY_0: 'url.https://some-token@github.com/.insteadOf',
+              GIT_CONFIG_KEY_1:
+                'url.https://some-token@api.github.com/.insteadOf',
+              GIT_CONFIG_KEY_2:
+                'url.https://some-enterprise-token@github.enterprise.com/.insteadOf',
+              GIT_CONFIG_KEY_3:
+                'url.https://some-gitlab-token@gitlab.enterprise.com/.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://github.com/',
+              GIT_CONFIG_VALUE_1: 'https://api.github.com/',
+              GIT_CONFIG_VALUE_2: 'https://github.enterprise.com/',
+              GIT_CONFIG_VALUE_3: 'https://gitlab.enterprise.com/',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
+  it('supports docker mode and ignores non git credentials', async () => {
+    setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
+    hostRules.find.mockReturnValueOnce({
+      token: 'some-token',
+    });
+    hostRules.getAll.mockReturnValueOnce([
+      {
+        token: 'some-enterprise-token',
+        matchHost: 'github.enterprise.com',
+        hostType: 'npm',
+      },
+    ]);
+    fs.readFile.mockResolvedValueOnce('Current go.sum' as any);
+    fs.readFile.mockResolvedValueOnce(null as any); // vendor modules filename
+    const execSnapshots = mockExecAll(exec);
+    git.getRepoStatus.mockResolvedValueOnce({
+      modified: ['go.sum'],
+    } as StatusResult);
+    fs.readFile.mockResolvedValueOnce('New go.sum' as any);
+    expect(
+      await gomod.updateArtifacts({
+        packageFileName: 'go.mod',
+        updatedDeps: [],
+        newPackageFileContent: gomod1,
+        config,
+      })
+    ).not.toBeNull();
+    expect(execSnapshots).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          options: expect.objectContaining({
+            env: expect.objectContaining({
+              GIT_CONFIG_COUNT: '1',
+              GIT_CONFIG_KEY_0: 'url.https://some-token@github.com/.insteadOf',
+              GIT_CONFIG_VALUE_0: 'https://github.com/',
+            }),
+          }),
+        }),
+      ])
+    );
+  });
+
   it('supports docker mode with goModTidy', async () => {
     setGlobalConfig({ ...adminConfig, binarySource: 'docker' });
     hostRules.find.mockReturnValueOnce({});

lib/manager/gomod/artifacts.ts

@@ -8,8 +8,9 @@ import { ExecOptions, exec } from '../../util/exec';
 import { ensureCacheDir, readLocalFile, writeLocalFile } from '../../util/fs';
 import { getRepoStatus } from '../../util/git';
 import { getGitAuthenticatedEnvironmentVariables } from '../../util/git/auth';
-import { find } from '../../util/host-rules';
+import { find, getAll } from '../../util/host-rules';
 import { regEx } from '../../util/regex';
+import { createURLFromHostOrURL, validateUrl } from '../../util/url';
 import { isValid } from '../../versioning/semver';
 import type {
   PackageDependency,
@@ -21,6 +22,7 @@ import type {
 function getGitEnvironmentVariables(): NodeJS.ProcessEnv {
   let environmentVariables: NodeJS.ProcessEnv = {};
 
+  // hard-coded logic to use authentication for github.com based on the credentials for api.github.com
   const credentials = find({
     hostType: PlatformId.Github,
     url: 'https://api.github.com/',
@@ -33,6 +35,45 @@ function getGitEnvironmentVariables(): NodeJS.ProcessEnv {
     );
   }
 
+  // get extra host rules for other git-based Go Module hosts
+  const hostRules = getAll() || [];
+
+  const goGitAllowedHostType: string[] = [
+    // All known git platforms
+    PlatformId.Azure,
+    PlatformId.Bitbucket,
+    PlatformId.BitbucketServer,
+    PlatformId.Gitea,
+    PlatformId.Github,
+    PlatformId.Gitlab,
+    // plus all without a host type (=== undefined)
+    undefined,
+  ];
+
+  // for each hostRule we add additional authentication variables to the environmentVariables
+  for (const hostRule of hostRules) {
+    if (
+      hostRule?.token &&
+      hostRule.matchHost &&
+      goGitAllowedHostType.includes(hostRule.hostType)
+    ) {
+      const httpUrl = createURLFromHostOrURL(hostRule.matchHost).toString();
+      if (validateUrl(httpUrl)) {
+        logger.debug(
+          `Adding Git authentication for Go Module retrieval for ${httpUrl} using token auth.`
+        );
+        environmentVariables = getGitAuthenticatedEnvironmentVariables(
+          httpUrl,
+          hostRule.token,
+          environmentVariables
+        );
+      } else {
+        logger.warn(
+          `Could not parse registryUrl ${hostRule.matchHost} or not using http(s). Ignoring`
+        );
+      }
+    }
+  }
   return environmentVariables;
 }
 

lib/util/git/auth.ts

@@ -1,9 +1,10 @@
 import { logger } from '../../logger';
 import { getHttpUrl } from './url';
 
-/*
-    Add authorization to a Git Url and returns the updated environment variables
-*/
+/**
+ * Add authorization to a Git Url and returns a new environment variables object
+ * @returns a new NodeJS.ProcessEnv object without modifying any input parameters
+ */
 export function getGitAuthenticatedEnvironmentVariables(
   gitUrl: string,
   token: string,