From 3c592a236c92fcc47010d9c6b2d23dd6fe8422f1 Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Tue, 3 Oct 2023 14:52:55 +0200 Subject: [PATCH] feat(cargo): support private crate (#24704) Co-authored-by: Sebastian Poxhofer Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Co-authored-by: Rhys Arkins --- docs/usage/rust.md | 29 +- .../__snapshots__/artifacts.spec.ts.snap | 10 +- lib/modules/manager/cargo/artifacts.spec.ts | 321 +++++++++++++++++- lib/modules/manager/cargo/artifacts.ts | 6 +- lib/modules/manager/cargo/readme.md | 17 + 5 files changed, 367 insertions(+), 16 deletions(-) diff --git a/docs/usage/rust.md b/docs/usage/rust.md index 53813f288d628f..8854b5c04a7a12 100644 --- a/docs/usage/rust.md +++ b/docs/usage/rust.md @@ -31,9 +31,26 @@ Read the [Rust environment variables docs](https://doc.rust-lang.org/cargo/refer ## Private crate registries and private Git dependencies -If any dependencies are hosted in private Git repositories, [Git Authentication for cargo](https://doc.rust-lang.org/cargo/appendix/git-authentication.html) must be set up. - -If any dependencies are hosted on private crate registries (i.e., not on `crates.io`), then credentials should be set up in such a way that the Git command-line is able to clone the registry index. -Third-party crate registries usually provide instructions to achieve this. - -Both of these are currently only possible when running Renovate self-hosted. +You as user can set authentication for private crates by adding a `hostRules` configuration to your `renovate.json` file. + +All token `hostRules` with a `hostType` (e.g. `github`, `gitlab`, `bitbucket`, etc.) and host rules without a `hostType` will be automatically setup for authentication. +You can also configure a `hostRules` that's only for Cargo authentication (e.g. `hostType: 'cargo'`). + +Here's an example of authentication for a private GitHub and Cargo registry: + +```js +module.exports = { + hostRules: [ + { + matchHost: 'github.enterprise.com', + token: process.env.GITHUB_TOKEN, + hostType: 'github', + }, + { + matchHost: 'someGitHost.enterprise.com', + token: process.env.CARGO_GIT_TOKEN, + hostType: 'cargo', + }, + ], +}; +``` diff --git a/lib/modules/manager/cargo/__snapshots__/artifacts.spec.ts.snap b/lib/modules/manager/cargo/__snapshots__/artifacts.spec.ts.snap index 5a5700f2dc8521..b751a1d0a7525e 100644 --- a/lib/modules/manager/cargo/__snapshots__/artifacts.spec.ts.snap +++ b/lib/modules/manager/cargo/__snapshots__/artifacts.spec.ts.snap @@ -3,7 +3,7 @@ exports[`modules/manager/cargo/artifacts returns null if unchanged 1`] = ` [ { - "cmd": "cargo update --manifest-path Cargo.toml --workspace", + "cmd": "cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace", "options": { "cwd": "/tmp/github/some/repo", "encoding": "utf-8", @@ -26,7 +26,7 @@ exports[`modules/manager/cargo/artifacts returns null if unchanged 1`] = ` exports[`modules/manager/cargo/artifacts returns updated Cargo.lock 1`] = ` [ { - "cmd": "cargo update --manifest-path Cargo.toml --workspace", + "cmd": "cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace", "options": { "cwd": "/tmp/github/some/repo", "encoding": "utf-8", @@ -49,7 +49,7 @@ exports[`modules/manager/cargo/artifacts returns updated Cargo.lock 1`] = ` exports[`modules/manager/cargo/artifacts returns updated Cargo.lock for lockfile maintenance 1`] = ` [ { - "cmd": "cargo update --manifest-path Cargo.toml", + "cmd": "cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml", "options": { "cwd": "/tmp/github/some/repo", "encoding": "utf-8", @@ -72,7 +72,7 @@ exports[`modules/manager/cargo/artifacts returns updated Cargo.lock for lockfile exports[`modules/manager/cargo/artifacts returns updated workspace Cargo.lock 1`] = ` [ { - "cmd": "cargo update --manifest-path crates/one/Cargo.toml --workspace", + "cmd": "cargo update --config net.git-fetch-with-cli=true --manifest-path crates/one/Cargo.toml --workspace", "options": { "cwd": "/tmp/github/some/repo", "encoding": "utf-8", @@ -95,7 +95,7 @@ exports[`modules/manager/cargo/artifacts returns updated workspace Cargo.lock 1` exports[`modules/manager/cargo/artifacts updates Cargo.lock based on the packageName, when given 1`] = ` [ { - "cmd": "cargo update --manifest-path Cargo.toml --workspace", + "cmd": "cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace", "options": { "cwd": "/tmp/github/some/repo", "encoding": "utf-8", diff --git a/lib/modules/manager/cargo/artifacts.spec.ts b/lib/modules/manager/cargo/artifacts.spec.ts index f035d46a6e3821..44b53dc198bc83 100644 --- a/lib/modules/manager/cargo/artifacts.spec.ts +++ b/lib/modules/manager/cargo/artifacts.spec.ts @@ -1,19 +1,22 @@ +import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../../test/exec-util'; -import { env, fs, git } from '../../../../test/util'; +import { env, fs, git, mocked } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; import * as docker from '../../../util/exec/docker'; +import * as _hostRules from '../../../util/host-rules'; import type { UpdateArtifactsConfig } from '../types'; import * as cargo from '.'; jest.mock('../../../util/exec/env'); jest.mock('../../../util/git'); +jest.mock('../../../util/host-rules', () => mockDeep()); jest.mock('../../../util/http'); jest.mock('../../../util/fs'); process.env.CONTAINERBASE = 'true'; - +const hostRules = mocked(_hostRules); const config: UpdateArtifactsConfig = {}; const adminConfig: RepoGlobalConfig = { @@ -29,6 +32,7 @@ describe('modules/manager/cargo/artifacts', () => { env.getChildProcessEnv.mockReturnValue(envMock.basic); GlobalConfig.set(adminConfig); docker.resetPrefetchedImages(); + hostRules.getAll.mockReturnValue([]); }); afterEach(() => { @@ -221,18 +225,327 @@ describe('modules/manager/cargo/artifacts', () => { 'bash -l -c "' + 'install-tool rust 1.65.0' + ' && ' + - 'cargo update --manifest-path Cargo.toml --workspace' + + 'cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace' + + '"', + options: { + cwd: '/tmp/github/some/repo', + env: { + CONTAINERBASE_CACHE_DIR: '/tmp/cache/containerbase', + }, + }, + }, + ]); + }); + + it('supports docker mode with credentials', async () => { + fs.statLocalFile.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); + GlobalConfig.set({ ...adminConfig, binarySource: 'docker' }); + hostRules.find.mockReturnValueOnce({ + token: 'some-token', + }); + hostRules.getAll.mockReturnValueOnce([ + { + token: 'some-token', + hostType: 'github', + matchHost: 'api.github.com', + }, + { token: 'some-other-token', matchHost: 'https://gitea.com' }, + ]); + git.getFile.mockResolvedValueOnce('Old Cargo.lock'); + const execSnapshots = mockExecAll(); + fs.findLocalSiblingOrParent.mockResolvedValueOnce('Cargo.lock'); + fs.readLocalFile.mockResolvedValueOnce('New Cargo.lock'); + const updatedDeps = [ + { + depName: 'dep1', + }, + ]; + expect( + await cargo.updateArtifacts({ + packageFileName: 'Cargo.toml', + updatedDeps, + newPackageFileContent: '{}', + config: { ...config, constraints: { rust: '1.65.0' } }, + }) + ).toEqual([ + { + file: { + contents: undefined, + path: 'Cargo.lock', + type: 'addition', + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { cmd: 'docker pull ghcr.io/containerbase/sidecar' }, + {}, + { + cmd: + 'docker run --rm --name=renovate_sidecar --label=renovate_child ' + + '-v "/tmp/github/some/repo":"/tmp/github/some/repo" ' + + '-v "/tmp/cache":"/tmp/cache" ' + + '-e GIT_CONFIG_KEY_0 ' + + '-e GIT_CONFIG_VALUE_0 ' + + '-e GIT_CONFIG_KEY_1 ' + + '-e GIT_CONFIG_VALUE_1 ' + + '-e GIT_CONFIG_KEY_2 ' + + '-e GIT_CONFIG_VALUE_2 ' + + '-e GIT_CONFIG_COUNT ' + + '-e GIT_CONFIG_KEY_3 ' + + '-e GIT_CONFIG_VALUE_3 ' + + '-e GIT_CONFIG_KEY_4 ' + + '-e GIT_CONFIG_VALUE_4 ' + + '-e GIT_CONFIG_KEY_5 ' + + '-e GIT_CONFIG_VALUE_5 ' + + '-e CONTAINERBASE_CACHE_DIR ' + + '-w "/tmp/github/some/repo" ' + + 'ghcr.io/containerbase/sidecar ' + + 'bash -l -c "' + + 'install-tool rust 1.65.0' + + ' && ' + + 'cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace' + '"', options: { cwd: '/tmp/github/some/repo', env: { CONTAINERBASE_CACHE_DIR: '/tmp/cache/containerbase', + GIT_CONFIG_COUNT: '6', + GIT_CONFIG_KEY_0: + 'url.https://ssh:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_1: + 'url.https://git:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_2: 'url.https://some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_3: + 'url.https://ssh:some-other-token@gitea.com/.insteadOf', + GIT_CONFIG_KEY_4: + 'url.https://git:some-other-token@gitea.com/.insteadOf', + GIT_CONFIG_KEY_5: + 'url.https://some-other-token@gitea.com/.insteadOf', + GIT_CONFIG_VALUE_0: 'ssh://git@github.com/', + GIT_CONFIG_VALUE_1: 'git@github.com:', + GIT_CONFIG_VALUE_2: 'https://github.com/', + GIT_CONFIG_VALUE_3: 'ssh://git@gitea.com/', + GIT_CONFIG_VALUE_4: 'git@gitea.com:', + GIT_CONFIG_VALUE_5: 'https://gitea.com/', }, }, }, ]); }); + it('supports docker mode with many credentials', async () => { + fs.statLocalFile.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); + GlobalConfig.set({ ...adminConfig, binarySource: 'docker' }); + hostRules.find.mockReturnValueOnce({ + token: 'some-token', + }); + hostRules.getAll.mockReturnValueOnce([ + { + token: 'some-token', + matchHost: 'api.github.com', + hostType: 'github', + }, + { + token: 'some-enterprise-token', + matchHost: 'github.enterprise.com', + hostType: 'github', + }, + { + token: 'some-gitlab-token', + matchHost: 'gitlab.enterprise.com', + hostType: 'gitlab', + }, + ]); + git.getFile.mockResolvedValueOnce('Old Cargo.lock'); + const execSnapshots = mockExecAll(); + fs.findLocalSiblingOrParent.mockResolvedValueOnce('Cargo.lock'); + fs.readLocalFile.mockResolvedValueOnce('New Cargo.lock'); + const updatedDeps = [ + { + depName: 'dep1', + }, + ]; + expect( + await cargo.updateArtifacts({ + packageFileName: 'Cargo.toml', + updatedDeps, + newPackageFileContent: '{}', + config: { ...config, constraints: { rust: '1.65.0' } }, + }) + ).toEqual([ + { + file: { + contents: undefined, + path: 'Cargo.lock', + type: 'addition', + }, + }, + ]); + expect(execSnapshots).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + options: expect.objectContaining({ + env: expect.objectContaining({ + GIT_CONFIG_COUNT: '9', + GIT_CONFIG_KEY_0: + 'url.https://ssh:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_1: + 'url.https://git:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_2: 'url.https://some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_3: + 'url.https://ssh:some-enterprise-token@github.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_4: + 'url.https://git:some-enterprise-token@github.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_5: + 'url.https://some-enterprise-token@github.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_6: + 'url.https://gitlab-ci-token:some-gitlab-token@gitlab.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_7: + 'url.https://gitlab-ci-token:some-gitlab-token@gitlab.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_8: + 'url.https://gitlab-ci-token:some-gitlab-token@gitlab.enterprise.com/.insteadOf', + GIT_CONFIG_VALUE_0: 'ssh://git@github.com/', + GIT_CONFIG_VALUE_1: 'git@github.com:', + GIT_CONFIG_VALUE_2: 'https://github.com/', + GIT_CONFIG_VALUE_3: 'ssh://git@github.enterprise.com/', + GIT_CONFIG_VALUE_4: 'git@github.enterprise.com:', + GIT_CONFIG_VALUE_5: 'https://github.enterprise.com/', + GIT_CONFIG_VALUE_6: 'ssh://git@gitlab.enterprise.com/', + GIT_CONFIG_VALUE_7: 'git@gitlab.enterprise.com:', + }), + }), + }), + ]) + ); + }); + + it('supports docker mode and ignores non git credentials', async () => { + fs.statLocalFile.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); + GlobalConfig.set({ ...adminConfig, binarySource: 'docker' }); + hostRules.find.mockReturnValueOnce({ + token: 'some-token', + }); + hostRules.getAll.mockReturnValueOnce([ + { + token: 'some-enterprise-token', + matchHost: 'github.enterprise.com', + hostType: 'npm', + }, + ]); + git.getFile.mockResolvedValueOnce('Old Cargo.lock'); + const execSnapshots = mockExecAll(); + fs.findLocalSiblingOrParent.mockResolvedValueOnce('Cargo.lock'); + fs.readLocalFile.mockResolvedValueOnce('New Cargo.lock'); + const updatedDeps = [ + { + depName: 'dep1', + }, + ]; + expect( + await cargo.updateArtifacts({ + packageFileName: 'Cargo.toml', + updatedDeps, + newPackageFileContent: '{}', + config: { ...config, constraints: { rust: '1.65.0' } }, + }) + ).toEqual([ + { + file: { + contents: undefined, + path: 'Cargo.lock', + type: 'addition', + }, + }, + ]); + expect(execSnapshots).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + options: expect.objectContaining({ + env: expect.objectContaining({ + GIT_CONFIG_COUNT: '3', + GIT_CONFIG_KEY_0: + 'url.https://ssh:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_1: + 'url.https://git:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_2: 'url.https://some-token@github.com/.insteadOf', + GIT_CONFIG_VALUE_0: 'ssh://git@github.com/', + GIT_CONFIG_VALUE_1: 'git@github.com:', + GIT_CONFIG_VALUE_2: 'https://github.com/', + }), + }), + }), + ]) + ); + }); + + it('supports docker mode with Cargo specific credential', async () => { + fs.statLocalFile.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); + GlobalConfig.set({ ...adminConfig, binarySource: 'docker' }); + hostRules.find.mockReturnValueOnce({ + token: 'some-token', + }); + hostRules.getAll.mockReturnValueOnce([ + { + token: 'some-enterprise-token-cargo', + matchHost: 'github.enterprise.com', + hostType: 'cargo', + }, + ]); + git.getFile.mockResolvedValueOnce('Old Cargo.lock'); + const execSnapshots = mockExecAll(); + fs.findLocalSiblingOrParent.mockResolvedValueOnce('Cargo.lock'); + fs.readLocalFile.mockResolvedValueOnce('New Cargo.lock'); + const updatedDeps = [ + { + depName: 'dep1', + }, + ]; + expect( + await cargo.updateArtifacts({ + packageFileName: 'Cargo.toml', + updatedDeps, + newPackageFileContent: '{}', + config: { ...config, constraints: { rust: '1.65.0' } }, + }) + ).toEqual([ + { + file: { + contents: undefined, + path: 'Cargo.lock', + type: 'addition', + }, + }, + ]); + expect(execSnapshots).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + options: expect.objectContaining({ + env: expect.objectContaining({ + GIT_CONFIG_COUNT: '6', + GIT_CONFIG_KEY_0: + 'url.https://ssh:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_1: + 'url.https://git:some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_2: 'url.https://some-token@github.com/.insteadOf', + GIT_CONFIG_KEY_3: + 'url.https://ssh:some-enterprise-token-cargo@github.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_4: + 'url.https://git:some-enterprise-token-cargo@github.enterprise.com/.insteadOf', + GIT_CONFIG_KEY_5: + 'url.https://some-enterprise-token-cargo@github.enterprise.com/.insteadOf', + GIT_CONFIG_VALUE_0: 'ssh://git@github.com/', + GIT_CONFIG_VALUE_1: 'git@github.com:', + GIT_CONFIG_VALUE_2: 'https://github.com/', + GIT_CONFIG_VALUE_3: 'ssh://git@github.enterprise.com/', + GIT_CONFIG_VALUE_4: 'git@github.enterprise.com:', + GIT_CONFIG_VALUE_5: 'https://github.enterprise.com/', + }), + }), + }), + ]) + ); + }); + it('supports install mode', async () => { fs.statLocalFile.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); GlobalConfig.set({ ...adminConfig, binarySource: 'install' }); @@ -273,7 +586,7 @@ describe('modules/manager/cargo/artifacts', () => { }, }, { - cmd: 'cargo update --manifest-path Cargo.toml --workspace', + cmd: 'cargo update --config net.git-fetch-with-cli=true --manifest-path Cargo.toml --workspace', options: { cwd: '/tmp/github/some/repo', env: { diff --git a/lib/modules/manager/cargo/artifacts.ts b/lib/modules/manager/cargo/artifacts.ts index 666219493068bf..3c9b4222e35e63 100644 --- a/lib/modules/manager/cargo/artifacts.ts +++ b/lib/modules/manager/cargo/artifacts.ts @@ -8,6 +8,7 @@ import { readLocalFile, writeLocalFile, } from '../../../util/fs'; +import { getGitEnvironmentVariables } from '../../../util/git/auth'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; async function cargoUpdate( @@ -15,7 +16,9 @@ async function cargoUpdate( isLockFileMaintenance: boolean, constraint: string | undefined ): Promise { - let cmd = `cargo update --manifest-path ${quote(manifestPath)}`; + let cmd = `cargo update --config net.git-fetch-with-cli=true --manifest-path ${quote( + manifestPath + )}`; // If we're updating a specific crate, `cargo-update` requires `--workspace` // for more information, see: https://github.com/renovatebot/renovate/issues/12332 if (!isLockFileMaintenance) { @@ -23,6 +26,7 @@ async function cargoUpdate( } const execOptions: ExecOptions = { + extraEnv: { ...getGitEnvironmentVariables(['cargo']) }, docker: {}, toolConstraints: [{ toolName: 'rust', constraint }], }; diff --git a/lib/modules/manager/cargo/readme.md b/lib/modules/manager/cargo/readme.md index 5e05f6a2283948..46195044aebb2a 100644 --- a/lib/modules/manager/cargo/readme.md +++ b/lib/modules/manager/cargo/readme.md @@ -4,3 +4,20 @@ When using the default rangeStrategy=auto: - If a "less than" instruction is found (e.g. `<2`) then `rangeStrategy=widen` will be selected, - Otherwise, `rangeStrategy=bump` will be selected. + +### Private Modules Authentication + +Before running the `cargo` commands to update the `cargo.lock`, Renovate exports `git` [`insteadOf`](https://git-scm.com/docs/git-config#Documentation/git-config.txt-urlltbasegtinsteadOf) directives in environment variables. + +Renovate uses this logic before it updates any "artifacts": + +The token from the `hostRules` entry matching `hostType=github` and `matchHost=api.github.com` is added as the default authentication for `github.com`. +For those running against `github.com`, this token will be the default platform token. + +Next, all `hostRules` with both a token or username/password and `matchHost` will be fetched, except for any `github.com` one from above. + +Rules from this list are converted to environment variable directives if they match _any_ of these characteristics: + +- No `hostType` is defined, or +- `hostType` is `cargo`, or +- `hostType` is a platform (`github`, `gitlab`, `azure`, etc.)