diff --git a/docs/usage/getting-started/private-packages.md b/docs/usage/getting-started/private-packages.md index 601666924635b7..4f0761bd1d36bf 100644 --- a/docs/usage/getting-started/private-packages.md +++ b/docs/usage/getting-started/private-packages.md @@ -442,6 +442,19 @@ For example: } ``` +### pipenv + +If a `Pipfile` contains a `source` with `USERNAME` or `PASSWORD` environment variables and there is a `hostRules` entry with a matching host plus `username` and `password` fields, then these variables would be passed to `pipenv lock`. + +For example: + +```ini +[[source]] +url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "pypi" +``` + ### poetry For every Poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`, where `X` represents the normalized name of the source in `pyproject.toml`. diff --git a/lib/modules/manager/pipenv/__fixtures__/Pipfile6 b/lib/modules/manager/pipenv/__fixtures__/Pipfile6 new file mode 100644 index 00000000000000..be5a65f82ee950 --- /dev/null +++ b/lib/modules/manager/pipenv/__fixtures__/Pipfile6 @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "private" + +[packages] +requests = {version = "==0.21.0", index = "private"} diff --git a/lib/modules/manager/pipenv/__fixtures__/Pipfile7 b/lib/modules/manager/pipenv/__fixtures__/Pipfile7 new file mode 100644 index 00000000000000..93a7a27dff7493 --- /dev/null +++ b/lib/modules/manager/pipenv/__fixtures__/Pipfile7 @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://$USERNAME_FOO:${PAZZWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "private" + +[packages] +requests = {version = "==0.21.0", index = "private"} diff --git a/lib/modules/manager/pipenv/artifacts.spec.ts b/lib/modules/manager/pipenv/artifacts.spec.ts index ede5e04b9141c9..059f7fd61916a6 100644 --- a/lib/modules/manager/pipenv/artifacts.spec.ts +++ b/lib/modules/manager/pipenv/artifacts.spec.ts @@ -1,6 +1,7 @@ import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../../test/exec-util'; +import { Fixtures } from '../../../../test/fixtures'; import { env, fs, @@ -13,12 +14,14 @@ import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; import * as docker from '../../../util/exec/docker'; import type { StatusResult } from '../../../util/git/types'; +import { find as _find } from '../../../util/host-rules'; import { getPkgReleases as _getPkgReleases } from '../../datasource'; import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; import * as pipenv from '.'; const datasource = mocked(_datasource); +const find = mockedFunction(_find); jest.mock('../../../util/exec/env'); jest.mock('../../../util/git'); @@ -328,4 +331,121 @@ describe('modules/manager/pipenv/artifacts', () => { ).not.toBeNull(); expect(execSnapshots).toMatchSnapshot(); }); + + it('passes private credential environment vars', async () => { + fs.ensureCacheDir.mockResolvedValueOnce( + '/tmp/renovate/cache/others/pipenv', + ); + fs.readLocalFile.mockResolvedValueOnce('current pipfile.lock'); + const execSnapshots = mockExecAll(); + git.getRepoStatus.mockResolvedValue( + partial({ + modified: ['Pipfile.lock'], + }), + ); + fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock'); + + find.mockReturnValueOnce({ + username: 'usernameOne', + password: 'passwordTwo', + }); + + const pipfile = Fixtures.get('Pipfile6'); + + expect( + await pipenv.updateArtifacts({ + packageFileName: 'Pipfile', + updatedDeps: [], + newPackageFileContent: pipfile, + config: { ...config, constraints: { python: '== 3.8.*' } }, + }), + ).toEqual([ + { + file: { + contents: 'New Pipfile.lock', + path: 'Pipfile.lock', + type: 'addition', + }, + }, + ]); + + expect(execSnapshots).toMatchObject([ + { + cmd: 'pipenv lock', + options: { + cwd: '/tmp/github/some/repo', + encoding: 'utf-8', + env: { + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PASSWORD: 'passwordTwo', + PATH: '/tmp/path', + PIPENV_CACHE_DIR: '/tmp/renovate/cache/others/pipenv', + USERNAME: 'usernameOne', + }, + maxBuffer: 10485760, + timeout: 900000, + }, + }, + ]); + }); + + it('does not pass private credential environment vars if variable names differ from allowed', async () => { + fs.ensureCacheDir.mockResolvedValueOnce( + '/tmp/renovate/cache/others/pipenv', + ); + fs.readLocalFile.mockResolvedValueOnce('current pipfile.lock'); + const execSnapshots = mockExecAll(); + git.getRepoStatus.mockResolvedValue( + partial({ + modified: ['Pipfile.lock'], + }), + ); + fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock'); + + const pipfile = Fixtures.get('Pipfile7'); + + expect( + await pipenv.updateArtifacts({ + packageFileName: 'Pipfile', + updatedDeps: [], + newPackageFileContent: pipfile, + config: { ...config, constraints: { python: '== 3.8.*' } }, + }), + ).toEqual([ + { + file: { + contents: 'New Pipfile.lock', + path: 'Pipfile.lock', + type: 'addition', + }, + }, + ]); + + expect(execSnapshots).toMatchObject([ + { + cmd: 'pipenv lock', + options: { + cwd: '/tmp/github/some/repo', + encoding: 'utf-8', + env: { + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', + PIPENV_CACHE_DIR: '/tmp/renovate/cache/others/pipenv', + }, + maxBuffer: 10485760, + timeout: 900000, + }, + }, + ]); + }); }); diff --git a/lib/modules/manager/pipenv/artifacts.ts b/lib/modules/manager/pipenv/artifacts.ts index 347d3ad61b1618..3d74191244ebed 100644 --- a/lib/modules/manager/pipenv/artifacts.ts +++ b/lib/modules/manager/pipenv/artifacts.ts @@ -1,7 +1,8 @@ import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; +import type { HostRule } from '../../../types'; import { exec } from '../../../util/exec'; -import type { ExecOptions } from '../../../util/exec/types'; +import type { ExecOptions, ExtraEnv, Opt } from '../../../util/exec/types'; import { deleteLocalFile, ensureCacheDir, @@ -9,11 +10,14 @@ import { writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; +import { find } from '../../../util/host-rules'; +import { PypiDatasource } from '../../datasource/pypi'; import type { UpdateArtifact, UpdateArtifactsConfig, UpdateArtifactsResult, } from '../types'; +import { extractPackageFile } from './extract'; import { PipfileLockSchema } from './schema'; export function getPythonConstraint( @@ -78,6 +82,37 @@ export function getPipenvConstraint( return ''; } +function getMatchingHostRule(url: string): HostRule { + return find({ hostType: PypiDatasource.id, url }); +} + +async function findPipfileSourceUrlWithCredentials( + pipfileContent: string, + pipfileName: string, +): Promise { + const pipfile = await extractPackageFile(pipfileContent, pipfileName); + if (!pipfile) { + logger.debug('Error parsing Pipfile'); + return null; + } + + const credentialTokens = [ + '$USERNAME:', + // eslint-disable-next-line no-template-curly-in-string + '${USERNAME}', + '$PASSWORD@', + // eslint-disable-next-line no-template-curly-in-string + '${PASSWORD}', + ]; + + const sourceWithCredentials = pipfile.registryUrls?.find((url) => + credentialTokens.some((token) => url.includes(token)), + ); + + // Only one source is currently supported + return sourceWithCredentials ?? null; +} + export async function updateArtifacts({ packageFileName: pipfileName, newPackageFileContent: newPipfileContent, @@ -102,12 +137,12 @@ export async function updateArtifacts({ existingLockFileContent, config, ); + const extraEnv: Opt = { + PIPENV_CACHE_DIR: await ensureCacheDir('pipenv'), + PIP_CACHE_DIR: await ensureCacheDir('pip'), + }; const execOptions: ExecOptions = { cwdFile: pipfileName, - extraEnv: { - PIPENV_CACHE_DIR: await ensureCacheDir('pipenv'), - PIP_CACHE_DIR: await ensureCacheDir('pip'), - }, docker: {}, toolConstraints: [ { @@ -120,6 +155,28 @@ export async function updateArtifacts({ }, ], }; + + const sourceUrl = await findPipfileSourceUrlWithCredentials( + newPipfileContent, + pipfileName, + ); + if (sourceUrl) { + logger.debug({ sourceUrl }, 'Pipfile contains credentials'); + const hostRule = getMatchingHostRule(sourceUrl); + if (hostRule) { + logger.debug('Found matching hostRule for Pipfile credentials'); + if (hostRule.username) { + logger.debug('Adding USERNAME environment variable for pipenv'); + extraEnv.USERNAME = hostRule.username; + } + if (hostRule.password) { + logger.debug('Adding PASSWORD environment variable for pipenv'); + extraEnv.PASSWORD = hostRule.password; + } + } + } + execOptions.extraEnv = extraEnv; + logger.trace({ cmd }, 'pipenv lock command'); await exec(cmd, execOptions); const status = await getRepoStatus();