feat(pipenv): add support for auth (#24581)

Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

docs/usage/getting-started/private-packages.md

@@ -442,6 +442,19 @@ For example:
 }
 ```
 
+### pipenv
+
+If a `Pipfile` contains a `source` with `USERNAME` or `PASSWORD` environment variables and there is a `hostRules` entry with a matching host plus `username` and `password` fields, then these variables would be passed to `pipenv lock`.
+
+For example:
+
+```ini
+[[source]]
+url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple"
+verify_ssl = true
+name = "pypi"
+```
+
 ### poetry
 
 For every Poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`, where `X` represents the normalized name of the source in `pyproject.toml`.

lib/modules/manager/pipenv/__fixtures__/Pipfile6

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple"
+verify_ssl = true
+name = "private"
+
+[packages]
+requests = {version = "==0.21.0", index = "private"}

lib/modules/manager/pipenv/__fixtures__/Pipfile7

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://$USERNAME_FOO:${PAZZWORD}@mypypi.example.com/simple"
+verify_ssl = true
+name = "private"
+
+[packages]
+requests = {version = "==0.21.0", index = "private"}

lib/modules/manager/pipenv/artifacts.spec.ts

@@ -1,6 +1,7 @@
 import { mockDeep } from 'jest-mock-extended';
 import { join } from 'upath';
 import { envMock, mockExecAll } from '../../../../test/exec-util';
+import { Fixtures } from '../../../../test/fixtures';
 import {
   env,
   fs,
@@ -13,12 +14,14 @@ import { GlobalConfig } from '../../../config/global';
 import type { RepoGlobalConfig } from '../../../config/types';
 import * as docker from '../../../util/exec/docker';
 import type { StatusResult } from '../../../util/git/types';
+import { find as _find } from '../../../util/host-rules';
 import { getPkgReleases as _getPkgReleases } from '../../datasource';
 import * as _datasource from '../../datasource';
 import type { UpdateArtifactsConfig } from '../types';
 import * as pipenv from '.';
 
 const datasource = mocked(_datasource);
+const find = mockedFunction(_find);
 
 jest.mock('../../../util/exec/env');
 jest.mock('../../../util/git');
@@ -328,4 +331,121 @@ describe('modules/manager/pipenv/artifacts', () => {
     ).not.toBeNull();
     expect(execSnapshots).toMatchSnapshot();
   });
+
+  it('passes private credential environment vars', async () => {
+    fs.ensureCacheDir.mockResolvedValueOnce(
+      '/tmp/renovate/cache/others/pipenv',
+    );
+    fs.readLocalFile.mockResolvedValueOnce('current pipfile.lock');
+    const execSnapshots = mockExecAll();
+    git.getRepoStatus.mockResolvedValue(
+      partial<StatusResult>({
+        modified: ['Pipfile.lock'],
+      }),
+    );
+    fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock');
+
+    find.mockReturnValueOnce({
+      username: 'usernameOne',
+      password: 'passwordTwo',
+    });
+
+    const pipfile = Fixtures.get('Pipfile6');
+
+    expect(
+      await pipenv.updateArtifacts({
+        packageFileName: 'Pipfile',
+        updatedDeps: [],
+        newPackageFileContent: pipfile,
+        config: { ...config, constraints: { python: '== 3.8.*' } },
+      }),
+    ).toEqual([
+      {
+        file: {
+          contents: 'New Pipfile.lock',
+          path: 'Pipfile.lock',
+          type: 'addition',
+        },
+      },
+    ]);
+
+    expect(execSnapshots).toMatchObject([
+      {
+        cmd: 'pipenv lock',
+        options: {
+          cwd: '/tmp/github/some/repo',
+          encoding: 'utf-8',
+          env: {
+            HOME: '/home/user',
+            HTTPS_PROXY: 'https://example.com',
+            HTTP_PROXY: 'http://example.com',
+            LANG: 'en_US.UTF-8',
+            LC_ALL: 'en_US',
+            NO_PROXY: 'localhost',
+            PASSWORD: 'passwordTwo',
+            PATH: '/tmp/path',
+            PIPENV_CACHE_DIR: '/tmp/renovate/cache/others/pipenv',
+            USERNAME: 'usernameOne',
+          },
+          maxBuffer: 10485760,
+          timeout: 900000,
+        },
+      },
+    ]);
+  });
+
+  it('does not pass private credential environment vars if variable names differ from allowed', async () => {
+    fs.ensureCacheDir.mockResolvedValueOnce(
+      '/tmp/renovate/cache/others/pipenv',
+    );
+    fs.readLocalFile.mockResolvedValueOnce('current pipfile.lock');
+    const execSnapshots = mockExecAll();
+    git.getRepoStatus.mockResolvedValue(
+      partial<StatusResult>({
+        modified: ['Pipfile.lock'],
+      }),
+    );
+    fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock');
+
+    const pipfile = Fixtures.get('Pipfile7');
+
+    expect(
+      await pipenv.updateArtifacts({
+        packageFileName: 'Pipfile',
+        updatedDeps: [],
+        newPackageFileContent: pipfile,
+        config: { ...config, constraints: { python: '== 3.8.*' } },
+      }),
+    ).toEqual([
+      {
+        file: {
+          contents: 'New Pipfile.lock',
+          path: 'Pipfile.lock',
+          type: 'addition',
+        },
+      },
+    ]);
+
+    expect(execSnapshots).toMatchObject([
+      {
+        cmd: 'pipenv lock',
+        options: {
+          cwd: '/tmp/github/some/repo',
+          encoding: 'utf-8',
+          env: {
+            HOME: '/home/user',
+            HTTPS_PROXY: 'https://example.com',
+            HTTP_PROXY: 'http://example.com',
+            LANG: 'en_US.UTF-8',
+            LC_ALL: 'en_US',
+            NO_PROXY: 'localhost',
+            PATH: '/tmp/path',
+            PIPENV_CACHE_DIR: '/tmp/renovate/cache/others/pipenv',
+          },
+          maxBuffer: 10485760,
+          timeout: 900000,
+        },
+      },
+    ]);
+  });
 });

lib/modules/manager/pipenv/artifacts.ts

@@ -1,19 +1,23 @@
 import { TEMPORARY_ERROR } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
+import type { HostRule } from '../../../types';
 import { exec } from '../../../util/exec';
-import type { ExecOptions } from '../../../util/exec/types';
+import type { ExecOptions, ExtraEnv, Opt } from '../../../util/exec/types';
 import {
   deleteLocalFile,
   ensureCacheDir,
   readLocalFile,
   writeLocalFile,
 } from '../../../util/fs';
 import { getRepoStatus } from '../../../util/git';
+import { find } from '../../../util/host-rules';
+import { PypiDatasource } from '../../datasource/pypi';
 import type {
   UpdateArtifact,
   UpdateArtifactsConfig,
   UpdateArtifactsResult,
 } from '../types';
+import { extractPackageFile } from './extract';
 import { PipfileLockSchema } from './schema';
 
 export function getPythonConstraint(
@@ -78,6 +82,37 @@ export function getPipenvConstraint(
   return '';
 }
 
+function getMatchingHostRule(url: string): HostRule {
+  return find({ hostType: PypiDatasource.id, url });
+}
+
+async function findPipfileSourceUrlWithCredentials(
+  pipfileContent: string,
+  pipfileName: string,
+): Promise<string | null> {
+  const pipfile = await extractPackageFile(pipfileContent, pipfileName);
+  if (!pipfile) {
+    logger.debug('Error parsing Pipfile');
+    return null;
+  }
+
+  const credentialTokens = [
+    '$USERNAME:',
+    // eslint-disable-next-line no-template-curly-in-string
+    '${USERNAME}',
+    '$PASSWORD@',
+    // eslint-disable-next-line no-template-curly-in-string
+    '${PASSWORD}',
+  ];
+
+  const sourceWithCredentials = pipfile.registryUrls?.find((url) =>
+    credentialTokens.some((token) => url.includes(token)),
+  );
+
+  // Only one source is currently supported
+  return sourceWithCredentials ?? null;
+}
+
 export async function updateArtifacts({
   packageFileName: pipfileName,
   newPackageFileContent: newPipfileContent,
@@ -102,12 +137,12 @@ export async function updateArtifacts({
       existingLockFileContent,
       config,
     );
+    const extraEnv: Opt<ExtraEnv> = {
+      PIPENV_CACHE_DIR: await ensureCacheDir('pipenv'),
+      PIP_CACHE_DIR: await ensureCacheDir('pip'),
+    };
     const execOptions: ExecOptions = {
       cwdFile: pipfileName,
-      extraEnv: {
-        PIPENV_CACHE_DIR: await ensureCacheDir('pipenv'),
-        PIP_CACHE_DIR: await ensureCacheDir('pip'),
-      },
       docker: {},
       toolConstraints: [
         {
@@ -120,6 +155,28 @@ export async function updateArtifacts({
         },
       ],
     };
+
+    const sourceUrl = await findPipfileSourceUrlWithCredentials(
+      newPipfileContent,
+      pipfileName,
+    );
+    if (sourceUrl) {
+      logger.debug({ sourceUrl }, 'Pipfile contains credentials');
+      const hostRule = getMatchingHostRule(sourceUrl);
+      if (hostRule) {
+        logger.debug('Found matching hostRule for Pipfile credentials');
+        if (hostRule.username) {
+          logger.debug('Adding USERNAME environment variable for pipenv');
+          extraEnv.USERNAME = hostRule.username;
+        }
+        if (hostRule.password) {
+          logger.debug('Adding PASSWORD environment variable for pipenv');
+          extraEnv.PASSWORD = hostRule.password;
+        }
+      }
+    }
+    execOptions.extraEnv = extraEnv;
+
     logger.trace({ cmd }, 'pipenv lock command');
     await exec(cmd, execOptions);
     const status = await getRepoStatus();