feat(poetry): Add private package index support to Poetry (#6994)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/manager/poetry/__fixtures__/pyproject.10.toml

@@ -0,0 +1,17 @@
+[[tool.poetry.source]]
+name = "one"
+url = "some.url"
+
+[[tool.poetry.source]]
+name = "two"
+url = "another.url"
+
+[[tool.poetry.source]]
+url = "missingname.url"
+
+[[tool.poetry.source]]
+name = "four"
+url = "last.url"
+
+[[tool.poetry.source]]
+name = "five"

lib/manager/poetry/__snapshots__/artifacts.spec.ts.snap

@@ -12,6 +12,32 @@ undefined",
 ]
 `;
 
+exports[`.updateArtifacts() passes private credential environment vars 1`] = `
+Array [
+  Object {
+    "cmd": "poetry update --lock --no-interaction dep1",
+    "options": Object {
+      "cwd": "/tmp/github/some/repo",
+      "encoding": "utf-8",
+      "env": Object {
+        "HOME": "/home/user",
+        "HTTPS_PROXY": "https://example.com",
+        "HTTP_PROXY": "http://example.com",
+        "LANG": "en_US.UTF-8",
+        "LC_ALL": "en_US",
+        "NO_PROXY": "localhost",
+        "PATH": "/tmp/path",
+        "POETRY_HTTP_BASIC_FOUR_PASSWORD": "passwordFour",
+        "POETRY_HTTP_BASIC_ONE_PASSWORD": "passwordOne",
+        "POETRY_HTTP_BASIC_ONE_USERNAME": "usernameOne",
+        "POETRY_HTTP_BASIC_TWO_USERNAME": "usernameTwo",
+      },
+      "timeout": 900000,
+    },
+  },
+]
+`;
+
 exports[`.updateArtifacts() returns null if unchanged 1`] = `
 Array [
   Object {

lib/manager/poetry/artifacts.spec.ts

@@ -1,4 +1,5 @@
 import { exec as _exec } from 'child_process';
+import { readFileSync } from 'fs';
 import _fs from 'fs-extra';
 import { join } from 'upath';
 import { envMock, mockExecAll } from '../../../test/execUtil';
@@ -8,17 +9,25 @@ import { setExecConfig } from '../../util/exec';
 import { BinarySource } from '../../util/exec/common';
 import * as docker from '../../util/exec/docker';
 import * as _env from '../../util/exec/env';
+import * as _hostRules from '../../util/host-rules';
 import { updateArtifacts } from './artifacts';
 
+const pyproject10toml = readFileSync(
+  'lib/manager/poetry/__fixtures__/pyproject.10.toml',
+  'utf8'
+);
+
 jest.mock('fs-extra');
 jest.mock('child_process');
 jest.mock('../../util/exec/env');
 jest.mock('../../datasource');
+jest.mock('../../util/host-rules');
 
 const fs: jest.Mocked<typeof _fs> = _fs as any;
 const exec: jest.Mock<typeof _exec> = _exec as any;
 const env = mocked(_env);
 const datasource = mocked(_datasource);
+const hostRules = mocked(_hostRules);
 
 const config = {
   localDir: join('/tmp/github/some/repo'),
@@ -82,7 +91,29 @@ describe('.updateArtifacts()', () => {
     ).not.toBeNull();
     expect(execSnapshots).toMatchSnapshot();
   });
-
+  it('passes private credential environment vars', async () => {
+    fs.readFile.mockResolvedValueOnce(null);
+    fs.readFile.mockResolvedValueOnce('[metadata]\n' as never);
+    const execSnapshots = mockExecAll(exec);
+    fs.readFile.mockReturnValueOnce('New poetry.lock' as any);
+    hostRules.find.mockReturnValueOnce({
+      username: 'usernameOne',
+      password: 'passwordOne',
+    });
+    hostRules.find.mockReturnValueOnce({ username: 'usernameTwo' });
+    hostRules.find.mockReturnValueOnce({ password: 'passwordFour' });
+    const updatedDeps = ['dep1'];
+    expect(
+      await updateArtifacts({
+        packageFileName: 'pyproject.toml',
+        updatedDeps,
+        newPackageFileContent: pyproject10toml,
+        config,
+      })
+    ).not.toBeNull();
+    expect(hostRules.find.mock.calls).toHaveLength(3);
+    expect(execSnapshots).toMatchSnapshot();
+  });
   it('returns updated pyproject.lock', async () => {
     fs.readFile.mockResolvedValueOnce(null);
     fs.readFile.mockResolvedValueOnce('[metadata]\n' as never);

lib/manager/poetry/artifacts.ts

@@ -9,11 +9,13 @@ import {
   readLocalFile,
   writeLocalFile,
 } from '../../util/fs';
+import { find } from '../../util/host-rules';
 import {
   UpdateArtifact,
   UpdateArtifactsConfig,
   UpdateArtifactsResult,
 } from '../common';
+import { PoetryFile, PoetrySource } from './types';
 
 function getPythonConstraint(
   existingLockFileContent: string,
@@ -37,6 +39,50 @@ function getPythonConstraint(
   return undefined;
 }
 
+function getPoetrySources(content: string, fileName: string): PoetrySource[] {
+  let pyprojectFile: PoetryFile;
+  try {
+    pyprojectFile = parse(content);
+  } catch (err) {
+    logger.debug({ err }, 'Error parsing pyproject.toml file');
+    return [];
+  }
+  if (!pyprojectFile.tool?.poetry) {
+    logger.debug(`{$fileName} contains no poetry section`);
+    return [];
+  }
+
+  const sources = pyprojectFile.tool?.poetry?.source;
+  const sourceArray: PoetrySource[] = [];
+  for (const source of sources) {
+    if (source.name && source.url) {
+      sourceArray.push({ name: source.name, url: source.url });
+    }
+  }
+  return sourceArray;
+}
+
+function getSourceCredentialVars(
+  pyprojectContent: string,
+  packageFileName: string
+): Record<string, string> {
+  const poetrySources = getPoetrySources(pyprojectContent, packageFileName);
+  const envVars: Record<string, string> = {};
+
+  for (const source of poetrySources) {
+    const matchingHostRule = find({ url: source.url });
+    const formattedSourceName = source.name.toUpperCase();
+    if (matchingHostRule.username) {
+      envVars[`POETRY_HTTP_BASIC_${formattedSourceName}_USERNAME`] =
+        matchingHostRule.username;
+    }
+    if (matchingHostRule.password) {
+      envVars[`POETRY_HTTP_BASIC_${formattedSourceName}_PASSWORD`] =
+        matchingHostRule.password;
+    }
+  }
+  return envVars;
+}
 export async function updateArtifacts({
   packageFileName,
   updatedDeps,
@@ -76,8 +122,14 @@ export async function updateArtifacts({
     const tagConstraint = getPythonConstraint(existingLockFileContent, config);
     const poetryRequirement = config.compatibility?.poetry || 'poetry';
     const poetryInstall = 'pip install ' + quote(poetryRequirement);
+    const extraEnv = getSourceCredentialVars(
+      newPackageFileContent,
+      packageFileName
+    );
+
     const execOptions: ExecOptions = {
       cwdFile: packageFileName,
+      extraEnv,
       docker: {
         image: 'renovate/python',
         tagConstraint,