From 748f3e69077e336e78c6f472e88262c34b4155fe Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Mon, 27 Jun 2022 17:39:14 +0300 Subject: [PATCH 01/10] Use correct Poetry config when collecting Poetry projects When collecting Poetry projects for caching, a '**/poetry.lock' glob is used. However, in order to process the Poetry configuration, the "poetry" command is run from the repo's root directory; this causes Poetry to return an invalid configuration when there is a Poetry project inside an inner directory. Instead of running a single Poetry command, glob for the same pattern, and run a Poetry command for every discovered project. --- dist/setup/index.js | 42 ++++++++++++++++++------- src/cache-distributions/poetry-cache.ts | 35 ++++++++++++--------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/dist/setup/index.js b/dist/setup/index.js index 6cf4871db..114689434 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -66078,6 +66078,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; @@ -66096,13 +66103,29 @@ class PoetryCache extends cache_distributor_1.default { this.patterns = patterns; } getCacheGlobalDirectories() { + var e_1, _a; return __awaiter(this, void 0, void 0, function* () { - const poetryConfig = yield this.getPoetryConfiguration(); - const cacheDir = poetryConfig['cache-dir']; - const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); - const paths = [virtualenvsPath]; - if (poetryConfig['virtualenvs.in-project'] === true) { - paths.push(path.join(process.cwd(), '.venv')); + const paths = []; + const globber = yield glob.create(this.patterns); + try { + for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { + const file = _c.value; + const basedir = path.dirname(file); + const poetryConfig = yield this.getPoetryConfiguration(basedir); + const cacheDir = poetryConfig['cache-dir']; + const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); + paths.push(virtualenvsPath); + if (poetryConfig['virtualenvs.in-project'] === true) { + paths.push(path.join(basedir, '.venv')); + } + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b); + } + finally { if (e_1) throw e_1.error; } } const pythonLocation = yield io.which('python'); if (pythonLocation) { @@ -66129,12 +66152,9 @@ class PoetryCache extends cache_distributor_1.default { }; }); } - getPoetryConfiguration() { + getPoetryConfiguration(basedir) { return __awaiter(this, void 0, void 0, function* () { - const { stdout, stderr, exitCode } = yield exec.getExecOutput('poetry', [ - 'config', - '--list' - ]); + const { stdout, stderr, exitCode } = yield exec.getExecOutput('poetry', ['config', '--list'], { cwd: basedir }); if (exitCode && stderr) { throw new Error('Could not get cache folder path for poetry package manager'); } diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index 60ecea2b1..46e239a80 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -16,18 +16,24 @@ class PoetryCache extends CacheDistributor { } protected async getCacheGlobalDirectories() { - const poetryConfig = await this.getPoetryConfiguration(); + const paths = []; + const globber = await glob.create(this.patterns); - const cacheDir = poetryConfig['cache-dir']; - const virtualenvsPath = poetryConfig['virtualenvs.path'].replace( - '{cache-dir}', - cacheDir - ); + for await (const file of globber.globGenerator()) { + const basedir = path.dirname(file); + const poetryConfig = await this.getPoetryConfiguration(basedir); + + const cacheDir = poetryConfig['cache-dir']; + const virtualenvsPath = poetryConfig['virtualenvs.path'].replace( + '{cache-dir}', + cacheDir + ); - const paths = [virtualenvsPath]; + paths.push(virtualenvsPath); - if (poetryConfig['virtualenvs.in-project'] === true) { - paths.push(path.join(process.cwd(), '.venv')); + if (poetryConfig['virtualenvs.in-project'] === true) { + paths.push(path.join(basedir, '.venv')); + } } const pythonLocation = await io.which('python'); @@ -63,11 +69,12 @@ class PoetryCache extends CacheDistributor { }; } - private async getPoetryConfiguration() { - const {stdout, stderr, exitCode} = await exec.getExecOutput('poetry', [ - 'config', - '--list' - ]); + private async getPoetryConfiguration(basedir: string) { + const {stdout, stderr, exitCode} = await exec.getExecOutput( + 'poetry', + ['config', '--list'], + {cwd: basedir} + ); if (exitCode && stderr) { throw new Error( From 157f5da699c373e908dfd339049520962d17bb0b Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Sat, 8 Oct 2022 11:45:46 +0300 Subject: [PATCH 02/10] Fix typo: saveSatetSpy -> saveStateSpy --- __tests__/cache-restore.test.ts | 6 +++--- __tests__/cache-save.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/__tests__/cache-restore.test.ts b/__tests__/cache-restore.test.ts index 81caabad0..1d062e717 100644 --- a/__tests__/cache-restore.test.ts +++ b/__tests__/cache-restore.test.ts @@ -27,7 +27,7 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py let infoSpy: jest.SpyInstance; let warningSpy: jest.SpyInstance; let debugSpy: jest.SpyInstance; - let saveSatetSpy: jest.SpyInstance; + let saveStateSpy: jest.SpyInstance; let getStateSpy: jest.SpyInstance; let setOutputSpy: jest.SpyInstance; @@ -52,8 +52,8 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py debugSpy = jest.spyOn(core, 'debug'); debugSpy.mockImplementation(input => undefined); - saveSatetSpy = jest.spyOn(core, 'saveState'); - saveSatetSpy.mockImplementation(input => undefined); + saveStateSpy = jest.spyOn(core, 'saveState'); + saveStateSpy.mockImplementation(input => undefined); getStateSpy = jest.spyOn(core, 'getState'); getStateSpy.mockImplementation(input => undefined); diff --git a/__tests__/cache-save.test.ts b/__tests__/cache-save.test.ts index f90ca7f6a..33809ba8d 100644 --- a/__tests__/cache-save.test.ts +++ b/__tests__/cache-save.test.ts @@ -18,7 +18,7 @@ describe('run', () => { let infoSpy: jest.SpyInstance; let warningSpy: jest.SpyInstance; let debugSpy: jest.SpyInstance; - let saveSatetSpy: jest.SpyInstance; + let saveStateSpy: jest.SpyInstance; let getStateSpy: jest.SpyInstance; let getInputSpy: jest.SpyInstance; let setFailedSpy: jest.SpyInstance; @@ -43,8 +43,8 @@ describe('run', () => { debugSpy = jest.spyOn(core, 'debug'); debugSpy.mockImplementation(input => undefined); - saveSatetSpy = jest.spyOn(core, 'saveState'); - saveSatetSpy.mockImplementation(input => undefined); + saveStateSpy = jest.spyOn(core, 'saveState'); + saveStateSpy.mockImplementation(input => undefined); getStateSpy = jest.spyOn(core, 'getState'); getStateSpy.mockImplementation(input => { From dc45b1e4e099d3a97f140f3216f45892eb9a089b Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Sat, 8 Oct 2022 11:46:59 +0300 Subject: [PATCH 03/10] poetry: Support same virtualenv appearing in multiple projects --- dist/setup/index.js | 9 +++++---- src/cache-distributions/poetry-cache.ts | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dist/setup/index.js b/dist/setup/index.js index 114689434..804d6a320 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -66105,7 +66105,8 @@ class PoetryCache extends cache_distributor_1.default { getCacheGlobalDirectories() { var e_1, _a; return __awaiter(this, void 0, void 0, function* () { - const paths = []; + // Same virtualenvs path may appear for different projects, hence we use a Set + const paths = new Set(); const globber = yield glob.create(this.patterns); try { for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { @@ -66114,9 +66115,9 @@ class PoetryCache extends cache_distributor_1.default { const poetryConfig = yield this.getPoetryConfiguration(basedir); const cacheDir = poetryConfig['cache-dir']; const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); - paths.push(virtualenvsPath); + paths.add(virtualenvsPath); if (poetryConfig['virtualenvs.in-project'] === true) { - paths.push(path.join(basedir, '.venv')); + paths.add(path.join(basedir, '.venv')); } } } @@ -66138,7 +66139,7 @@ class PoetryCache extends cache_distributor_1.default { else { utils_1.logWarning('python binaries were not found in PATH'); } - return paths; + return [...paths]; }); } computeKeys() { diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index 46e239a80..a0666d445 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -16,7 +16,8 @@ class PoetryCache extends CacheDistributor { } protected async getCacheGlobalDirectories() { - const paths = []; + // Same virtualenvs path may appear for different projects, hence we use a Set + const paths = new Set(); const globber = await glob.create(this.patterns); for await (const file of globber.globGenerator()) { @@ -29,10 +30,10 @@ class PoetryCache extends CacheDistributor { cacheDir ); - paths.push(virtualenvsPath); + paths.add(virtualenvsPath); if (poetryConfig['virtualenvs.in-project'] === true) { - paths.push(path.join(basedir, '.venv')); + paths.add(path.join(basedir, '.venv')); } } @@ -56,7 +57,7 @@ class PoetryCache extends CacheDistributor { logWarning('python binaries were not found in PATH'); } - return paths; + return [...paths]; } protected async computeKeys() { From f35052820fa209b5b162e650b4ee6cad3df820c7 Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Sat, 8 Oct 2022 12:22:27 +0300 Subject: [PATCH 04/10] Add nested Poetry projects test --- __tests__/cache-restore.test.ts | 62 ++++++++++++++++++++++++----- __tests__/data/inner/poetry.lock | 1 + __tests__/data/inner/pyproject.toml | 1 + 3 files changed, 55 insertions(+), 9 deletions(-) create mode 120000 __tests__/data/inner/poetry.lock create mode 120000 __tests__/data/inner/pyproject.toml diff --git a/__tests__/cache-restore.test.ts b/__tests__/cache-restore.test.ts index 1d062e717..d19cb1e61 100644 --- a/__tests__/cache-restore.test.ts +++ b/__tests__/cache-restore.test.ts @@ -1,8 +1,10 @@ +import * as path from 'path'; import * as core from '@actions/core'; import * as cache from '@actions/cache'; import * as exec from '@actions/exec'; import * as io from '@actions/io'; import {getCacheDistributor} from '../src/cache-distributions/cache-factory'; +import {State} from '../src/cache-distributions/cache-distributor'; import * as utils from './../src/utils'; describe('restore-cache', () => { @@ -13,7 +15,7 @@ describe('restore-cache', () => { const requirementsLinuxHash = '2d0ff7f46b0e120e3d3294db65768b474934242637b9899b873e6283dfd16d7c'; const poetryLockHash = - '571bf984f8d210e6a97f854e479fdd4a2b5af67b5fdac109ec337a0ea16e7836'; + 'f24ea1ad73968e6c8d80c16a093ade72d9332c433aeef979a0dd943e6a99b2ab'; const poetryConfigOutput = ` cache-dir = "/Users/patrick/Library/Caches/pypoetry" experimental.new-installer = false @@ -100,21 +102,56 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py describe('Restore dependencies', () => { it.each([ - ['pip', '3.8.12', undefined, requirementsHash], - ['pip', '3.8.12', '**/requirements-linux.txt', requirementsLinuxHash], + ['pip', '3.8.12', undefined, requirementsHash, undefined], + [ + 'pip', + '3.8.12', + '**/requirements-linux.txt', + requirementsLinuxHash, + undefined + ], [ 'pip', '3.8.12', '__tests__/data/requirements-linux.txt', - requirementsLinuxHash + requirementsLinuxHash, + undefined ], - ['pip', '3.8.12', '__tests__/data/requirements.txt', requirementsHash], - ['pipenv', '3.9.1', undefined, pipFileLockHash], - ['pipenv', '3.9.12', '__tests__/data/requirements.txt', requirementsHash], - ['poetry', '3.9.1', undefined, poetryLockHash] + [ + 'pip', + '3.8.12', + '__tests__/data/requirements.txt', + requirementsHash, + undefined + ], + ['pipenv', '3.9.1', undefined, pipFileLockHash, undefined], + [ + 'pipenv', + '3.9.12', + '__tests__/data/requirements.txt', + requirementsHash, + undefined + ], + [ + 'poetry', + '3.9.1', + undefined, + poetryLockHash, + [ + '/Users/patrick/Library/Caches/pypoetry/virtualenvs', + path.join(__dirname, 'data', 'inner', '.venv'), + path.join(__dirname, 'data', '.venv') + ] + ] ])( 'restored dependencies for %s by primaryKey', - async (packageManager, pythonVersion, dependencyFile, fileHash) => { + async ( + packageManager, + pythonVersion, + dependencyFile, + fileHash, + cachePaths + ) => { const cacheDistributor = getCacheDistributor( packageManager, pythonVersion, @@ -123,6 +160,13 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py await cacheDistributor.restoreCache(); + if (cachePaths !== undefined) { + expect(saveStateSpy).toHaveBeenCalledWith( + State.CACHE_PATHS, + cachePaths + ); + } + if (process.platform === 'linux' && packageManager === 'pip') { expect(infoSpy).toHaveBeenCalledWith( `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-20.04-Ubuntu-python-${pythonVersion}-${packageManager}-${fileHash}` diff --git a/__tests__/data/inner/poetry.lock b/__tests__/data/inner/poetry.lock new file mode 120000 index 000000000..99fe86c2a --- /dev/null +++ b/__tests__/data/inner/poetry.lock @@ -0,0 +1 @@ +../poetry.lock \ No newline at end of file diff --git a/__tests__/data/inner/pyproject.toml b/__tests__/data/inner/pyproject.toml new file mode 120000 index 000000000..1e11d7825 --- /dev/null +++ b/__tests__/data/inner/pyproject.toml @@ -0,0 +1 @@ +../pyproject.toml \ No newline at end of file From 727a4d2ef2b6c372d34d2f519048a24901ecf238 Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Sat, 8 Oct 2022 13:00:06 +0300 Subject: [PATCH 05/10] poetry: Set up environment for each project individually --- dist/setup/index.js | 25 ++++++++++-------- src/cache-distributions/poetry-cache.ts | 35 +++++++++++++------------ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/dist/setup/index.js b/dist/setup/index.js index 804d6a320..d5793b2da 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -66108,10 +66108,18 @@ class PoetryCache extends cache_distributor_1.default { // Same virtualenvs path may appear for different projects, hence we use a Set const paths = new Set(); const globber = yield glob.create(this.patterns); + const pythonLocation = yield io.which('python'); + if (pythonLocation) { + core.debug(`pythonLocation is ${pythonLocation}`); + } + else { + utils_1.logWarning('python binaries were not found in PATH'); + } try { for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { const file = _c.value; const basedir = path.dirname(file); + core.debug(`Processing Poetry project at ${basedir}`); const poetryConfig = yield this.getPoetryConfiguration(basedir); const cacheDir = poetryConfig['cache-dir']; const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); @@ -66119,6 +66127,12 @@ class PoetryCache extends cache_distributor_1.default { if (poetryConfig['virtualenvs.in-project'] === true) { paths.add(path.join(basedir, '.venv')); } + if (pythonLocation) { + const { exitCode, stderr } = yield exec.getExecOutput('poetry', ['env', 'use', pythonLocation], { ignoreReturnCode: true, cwd: basedir }); + if (exitCode) { + utils_1.logWarning(stderr); + } + } } } catch (e_1_1) { e_1 = { error: e_1_1 }; } @@ -66128,17 +66142,6 @@ class PoetryCache extends cache_distributor_1.default { } finally { if (e_1) throw e_1.error; } } - const pythonLocation = yield io.which('python'); - if (pythonLocation) { - core.debug(`pythonLocation is ${pythonLocation}`); - const { exitCode, stderr } = yield exec.getExecOutput(`poetry env use ${pythonLocation}`, undefined, { ignoreReturnCode: true }); - if (exitCode) { - utils_1.logWarning(stderr); - } - } - else { - utils_1.logWarning('python binaries were not found in PATH'); - } return [...paths]; }); } diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index a0666d445..ac3946457 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -20,8 +20,17 @@ class PoetryCache extends CacheDistributor { const paths = new Set(); const globber = await glob.create(this.patterns); + const pythonLocation = await io.which('python'); + if (pythonLocation) { + core.debug(`pythonLocation is ${pythonLocation}`); + } else { + logWarning('python binaries were not found in PATH'); + } + for await (const file of globber.globGenerator()) { const basedir = path.dirname(file); + core.debug(`Processing Poetry project at ${basedir}`); + const poetryConfig = await this.getPoetryConfiguration(basedir); const cacheDir = poetryConfig['cache-dir']; @@ -35,26 +44,18 @@ class PoetryCache extends CacheDistributor { if (poetryConfig['virtualenvs.in-project'] === true) { paths.add(path.join(basedir, '.venv')); } - } - - const pythonLocation = await io.which('python'); - if (pythonLocation) { - core.debug(`pythonLocation is ${pythonLocation}`); - const { - exitCode, - stderr - } = await exec.getExecOutput( - `poetry env use ${pythonLocation}`, - undefined, - {ignoreReturnCode: true} - ); + if (pythonLocation) { + const {exitCode, stderr} = await exec.getExecOutput( + 'poetry', + ['env', 'use', pythonLocation], + {ignoreReturnCode: true, cwd: basedir} + ); - if (exitCode) { - logWarning(stderr); + if (exitCode) { + logWarning(stderr); + } } - } else { - logWarning('python binaries were not found in PATH'); } return [...paths]; From 95696f70286dc21ff2a1f139e05c66e68af3dbd9 Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Fri, 14 Oct 2022 15:32:30 +0300 Subject: [PATCH 06/10] tests/cache-restore: Do not look for dependency files outside `data` When the default dependency path is used for cache distributors, they are looking for the dependency file in the project's root (including the source code), which leads to tests taking a significant amount of time, especially on Windows runners. We thus hit sporadic test failures. Change the test cases such that dependency files are always searched for inside of `__tests__/data`, ignoring the rest of the project. --- __tests__/cache-restore.test.ts | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/__tests__/cache-restore.test.ts b/__tests__/cache-restore.test.ts index d19cb1e61..44b400eef 100644 --- a/__tests__/cache-restore.test.ts +++ b/__tests__/cache-restore.test.ts @@ -102,11 +102,17 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py describe('Restore dependencies', () => { it.each([ - ['pip', '3.8.12', undefined, requirementsHash, undefined], [ 'pip', '3.8.12', - '**/requirements-linux.txt', + '__tests__/data/**/requirements.txt', + requirementsHash, + undefined + ], + [ + 'pip', + '3.8.12', + '__tests__/data/**/requirements-linux.txt', requirementsLinuxHash, undefined ], @@ -124,7 +130,13 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py requirementsHash, undefined ], - ['pipenv', '3.9.1', undefined, pipFileLockHash, undefined], + [ + 'pipenv', + '3.9.1', + '__tests__/data/**/Pipfile.lock', + pipFileLockHash, + undefined + ], [ 'pipenv', '3.9.12', @@ -135,7 +147,7 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py [ 'poetry', '3.9.1', - undefined, + '__tests__/data/**/poetry.lock', poetryLockHash, [ '/Users/patrick/Library/Caches/pypoetry/virtualenvs', @@ -208,8 +220,13 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py describe('Dependencies changed', () => { it.each([ - ['pip', '3.8.12', undefined, pipFileLockHash], - ['pip', '3.8.12', '**/requirements-linux.txt', pipFileLockHash], + ['pip', '3.8.12', '__tests__/data/**/requirements.txt', pipFileLockHash], + [ + 'pip', + '3.8.12', + '__tests__/data/**/requirements-linux.txt', + pipFileLockHash + ], [ 'pip', '3.8.12', @@ -217,9 +234,9 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py pipFileLockHash ], ['pip', '3.8.12', '__tests__/data/requirements.txt', pipFileLockHash], - ['pipenv', '3.9.1', undefined, requirementsHash], + ['pipenv', '3.9.1', '__tests__/data/**/Pipfile.lock', requirementsHash], ['pipenv', '3.9.12', '__tests__/data/requirements.txt', requirementsHash], - ['poetry', '3.9.1', undefined, requirementsHash] + ['poetry', '3.9.1', '__tests__/data/**/poetry.lock', requirementsHash] ])( 'restored dependencies for %s by primaryKey', async (packageManager, pythonVersion, dependencyFile, fileHash) => { From bb9c5ac469fa32ffeb379d753bb9515addc7b2d4 Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Tue, 15 Nov 2022 13:13:07 -0500 Subject: [PATCH 07/10] poetry: Simplify `virtualenvs.in-project` boolean check --- dist/setup/index.js | 2 +- src/cache-distributions/poetry-cache.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/setup/index.js b/dist/setup/index.js index d5793b2da..28e3cee72 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -66124,7 +66124,7 @@ class PoetryCache extends cache_distributor_1.default { const cacheDir = poetryConfig['cache-dir']; const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); paths.add(virtualenvsPath); - if (poetryConfig['virtualenvs.in-project'] === true) { + if (poetryConfig['virtualenvs.in-project']) { paths.add(path.join(basedir, '.venv')); } if (pythonLocation) { diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index ac3946457..24a96b13b 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -41,7 +41,7 @@ class PoetryCache extends CacheDistributor { paths.add(virtualenvsPath); - if (poetryConfig['virtualenvs.in-project'] === true) { + if (poetryConfig['virtualenvs.in-project']) { paths.add(path.join(basedir, '.venv')); } From bc3992ed3081de07e31f07f4ccd9fda82112c36c Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Tue, 15 Nov 2022 13:55:18 -0500 Subject: [PATCH 08/10] README: Explain that poetry might create multiple caches --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b21c77bd..08619c6a3 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The action defaults to searching for a dependency file (`requirements.txt` for p - For `pip`, the action will cache the global cache directory - For `pipenv`, the action will cache virtualenv directory - - For `poetry`, the action will cache virtualenv directory + - For `poetry`, the action will cache virtualenv directories -- one for each poetry project found **Caching pip dependencies:** From 881ca6e47731b87e35fd347eb1a7815ab033bb1c Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Fri, 9 Dec 2022 17:48:00 +0200 Subject: [PATCH 09/10] poetry: Run `poetry env use` only after cache is loaded The virtualenv cache might contain invalid entries, such as virtualenvs built in previous, buggy versions of this action. The `poetry env use` command will recreate virtualenvs in case they are invalid, but it has to be run only *after* the cache is loaded. Refactor `CacheDistributor` a bit such that the validation (and possible recreation) of virtualenvs happens only after the cache is loaded. --- dist/cache-save/index.js | 4 ++ dist/setup/index.js | 45 ++++++++++++------ src/cache-distributions/cache-distributor.ts | 3 ++ src/cache-distributions/poetry-cache.ts | 50 ++++++++++++-------- 4 files changed, 68 insertions(+), 34 deletions(-) diff --git a/dist/cache-save/index.js b/dist/cache-save/index.js index da8fdef70..09a9fc6ec 100644 --- a/dist/cache-save/index.js +++ b/dist/cache-save/index.js @@ -59711,6 +59711,9 @@ class CacheDistributor { this.cacheDependencyPath = cacheDependencyPath; this.CACHE_KEY_PREFIX = 'setup-python'; } + handleLoadedCache() { + return __awaiter(this, void 0, void 0, function* () { }); + } restoreCache() { return __awaiter(this, void 0, void 0, function* () { const { primaryKey, restoreKey } = yield this.computeKeys(); @@ -59723,6 +59726,7 @@ class CacheDistributor { core.saveState(State.CACHE_PATHS, cachePath); core.saveState(State.STATE_CACHE_PRIMARY_KEY, primaryKey); const matchedKey = yield cache.restoreCache(cachePath, primaryKey, restoreKey); + yield this.handleLoadedCache(); this.handleMatchResult(matchedKey, primaryKey); }); } diff --git a/dist/setup/index.js b/dist/setup/index.js index 28e3cee72..41ed58a1f 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -65787,6 +65787,9 @@ class CacheDistributor { this.cacheDependencyPath = cacheDependencyPath; this.CACHE_KEY_PREFIX = 'setup-python'; } + handleLoadedCache() { + return __awaiter(this, void 0, void 0, function* () { }); + } restoreCache() { return __awaiter(this, void 0, void 0, function* () { const { primaryKey, restoreKey } = yield this.computeKeys(); @@ -65799,6 +65802,7 @@ class CacheDistributor { core.saveState(State.CACHE_PATHS, cachePath); core.saveState(State.STATE_CACHE_PRIMARY_KEY, primaryKey); const matchedKey = yield cache.restoreCache(cachePath, primaryKey, restoreKey); + yield this.handleLoadedCache(); this.handleMatchResult(matchedKey, primaryKey); }); } @@ -66097,10 +66101,11 @@ const core = __importStar(__nccwpck_require__(2186)); const cache_distributor_1 = __importDefault(__nccwpck_require__(8953)); const utils_1 = __nccwpck_require__(1314); class PoetryCache extends cache_distributor_1.default { - constructor(pythonVersion, patterns = '**/poetry.lock') { + constructor(pythonVersion, patterns = '**/poetry.lock', poetryProjects = new Set()) { super('poetry', patterns); this.pythonVersion = pythonVersion; this.patterns = patterns; + this.poetryProjects = poetryProjects; } getCacheGlobalDirectories() { var e_1, _a; @@ -66108,18 +66113,12 @@ class PoetryCache extends cache_distributor_1.default { // Same virtualenvs path may appear for different projects, hence we use a Set const paths = new Set(); const globber = yield glob.create(this.patterns); - const pythonLocation = yield io.which('python'); - if (pythonLocation) { - core.debug(`pythonLocation is ${pythonLocation}`); - } - else { - utils_1.logWarning('python binaries were not found in PATH'); - } try { for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { const file = _c.value; const basedir = path.dirname(file); core.debug(`Processing Poetry project at ${basedir}`); + this.poetryProjects.add(basedir); const poetryConfig = yield this.getPoetryConfiguration(basedir); const cacheDir = poetryConfig['cache-dir']; const virtualenvsPath = poetryConfig['virtualenvs.path'].replace('{cache-dir}', cacheDir); @@ -66127,12 +66126,6 @@ class PoetryCache extends cache_distributor_1.default { if (poetryConfig['virtualenvs.in-project']) { paths.add(path.join(basedir, '.venv')); } - if (pythonLocation) { - const { exitCode, stderr } = yield exec.getExecOutput('poetry', ['env', 'use', pythonLocation], { ignoreReturnCode: true, cwd: basedir }); - if (exitCode) { - utils_1.logWarning(stderr); - } - } } } catch (e_1_1) { e_1 = { error: e_1_1 }; } @@ -66156,6 +66149,30 @@ class PoetryCache extends cache_distributor_1.default { }; }); } + handleLoadedCache() { + const _super = Object.create(null, { + handleLoadedCache: { get: () => super.handleLoadedCache } + }); + return __awaiter(this, void 0, void 0, function* () { + yield _super.handleLoadedCache.call(this); + // After the cache is loaded -- make sure virtualenvs use the correct Python version (the one that we have just installed). + // This will handle invalid caches, recreating virtualenvs if necessary. + const pythonLocation = yield io.which('python'); + if (pythonLocation) { + core.debug(`pythonLocation is ${pythonLocation}`); + } + else { + utils_1.logWarning('python binaries were not found in PATH'); + return; + } + for (const poetryProject of this.poetryProjects) { + const { exitCode, stderr } = yield exec.getExecOutput('poetry', ['env', 'use', pythonLocation], { ignoreReturnCode: true, cwd: poetryProject }); + if (exitCode) { + utils_1.logWarning(stderr); + } + } + }); + } getPoetryConfiguration(basedir) { return __awaiter(this, void 0, void 0, function* () { const { stdout, stderr, exitCode } = yield exec.getExecOutput('poetry', ['config', '--list'], { cwd: basedir }); diff --git a/src/cache-distributions/cache-distributor.ts b/src/cache-distributions/cache-distributor.ts index f24c78dab..2e46c961d 100644 --- a/src/cache-distributions/cache-distributor.ts +++ b/src/cache-distributions/cache-distributor.ts @@ -19,6 +19,7 @@ abstract class CacheDistributor { primaryKey: string; restoreKey: string[] | undefined; }>; + protected async handleLoadedCache() {} public async restoreCache() { const {primaryKey, restoreKey} = await this.computeKeys(); @@ -41,6 +42,8 @@ abstract class CacheDistributor { restoreKey ); + await this.handleLoadedCache(); + this.handleMatchResult(matchedKey, primaryKey); } diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index 24a96b13b..ebbffbacf 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -10,7 +10,8 @@ import {logWarning} from '../utils'; class PoetryCache extends CacheDistributor { constructor( private pythonVersion: string, - protected patterns: string = '**/poetry.lock' + protected patterns: string = '**/poetry.lock', + protected poetryProjects: Set = new Set() ) { super('poetry', patterns); } @@ -20,16 +21,10 @@ class PoetryCache extends CacheDistributor { const paths = new Set(); const globber = await glob.create(this.patterns); - const pythonLocation = await io.which('python'); - if (pythonLocation) { - core.debug(`pythonLocation is ${pythonLocation}`); - } else { - logWarning('python binaries were not found in PATH'); - } - for await (const file of globber.globGenerator()) { const basedir = path.dirname(file); core.debug(`Processing Poetry project at ${basedir}`); + this.poetryProjects.add(basedir); const poetryConfig = await this.getPoetryConfiguration(basedir); @@ -44,18 +39,6 @@ class PoetryCache extends CacheDistributor { if (poetryConfig['virtualenvs.in-project']) { paths.add(path.join(basedir, '.venv')); } - - if (pythonLocation) { - const {exitCode, stderr} = await exec.getExecOutput( - 'poetry', - ['env', 'use', pythonLocation], - {ignoreReturnCode: true, cwd: basedir} - ); - - if (exitCode) { - logWarning(stderr); - } - } } return [...paths]; @@ -71,6 +54,33 @@ class PoetryCache extends CacheDistributor { }; } + protected async handleLoadedCache() { + await super.handleLoadedCache(); + + // After the cache is loaded -- make sure virtualenvs use the correct Python version (the one that we have just installed). + // This will handle invalid caches, recreating virtualenvs if necessary. + + const pythonLocation = await io.which('python'); + if (pythonLocation) { + core.debug(`pythonLocation is ${pythonLocation}`); + } else { + logWarning('python binaries were not found in PATH'); + return; + } + + for (const poetryProject of this.poetryProjects) { + const {exitCode, stderr} = await exec.getExecOutput( + 'poetry', + ['env', 'use', pythonLocation], + {ignoreReturnCode: true, cwd: poetryProject} + ); + + if (exitCode) { + logWarning(stderr); + } + } + } + private async getPoetryConfiguration(basedir: string) { const {stdout, stderr, exitCode} = await exec.getExecOutput( 'poetry', From b69ad351aa3daed07de3f4d0e62a057b6f8c4132 Mon Sep 17 00:00:00 2001 From: Oran Avraham Date: Sat, 24 Dec 2022 16:29:40 +0200 Subject: [PATCH 10/10] poetry: Bump cache primary key --- __tests__/cache-restore.test.ts | 4 ++++ dist/setup/index.js | 3 ++- src/cache-distributions/poetry-cache.ts | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/__tests__/cache-restore.test.ts b/__tests__/cache-restore.test.ts index 44b400eef..d6ed8320b 100644 --- a/__tests__/cache-restore.test.ts +++ b/__tests__/cache-restore.test.ts @@ -183,6 +183,10 @@ virtualenvs.path = "{cache-dir}/virtualenvs" # /Users/patrick/Library/Caches/py expect(infoSpy).toHaveBeenCalledWith( `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-20.04-Ubuntu-python-${pythonVersion}-${packageManager}-${fileHash}` ); + } else if (packageManager === 'poetry') { + expect(infoSpy).toHaveBeenCalledWith( + `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-python-${pythonVersion}-${packageManager}-v2-${fileHash}` + ); } else { expect(infoSpy).toHaveBeenCalledWith( `Cache restored from key: setup-python-${process.env['RUNNER_OS']}-python-${pythonVersion}-${packageManager}-${fileHash}` diff --git a/dist/setup/index.js b/dist/setup/index.js index 41ed58a1f..266dce5f4 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -66141,7 +66141,8 @@ class PoetryCache extends cache_distributor_1.default { computeKeys() { return __awaiter(this, void 0, void 0, function* () { const hash = yield glob.hashFiles(this.patterns); - const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-${hash}`; + // "v2" is here to invalidate old caches of this cache distributor, which were created broken: + const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-v2-${hash}`; const restoreKey = undefined; return { primaryKey, diff --git a/src/cache-distributions/poetry-cache.ts b/src/cache-distributions/poetry-cache.ts index ebbffbacf..c31fb05d4 100644 --- a/src/cache-distributions/poetry-cache.ts +++ b/src/cache-distributions/poetry-cache.ts @@ -46,7 +46,8 @@ class PoetryCache extends CacheDistributor { protected async computeKeys() { const hash = await glob.hashFiles(this.patterns); - const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-${hash}`; + // "v2" is here to invalidate old caches of this cache distributor, which were created broken: + const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-v2-${hash}`; const restoreKey = undefined; return { primaryKey,