diff --git a/packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/index.ts b/packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/index.ts index 5bb037503a1..c7caf593557 100644 --- a/packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/index.ts +++ b/packages/snyk-fix/src/plugins/python/handlers/pipenv-pipfile/update-dependencies/index.ts @@ -6,19 +6,23 @@ import { FixChangesSummary, FixOptions, } from '../../../../../types'; -import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied'; import { ensureHasUpdates } from '../../ensure-has-updates'; -import { pipenvAdd } from './pipenv-add'; +import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied'; import { generateUpgrades } from './generate-upgrades'; +import { pipenvAdd } from './pipenv-add'; const debug = debugLib('snyk-fix:python:Pipfile'); +function chooseFixStrategy(options: FixOptions) { + return options.sequentialFix ? fixSequentially : fixAll; +} + export async function updateDependencies( entity: EntityToFix, options: FixOptions, ): Promise { - const handlerResult = await fixAll(entity, options); + const handlerResult = await chooseFixStrategy(options)(entity, options); return handlerResult; } @@ -66,3 +70,51 @@ async function fixAll( } return handlerResult; } + +async function fixSequentially( + entity: EntityToFix, + options: FixOptions, +): Promise { + const handlerResult: PluginFixResponse = { + succeeded: [], + failed: [], + skipped: [], + }; + const { upgrades } = await generateUpgrades(entity); + // TODO: for better support we need to: + // 1. parse the manifest and extract original requirements, version spec etc + // 2. swap out only the version and retain original spec + // 3. re-lock the lockfile + const changes: FixChangesSummary[] = []; + + try { + if (!upgrades.length) { + throw new NoFixesCouldBeAppliedError( + 'Failed to calculate package updates to apply', + ); + } + // update prod dependencies first + if (upgrades.length) { + for (const upgrade of upgrades) { + changes.push(...(await pipenvAdd(entity, options, [upgrade]))); + } + } + + ensureHasUpdates(changes); + + handlerResult.succeeded.push({ + original: entity, + changes, + }); + } catch (error) { + debug( + `Failed to fix ${entity.scanResult.identity.targetFile}.\nERROR: ${error}`, + ); + handlerResult.failed.push({ + original: entity, + tip: error.tip, + error, + }); + } + return handlerResult; +} diff --git a/packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/index.ts b/packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/index.ts index 717f10454a6..a146a685d30 100644 --- a/packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/index.ts +++ b/packages/snyk-fix/src/plugins/python/handlers/poetry/update-dependencies/index.ts @@ -6,10 +6,10 @@ import { FixChangesSummary, FixOptions, } from '../../../../../types'; -import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied'; import { generateUpgrades } from './generate-upgrades'; import { poetryAdd } from './poetry-add'; import { ensureHasUpdates } from '../../ensure-has-updates'; +import { NoFixesCouldBeAppliedError } from '../../../../../lib/errors/no-fixes-applied'; const debug = debugLib('snyk-fix:python:Poetry'); diff --git a/packages/snyk-fix/test/acceptance/plugins/python/handlers/pipenv-pipfile/update-dependencies.spec.ts b/packages/snyk-fix/test/acceptance/plugins/python/handlers/pipenv-pipfile/update-dependencies.spec.ts index 4355c65ea1c..f3efc29f7fb 100644 --- a/packages/snyk-fix/test/acceptance/plugins/python/handlers/pipenv-pipfile/update-dependencies.spec.ts +++ b/packages/snyk-fix/test/acceptance/plugins/python/handlers/pipenv-pipfile/update-dependencies.spec.ts @@ -176,13 +176,13 @@ describe('fix Pipfile Python projects', () => { ); expect(result.fixSummary).toContain('✖ No successful fixes'); expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); - expect(pipenvPipfileFixStub.mock.calls[0]).toEqual([ + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( pathLib.resolve(workspacesPath, 'with-dev-deps'), ['django==2.0.1', 'transitive==1.1.1'], { python: 'python3', }, - ]); + ); }); it('applies expected changes to Pipfile when install fails', async () => { @@ -259,13 +259,455 @@ describe('fix Pipfile Python projects', () => { ); expect(result.fixSummary).toContain('✖ No successful fixes'); expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); - expect(pipenvPipfileFixStub.mock.calls[0]).toEqual([ + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( pathLib.resolve(workspacesPath, 'with-dev-deps'), ['django==2.0.1', 'transitive==1.1.1'], { python: 'python3', }, - ]); + ); + }); + + it('applies expected changes to Pipfile (100% success)', async () => { + jest.spyOn(pipenvPipfileFix, 'pipenvInstall').mockResolvedValue({ + exitCode: 0, + stdout: '', + stderr: '', + command: 'pipenv install django==2.0.1', + duration: 123, + }); + // Arrange + const targetFile = 'with-django-upgrade/Pipfile'; + const testResult = { + ...generateTestResult(), + remediation: { + unresolved: [], + upgrade: {}, + patch: {}, + ignore: {}, + pin: { + 'django@1.6.1': { + upgradeTo: 'django@2.0.1', + vulns: ['vuln-id'], + isTransitive: false, + }, + }, + }, + }; + + const entityToFix = generateEntityToFixWithFileReadWrite( + workspacesPath, + targetFile, + testResult, + ); + + // Act + const result = await snykFix.fix([entityToFix], { + quiet: true, + stripAnsi: true, + }); + // Assert + expect(result).toMatchObject({ + exceptions: {}, + results: { + python: { + failed: [], + skipped: [], + succeeded: [ + { + original: entityToFix, + changes: [ + { + success: true, + userMessage: 'Upgraded django from 1.6.1 to 2.0.1', + }, + ], + }, + ], + }, + }, + }); + expect(result.fixSummary).toContain( + '✔ Upgraded django from 1.6.1 to 2.0.1', + ); + expect(result.fixSummary).toContain('1 items were successfully fixed'); + expect(result.fixSummary).toContain('1 issues were successfully fixed'); + expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-django-upgrade'), + ['django==2.0.1'], + { + python: 'python3', + }, + ); + }); + + it('passes down custom --python if the project was tested with this (--command) from CLI', async () => { + // Arrange + const targetFile = 'with-django-upgrade/Pipfile'; + const testResult = { + ...generateTestResult(), + remediation: { + unresolved: [], + upgrade: {}, + patch: {}, + ignore: {}, + pin: { + 'django@1.6.1': { + upgradeTo: 'django@2.0.1', + vulns: ['vuln-id'], + isTransitive: false, + }, + }, + }, + }; + + const entityToFix = generateEntityToFixWithFileReadWrite( + workspacesPath, + targetFile, + testResult, + ); + + entityToFix.options.command = 'python2'; + // Act + const result = await snykFix.fix([entityToFix], { + quiet: true, + stripAnsi: true, + }); + + // Assert + expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-django-upgrade'), + ['django==2.0.1'], + { + python: 'python2', + }, + ); + + expect(result).toMatchObject({ + exceptions: {}, + results: { + python: { + failed: [], + skipped: [], + succeeded: [ + { + original: entityToFix, + changes: [ + { + success: true, + userMessage: 'Upgraded django from 1.6.1 to 2.0.1', + }, + ], + }, + ], + }, + }, + }); + expect(result.fixSummary).toContain( + '✔ Upgraded django from 1.6.1 to 2.0.1', + ); + expect(result.fixSummary).toContain('1 items were successfully fixed'); + expect(result.fixSummary).toContain('1 issues were successfully fixed'); + }); +}); + +describe('fix Pipfile Python projects (fix sequentially)', () => { + let pipenvPipfileFixStub: jest.SpyInstance; + beforeAll(() => { + jest.spyOn(pipenvPipfileFix, 'isPipenvSupportedVersion').mockReturnValue({ + supported: true, + versions: ['123.123.123'], + }); + jest.spyOn(pipenvPipfileFix, 'isPipenvInstalled').mockResolvedValue({ + version: '123.123.123', + }); + }); + + beforeEach(() => { + pipenvPipfileFixStub = jest.spyOn(pipenvPipfileFix, 'pipenvInstall'); + }); + + afterEach(() => { + pipenvPipfileFixStub.mockClear(); + }); + + const workspacesPath = pathLib.resolve(__dirname, 'workspaces'); + + it('shows expected changes with lockfile in --dry-run mode', async () => { + jest.spyOn(pipenvPipfileFix, 'pipenvInstall').mockResolvedValue({ + exitCode: 0, + stdout: '', + stderr: '', + command: 'pipenv install', + duration: 123, + }); + // Arrange + const targetFile = 'with-dev-deps/Pipfile'; + + const testResult = { + ...generateTestResult(), + remediation: { + unresolved: [], + upgrade: {}, + patch: {}, + ignore: {}, + pin: { + 'django@1.6.1': { + upgradeTo: 'django@2.0.1', + vulns: [], + isTransitive: false, + }, + 'transitive@1.0.0': { + upgradeTo: 'transitive@1.1.1', + vulns: [], + isTransitive: true, + }, + }, + }, + }; + + const entityToFix = generateEntityToFixWithFileReadWrite( + workspacesPath, + targetFile, + testResult, + ); + + // Act + const result = await snykFix.fix([entityToFix], { + quiet: true, + stripAnsi: true, + dryRun: true, + sequentialFix: true, + }); + // Assert + expect(result).toMatchObject({ + exceptions: {}, + results: { + python: { + failed: [], + skipped: [], + succeeded: [ + { + original: entityToFix, + changes: [ + { + success: true, + userMessage: 'Upgraded django from 1.6.1 to 2.0.1', + }, + { + success: true, + userMessage: 'Pinned transitive from 1.0.0 to 1.1.1', + }, + ], + }, + ], + }, + }, + }); + expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(0); + }); + + // FYI: on later pipenv versions the Pipfile changes are also not present of locking failed + it('applies expected changes to Pipfile when locking fails', async () => { + jest.spyOn(pipenvPipfileFix, 'pipenvInstall').mockResolvedValueOnce({ + exitCode: 1, + stdout: '', + stderr: 'Locking failed', + command: 'pipenv install django==2.0.1', + duration: 123, + }); + + jest.spyOn(pipenvPipfileFix, 'pipenvInstall').mockResolvedValueOnce({ + exitCode: 0, + stdout: '', + stderr: '', + command: 'poetry install transitive==1.1.1', + duration: 123, + }); + // Arrange + const targetFile = 'with-dev-deps/Pipfile'; + const testResult = { + ...generateTestResult(), + remediation: { + unresolved: [], + upgrade: {}, + patch: {}, + ignore: {}, + pin: { + 'django@1.6.1': { + upgradeTo: 'django@2.0.1', + vulns: ['vuln-id'], + isTransitive: false, + }, + 'transitive@1.0.0': { + upgradeTo: 'transitive@1.1.1', + vulns: [], + isTransitive: true, + }, + }, + }, + }; + + const entityToFix = generateEntityToFixWithFileReadWrite( + workspacesPath, + targetFile, + testResult, + ); + + // Act + const result = await snykFix.fix([entityToFix], { + quiet: true, + stripAnsi: true, + sequentialFix: true, + }); + // Assert + expect(result).toMatchObject({ + exceptions: {}, + results: { + python: { + failed: [], + skipped: [], + succeeded: [ + { + original: entityToFix, + changes: [ + { + from: 'django@1.6.1', + issueIds: ['vuln-id'], + reason: 'Locking failed', + success: false, + tip: 'Try running `pipenv install django==2.0.1`', + to: 'django@2.0.1', + userMessage: 'Failed to upgrade django from 1.6.1 to 2.0.1', + }, + { + from: 'transitive@1.0.0', + issueIds: [], + success: true, + to: 'transitive@1.1.1', + userMessage: 'Pinned transitive from 1.0.0 to 1.1.1', + }, + ], + }, + ], + }, + }, + }); + expect(result.fixSummary).toContain('Locking failed'); + expect(result.fixSummary).toContain( + 'Tip: Try running `pipenv install django==2.0.1`', + ); + expect(result.fixSummary).toContain( + '✔ Pinned transitive from 1.0.0 to 1.1.1', + ); + expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(2); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-dev-deps'), + ['django==2.0.1'], + { + python: 'python3', + }, + ); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-dev-deps'), + ['transitive==1.1.1'], + { + python: 'python3', + }, + ); + }); + + it('applies expected changes to Pipfile when install fails', async () => { + jest.spyOn(pipenvPipfileFix, 'pipenvInstall').mockResolvedValue({ + exitCode: 1, + stdout: '', + stderr: `Updating dependenciesResolving dependencies... (1.1s) + + SolverProblemError + + Because django (2.6) depends on numpy (>=1.19)and tensorflow (2.2.1) depends on numpy (>=1.16.0,<1.19.0), django (2.6) is incompatible with tensorflow (2.2.1).So, because pillow depends on both tensorflow (2.2.1) and django (2.6), version solving failed.`, + command: 'pipenv install django==2.0.1 transitive==1.1.1', + duration: 123, + }); + + // Arrange + const targetFile = 'with-dev-deps/Pipfile'; + const testResult = { + ...generateTestResult(), + remediation: { + unresolved: [], + upgrade: {}, + patch: {}, + ignore: {}, + pin: { + 'django@1.6.1': { + upgradeTo: 'django@2.0.1', + vulns: ['vuln-id'], + isTransitive: false, + }, + 'transitive@1.0.0': { + upgradeTo: 'transitive@1.1.1', + vulns: [], + isTransitive: true, + }, + }, + }, + }; + + const entityToFix = generateEntityToFixWithFileReadWrite( + workspacesPath, + targetFile, + testResult, + ); + + // Act + const result = await snykFix.fix([entityToFix], { + quiet: true, + stripAnsi: true, + sequentialFix: true, + }); + // Assert + expect(result).toMatchObject({ + exceptions: {}, + results: { + python: { + failed: [ + { + original: entityToFix, + error: expect.objectContaining({ + name: 'NoFixesCouldBeAppliedError', + }), + tip: + 'Try running `pipenv install django==2.0.1 transitive==1.1.1`', + }, + ], + skipped: [], + succeeded: [], + }, + }, + }); + expect(result.fixSummary).toContain('version solving failed'); + expect(result.fixSummary).toContain( + 'Tip: Try running `pipenv install django==2.0.1 transitive==1.1.1`', + ); + expect(result.fixSummary).toContain('✖ No successful fixes'); + expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(2); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-dev-deps'), + ['django==2.0.1'], + { + python: 'python3', + }, + ); + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( + pathLib.resolve(workspacesPath, 'with-dev-deps'), + ['transitive==1.1.1'], + { + python: 'python3', + }, + ); }); it('applies expected changes to Pipfile (100% success)', async () => { @@ -305,6 +747,7 @@ describe('fix Pipfile Python projects', () => { const result = await snykFix.fix([entityToFix], { quiet: true, stripAnsi: true, + sequentialFix: true, }); // Assert expect(result).toMatchObject({ @@ -333,13 +776,13 @@ describe('fix Pipfile Python projects', () => { expect(result.fixSummary).toContain('1 items were successfully fixed'); expect(result.fixSummary).toContain('1 issues were successfully fixed'); expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); - expect(pipenvPipfileFixStub.mock.calls[0]).toEqual([ + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( pathLib.resolve(workspacesPath, 'with-django-upgrade'), ['django==2.0.1'], { python: 'python3', }, - ]); + ); }); it('passes down custom --python if the project was tested with this (--command) from CLI', async () => { @@ -373,17 +816,18 @@ describe('fix Pipfile Python projects', () => { const result = await snykFix.fix([entityToFix], { quiet: true, stripAnsi: true, + sequentialFix: true, }); // Assert expect(pipenvPipfileFixStub).toHaveBeenCalledTimes(1); - expect(pipenvPipfileFixStub.mock.calls[0]).toEqual([ + expect(pipenvPipfileFixStub).toHaveBeenCalledWith( pathLib.resolve(workspacesPath, 'with-django-upgrade'), ['django==2.0.1'], { python: 'python2', }, - ]); + ); expect(result).toMatchObject({ exceptions: {}, diff --git a/packages/snyk-fix/test/unit/plugins/python/handlers/pipenv-pipfile/update-dependencies/update-dependencies.ts b/packages/snyk-fix/test/unit/plugins/python/handlers/pipenv-pipfile/update-dependencies/update-dependencies.ts index 29293b4b4ce..d38925db282 100644 --- a/packages/snyk-fix/test/unit/plugins/python/handlers/pipenv-pipfile/update-dependencies/update-dependencies.ts +++ b/packages/snyk-fix/test/unit/plugins/python/handlers/pipenv-pipfile/update-dependencies/update-dependencies.ts @@ -1,10 +1,10 @@ -import { - generateSuccessfulChanges, - generateUpgrades, -} from '../../../../../../../src/plugins/python/handlers/pipenv-pipfile/update-dependencies'; +import { generateSuccessfulChanges } from '../../../../../../../src/plugins/python/handlers/attempted-changes-summary'; +import { generateUpgrades } from '../../../../../../../src/plugins/python/handlers/pipenv-pipfile/update-dependencies/generate-upgrades'; +import { generateEntityToFix } from '../../../../../../helpers/generate-entity-to-fix'; describe('generateUpgrades', () => { it('generates upgrades as expected', async () => { + const entityToFix = generateEntityToFix('pip', 'Pipfile', ''); // Arrange const pinRemediation = { 'django@1.6.1': { @@ -18,18 +18,50 @@ describe('generateUpgrades', () => { isTransitive: true, }, }; + (entityToFix.testResult as any).remediation = { + ignore: {}, + patch: {}, + pin: pinRemediation, + unresolved: [], + // only pins are supported for Python + upgrade: { + 'json-api@0.1.21': { + upgradeTo: 'json-api@0.1.22', + upgrades: ['json-api@0.1.22'], + vulns: ['pip:json-api:20170213'], + isTransitive: false, + }, + }, + }; // Act - const res = await generateUpgrades(pinRemediation); + const { upgrades } = await generateUpgrades(entityToFix); // Assert - expect(res).toEqual(['django>=2.0.1', 'transitive>=1.1.1']); + expect(upgrades).toEqual(['django>=2.0.1', 'transitive>=1.1.1']); }); it('returns [] when no pins available', async () => { // Arrange + const entityToFix = generateEntityToFix('pip', 'Pipfile', ''); + // Arrange + (entityToFix.testResult as any).remediation = { + ignore: {}, + patch: {}, + pin: {}, + unresolved: [], + // only pins are supported for Python + upgrade: { + 'json-api@0.1.21': { + upgradeTo: 'json-api@0.1.22', + upgrades: ['json-api@0.1.22'], + vulns: ['pip:json-api:20170213'], + isTransitive: false, + }, + }, + }; // Act - const res = await generateUpgrades({}); + const { upgrades } = await generateUpgrades(entityToFix); // Assert - expect(res).toEqual([]); + expect(upgrades).toEqual([]); }); }); @@ -50,7 +82,10 @@ describe('generateSuccessfulChanges', () => { }; // Act - const res = await generateSuccessfulChanges(pinRemediation); + const res = await generateSuccessfulChanges( + ['django===2.0.1', 'transitive==1.1.1'], + pinRemediation, + ); // Assert expect(res).toEqual([ { @@ -72,7 +107,7 @@ describe('generateSuccessfulChanges', () => { it('returns [] when no pins available', async () => { // Arrange // Act - const res = await generateSuccessfulChanges({}); + const res = await generateSuccessfulChanges([], {}); // Assert expect(res).toEqual([]); }); diff --git a/packages/snyk-fix/test/unit/plugins/python/handlers/poetry/generate-upgrades.spec.ts b/packages/snyk-fix/test/unit/plugins/python/handlers/poetry/generate-upgrades.spec.ts index 7b6062381d6..0a5ae2cc7ec 100644 --- a/packages/snyk-fix/test/unit/plugins/python/handlers/poetry/generate-upgrades.spec.ts +++ b/packages/snyk-fix/test/unit/plugins/python/handlers/poetry/generate-upgrades.spec.ts @@ -25,8 +25,7 @@ describe('generateUpgrades', () => { manifestContents, ); - // @ts-ignore: for test purpose need specific remediation - entityToFix.testResult.remediation = { + (entityToFix.testResult as any).remediation = { ignore: {}, patch: {}, pin: {}, @@ -70,12 +69,10 @@ describe('generateUpgrades', () => { 'pyproject.toml', manifestContents, ); - // @ts-ignore: for test purpose need specific remediation - entityToFix.options = { + (entityToFix as any).options = { dev: true, }; - // @ts-ignore: for test purpose need specific remediation - entityToFix.testResult.remediation = { + (entityToFix.testResult as any).remediation = { ignore: {}, patch: {}, pin: { @@ -120,8 +117,7 @@ describe('generateUpgrades', () => { 'pyproject.toml', manifestContents, ); - // @ts-ignore: for test purpose need specific remediation - entityToFix.testResult.remediation = { + (entityToFix.testResult as any).remediation = { ignore: {}, patch: {}, pin: { @@ -165,8 +161,7 @@ describe('generateUpgrades', () => { 'pyproject.toml', manifestContents, ); - // @ts-ignore: for test purpose need specific remediation - entityToFix.testResult.remediation = { + (entityToFix.testResult as any).remediation = { ignore: {}, patch: {}, pin: { @@ -212,12 +207,10 @@ describe('generateUpgrades', () => { 'pyproject.toml', manifestContents, ); - // @ts-ignore: for test purpose need specific remediation - entityToFix.options = { + (entityToFix as any).options = { dev: true, }; - // @ts-ignore: for test purpose need specific remediation - entityToFix.testResult.remediation = { + (entityToFix.testResult as any).remediation = { ignore: {}, patch: {}, pin: {