From bbd3c607e89971273ec8d665eff5d4d383464172 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 6 Jun 2023 10:12:43 +0200 Subject: [PATCH] feat(manager/npm): support parsing lockfileVersion=3 (#22281) Co-authored-by: Rhys Arkins Co-authored-by: Michael Kriese Co-authored-by: Sergei Zharinov --- .../npm/__fixtures__/npm9/package-lock.json | 80 ++++++++++++ .../npm/__fixtures__/npm9/package.json | 15 +++ .../extract/__snapshots__/npm.spec.ts.snap | 31 ----- .../npm/extract/locked-versions.spec.ts | 116 ++++++++++++++++++ .../manager/npm/extract/locked-versions.ts | 18 ++- lib/modules/manager/npm/extract/npm.spec.ts | 57 +++++++-- lib/modules/manager/npm/extract/npm.ts | 31 ++--- lib/modules/manager/npm/extract/schema.ts | 33 +++++ 8 files changed, 327 insertions(+), 54 deletions(-) create mode 100644 lib/modules/manager/npm/__fixtures__/npm9/package-lock.json create mode 100644 lib/modules/manager/npm/__fixtures__/npm9/package.json delete mode 100644 lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap create mode 100644 lib/modules/manager/npm/extract/schema.ts diff --git a/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json b/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json new file mode 100644 index 00000000000000..d637c965ca07ed --- /dev/null +++ b/lib/modules/manager/npm/__fixtures__/npm9/package-lock.json @@ -0,0 +1,80 @@ +{ + "name": "npm9", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "npm9", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "chalk": "^2.4.1" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + } + } +} diff --git a/lib/modules/manager/npm/__fixtures__/npm9/package.json b/lib/modules/manager/npm/__fixtures__/npm9/package.json new file mode 100644 index 00000000000000..54614b8a2649fe --- /dev/null +++ b/lib/modules/manager/npm/__fixtures__/npm9/package.json @@ -0,0 +1,15 @@ +{ + "name": "npm9", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "chalk": "^2.4.1" + } +} diff --git a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap b/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap deleted file mode 100644 index 91caae831ab088..00000000000000 --- a/lib/modules/manager/npm/extract/__snapshots__/npm.spec.ts.snap +++ /dev/null @@ -1,31 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`modules/manager/npm/extract/npm .getNpmLock() extracts 1`] = ` -{ - "lockedVersions": { - "ansi-styles": "3.2.1", - "chalk": "2.4.1", - "color-convert": "1.9.1", - "color-name": "1.1.3", - "escape-string-regexp": "1.0.5", - "has-flag": "3.0.0", - "supports-color": "5.4.0", - }, - "lockfileVersion": 1, -} -`; - -exports[`modules/manager/npm/extract/npm .getNpmLock() extracts npm 7 lockfile 1`] = ` -{ - "lockedVersions": { - "ansi-styles": "3.2.1", - "chalk": "2.4.1", - "color-convert": "1.9.1", - "color-name": "1.1.3", - "escape-string-regexp": "1.0.5", - "has-flag": "3.0.0", - "supports-color": "5.4.0", - }, - "lockfileVersion": 2, -} -`; diff --git a/lib/modules/manager/npm/extract/locked-versions.spec.ts b/lib/modules/manager/npm/extract/locked-versions.spec.ts index 9891ec0ffeca5c..18748268d0d6b4 100644 --- a/lib/modules/manager/npm/extract/locked-versions.spec.ts +++ b/lib/modules/manager/npm/extract/locked-versions.spec.ts @@ -1,3 +1,4 @@ +import { logger } from '../../../../../test/util'; import type { PackageFile } from '../../types'; import type { NpmManagerData } from '../types'; import { getLockedVersions } from './locked-versions'; @@ -550,4 +551,119 @@ describe('modules/manager/npm/extract/locked-versions', () => { }, ]); }); + + it('should log warning if unsupported lockfileVersion is found', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: {}, + lockfileVersion: 99, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: {}, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual(packageFiles); + expect(logger.logger.warn).toHaveBeenCalledWith( + { + lockfileVersion: 99, + npmLock: 'package-lock.json', + }, + 'Found unsupported npm lockfile version' + ); + }); + + describe('lockfileVersion 3', () => { + it('uses package-lock.json with npm v9.0.0', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 3, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: {}, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { + npm: '>=7', + }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + packageFile: 'some-file', + lockFiles: ['package-lock.json'], + managerData: { + npmLock: 'package-lock.json', + }, + }, + ]); + }); + + it('uses package-lock.json with npm v7.0.0', async () => { + npm.getNpmLock.mockReturnValue({ + lockedVersions: { + a: '1.0.0', + b: '2.0.0', + c: '3.0.0', + }, + lockfileVersion: 3, + }); + const packageFiles = [ + { + managerData: { + npmLock: 'package-lock.json', + }, + extractedConstraints: { + npm: '^9.0.0', + }, + deps: [ + { depName: 'a', currentValue: '1.0.0' }, + { depName: 'b', currentValue: '2.0.0' }, + ], + packageFile: 'some-file', + }, + ]; + await getLockedVersions(packageFiles); + expect(packageFiles).toEqual([ + { + extractedConstraints: { + npm: '^9.0.0', + }, + deps: [ + { currentValue: '1.0.0', depName: 'a', lockedVersion: '1.0.0' }, + { currentValue: '2.0.0', depName: 'b', lockedVersion: '2.0.0' }, + ], + packageFile: 'some-file', + lockFiles: ['package-lock.json'], + managerData: { + npmLock: 'package-lock.json', + }, + }, + ]); + }); + }); }); diff --git a/lib/modules/manager/npm/extract/locked-versions.ts b/lib/modules/manager/npm/extract/locked-versions.ts index 93ce3adb0e73d1..17caf3d5080a9c 100644 --- a/lib/modules/manager/npm/extract/locked-versions.ts +++ b/lib/modules/manager/npm/extract/locked-versions.ts @@ -61,7 +61,13 @@ export async function getLockedVersions( lockFiles.push(npmLock); if (!lockFileCache[npmLock]) { logger.trace('Retrieving/parsing ' + npmLock); - lockFileCache[npmLock] = await getNpmLock(npmLock); + const cache = await getNpmLock(npmLock); + // istanbul ignore if + if (!cache) { + logger.warn({ npmLock }, 'Npm: unable to get lockfile'); + return; + } + lockFileCache[npmLock] = cache; } const { lockfileVersion } = lockFileCache[npmLock]; @@ -88,6 +94,16 @@ export async function getLockedVersions( } else { npm = '<9'; } + } else if (lockfileVersion === 3) { + if (!packageFile.extractedConstraints?.npm) { + npm = '>=7'; + } + } else { + logger.warn( + { lockfileVersion, npmLock }, + 'Found unsupported npm lockfile version' + ); + return; } if (npm) { packageFile.extractedConstraints ??= {}; diff --git a/lib/modules/manager/npm/extract/npm.spec.ts b/lib/modules/manager/npm/extract/npm.spec.ts index 46a2787a0735fd..e169aa57aefa86 100644 --- a/lib/modules/manager/npm/extract/npm.spec.ts +++ b/lib/modules/manager/npm/extract/npm.spec.ts @@ -6,7 +6,7 @@ jest.mock('../../../../util/fs'); describe('modules/manager/npm/extract/npm', () => { describe('.getNpmLock()', () => { - it('returns empty if failed to parse', async () => { + it('returns null if failed to parse', async () => { fs.readLocalFile.mockResolvedValueOnce('abcd'); const res = await getNpmLock('package.json'); expect(Object.keys(res.lockedVersions!)).toHaveLength(0); @@ -16,23 +16,66 @@ describe('modules/manager/npm/extract/npm', () => { const plocktest1Lock = Fixtures.get('plocktest1/package-lock.json', '..'); fs.readLocalFile.mockResolvedValueOnce(plocktest1Lock as never); const res = await getNpmLock('package.json'); - expect(res).toMatchSnapshot(); - expect(Object.keys(res.lockedVersions!)).toHaveLength(7); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.1', + 'color-convert': '1.9.1', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.4.0', + }, + lockfileVersion: 1, + }); }); it('extracts npm 7 lockfile', async () => { const npm7Lock = Fixtures.get('npm7/package-lock.json', '..'); fs.readLocalFile.mockResolvedValueOnce(npm7Lock as never); const res = await getNpmLock('package.json'); - expect(res).toMatchSnapshot(); - expect(Object.keys(res.lockedVersions!)).toHaveLength(7); - expect(res.lockfileVersion).toBe(2); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.1', + 'color-convert': '1.9.1', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.4.0', + }, + lockfileVersion: 2, + }); }); - it('returns empty if no deps', async () => { + it('extracts npm 9 lockfile', async () => { + const npm9Lock = Fixtures.get('npm9/package-lock.json', '..'); + fs.readLocalFile.mockResolvedValueOnce(npm9Lock); + const res = await getNpmLock('package.json'); + expect(res).toEqual({ + lockedVersions: { + 'ansi-styles': '3.2.1', + chalk: '2.4.2', + 'color-convert': '1.9.3', + 'color-name': '1.1.3', + 'escape-string-regexp': '1.0.5', + 'has-flag': '3.0.0', + 'supports-color': '5.5.0', + }, + lockfileVersion: 3, + }); + }); + + it('returns null if no deps', async () => { fs.readLocalFile.mockResolvedValueOnce('{}'); const res = await getNpmLock('package.json'); expect(Object.keys(res.lockedVersions!)).toHaveLength(0); }); + + it('returns null on read error', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + const res = await getNpmLock('package.json'); + expect(Object.keys(res.lockedVersions!)).toHaveLength(0); + }); }); }); diff --git a/lib/modules/manager/npm/extract/npm.ts b/lib/modules/manager/npm/extract/npm.ts index 7a756734b92ef3..862cabfcbe63ae 100644 --- a/lib/modules/manager/npm/extract/npm.ts +++ b/lib/modules/manager/npm/extract/npm.ts @@ -1,22 +1,23 @@ import { logger } from '../../../../logger'; import { readLocalFile } from '../../../../util/fs'; -import type { LockFile, LockFileEntry } from './types'; +import { PackageLock } from './schema'; +import type { LockFile } from './types'; export async function getNpmLock(filePath: string): Promise { - // TODO #7154 - const lockRaw = (await readLocalFile(filePath, 'utf8'))!; - try { - const lockParsed = JSON.parse(lockRaw); - const lockedVersions: Record = {}; - for (const [entry, val] of Object.entries( - (lockParsed.dependencies || {}) as LockFileEntry - )) { - logger.trace({ entry, version: val.version }); - lockedVersions[entry] = val.version; - } - return { lockedVersions, lockfileVersion: lockParsed.lockfileVersion }; - } catch (err) { - logger.debug({ filePath, err }, 'Warning: Exception parsing npm lock file'); + const lockfileContent = await readLocalFile(filePath, 'utf8'); + if (!lockfileContent) { + logger.debug({ filePath }, 'Npm: unable to read lockfile'); return { lockedVersions: {} }; } + + const parsedLockfile = PackageLock.safeParse(lockfileContent); + if (!parsedLockfile.success) { + logger.debug( + { filePath, err: parsedLockfile.error }, + 'Npm: unable to parse lockfile' + ); + return { lockedVersions: {} }; + } + + return parsedLockfile.data; } diff --git a/lib/modules/manager/npm/extract/schema.ts b/lib/modules/manager/npm/extract/schema.ts new file mode 100644 index 00000000000000..96cde78ca64e19 --- /dev/null +++ b/lib/modules/manager/npm/extract/schema.ts @@ -0,0 +1,33 @@ +import { z } from 'zod'; +import { Json, LooseRecord } from '../../../../util/schema-utils'; + +export const PackageLockV3Schema = z.object({ + lockfileVersion: z.literal(3), + packages: LooseRecord( + z + .string() + .transform((x) => x.replace(/^node_modules\//, '')) + .refine((x) => x.trim() !== ''), + z.object({ version: z.string() }) + ), +}); + +export const PackageLockPreV3Schema = z + .object({ + lockfileVersion: z.union([z.literal(2), z.literal(1)]), + dependencies: LooseRecord(z.object({ version: z.string() })), + }) + .transform(({ lockfileVersion, dependencies: packages }) => ({ + lockfileVersion, + packages, + })); + +export const PackageLock = Json.pipe( + z.union([PackageLockV3Schema, PackageLockPreV3Schema]) +).transform(({ packages, lockfileVersion }) => { + const lockedVersions: Record = {}; + for (const [entry, val] of Object.entries(packages)) { + lockedVersions[entry] = val.version; + } + return { lockedVersions, lockfileVersion }; +});