feat(cli): validate yarn.lock in 'start' and 'build' commands and add 'deduplicate' command

cli/package.json

@@ -31,6 +31,7 @@
         "@dhis2/app-shell": "8.1.1",
         "@dhis2/cli-helpers-engine": "^3.0.0",
         "@jest/core": "^27.0.6",
+        "@yarnpkg/lockfile": "^1.1.0",
         "archiver": "^3.1.1",
         "axios": "^0.20.0",
         "babel-jest": "^27.0.6",
@@ -66,5 +67,8 @@
             "src/commands/test.js"
         ],
         "testEnvironment": "node"
+    },
+    "devDependencies": {
+        "outdent": "^0.8.0"
     }
 }

cli/src/commands/deduplicate.js

@@ -0,0 +1,36 @@
+const fs = require('fs')
+const { reporter, chalk } = require('@dhis2/cli-helpers-engine')
+const makePaths = require('../lib/paths')
+const { listDuplicates, fixDuplicates } = require('../lib/yarnDeduplicate')
+
+const handler = async ({ cwd }) => {
+    const paths = makePaths(cwd)
+
+    if (paths.yarnLock === null) {
+        reporter.error('Could not find yarn.lock')
+        process.exit(1)
+    }
+
+    const yarnLock = fs.readFileSync(paths.yarnLock, 'utf8')
+    const deduped = fixDuplicates(yarnLock)
+    fs.writeFileSync(paths.yarnLock, deduped)
+
+    const duplicates = listDuplicates(deduped)
+    if (duplicates.size > 0) {
+        reporter.error('Failed to deduplicate the following packages:')
+        for (const [name, versions] of duplicates) {
+            reporter.error(
+                ` * ${chalk.bold(name)} (found versions ${versions.join(', ')})`
+            )
+        }
+        process.exit(1)
+    }
+}
+
+const command = {
+    command: 'deduplicate',
+    desc: 'Deduplicate dependencies found in yarn.lock',
+    handler,
+}
+
+module.exports = command

cli/src/lib/paths.js

@@ -1,3 +1,4 @@
+const fs = require('fs')
 const path = require('path')
 const { reporter } = require('@dhis2/cli-helpers-engine')
 
@@ -6,6 +7,18 @@ const shellSource = path.dirname(
 )
 const shellAppDirname = 'src/D2App'
 
+const findYarnLock = base => {
+    if (base === '/') {
+        return null
+    }
+
+    const yarnLock = path.join(base, './yarn.lock')
+    if (fs.existsSync(yarnLock)) {
+        return yarnLock
+    }
+    return findYarnLock(path.dirname(base))
+}
+
 module.exports = (cwd = process.cwd()) => {
     const base = path.resolve(cwd)
     const paths = {
@@ -31,6 +44,7 @@ module.exports = (cwd = process.cwd()) => {
 
         base,
         package: path.join(base, './package.json'),
+        yarnLock: findYarnLock(base),
         dotenv: path.join(base, './.env'),
         config: path.join(base, './d2.config.js'),
         readme: path.join(base, './README.md'),

cli/src/lib/validatePackage.js

@@ -1,5 +1,6 @@
 const { reporter, chalk } = require('@dhis2/cli-helpers-engine')
 const fs = require('fs-extra')
+const { validateLockfile } = require('./validators/validateLockfile')
 const {
     validatePackageExports,
 } = require('./validators/validatePackageExports')
@@ -25,5 +26,11 @@ module.exports.validatePackage = async ({
 
     reporter.debug('Validating package...', { pkg, offerFix, noVerify })
 
-    return await validatePackageExports(pkg, { config, paths, offerFix })
+    const validators = [validatePackageExports, validateLockfile]
+    for (const validator of validators) {
+        if (!(await validator(pkg, { config, paths, offerFix }))) {
+            return false
+        }
+    }
+    return true
 }

cli/src/lib/validators/validateLockfile.js

@@ -0,0 +1,59 @@
+const fs = require('fs')
+const { reporter, prompt } = require('@dhis2/cli-helpers-engine')
+const { listDuplicates, fixDuplicates } = require('../yarnDeduplicate')
+
+const singletonDependencies = [
+    '@dhis2/app-runtime',
+    '@dhis2/d2-i18n',
+    '@dhis2/ui',
+    'react',
+    'styled-jsx',
+]
+
+const listSingletonDuplicates = yarnLock =>
+    listDuplicates(yarnLock, {
+        includePackages: singletonDependencies,
+        singleton: true,
+    })
+
+exports.validateLockfile = async (pkg, { paths, offerFix = false }) => {
+    if (paths.yarnLock === null) {
+        reporter.warn('Could not find yarn.lock')
+        return false
+    }
+
+    const yarnLock = fs.readFileSync(paths.yarnLock, 'utf8')
+
+    // Yarn v2 and above deduplicate dependencies automatically
+    if (!yarnLock.includes('# yarn lockfile v1')) {
+        return true
+    }
+
+    const singletonDuplicates = listSingletonDuplicates(yarnLock)
+    for (const [name, versions] of singletonDuplicates) {
+        reporter.warn(
+            `Found ${
+                versions.length
+            } versions of '${name}' in yarn.lock: ${versions.join(', ')}`
+        )
+    }
+
+    const valid = singletonDuplicates.size === 0
+    if (!valid && offerFix) {
+        const { fix } = await prompt({
+            name: 'fix',
+            type: 'confirm',
+            message:
+                'There are duplicate dependencies in yarn.lock, would you like to correct them now?',
+        })
+
+        if (!fix) {
+            return false
+        }
+        const dedupedYarnLock = fixDuplicates(yarnLock)
+        fs.writeFileSync(paths.yarnLock, dedupedYarnLock)
+        return listSingletonDuplicates(dedupedYarnLock).size === 0
+    }
+
+    return valid
+}

cli/src/lib/validators/validateLockfile.test.js

@@ -0,0 +1,203 @@
+const fs = require('fs')
+const { reporter, prompt } = require('@dhis2/cli-helpers-engine')
+const { validateLockfile } = require('./validateLockfile')
+
+jest.mock('fs')
+jest.mock('@dhis2/cli-helpers-engine')
+
+describe('validateLockfile', () => {
+    afterEach(() => {
+        jest.clearAllMocks()
+    })
+
+    const mockYarnLock = yarnLock => {
+        fs.readFileSync.mockReturnValue(yarnLock)
+    }
+
+    it('returns true if there are no duplicates', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"some-dependency":
+  version "1.0.0"
+
+"other-dependency":
+  version "1.0.0"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(true)
+        expect(reporter.warn).toHaveBeenCalledTimes(0)
+    })
+
+    it('returns true if duplicates are solely for non-sensitive dependencies', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"some-dependency":
+  version "1.0.0"
+
+"some-dependency":
+  version "1.2.0"
+
+"some-dependency":
+  version "2.0.0"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(true)
+        expect(reporter.warn).toHaveBeenCalledTimes(0)
+    })
+
+    it('detects minor version duplicates of sensitive dependencies', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@dhis2/ui@6.1.2":
+  version "6.1.2"
+
+"@dhis2/ui@6.1.3":
+  version "6.1.3"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(false)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(reporter.warn).toHaveBeenLastCalledWith(
+            `Found 2 versions of '@dhis2/ui' in yarn.lock: 6.1.2, 6.1.3`
+        )
+    })
+
+    it('detects mjaor version duplicates of sensitive dependencies', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@dhis2/ui@7.0.0":
+  version "7.0.0"
+
+"@dhis2/ui@6.1.3":
+  version "6.1.3"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(false)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(reporter.warn).toHaveBeenLastCalledWith(
+            `Found 2 versions of '@dhis2/ui' in yarn.lock: 7.0.0, 6.1.3`
+        )
+    })
+
+    it('only outputs warnings for sensitive dependencies', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"foo@^1.4.5":
+  version "1.4.5"
+
+"foo@1.2.3":
+  version "1.2.3"
+  dependencies:
+    "@dhis2/ui" "^6.25.0"
+
+"@dhis2/ui@^6.23.0":
+  version "6.23.0"
+
+"@dhis2/ui@^6.25.0":
+  version "6.25.0"
+
+"@dhis2/ui@^7.2.0":
+  version "7.2.1"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(false)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(reporter.warn).toHaveBeenLastCalledWith(
+            `Found 3 versions of '@dhis2/ui' in yarn.lock: 6.23.0, 6.25.0, 7.2.1`
+        )
+    })
+
+    it('can fix lockfiles with duplicate minor versions', async () => {
+        mockYarnLock(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@dhis2/ui@^6.0.0":
+  version "6.0.0"
+
+"@dhis2/ui@^6.2.3":
+  version "6.2.3"
+
+"@dhis2/ui@^6.1.2":
+  version "6.1.2"
+`)
+
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: false,
+            })
+        ).toBe(false)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(prompt).toHaveBeenCalledTimes(0)
+
+        reporter.warn.mockClear()
+        prompt.mockResolvedValue({ fix: false })
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: true,
+            })
+        ).toBe(false)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(prompt).toHaveBeenCalledTimes(1)
+
+        reporter.warn.mockClear()
+        prompt.mockClear()
+        prompt.mockResolvedValue({ fix: true })
+        expect(
+            await validateLockfile(null, {
+                paths: { yarnLock: 'yarn.lock' },
+                offerFix: true,
+            })
+        ).toBe(true)
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
+        expect(prompt).toHaveBeenCalledTimes(1)
+        expect(fs.writeFileSync).toHaveBeenCalledTimes(1)
+        expect(fs.writeFileSync).toHaveBeenLastCalledWith(
+            'yarn.lock',
+            `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@dhis2/ui@^6.0.0", "@dhis2/ui@^6.1.2", "@dhis2/ui@^6.2.3":
+  version "6.2.3"
+`
+        )
+    })
+})