From 46315d603e7e560cc15dd3b212635596d97dd123 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 11:24:45 -0400 Subject: [PATCH 01/14] failing test for peerDep corrections --- test.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test.ts b/test.ts index 2252abf..ca7cae4 100644 --- a/test.ts +++ b/test.ts @@ -480,6 +480,33 @@ describe('Project', function () { ); }); + it('adjusts peerDependencies of linked dependencies', function () { + let baseProject = new Project('base'); + let alpha = baseProject.addDependency('alpha', { + files: { + 'index.js': ` + module.exports = function() { + return require('beta/package.json').version; + } + `, + }, + }); + alpha.pkg.peerDependencies = { beta: '^1.0.0' }; + baseProject.addDependency('beta', { version: '1.1.0' }); + baseProject.writeSync(); + + // precondition: in the baseProject, alpha sees its beta peerDep as beta@1.1.0 + expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] }))()).to.eql('1.1.0'); + + let project = new Project('my-app'); + project.linkDependency('alpha', { baseDir: baseProject.baseDir }); + project.addDependency('beta', { version: '1.2.0' }); + project.writeSync(); + + // in our linked project, alpha sees its beta peerDep as beta@1.2.0 + expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); + }); + it('adds linked dependencies to package.json', function () { let baseProject = new Project('base'); baseProject.addDependency('moment', '1.2.3'); From 4a975719ecb9f0b7e9b1a9c96c32f75484e483e8 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 15:17:43 -0400 Subject: [PATCH 02/14] testing a hardlinking strategy --- index.ts | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/index.ts b/index.ts index 75e38c5..eed2ee6 100644 --- a/index.ts +++ b/index.ts @@ -4,6 +4,7 @@ import fs = require('fs-extra'); import path = require('path'); import resolvePackagePath = require('resolve-package-path'); import { PackageJson } from 'type-fest'; +import { readdirSync, statSync } from 'fs'; tmp.setGracefulCleanup(); @@ -246,10 +247,6 @@ export class Project { this.autoBaseDir(); fixturify.writeSync(this.baseDir, this.files); fs.outputJSONSync(path.join(this.baseDir, 'package.json'), this.pkgJSONWithDeps(), { spaces: 2 }); - - for (let [name, { dir: target }] of this.dependencyLinks) { - fs.ensureSymlinkSync(target, path.join(this.baseDir, 'node_modules', name), 'dir'); - } for (let dep of this.dependencyProjects()) { dep.baseDir = path.join(this.baseDir, 'node_modules', dep.name); dep.writeSync(); @@ -258,6 +255,67 @@ export class Project { dep.baseDir = path.join(this.baseDir, 'node_modules', dep.name); dep.writeSync(); } + for (let [name, { dir: target }] of this.dependencyLinks) { + this.writeLinkedPackage(name, target); + } + } + + private writeLinkedPackage(name: string, target: string) { + let targetPkg = require(path.join(target, 'package.json')); + let { peerDependencies } = targetPkg; + let overriddenPeers = new Map(); + if (peerDependencies) { + for (let peerName of Object.keys(peerDependencies)) { + let theirTarget = resolvePackagePath(peerName, target); + let ourTarget = resolvePackagePath(peerName, this.baseDir); + if (theirTarget !== ourTarget) { + overriddenPeers.set(peerName, ourTarget); + } + } + } + + let destination = path.join(this.baseDir, 'node_modules', name); + + if (overriddenPeers.size === 0) { + // no peerDeps, so we can just symlink the whole package + fs.ensureSymlinkSync(target, destination, 'dir'); + return; + } + + // need to reproduce the package structure in our own location + this.hardLinkContents(target, destination); + + for (let section of ['dependencies', 'peerDependencies']) { + if (targetPkg[section]) { + for (let depName of Object.keys(targetPkg[section])) { + let depTarget = overriddenPeers.get(depName); + if (!depTarget) { + depTarget = resolvePackagePath(depName, target); + } + if (!depTarget) { + throw new Error(`package ${name} in ${target} depends on ${depName} but we could not resolve it`); + } + fs.ensureSymlinkSync( + depTarget.slice(0, -1 * '/package.json'.length), + path.join(destination, 'node_modules', depName) + ); + } + } + } + } + + private hardLinkContents(target: string, destination: string, exclude = 'node_modules') { + for (let name of readdirSync(target)) { + if (name === exclude) { + continue; + } + let stat = statSync(path.join(target, name)); + if (stat.isDirectory()) { + this.hardLinkContents(path.join(target, name), path.join(destination, name)); + } else { + fs.ensureLinkSync(path.join(target, name), path.join(destination, name)); + } + } } static fromDir(root: string, opts?: ReadDirOpts): Project { From 376168d23c54036b5452dd2ff22d12233184ccf6 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 15:35:50 -0400 Subject: [PATCH 03/14] support undeclaredPeerDeps --- README.md | 8 ++++++-- index.ts | 28 +++++++++++++--------------- test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 180b299..94e0d7d 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,11 @@ project.linkDependency('c', { baseDir: '/example' }); // this will follow node resolution rules to lookup "my-aliased-name" from "../elsewhere" project.linkDependency('d', { baseDir: '/example', resolveName: 'my-aliased-name' }); +// if the package you're linking to is misbehaved and depends on your other dependencies +// without declaring them as peerDependencies, you can provide a hint so we will still +// link everything up correctly. +project.linkDependency('e', { baseDir: '/example', undeclaredPeerDeps: ['some-dep'] }); + project.writeSync(); ``` @@ -109,11 +114,10 @@ When constructing a whole Project from a directory, you can choose to link all dependencies instead of copying them in as Projects: ```js -let project = Project.fromDir("./sample-project", { linkDeps: true }); +let project = Project.fromDir('./sample-project', { linkDeps: true }); project.files['extra.js'] = '// stuff'; project.write(); ``` This will generate a new copy of sample-project, with symlinks to all its original dependencies, but with "extra.js" added. - diff --git a/index.ts b/index.ts index eed2ee6..c766b48 100644 --- a/index.ts +++ b/index.ts @@ -124,7 +124,8 @@ export class Project { // will appear as within the parent's package.json private requestedRange: string; - private dependencyLinks: Map = new Map(); + private dependencyLinks: Map = + new Map(); private linkIsDevDependency: Set = new Set(); constructor( @@ -255,22 +256,19 @@ export class Project { dep.baseDir = path.join(this.baseDir, 'node_modules', dep.name); dep.writeSync(); } - for (let [name, { dir: target }] of this.dependencyLinks) { - this.writeLinkedPackage(name, target); + for (let [name, { dir: target, undeclaredPeerDeps }] of this.dependencyLinks) { + this.writeLinkedPackage(name, target, undeclaredPeerDeps); } } - private writeLinkedPackage(name: string, target: string) { + private writeLinkedPackage(name: string, target: string, undeclaredPeerDeps: string[]) { let targetPkg = require(path.join(target, 'package.json')); - let { peerDependencies } = targetPkg; let overriddenPeers = new Map(); - if (peerDependencies) { - for (let peerName of Object.keys(peerDependencies)) { - let theirTarget = resolvePackagePath(peerName, target); - let ourTarget = resolvePackagePath(peerName, this.baseDir); - if (theirTarget !== ourTarget) { - overriddenPeers.set(peerName, ourTarget); - } + for (let peerName of [...undeclaredPeerDeps, ...Object.keys(targetPkg.peerDependencies ?? {})]) { + let theirTarget = resolvePackagePath(peerName, target); + let ourTarget = resolvePackagePath(peerName, this.baseDir); + if (theirTarget !== ourTarget) { + overriddenPeers.set(peerName, ourTarget); } } @@ -484,8 +482,8 @@ export class Project { linkDependency( name: string, opts: - | { baseDir: string; resolveName?: string; requestedRange?: string } - | { target: string; requestedRange?: string } + | { baseDir: string; resolveName?: string; requestedRange?: string; undeclaredPeerDeps?: string[] } + | { target: string; requestedRange?: string; undeclaredPeerDeps?: string[] } ) { this.removeDependency(name); this.removeDevDependency(name); @@ -500,7 +498,7 @@ export class Project { dir = opts.target; } let requestedRange = opts?.requestedRange ?? fs.readJsonSync(path.join(dir, 'package.json')).version; - this.dependencyLinks.set(name, { dir, requestedRange }); + this.dependencyLinks.set(name, { dir, requestedRange, undeclaredPeerDeps: opts?.undeclaredPeerDeps || [] }); } linkDevDependency(name: string, opts: { baseDir: string; resolveName?: string } | { target: string }) { diff --git a/test.ts b/test.ts index ca7cae4..135ac39 100644 --- a/test.ts +++ b/test.ts @@ -507,6 +507,32 @@ describe('Project', function () { expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); }); + it('supports undeclaredPeerDeps', function () { + let baseProject = new Project('base'); + baseProject.addDependency('alpha', { + files: { + 'index.js': ` + module.exports = function() { + return require('beta/package.json').version; + } + `, + }, + }); + baseProject.addDependency('beta', { version: '1.1.0' }); + baseProject.writeSync(); + + // precondition: in the baseProject, alpha sees its beta peerDep as beta@1.1.0 + expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] }))()).to.eql('1.1.0'); + + let project = new Project('my-app'); + project.linkDependency('alpha', { baseDir: baseProject.baseDir, undeclaredPeerDeps: ['beta'] }); + project.addDependency('beta', { version: '1.2.0' }); + project.writeSync(); + + // in our linked project, alpha sees its beta peerDep as beta@1.2.0 + expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); + }); + it('adds linked dependencies to package.json', function () { let baseProject = new Project('base'); baseProject.addDependency('moment', '1.2.3'); From cad83c98c9396f7057957223744b3fac628b8a9b Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 15:41:59 -0400 Subject: [PATCH 04/14] add test coverage for deeper modules --- test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test.ts b/test.ts index 135ac39..fd3da03 100644 --- a/test.ts +++ b/test.ts @@ -489,6 +489,13 @@ describe('Project', function () { return require('beta/package.json').version; } `, + deeper: { + 'index.js': ` + module.exports = function() { + return 'inner' + require('beta/package.json').version; + } + `, + }, }, }); alpha.pkg.peerDependencies = { beta: '^1.0.0' }; @@ -505,6 +512,9 @@ describe('Project', function () { // in our linked project, alpha sees its beta peerDep as beta@1.2.0 expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); + + // deeper modules in our package also work correctly + expect(require(require.resolve('alpha/deeper', { paths: [project.baseDir] }))()).to.eql('inner1.2.0'); }); it('supports undeclaredPeerDeps', function () { @@ -521,7 +531,8 @@ describe('Project', function () { baseProject.addDependency('beta', { version: '1.1.0' }); baseProject.writeSync(); - // precondition: in the baseProject, alpha sees its beta peerDep as beta@1.1.0 + // precondition: in the baseProject, alpha sees its undeclared beta peerDep + // as beta@1.1.0 expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] }))()).to.eql('1.1.0'); let project = new Project('my-app'); @@ -529,7 +540,7 @@ describe('Project', function () { project.addDependency('beta', { version: '1.2.0' }); project.writeSync(); - // in our linked project, alpha sees its beta peerDep as beta@1.2.0 + // in our linked project, alpha sees its undeclared beta peerDep as beta@1.2.0 expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); }); From b5bd2d0e716ee0830a127d26486de93fe8876a63 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 15:46:00 -0400 Subject: [PATCH 05/14] expand test coverage for non-overridden deps --- test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test.ts b/test.ts index fd3da03..023a08f 100644 --- a/test.ts +++ b/test.ts @@ -485,13 +485,16 @@ describe('Project', function () { let alpha = baseProject.addDependency('alpha', { files: { 'index.js': ` - module.exports = function() { + exports.betaVersion = function() { return require('beta/package.json').version; } + exports.gammaLocation = function() { + return require.resolve('gamma/package.json'); + } `, deeper: { 'index.js': ` - module.exports = function() { + exports.betaVersion = function() { return 'inner' + require('beta/package.json').version; } `, @@ -499,11 +502,12 @@ describe('Project', function () { }, }); alpha.pkg.peerDependencies = { beta: '^1.0.0' }; + alpha.addDependency('gamma'); baseProject.addDependency('beta', { version: '1.1.0' }); baseProject.writeSync(); // precondition: in the baseProject, alpha sees its beta peerDep as beta@1.1.0 - expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] }))()).to.eql('1.1.0'); + expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] })).betaVersion()).to.eql('1.1.0'); let project = new Project('my-app'); project.linkDependency('alpha', { baseDir: baseProject.baseDir }); @@ -511,10 +515,15 @@ describe('Project', function () { project.writeSync(); // in our linked project, alpha sees its beta peerDep as beta@1.2.0 - expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); + expect(require(require.resolve('alpha', { paths: [project.baseDir] })).betaVersion()).to.eql('1.2.0'); // deeper modules in our package also work correctly - expect(require(require.resolve('alpha/deeper', { paths: [project.baseDir] }))()).to.eql('inner1.2.0'); + expect(require(require.resolve('alpha/deeper', { paths: [project.baseDir] })).betaVersion()).to.eql('inner1.2.0'); + + // unrelated dependencies are still shared + expect(require(require.resolve('alpha', { paths: [project.baseDir] })).gammaLocation()).to.eql( + require(require.resolve('alpha', { paths: [baseProject.baseDir] })).gammaLocation() + ); }); it('supports undeclaredPeerDeps', function () { From 5701f9034ba21c8305fcf42f13aad14adf5749fc Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 10 Sep 2021 17:22:15 -0400 Subject: [PATCH 06/14] drop undeclaredPeerDeps, fix requestedRange, don't link devDeps for libraries --- README.md | 9 ++++----- index.ts | 41 ++++++++++++++++--------------------- test.ts | 60 ++++++++++++++++++++++++++++++------------------------- 3 files changed, 54 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 94e0d7d..250efb2 100644 --- a/README.md +++ b/README.md @@ -91,11 +91,6 @@ project.linkDependency('c', { baseDir: '/example' }); // this will follow node resolution rules to lookup "my-aliased-name" from "../elsewhere" project.linkDependency('d', { baseDir: '/example', resolveName: 'my-aliased-name' }); -// if the package you're linking to is misbehaved and depends on your other dependencies -// without declaring them as peerDependencies, you can provide a hint so we will still -// link everything up correctly. -project.linkDependency('e', { baseDir: '/example', undeclaredPeerDeps: ['some-dep'] }); - project.writeSync(); ``` @@ -121,3 +116,7 @@ project.write(); This will generate a new copy of sample-project, with symlinks to all its original dependencies, but with "extra.js" added. + +By default, `linkDeps` will only link up `dependencies` (which is appropriate +for libraries). If you want to also include `devDependencies` (which is +appropriate for apps) you can use `linkDevDeps` instead. diff --git a/index.ts b/index.ts index c766b48..acf9cc7 100644 --- a/index.ts +++ b/index.ts @@ -91,6 +91,7 @@ function getPackageVersion(pkg: PackageJson): string { interface ReadDirOpts { linkDeps?: boolean; + linkDevDeps?: boolean; } // This only shallow-merges with any user-provided files, which is OK right now @@ -124,8 +125,7 @@ export class Project { // will appear as within the parent's package.json private requestedRange: string; - private dependencyLinks: Map = - new Map(); + private dependencyLinks: Map = new Map(); private linkIsDevDependency: Set = new Set(); constructor( @@ -256,25 +256,17 @@ export class Project { dep.baseDir = path.join(this.baseDir, 'node_modules', dep.name); dep.writeSync(); } - for (let [name, { dir: target, undeclaredPeerDeps }] of this.dependencyLinks) { - this.writeLinkedPackage(name, target, undeclaredPeerDeps); + for (let [name, { dir: target }] of this.dependencyLinks) { + this.writeLinkedPackage(name, target); } } - private writeLinkedPackage(name: string, target: string, undeclaredPeerDeps: string[]) { + private writeLinkedPackage(name: string, target: string) { let targetPkg = require(path.join(target, 'package.json')); - let overriddenPeers = new Map(); - for (let peerName of [...undeclaredPeerDeps, ...Object.keys(targetPkg.peerDependencies ?? {})]) { - let theirTarget = resolvePackagePath(peerName, target); - let ourTarget = resolvePackagePath(peerName, this.baseDir); - if (theirTarget !== ourTarget) { - overriddenPeers.set(peerName, ourTarget); - } - } - + let peers = new Set(Object.keys(targetPkg.peerDependencies ?? {})); let destination = path.join(this.baseDir, 'node_modules', name); - if (overriddenPeers.size === 0) { + if (peers.size === 0) { // no peerDeps, so we can just symlink the whole package fs.ensureSymlinkSync(target, destination, 'dir'); return; @@ -286,10 +278,10 @@ export class Project { for (let section of ['dependencies', 'peerDependencies']) { if (targetPkg[section]) { for (let depName of Object.keys(targetPkg[section])) { - let depTarget = overriddenPeers.get(depName); - if (!depTarget) { - depTarget = resolvePackagePath(depName, target); + if (peers.has(depName)) { + continue; } + let depTarget = resolvePackagePath(depName, target); if (!depTarget) { throw new Error(`package ${name} in ${target} depends on ${depName} but we could not resolve it`); } @@ -325,20 +317,21 @@ export class Project { private readSync(root: string, opts?: ReadDirOpts): void { const files = fixturify.readSync(root, { // when linking deps, we don't need to crawl all of node_modules - ignore: opts?.linkDeps ? ['node_modules'] : [], + ignore: opts?.linkDeps || opts?.linkDevDeps ? ['node_modules'] : [], }); this.pkg = deserializePackageJson(getFile(files, 'package.json')); + this.requestedRange = this.version; delete files['package.json']; this.files = files; - if (opts?.linkDeps) { + if (opts?.linkDeps || opts?.linkDevDeps) { if (this.pkg.dependencies) { for (let dep of Object.keys(this.pkg.dependencies)) { this.linkDependency(dep, { baseDir: path.join(root, this.name) }); } } - if (this.pkg.devDependencies) { + if (this.pkg.devDependencies && opts.linkDevDeps) { for (let dep of Object.keys(this.pkg.devDependencies)) { this.linkDevDependency(dep, { baseDir: path.join(root, this.name) }); } @@ -482,8 +475,8 @@ export class Project { linkDependency( name: string, opts: - | { baseDir: string; resolveName?: string; requestedRange?: string; undeclaredPeerDeps?: string[] } - | { target: string; requestedRange?: string; undeclaredPeerDeps?: string[] } + | { baseDir: string; resolveName?: string; requestedRange?: string } + | { target: string; requestedRange?: string } ) { this.removeDependency(name); this.removeDevDependency(name); @@ -498,7 +491,7 @@ export class Project { dir = opts.target; } let requestedRange = opts?.requestedRange ?? fs.readJsonSync(path.join(dir, 'package.json')).version; - this.dependencyLinks.set(name, { dir, requestedRange, undeclaredPeerDeps: opts?.undeclaredPeerDeps || [] }); + this.dependencyLinks.set(name, { dir, requestedRange }); } linkDevDependency(name: string, opts: { baseDir: string; resolveName?: string } | { target: string }) { diff --git a/test.ts b/test.ts index 023a08f..712877c 100644 --- a/test.ts +++ b/test.ts @@ -526,33 +526,6 @@ describe('Project', function () { ); }); - it('supports undeclaredPeerDeps', function () { - let baseProject = new Project('base'); - baseProject.addDependency('alpha', { - files: { - 'index.js': ` - module.exports = function() { - return require('beta/package.json').version; - } - `, - }, - }); - baseProject.addDependency('beta', { version: '1.1.0' }); - baseProject.writeSync(); - - // precondition: in the baseProject, alpha sees its undeclared beta peerDep - // as beta@1.1.0 - expect(require(require.resolve('alpha', { paths: [baseProject.baseDir] }))()).to.eql('1.1.0'); - - let project = new Project('my-app'); - project.linkDependency('alpha', { baseDir: baseProject.baseDir, undeclaredPeerDeps: ['beta'] }); - project.addDependency('beta', { version: '1.2.0' }); - project.writeSync(); - - // in our linked project, alpha sees its undeclared beta peerDep as beta@1.2.0 - expect(require(require.resolve('alpha', { paths: [project.baseDir] }))()).to.eql('1.2.0'); - }); - it('adds linked dependencies to package.json', function () { let baseProject = new Project('base'); baseProject.addDependency('moment', '1.2.3'); @@ -592,6 +565,7 @@ describe('Project', function () { // start with a template addon let addonTemplate = new Project('stock-addon'); addonTemplate.addDependency('helper-lib', '1.2.3'); + addonTemplate.addDevDependency('test-lib'); addonTemplate.files['hello.js'] = '// it works'; addonTemplate.writeSync(); @@ -608,10 +582,42 @@ describe('Project', function () { expect( fs.readlinkSync(path.join(myApp.baseDir, 'node_modules', 'custom-addon', 'node_modules', 'helper-lib')) ).to.eql(path.join(addonTemplate.baseDir, 'node_modules', 'helper-lib')); + + // dev dependencies not included by default + expect(fs.existsSync(path.join(myApp.baseDir, 'node_modules', 'custom-addon', 'node_modules', 'test-lib'))).to.eql( + false + ); + expect(fs.existsSync(path.join(myApp.baseDir, 'node_modules', 'custom-addon', 'hello.js'))).to.eql(true); expect(fs.existsSync(path.join(myApp.baseDir, 'node_modules', 'custom-addon', 'layered-extra.js'))).to.eql(true); }); + it('can read a project with linked dev dependencies', function () { + // start with a template app + let appTemplate = new Project('stock-app'); + appTemplate.addDependency('helper-lib', '1.2.3'); + appTemplate.addDevDependency('test-lib'); + appTemplate.files['hello.js'] = '// it works'; + appTemplate.writeSync(); + + // build a new addon from the template + let myApp = Project.fromDir(appTemplate.baseDir, { linkDevDeps: true }); + myApp.name = 'custom-addon'; + myApp.files['layered-extra.js'] = '// extra stuff'; + myApp.writeSync(); + + expect(fs.readlinkSync(path.join(myApp.baseDir, 'node_modules', 'helper-lib'))).to.eql( + path.join(appTemplate.baseDir, 'node_modules', 'helper-lib') + ); + + expect(fs.readlinkSync(path.join(myApp.baseDir, 'node_modules', 'test-lib'))).to.eql( + path.join(appTemplate.baseDir, 'node_modules', 'test-lib') + ); + + expect(fs.existsSync(path.join(myApp.baseDir, 'hello.js'))).to.eql(true); + expect(fs.existsSync(path.join(myApp.baseDir, 'layered-extra.js'))).to.eql(true); + }); + it('can override a linked dependency with a new Project dependency', function () { let baseProject = new Project('base'); baseProject.addDependency('moment', '1.2.3'); From b47657ed46c10ea1ef7c373605669182fc287c3a Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 16:04:14 -0400 Subject: [PATCH 07/14] use copying for cross-device situations --- index.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.ts b/index.ts index acf9cc7..4e3de69 100644 --- a/index.ts +++ b/index.ts @@ -127,6 +127,7 @@ export class Project { private dependencyLinks: Map = new Map(); private linkIsDevDependency: Set = new Set(); + private usingHardLinks = true; constructor( name?: string, @@ -303,11 +304,26 @@ export class Project { if (stat.isDirectory()) { this.hardLinkContents(path.join(target, name), path.join(destination, name)); } else { - fs.ensureLinkSync(path.join(target, name), path.join(destination, name)); + this.hardLinkFile(path.join(target, name), path.join(destination, name)); } } } + private hardLinkFile(source: string, destination: string) { + if (this.usingHardLinks) { + try { + fs.ensureLinkSync(source, destination); + return; + } catch (err: any) { + if (err.code !== 'EXDEV') { + throw err; + } + this.usingHardLinks = false; + } + } + fs.copySync(source, destination); + } + static fromDir(root: string, opts?: ReadDirOpts): Project { let project = new Project(); project.readSync(root, opts); From 0a9b9a843852983dda067ac2c232e2f967137c7c Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 16:07:42 -0400 Subject: [PATCH 08/14] don't cache package.jsons --- index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.ts b/index.ts index 4e3de69..0c9a74a 100644 --- a/index.ts +++ b/index.ts @@ -263,7 +263,7 @@ export class Project { } private writeLinkedPackage(name: string, target: string) { - let targetPkg = require(path.join(target, 'package.json')); + let targetPkg = fs.readJsonSync(`${target}/package.json`); let peers = new Set(Object.keys(targetPkg.peerDependencies ?? {})); let destination = path.join(this.baseDir, 'node_modules', name); From 3714af686b8cd15e16b7e345c8d9f34bd68309d9 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 16:11:27 -0400 Subject: [PATCH 09/14] keep separate resolution caches --- index.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/index.ts b/index.ts index 0c9a74a..327390e 100644 --- a/index.ts +++ b/index.ts @@ -3,6 +3,7 @@ import tmp = require('tmp'); import fs = require('fs-extra'); import path = require('path'); import resolvePackagePath = require('resolve-package-path'); +import CacheGroup = require('resolve-package-path/lib/cache-group'); import { PackageJson } from 'type-fest'; import { readdirSync, statSync } from 'fs'; @@ -129,6 +130,11 @@ export class Project { private linkIsDevDependency: Set = new Set(); private usingHardLinks = true; + // we keep our own package resolution cache because the default global one + // could get polluted by us resolving test-specific things that will change on + // subsequent tests. + private resolutionCache = new CacheGroup(); + constructor( name?: string, version?: string, @@ -282,7 +288,7 @@ export class Project { if (peers.has(depName)) { continue; } - let depTarget = resolvePackagePath(depName, target); + let depTarget = resolvePackagePath(depName, target, this.resolutionCache); if (!depTarget) { throw new Error(`package ${name} in ${target} depends on ${depName} but we could not resolve it`); } @@ -498,7 +504,7 @@ export class Project { this.removeDevDependency(name); let dir: string; if ('baseDir' in opts) { - let pkgJSONPath = resolvePackagePath(opts.resolveName || name, opts.baseDir); + let pkgJSONPath = resolvePackagePath(opts.resolveName || name, opts.baseDir, this.resolutionCache); if (!pkgJSONPath) { throw new Error(`failed to locate ${opts.resolveName || name} in ${opts.baseDir}`); } From 97543f0e5c7a59a97e42937cbac315599a2e1d39 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 16:56:01 -0400 Subject: [PATCH 10/14] use walkSync for cycle protection --- index.ts | 16 ++++++---------- package.json | 7 ++++--- yarn.lock | 12 +++++++++++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/index.ts b/index.ts index 327390e..f060a4b 100644 --- a/index.ts +++ b/index.ts @@ -5,7 +5,7 @@ import path = require('path'); import resolvePackagePath = require('resolve-package-path'); import CacheGroup = require('resolve-package-path/lib/cache-group'); import { PackageJson } from 'type-fest'; -import { readdirSync, statSync } from 'fs'; +import { entries } from 'walk-sync'; tmp.setGracefulCleanup(); @@ -301,16 +301,12 @@ export class Project { } } - private hardLinkContents(target: string, destination: string, exclude = 'node_modules') { - for (let name of readdirSync(target)) { - if (name === exclude) { - continue; - } - let stat = statSync(path.join(target, name)); - if (stat.isDirectory()) { - this.hardLinkContents(path.join(target, name), path.join(destination, name)); + private hardLinkContents(target: string, destination: string) { + for (let entry of entries(target, { ignore: ['node_modules'] })) { + if (entry.isDirectory()) { + this.hardLinkContents(entry.fullPath, path.join(destination, entry.relativePath)); } else { - this.hardLinkFile(path.join(target, name), path.join(destination, name)); + this.hardLinkFile(entry.fullPath, path.join(destination, entry.relativePath)); } } } diff --git a/package.json b/package.json index 5d5ee5c..9a8feba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "fixturify-project", - "version": "3.0.2", + "name": "@ef4/fixturify-project", + "version": "4.0.0-alpha.2", "main": "index.js", "repository": "git@github.com:stefanpenner/node-fixturify-project", "author": "Stefan Penner ", @@ -9,7 +9,8 @@ "fixturify": "^2.1.1", "resolve-package-path": "^3.1.0", "tmp": "^0.0.33", - "type-fest": "^0.11.0" + "type-fest": "^0.11.0", + "walk-sync": "^3.0.0" }, "devDependencies": { "@types/chai": "^4.2.18", diff --git a/yarn.lock b/yarn.lock index 03a93a1..456a7a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -29,7 +29,7 @@ "@types/minimatch" "*" "@types/node" "*" -"@types/minimatch@*", "@types/minimatch@^3.0.3": +"@types/minimatch@*", "@types/minimatch@^3.0.3", "@types/minimatch@^3.0.4": version "3.0.5" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== @@ -772,6 +772,16 @@ walk-sync@^2.0.2: matcher-collection "^2.0.0" minimatch "^3.0.4" +walk-sync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/walk-sync/-/walk-sync-3.0.0.tgz#67f882925021e20569a1edd560b8da31da8d171c" + integrity sha512-41TvKmDGVpm2iuH7o+DAOt06yyu/cSHpX3uzAwetzASvlNtVddgIjXIb2DfB/Wa20B1Jo86+1Dv1CraSU7hWdw== + dependencies: + "@types/minimatch" "^3.0.4" + ensure-posix-path "^1.1.0" + matcher-collection "^2.0.1" + minimatch "^3.0.4" + which@2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" From 658785b916514ce60c9ccebed0668b22c8bff86d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 16:57:50 -0400 Subject: [PATCH 11/14] label error better --- index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.ts b/index.ts index f060a4b..0dcb91f 100644 --- a/index.ts +++ b/index.ts @@ -290,7 +290,9 @@ export class Project { } let depTarget = resolvePackagePath(depName, target, this.resolutionCache); if (!depTarget) { - throw new Error(`package ${name} in ${target} depends on ${depName} but we could not resolve it`); + throw new Error( + `[FixturifyProject] package ${name} in ${target} depends on ${depName} but we could not resolve it` + ); } fs.ensureSymlinkSync( depTarget.slice(0, -1 * '/package.json'.length), From 2c5156d7bd0714a6d37012cdbf1fd86aa8446b0d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 17:06:24 -0400 Subject: [PATCH 12/14] incorporating feedback --- index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/index.ts b/index.ts index 0dcb91f..462b180 100644 --- a/index.ts +++ b/index.ts @@ -294,10 +294,7 @@ export class Project { `[FixturifyProject] package ${name} in ${target} depends on ${depName} but we could not resolve it` ); } - fs.ensureSymlinkSync( - depTarget.slice(0, -1 * '/package.json'.length), - path.join(destination, 'node_modules', depName) - ); + fs.ensureSymlinkSync(path.dirname(depTarget), path.join(destination, 'node_modules', depName)); } } } @@ -325,7 +322,7 @@ export class Project { this.usingHardLinks = false; } } - fs.copySync(source, destination); + fs.copyFileSync(source, destination, fs.constants.COPYFILE_FICLONE); } static fromDir(root: string, opts?: ReadDirOpts): Project { From 5e246abd56f31cec495de215d818ced324605e8e Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 17:16:07 -0400 Subject: [PATCH 13/14] unintentional test changes included --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 9a8feba..a6dadb8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "@ef4/fixturify-project", - "version": "4.0.0-alpha.2", + "name": "fixturify-project", + "version": "3.0.2", "main": "index.js", "repository": "git@github.com:stefanpenner/node-fixturify-project", "author": "Stefan Penner ", From 58bc0a24e601b09ab12ebb00b83938e6c59c7a11 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 13 Sep 2021 17:17:47 -0400 Subject: [PATCH 14/14] use exclusive flag --- index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.ts b/index.ts index 462b180..54ded0c 100644 --- a/index.ts +++ b/index.ts @@ -322,7 +322,7 @@ export class Project { this.usingHardLinks = false; } } - fs.copyFileSync(source, destination, fs.constants.COPYFILE_FICLONE); + fs.copyFileSync(source, destination, fs.constants.COPYFILE_FICLONE | fs.constants.COPYFILE_EXCL); } static fromDir(root: string, opts?: ReadDirOpts): Project {