diff --git a/e2e/affected.test.ts b/e2e/affected.test.ts index 8859a5fe240af..47fc018bd7790 100644 --- a/e2e/affected.test.ts +++ b/e2e/affected.test.ts @@ -75,7 +75,7 @@ forEachCli(() => { expect(affectedApps).not.toContain(`${myapp}-e2e`); const implicitlyAffectedApps = runCommand( - 'npm run affected:apps -- --files="package.json"' + 'npm run affected:apps -- --files="tsconfig.json"' ); expect(implicitlyAffectedApps).toContain(myapp); expect(implicitlyAffectedApps).toContain(myapp2); @@ -100,7 +100,7 @@ forEachCli(() => { expect(affectedLibs).not.toContain(mylib2); const implicitlyAffectedLibs = runCommand( - 'npm run affected:libs -- --files="package.json"' + 'npm run affected:libs -- --files="tsconfig.json"' ); expect(implicitlyAffectedLibs).toContain(mypublishablelib); expect(implicitlyAffectedLibs).toContain(mylib); diff --git a/packages/workspace/src/command-line/affected.ts b/packages/workspace/src/command-line/affected.ts index 99aaee614f4a7..ce76310911497 100644 --- a/packages/workspace/src/command-line/affected.ts +++ b/packages/workspace/src/command-line/affected.ts @@ -7,6 +7,7 @@ import { NxArgs, splitArgsIntoNxArgsAndOverrides } from './utils'; import { filterAffected } from '../core/affected-project-graph'; import { createProjectGraph, + onlyWorkspaceProjects, ProjectGraphNode, ProjectType, withDeps @@ -23,7 +24,9 @@ export function affected(command: string, parsedArgs: yargs.Arguments): void { const fileChanges = readFileChanges(nxArgs); let affectedGraph = filterAffected(projectGraph, fileChanges); if (parsedArgs.withDeps) { - affectedGraph = withDeps(projectGraph, Object.values(affectedGraph.nodes)); + affectedGraph = onlyWorkspaceProjects( + withDeps(projectGraph, Object.values(affectedGraph.nodes)) + ); } const affectedProjects = Object.values( parsedArgs.all ? projectGraph.nodes : affectedGraph.nodes diff --git a/packages/workspace/src/command-line/dep-graph.ts b/packages/workspace/src/command-line/dep-graph.ts index 30f012506373e..277d6631249e4 100644 --- a/packages/workspace/src/command-line/dep-graph.ts +++ b/packages/workspace/src/command-line/dep-graph.ts @@ -3,16 +3,18 @@ import * as http from 'http'; import * as opn from 'opn'; import { createProjectGraph, + onlyWorkspaceProjects, ProjectGraph, ProjectGraphNode } from '../core/project-graph'; import { output } from '../utils/output'; +import { join } from 'path'; export function generateGraph( args: { file?: string; filter?: string[]; exclude?: string[] }, affectedProjects: string[] ): void { - const graph = createProjectGraph(); + const graph = onlyWorkspaceProjects(createProjectGraph()); const renderProjects: ProjectGraphNode[] = filterProjects( graph, @@ -43,7 +45,9 @@ function startServer( graph: ProjectGraph, affected: string[] ) { - const f = readFileSync(__dirname + '/dep-graph/dep-graph.html').toString(); + const f = readFileSync( + join(__dirname, '../core/dep-graph/dep-graph.html') + ).toString(); const html = f .replace( `window.projects = null`, @@ -90,13 +94,17 @@ function filterProjects( return filteredProjects; } -function hasPath(graph, target, node, visited) { +function hasPath( + graph: ProjectGraph, + target: string, + node: string, + visited: string[] +) { if (target === node) return true; - for (let d of graph.nodes[node]) { - if (visited.indexOf(d.projectName) > -1) continue; - if (hasPath(graph, target, d.projectName, [...visited, d.projectName])) - return true; + for (let d of graph.dependencies[node] || []) { + if (visited.indexOf(d.target) > -1) continue; + if (hasPath(graph, target, d.target, [...visited, d.target])) return true; } return false; } diff --git a/packages/workspace/src/command-line/lint.ts b/packages/workspace/src/command-line/lint.ts index 1d11f5c7e93c7..f0d12bee21c67 100644 --- a/packages/workspace/src/command-line/lint.ts +++ b/packages/workspace/src/command-line/lint.ts @@ -1,4 +1,7 @@ -import { createProjectGraph } from '../core/project-graph'; +import { + createProjectGraph, + onlyWorkspaceProjects +} from '../core/project-graph'; import { WorkspaceIntegrityChecks } from './workspace-integrity-checks'; import * as path from 'path'; import { appRootPath } from '../utils/app-root'; @@ -6,7 +9,7 @@ import { allFilesInDir } from '../core/file-utils'; import { output } from '../utils/output'; export function workspaceLint() { - const graph = createProjectGraph(); + const graph = onlyWorkspaceProjects(createProjectGraph()); const cliErrorOutputConfigs = new WorkspaceIntegrityChecks( graph, diff --git a/packages/workspace/src/command-line/shared.ts b/packages/workspace/src/command-line/shared.ts index 5fe2d67594d3a..7b2c5df9ef942 100644 --- a/packages/workspace/src/command-line/shared.ts +++ b/packages/workspace/src/command-line/shared.ts @@ -1,5 +1,4 @@ import { execSync } from 'child_process'; -import * as fs from 'fs'; import { output } from '../utils/output'; import { createProjectGraph, ProjectGraphNode } from '../core/project-graph'; import { NxArgs } from './utils'; diff --git a/packages/workspace/src/command-line/utils.spec.ts b/packages/workspace/src/command-line/utils.spec.ts index 10bea57ab3254..b7ae3debeb481 100644 --- a/packages/workspace/src/command-line/utils.spec.ts +++ b/packages/workspace/src/command-line/utils.spec.ts @@ -10,6 +10,7 @@ describe('splitArgs', () => { $0: '' }).nxArgs ).toEqual({ + _: ['--override'], projects: [], files: [''] }); diff --git a/packages/workspace/src/command-line/utils.ts b/packages/workspace/src/command-line/utils.ts index 817023bcfa6a6..4b83dd325781d 100644 --- a/packages/workspace/src/command-line/utils.ts +++ b/packages/workspace/src/command-line/utils.ts @@ -30,7 +30,8 @@ const dummyOptions: NxArgs = { withDeps: false, 'with-deps': false, projects: [], - select: '' + select: '', + _: [] } as any; const nxSpecific = Object.keys(dummyOptions); diff --git a/packages/workspace/src/core/affected-project-graph/affected-project-graph-models.ts b/packages/workspace/src/core/affected-project-graph/affected-project-graph-models.ts index 0526cc1b5789c..69227beefff0a 100644 --- a/packages/workspace/src/core/affected-project-graph/affected-project-graph-models.ts +++ b/packages/workspace/src/core/affected-project-graph/affected-project-graph-models.ts @@ -2,6 +2,6 @@ import { NxJson } from '../shared-interfaces'; export interface AffectedProjectGraphContext { workspaceJson: any; - nxJson: NxJson; + nxJson: NxJson; touchedProjects: string[]; } diff --git a/packages/workspace/src/core/affected-project-graph/affected-project-graph.spec.ts b/packages/workspace/src/core/affected-project-graph/affected-project-graph.spec.ts index 8c0d1f8df6c7f..91d537bcd5bff 100644 --- a/packages/workspace/src/core/affected-project-graph/affected-project-graph.spec.ts +++ b/packages/workspace/src/core/affected-project-graph/affected-project-graph.spec.ts @@ -1,4 +1,5 @@ import { extname } from 'path'; +import { jsonDiff } from '../../utils/json-diff'; import { vol } from 'memfs'; import { stripIndents } from '@angular-devkit/core/src/utils/literals'; import { createProjectGraph } from '../project-graph'; @@ -22,11 +23,14 @@ describe('project graph', () => { beforeEach(() => { packageJson = { name: '@nrwl/workspace-src', + scripts: { + deploy: 'echo deploy' + }, dependencies: { 'happy-nrwl': '1.0.0' }, devDependencies: { - '@nrwl/workspace': '*' + '@nrwl/workspace': '8.0.0' } }; workspaceJson = { @@ -61,6 +65,14 @@ describe('project graph', () => { nxJson = { npmScope: 'nrwl', implicitDependencies: { + 'package.json': { + scripts: { + deploy: ['demo', 'api'] + }, + devDependencies: { + '@nrwl/workspace': '*' + } + }, 'something-for-api.txt': ['api'] }, projects: { @@ -183,4 +195,120 @@ describe('project graph', () => { } }); }); + + it('should create nodes and dependencies with npm packages', () => { + const graph = createProjectGraph(); + const updatedPackageJson = { + ...packageJson, + dependencies: { + 'happy-nrwl': '2.0.0' + } + }; + + const affected = filterAffected(graph, [ + { + file: 'package.json', + ext: '.json', + mtime: 1, + getChanges: () => jsonDiff(packageJson, updatedPackageJson) + } + ]); + + expect(affected).toEqual({ + nodes: { + 'happy-nrwl': { + type: 'npm', + name: 'happy-nrwl', + data: expect.anything() + }, + util: { + name: 'util', + type: 'lib', + data: expect.anything() + }, + ui: { + name: 'ui', + type: 'lib', + data: expect.anything() + }, + demo: { + name: 'demo', + type: 'app', + data: expect.anything() + }, + 'demo-e2e': { + name: 'demo-e2e', + type: 'e2e', + data: expect.anything() + } + }, + dependencies: { + 'demo-e2e': [ + { + type: 'implicit', + source: 'demo-e2e', + target: 'demo' + } + ], + demo: [ + { + type: 'static', + source: 'demo', + target: 'ui' + } + ], + ui: [{ type: 'static', source: 'ui', target: 'util' }], + util: [{ type: 'static', source: 'util', target: 'happy-nrwl' }] + } + }); + }); + + it('should support implicit JSON file dependencies (some projects)', () => { + const graph = createProjectGraph(); + const updatedPackageJson = { + ...packageJson, + scripts: { + deploy: 'echo deploy!!!' + } + }; + + const affected = filterAffected(graph, [ + { + file: 'package.json', + ext: '.json', + mtime: 1, + getChanges: () => jsonDiff(packageJson, updatedPackageJson) + } + ]); + + expect(Object.keys(affected.nodes)).toEqual(['demo', 'demo-e2e', 'api']); + }); + + it('should support implicit JSON file dependencies (all projects)', () => { + const graph = createProjectGraph(); + const updatedPackageJson = { + ...packageJson, + devDependencies: { + '@nrwl/workspace': '9.0.0' + } + }; + + const affected = filterAffected(graph, [ + { + file: 'package.json', + ext: '.json', + mtime: 1, + getChanges: () => jsonDiff(packageJson, updatedPackageJson) + } + ]); + + expect(Object.keys(affected.nodes)).toEqual([ + '@nrwl/workspace', + 'api', + 'demo', + 'demo-e2e', + 'ui', + 'util' + ]); + }); }); diff --git a/packages/workspace/src/core/affected-project-graph/affected-project-graph.ts b/packages/workspace/src/core/affected-project-graph/affected-project-graph.ts index 0619530ce91a1..54890324b4244 100644 --- a/packages/workspace/src/core/affected-project-graph/affected-project-graph.ts +++ b/packages/workspace/src/core/affected-project-graph/affected-project-graph.ts @@ -1,7 +1,14 @@ -import { ProjectGraph } from '../project-graph'; +import { ProjectGraph, ProjectGraphBuilder, reverse } from '../project-graph'; import { FileChange, readNxJson, readWorkspaceJson } from '../file-utils'; -import { filterAffectedProjects } from './filter-affected-projects'; import { NxJson } from '../shared-interfaces'; +import { + getImplicitlyTouchedProjects, + getTouchedProjects +} from './locators/workspace-projects'; +import { getTouchedNpmPackages } from './locators/npm-packages'; +import { getImplicitlyTouchedProjectsByJsonChanges } from './locators/implicit-json-changes'; +import { AffectedProjectGraphContext } from './affected-project-graph-models'; +import { normalizeNxJson } from '../normalize-nx-json'; export function filterAffected( graph: ProjectGraph, @@ -9,69 +16,69 @@ export function filterAffected( workspaceJson: any = readWorkspaceJson(), nxJson: NxJson = readNxJson() ): ProjectGraph { + const normalizedNxJson = normalizeNxJson(nxJson); // Additional affected logic should be in this array. const touchedProjectLocators = [ getTouchedProjects, - getImplicitlyTouchedProjects + getImplicitlyTouchedProjects, + getTouchedNpmPackages, + getImplicitlyTouchedProjectsByJsonChanges ]; const touchedProjects = touchedProjectLocators.reduce( (acc, f) => { - return acc.concat(f(workspaceJson, nxJson, touchedFiles)); + return acc.concat(f(workspaceJson, normalizedNxJson, touchedFiles)); }, [] as string[] ); return filterAffectedProjects(graph, { workspaceJson, - nxJson, + nxJson: normalizedNxJson, touchedProjects }); } -// --------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- -function getTouchedProjects( - workspaceJson: any, - nxJson: NxJson, - touchedFiles: FileChange[] -): string[] { - return touchedFiles - .map(f => { - return Object.keys(workspaceJson.projects).find(projectName => { - const p = workspaceJson.projects[projectName]; - return f.file.startsWith(p.root); - }); - }) - .filter(Boolean); +function filterAffectedProjects( + graph: ProjectGraph, + ctx: AffectedProjectGraphContext +): ProjectGraph { + const builder = new ProjectGraphBuilder(); + const reversed = reverse(graph); + ctx.touchedProjects.forEach(p => { + addAffectedNodes(p, reversed, builder); + addAffectedDependencies(p, reversed, builder); + }); + return builder.build(); } -function getImplicitlyTouchedProjects( - workspaceJson: any, - nxJson: NxJson, - fileChanges: FileChange[] -): string[] { - if (!nxJson.implicitDependencies) { - return []; +function addAffectedNodes( + startingProject: string, + reversed: ProjectGraph, + builder: ProjectGraphBuilder +): void { + builder.addNode(reversed.nodes[startingProject]); + const ds = reversed.dependencies[startingProject]; + if (ds) { + ds.forEach(({ target }) => addAffectedNodes(target, reversed, builder)); } +} - const touched = []; - - for (const [filePath, projects] of Object.entries( - nxJson.implicitDependencies - )) { - const implicitDependencyWasChanged = fileChanges.some( - f => f.file === filePath +function addAffectedDependencies( + startingProject: string, + reversed: ProjectGraph, + builder: ProjectGraphBuilder +): void { + if (reversed.dependencies[startingProject]) { + reversed.dependencies[startingProject].forEach(({ target }) => + addAffectedDependencies(target, reversed, builder) + ); + reversed.dependencies[startingProject].forEach( + ({ type, source, target }) => { + // Since source and target was reversed, + // we need to reverse it back to original direction. + builder.addDependency(type, target, source); + } ); - if (!implicitDependencyWasChanged) { - continue; - } - - // File change affects all projects, just return all projects. - if (projects === '*') { - return Object.keys(workspaceJson.projects); - } else { - touched.push(...projects); - } } - - return touched; } diff --git a/packages/workspace/src/core/affected-project-graph/filter-affected-projects.ts b/packages/workspace/src/core/affected-project-graph/filter-affected-projects.ts deleted file mode 100644 index 6d2849c475cb8..0000000000000 --- a/packages/workspace/src/core/affected-project-graph/filter-affected-projects.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { reverse, ProjectGraph, ProjectGraphBuilder } from '../project-graph'; -import { AffectedProjectGraphContext } from './affected-project-graph-models'; - -export function filterAffectedProjects( - graph: ProjectGraph, - ctx: AffectedProjectGraphContext -): ProjectGraph { - const builder = new ProjectGraphBuilder(); - const reversed = reverse(graph); - ctx.touchedProjects.forEach(p => { - addAffectedNodes(p, reversed, builder); - addAffectedDependencies(p, reversed, builder); - }); - return builder.build(); -} - -function addAffectedNodes( - startingProject: string, - reversed: ProjectGraph, - builder: ProjectGraphBuilder -): void { - builder.addNode(reversed.nodes[startingProject]); - const ds = reversed.dependencies[startingProject]; - if (ds) { - ds.forEach(({ target }) => addAffectedNodes(target, reversed, builder)); - } -} - -function addAffectedDependencies( - startingProject: string, - reversed: ProjectGraph, - builder: ProjectGraphBuilder -): void { - if (reversed.dependencies[startingProject]) { - reversed.dependencies[startingProject].forEach(({ target }) => - addAffectedDependencies(target, reversed, builder) - ); - reversed.dependencies[startingProject].forEach( - ({ type, source, target }) => { - // Since source and target was reversed, - // we need to reverse it back to original direction. - builder.addDependency(type, target, source); - } - ); - } -} diff --git a/packages/workspace/src/core/affected-project-graph/locators/implicit-json-changes.ts b/packages/workspace/src/core/affected-project-graph/locators/implicit-json-changes.ts new file mode 100644 index 0000000000000..3af47cbc8d437 --- /dev/null +++ b/packages/workspace/src/core/affected-project-graph/locators/implicit-json-changes.ts @@ -0,0 +1,44 @@ +import { NxJson } from '../../shared-interfaces'; +import { FileChange } from '../../file-utils'; +import { JsonValueDiff } from '../../../utils/json-diff'; + +export function getImplicitlyTouchedProjectsByJsonChanges( + workspaceJson: any, + nxJson: NxJson, + touchedFiles: FileChange[] +): string[] { + const { implicitDependencies } = nxJson; + + if (!implicitDependencies) { + return []; + } + + const touched = []; + + for (const f of touchedFiles) { + if (f.file.endsWith('.json') && implicitDependencies[f.file]) { + const changes: JsonValueDiff[] = f.getChanges(); + for (const c of changes) { + const projects = + getTouchedProjects(c.path, implicitDependencies[f.file]) || []; + projects.forEach(p => touched.push(p)); + } + } + } + + return touched; +} + +function getTouchedProjects(path: string[], implicitDependencyConfig: any) { + let curr = implicitDependencyConfig; + let found = true; + for (const key of path) { + if (curr[key]) { + curr = curr[key]; + } else { + found = false; + break; + } + } + return found ? curr : []; +} diff --git a/packages/workspace/src/core/affected-project-graph/locators/npm-packages.ts b/packages/workspace/src/core/affected-project-graph/locators/npm-packages.ts new file mode 100644 index 0000000000000..a0ffa044b0da4 --- /dev/null +++ b/packages/workspace/src/core/affected-project-graph/locators/npm-packages.ts @@ -0,0 +1,20 @@ +import { NxJson } from '../../shared-interfaces'; +import { FileChange } from '../../file-utils'; + +export function getTouchedNpmPackages( + workspaceJson: any, + nxJson: NxJson, + touchedFiles: FileChange[] +): string[] { + const packageJson = touchedFiles.find(f => f.file === 'package.json'); + const touched = []; + if (packageJson) { + const changes = packageJson.getChanges(); + changes.forEach(c => { + if (c.path[0] === 'dependencies' || c.path[0] === 'devDependencies') { + touched.push(c.path[1]); + } + }); + } + return touched; +} diff --git a/packages/workspace/src/core/affected-project-graph/locators/workspace-projects.ts b/packages/workspace/src/core/affected-project-graph/locators/workspace-projects.ts new file mode 100644 index 0000000000000..fda4fabe2107f --- /dev/null +++ b/packages/workspace/src/core/affected-project-graph/locators/workspace-projects.ts @@ -0,0 +1,49 @@ +import { NxJson } from '../../shared-interfaces'; +import { FileChange } from '../../file-utils'; + +export function getTouchedProjects( + workspaceJson: any, + nxJson: NxJson, + touchedFiles: FileChange[] +): string[] { + return touchedFiles + .map(f => { + return Object.keys(workspaceJson.projects).find(projectName => { + const p = workspaceJson.projects[projectName]; + return f.file.startsWith(p.root); + }); + }) + .filter(Boolean); +} + +export function getImplicitlyTouchedProjects( + workspaceJson: any, + nxJson: NxJson, + fileChanges: FileChange[] +): string[] { + if (!nxJson.implicitDependencies) { + return []; + } + + const touched = []; + + for (const [filePath, projects] of Object.entries( + nxJson.implicitDependencies + )) { + const implicitDependencyWasChanged = fileChanges.some( + f => f.file === filePath + ); + if (!implicitDependencyWasChanged) { + continue; + } + + // File change affects all projects, just return all projects. + if (projects === '*') { + return Object.keys(workspaceJson.projects); + } else if (Array.isArray(projects)) { + touched.push(...projects); + } + } + + return touched; +} diff --git a/packages/workspace/src/core/assert-workspace-validity.ts b/packages/workspace/src/core/assert-workspace-validity.ts index 43c35c9a79b72..d359a210bd36e 100644 --- a/packages/workspace/src/core/assert-workspace-validity.ts +++ b/packages/workspace/src/core/assert-workspace-validity.ts @@ -1,4 +1,5 @@ import { workspaceFileName } from './file-utils'; +import { ImplicitJsonSubsetDependency } from '@nrwl/workspace/src/core/shared-interfaces'; export function assertWorkspaceValidity(workspaceJson, nxJson) { const workspaceJsonProjects = Object.keys(workspaceJson.projects); @@ -29,8 +30,24 @@ export function assertWorkspaceValidity(workspaceJson, nxJson) { const invalidImplicitDependencies = new Map(); - Object.entries<'*' | string[]>(nxJson.implicitDependencies || {}) - .filter(([_, val]) => val !== '*') // These are valid since it is calculated + Object.entries<'*' | string[] | ImplicitJsonSubsetDependency>( + nxJson.implicitDependencies || {} + ) + .reduce((acc, entry) => { + function recur(value, acc = []) { + if (value === '*') { + // do nothing since '*' is calculated and always valid. + } else if (Array.isArray(value)) { + acc.push([entry[0], value]); + } else { + Object.values(value).forEach(v => { + recur(v, acc); + }); + } + } + recur(entry[1], acc); + return acc; + }, []) .reduce((map, [filename, projectNames]: [string, string[]]) => { detectAndSetInvalidProjectValues(map, filename, projectNames, projects); return map; diff --git a/packages/workspace/src/core/dep-graph/dep-graph.html b/packages/workspace/src/core/dep-graph/dep-graph.html index f17b93742e711..44b6c7df58e44 100644 --- a/packages/workspace/src/core/dep-graph/dep-graph.html +++ b/packages/workspace/src/core/dep-graph/dep-graph.html @@ -63,9 +63,9 @@ function hasPath(target, node, visited) { if (target === node) return true; - for (let d of window.graph.dependencies[node]) { - if (visited.indexOf(d.name) > -1) continue; - if (hasPath(target, d.name, [...visited, d.name])) return true; + for (let d of window.graph.dependencies[node] || []) { + if (visited.indexOf(d.target) > -1) continue; + if (hasPath(target, d.target, [...visited, d.target])) return true; } return false; } diff --git a/packages/workspace/src/core/file-graph/file-map.ts b/packages/workspace/src/core/file-graph/file-map.ts index 02a2704b585c7..b330d1c408955 100644 --- a/packages/workspace/src/core/file-graph/file-map.ts +++ b/packages/workspace/src/core/file-graph/file-map.ts @@ -5,7 +5,7 @@ export interface FileMap { } export function createFileMap(workspaceJson: any, files: FileData[]): FileMap { - const graph: FileMap = {}; + const fileMap: FileMap = {}; const seen = new Set(); // Sorting here so `apps/client-e2e` comes before `apps/client` and has // a chance to match prefix first. @@ -20,16 +20,16 @@ export function createFileMap(workspaceJson: any, files: FileData[]): FileMap { }) .forEach(projectName => { const p = workspaceJson.projects[projectName]; + fileMap[projectName] = fileMap[projectName] || []; files.forEach(f => { if (seen.has(f.file)) { return; } if (f.file.startsWith(p.root)) { - graph[projectName] = graph[projectName] || []; - graph[projectName].push(f); + fileMap[projectName].push(f); seen.add(f.file); } }); }); - return graph; + return fileMap; } diff --git a/packages/workspace/src/core/normalize-nx-json.spec.ts b/packages/workspace/src/core/normalize-nx-json.spec.ts new file mode 100644 index 0000000000000..32ac14dcc63e1 --- /dev/null +++ b/packages/workspace/src/core/normalize-nx-json.spec.ts @@ -0,0 +1,41 @@ +import { normalizeNxJson } from './normalize-nx-json'; + +describe('normalizeNxJson', () => { + it('should expand projects array', () => { + const result = normalizeNxJson({ + npmScope: 'nrwl', + projects: { + demo: { tags: [] }, + ui: { tags: [] }, + util: { tags: [] } + }, + implicitDependencies: { + 'package.json': '*', + 'whatever.json': { + a: { + b: { + c: ['demo'], + d: { + e: '*' + } + } + } + } + } + }); + + expect(result.implicitDependencies).toEqual({ + 'package.json': ['demo', 'ui', 'util'], + 'whatever.json': { + a: { + b: { + c: ['demo'], + d: { + e: ['demo', 'ui', 'util'] + } + } + } + } + }); + }); +}); diff --git a/packages/workspace/src/core/normalize-nx-json.ts b/packages/workspace/src/core/normalize-nx-json.ts new file mode 100644 index 0000000000000..ec91f6a6f88ba --- /dev/null +++ b/packages/workspace/src/core/normalize-nx-json.ts @@ -0,0 +1,28 @@ +import { NxJson } from './shared-interfaces'; + +export function normalizeNxJson(nxJson: NxJson): NxJson { + return nxJson.implicitDependencies + ? { + ...nxJson, + implicitDependencies: Object.entries( + nxJson.implicitDependencies + ).reduce((acc, [key, val]) => { + acc[key] = recur(val); + return acc; + + function recur(v: '*' | string[] | {}): string[] | {} { + if (v === '*') { + return Object.keys(nxJson.projects); + } else if (Array.isArray(v)) { + return v; + } else { + return Object.keys(v).reduce((xs, x) => { + xs[x] = recur(v[x]); + return xs; + }, {}); + } + } + }, {}) + } + : (nxJson as NxJson); +} diff --git a/packages/workspace/src/core/project-graph/build-dependencies/explicit-npm-dependencies.ts b/packages/workspace/src/core/project-graph/build-dependencies/explicit-npm-dependencies.ts new file mode 100644 index 0000000000000..2c8a1e1c67728 --- /dev/null +++ b/packages/workspace/src/core/project-graph/build-dependencies/explicit-npm-dependencies.ts @@ -0,0 +1,41 @@ +import { + AddProjectDependency, + DependencyType, + ProjectGraphContext, + ProjectGraphNodeRecords +} from '../project-graph-models'; +import { TypeScriptImportLocator } from './typescript-import-locator'; + +export function buildExplicitNpmDependencies( + ctx: ProjectGraphContext, + nodes: ProjectGraphNodeRecords, + addDependency: AddProjectDependency, + fileRead: (s: string) => string +) { + const importLocator = new TypeScriptImportLocator(fileRead); + + Object.keys(ctx.fileMap).forEach(source => { + Object.values(ctx.fileMap[source]).forEach(f => { + importLocator.fromFile( + f.file, + (importExpr: string, filePath: string, type: DependencyType) => { + const key = Object.keys(nodes).find(k => + isNpmPackageImport(k, importExpr) + ); + const target = nodes[key]; + if (source && target && target.type === 'npm') { + addDependency(type, source, target.name); + } + } + ); + }); + }); +} + +function isNpmPackageImport(p, e) { + const toMatch = e + .split(/[\/]/) + .slice(0, p.startsWith('@') ? 2 : 1) + .join('/'); + return toMatch === p; +} diff --git a/packages/workspace/src/core/project-graph/build-dependencies/index.ts b/packages/workspace/src/core/project-graph/build-dependencies/index.ts index 2c6c8843d02ac..7bfc04761edf3 100644 --- a/packages/workspace/src/core/project-graph/build-dependencies/index.ts +++ b/packages/workspace/src/core/project-graph/build-dependencies/index.ts @@ -1,3 +1,4 @@ export * from './build-dependencies'; export * from './implicit-project-dependencies'; export * from './explicit-project-dependencies'; +export * from './explicit-npm-dependencies'; diff --git a/packages/workspace/src/core/project-graph/build-nodes/index.ts b/packages/workspace/src/core/project-graph/build-nodes/index.ts index 3ebe4af8b0550..e8e4a7f3534b6 100644 --- a/packages/workspace/src/core/project-graph/build-nodes/index.ts +++ b/packages/workspace/src/core/project-graph/build-nodes/index.ts @@ -1,2 +1,3 @@ export * from './build-nodes'; export * from './workspace-projects'; +export * from './npm-packages'; diff --git a/packages/workspace/src/core/project-graph/build-nodes/npm-packages.ts b/packages/workspace/src/core/project-graph/build-nodes/npm-packages.ts new file mode 100644 index 0000000000000..60cf4e2876045 --- /dev/null +++ b/packages/workspace/src/core/project-graph/build-nodes/npm-packages.ts @@ -0,0 +1,24 @@ +import * as stripJsonComments from 'strip-json-comments'; +import { ProjectGraphContext, AddProjectNode } from '../project-graph-models'; + +export function buildNpmPackageNodes( + ctx: ProjectGraphContext, + addNode: AddProjectNode, + fileRead: (s: string) => string +) { + const packageJson = JSON.parse(stripJsonComments(fileRead('package.json'))); + const deps = { + ...packageJson.dependencies, + ...packageJson.devDependencies + }; + Object.keys(deps).forEach(d => { + addNode({ + type: 'npm', + name: d, + data: { + version: deps[d], + files: [] + } + }); + }); +} diff --git a/packages/workspace/src/core/project-graph/build-nodes/workspace-projects.ts b/packages/workspace/src/core/project-graph/build-nodes/workspace-projects.ts index 0a44eceaf2f85..5245c62d01466 100644 --- a/packages/workspace/src/core/project-graph/build-nodes/workspace-projects.ts +++ b/packages/workspace/src/core/project-graph/build-nodes/workspace-projects.ts @@ -1,15 +1,13 @@ -import { ProjectGraphContext, AddProjectNode } from '../project-graph-models'; +import { AddProjectNode, ProjectGraphContext } from '../project-graph-models'; export function buildWorkspaceProjectNodes( ctx: ProjectGraphContext, addNode: AddProjectNode, fileRead: (s: string) => string ) { - const workspaceJsonProjects = Object.keys(ctx.workspaceJson.projects); - const toAdd = []; - workspaceJsonProjects.forEach(key => { + Object.keys(ctx.fileMap).forEach(key => { const p = ctx.workspaceJson.projects[key]; const projectType = @@ -29,7 +27,7 @@ export function buildWorkspaceProjectNodes( data: { ...p, tags, - files: ctx.fileMap[key] || [] + files: ctx.fileMap[key] } }); }); diff --git a/packages/workspace/src/core/project-graph/operators.spec.ts b/packages/workspace/src/core/project-graph/operators.spec.ts index 477e2b8691f04..e93c15cb9e61e 100644 --- a/packages/workspace/src/core/project-graph/operators.spec.ts +++ b/packages/workspace/src/core/project-graph/operators.spec.ts @@ -1,5 +1,5 @@ import { DependencyType, ProjectGraph } from './project-graph-models'; -import { reverse, withDeps } from './operators'; +import { reverse, withDeps, filterNodes } from './operators'; const graph: ProjectGraph = { nodes: { @@ -171,3 +171,24 @@ describe('withDeps', () => { }); }); }); + +describe('filterNodes', () => { + it('filters out nodes based on predicate', () => { + const result = filterNodes(n => n.type === 'app')(graph); + expect(result).toEqual({ + nodes: { + 'app1-e2e': { name: 'app1-e2e', type: 'app', data: null }, + app1: { name: 'app1', type: 'app', data: null } + }, + dependencies: { + 'app1-e2e': [ + { + type: DependencyType.implicit, + source: 'app1-e2e', + target: 'app1' + } + ] + } + }); + }); +}); diff --git a/packages/workspace/src/core/project-graph/operators.ts b/packages/workspace/src/core/project-graph/operators.ts index 628fbdbc2d37c..26e25f95fdbfe 100644 --- a/packages/workspace/src/core/project-graph/operators.ts +++ b/packages/workspace/src/core/project-graph/operators.ts @@ -21,6 +21,33 @@ export function reverse(graph: ProjectGraph): ProjectGraph { return result; } +export function filterNodes( + predicate: (n: ProjectGraphNode) => boolean +): (p: ProjectGraph) => ProjectGraph { + return original => { + const builder = new ProjectGraphBuilder(); + const added = new Set(); + Object.values(original.nodes).forEach(n => { + if (predicate(n)) { + builder.addNode(n); + added.add(n.name); + } + }); + Object.values(original.dependencies).forEach(ds => { + ds.forEach(d => { + if (added.has(d.source) && added.has(d.target)) { + builder.addDependency(d.type, d.source, d.target); + } + }); + }); + return builder.build(); + }; +} + +export const onlyWorkspaceProjects = filterNodes( + n => n.type === 'app' || n.type === 'lib' || n.type === 'e2e' +); + export function withDeps( original: ProjectGraph, subsetNodes: ProjectGraphNode[] diff --git a/packages/workspace/src/core/project-graph/project-graph-builder.ts b/packages/workspace/src/core/project-graph/project-graph-builder.ts index 97481694071c0..c4ee3b22c4a8e 100644 --- a/packages/workspace/src/core/project-graph/project-graph-builder.ts +++ b/packages/workspace/src/core/project-graph/project-graph-builder.ts @@ -26,7 +26,7 @@ export class ProjectGraphBuilder { } addDependency( - type: DependencyType, + type: DependencyType | string, sourceProjectName: string, targetProjectName: string ) { diff --git a/packages/workspace/src/core/project-graph/project-graph-models.ts b/packages/workspace/src/core/project-graph/project-graph-models.ts index bd9242c3651ad..95dd525b00dad 100644 --- a/packages/workspace/src/core/project-graph/project-graph-models.ts +++ b/packages/workspace/src/core/project-graph/project-graph-models.ts @@ -24,13 +24,13 @@ export type ProjectGraphNodeRecords = Record; export type AddProjectNode = (node: ProjectGraphNode) => void; export interface ProjectGraphDependency { - type: DependencyType; + type: DependencyType | string; target: string; source: string; } export type AddProjectDependency = ( - type: DependencyType, + type: DependencyType | string, source: string, target: string ) => void; diff --git a/packages/workspace/src/core/project-graph/project-graph.spec.ts b/packages/workspace/src/core/project-graph/project-graph.spec.ts index 021e1bf08dfd9..bac8bdbd68693 100644 --- a/packages/workspace/src/core/project-graph/project-graph.spec.ts +++ b/packages/workspace/src/core/project-graph/project-graph.spec.ts @@ -1,5 +1,5 @@ import { extname } from 'path'; -import { vol } from 'memfs'; +import { vol, fs } from 'memfs'; import { stripIndents } from '@angular-devkit/core/src/utils/literals'; import { createProjectGraph } from './project-graph'; import { DependencyType } from './project-graph-models'; @@ -68,6 +68,13 @@ describe('project graph', () => { }; nxJson = { npmScope: 'nrwl', + implicitDependencies: { + 'package.json': { + scripts: { + deploy: '*' + } + } + }, projects: { api: { tags: [] }, demo: { tags: [], implicitDependencies: ['api'] }, @@ -106,7 +113,7 @@ describe('project graph', () => { import * as util from '@nrwl/shared/util'; `, './libs/shared/util/src/index.ts': stripIndents` - import * as happyNrwl from 'happy-nrwl'; + import * as happyNrwl from 'happy-nrwl/a/b/c'; `, './libs/shared/util/data/src/index.ts': stripIndents` export const SHARED_DATA = 'shared data'; @@ -137,7 +144,8 @@ describe('project graph', () => { ui: { name: 'ui', type: 'lib' }, 'shared-util': { name: 'shared-util', type: 'lib' }, 'shared-util-data': { name: 'shared-util-data', type: 'lib' }, - 'lazy-lib': { name: 'lazy-lib', type: 'lib' } + 'lazy-lib': { name: 'lazy-lib', type: 'lib' }, + 'happy-nrwl': { name: 'happy-nrwl', type: 'npm' } }); expect(graph.dependencies).toMatchObject({ 'demo-e2e': [ @@ -157,16 +165,24 @@ describe('project graph', () => { }, { type: DependencyType.implicit, source: 'demo', target: 'api' } ], - ui: [{ type: DependencyType.static, source: 'ui', target: 'shared-util' }] + ui: [ + { type: DependencyType.static, source: 'ui', target: 'shared-util' } + ], + 'shared-util': [ + { + type: DependencyType.static, + source: 'shared-util', + target: 'happy-nrwl' + } + ] }); }); it('should handle circular dependencies', () => { - filesJson['./libs/shared/util/src/index.ts'] = stripIndents` - import * as ui from '@nrwl/ui'; - import * as happyNrwl from 'happy-nrwl'; - `; - vol.fromJSON(filesJson, '/root'); + fs.writeFileSync( + '/root/libs/shared/util/src/index.ts', + `import * as ui from '@nrwl/ui';` + ); const graph = createProjectGraph(); diff --git a/packages/workspace/src/core/project-graph/project-graph.ts b/packages/workspace/src/core/project-graph/project-graph.ts index 1bd2d690e3592..0a1b439a7fd7a 100644 --- a/packages/workspace/src/core/project-graph/project-graph.ts +++ b/packages/workspace/src/core/project-graph/project-graph.ts @@ -1,5 +1,5 @@ import { mkdirSync, readFileSync } from 'fs'; -import { ProjectGraph, ProjectGraphContext } from './project-graph-models'; +import { ProjectGraph } from './project-graph-models'; import { ProjectGraphBuilder } from './project-graph-builder'; import { appRootPath } from '../../utils/app-root'; import { @@ -16,13 +16,19 @@ import { readWorkspaceJson } from '../file-utils'; import { createFileMap, FileMap } from '../file-graph'; -import { BuildNodes, buildWorkspaceProjectNodes } from './build-nodes'; +import { + BuildNodes, + buildNpmPackageNodes, + buildWorkspaceProjectNodes +} from './build-nodes'; import { BuildDependencies, + buildExplicitNpmDependencies, buildExplicitTypeScriptDependencies, buildImplicitProjectDependencies } from './build-dependencies'; import { assertWorkspaceValidity } from '../assert-workspace-validity'; +import { normalizeNxJson } from '../normalize-nx-json'; export function createProjectGraph( workspaceJson = readWorkspaceJson(), @@ -33,23 +39,26 @@ export function createProjectGraph( ): ProjectGraph { assertWorkspaceValidity(workspaceJson, nxJson); + const normalizedNxJson = normalizeNxJson(nxJson); + if (!cache || maxMTime(workspaceFiles) > cache.mtime) { - const builder = new ProjectGraphBuilder( - cache ? cache.data.projectGraph : undefined - ); - const buildNodesFns: BuildNodes[] = [buildWorkspaceProjectNodes]; - const buildDependenciesFns: BuildDependencies[] = [ - buildExplicitTypeScriptDependencies, - buildImplicitProjectDependencies - ]; - // TODO: File graph needs to be handled separately from project graph, or - // we need a way to support project extensions (e.g. npm project). const fileMap = createFileMap(workspaceJson, workspaceFiles); + const incremental = modifiedSinceCache(fileMap, cache); const ctx = { workspaceJson, - nxJson, - fileMap: modifiedSinceCache(fileMap, cache) + nxJson: normalizedNxJson, + fileMap: incremental.fileMap }; + const builder = new ProjectGraphBuilder(incremental.projectGraph); + const buildNodesFns: BuildNodes[] = [ + buildWorkspaceProjectNodes, + buildNpmPackageNodes + ]; + const buildDependenciesFns: BuildDependencies[] = [ + buildExplicitTypeScriptDependencies, + buildImplicitProjectDependencies, + buildExplicitNpmDependencies + ]; buildNodesFns.forEach(f => f(ctx, builder.addNode.bind(builder), fileRead)); @@ -113,38 +122,44 @@ function maxMTime(files: FileData[]) { function modifiedSinceCache( fileMap: FileMap, c: false | { data: ProjectGraphCache; mtime: number } -): FileMap { +): { fileMap: FileMap; projectGraph?: ProjectGraph } { // No cache -> compute entire graph if (!c) { - return fileMap; + return { fileMap }; } + const cachedFileMap = c.data.fileMap; const currentProjects = Object.keys(fileMap).sort(); - const previousProjects = Object.keys(c.data.fileMap).sort(); + const previousProjects = Object.keys(cachedFileMap).sort(); // Projects changed -> compute entire graph if ( currentProjects.length !== previousProjects.length || currentProjects.some((val, idx) => val !== previousProjects[idx]) ) { - return fileMap; + return { fileMap }; } - // Projects are same, only compute changed projects - const modifiedSince: FileMap = currentProjects.reduce((acc, k) => { - acc[k] = []; - return acc; - }, {}); + // Projects are same -> compute projects with file changes + const modifiedSince: FileMap = {}; currentProjects.forEach(p => { + let projectFilesChanged = false; for (const f of fileMap[p]) { - const fromCache = c.data.fileMap[p].find(x => x.file === f.file); - - if (!fromCache) { - modifiedSince[p].push(f); - } else if (f.mtime > fromCache.mtime) { - modifiedSince[p].push(f); + const fromCache = cachedFileMap[p].find(x => x.file === f.file); + if (!fromCache || f.mtime > fromCache.mtime) { + projectFilesChanged = true; + break; } } + if (projectFilesChanged) { + modifiedSince[p] = fileMap[p]; + } }); - return modifiedSince; + + // Re-compute nodes and dependencies for each project in file map. + Object.keys(modifiedSince).forEach(key => { + delete c.data.projectGraph.dependencies[key]; + }); + + return { fileMap: modifiedSince, projectGraph: c.data.projectGraph }; } diff --git a/packages/workspace/src/core/shared-interfaces.ts b/packages/workspace/src/core/shared-interfaces.ts index 736c802fd2df6..068bfae382f9d 100644 --- a/packages/workspace/src/core/shared-interfaces.ts +++ b/packages/workspace/src/core/shared-interfaces.ts @@ -1,7 +1,13 @@ -export type ImplicitDependencyEntry = { [key: string]: '*' | string[] }; +export type ImplicitDependencyEntry = { + [key: string]: T | ImplicitJsonSubsetDependency; +}; -export interface NxJson { - implicitDependencies?: ImplicitDependencyEntry; +export interface ImplicitJsonSubsetDependency { + [key: string]: T | ImplicitJsonSubsetDependency; +} + +export interface NxJson { + implicitDependencies?: ImplicitDependencyEntry; npmScope: string; projects: { [projectName: string]: NxJsonProjectConfig; diff --git a/packages/workspace/src/schematics/workspace/files/nx.json b/packages/workspace/src/schematics/workspace/files/nx.json index 846335099d34e..33c59889f3a2e 100644 --- a/packages/workspace/src/schematics/workspace/files/nx.json +++ b/packages/workspace/src/schematics/workspace/files/nx.json @@ -2,7 +2,10 @@ "npmScope": "<%= npmScope %>", "implicitDependencies": { "<%= workspaceFile %>.json": "*", - "package.json": "*", + "package.json": { + "dependencies": "*", + "devDependencies": "*" + }, "tsconfig.json": "*", "tslint.json": "*", "nx.json": "*" diff --git a/packages/workspace/src/schematics/workspace/workspace.spec.ts b/packages/workspace/src/schematics/workspace/workspace.spec.ts index f4c2b1c4caf63..2b2f77fcf3767 100644 --- a/packages/workspace/src/schematics/workspace/workspace.spec.ts +++ b/packages/workspace/src/schematics/workspace/workspace.spec.ts @@ -24,7 +24,10 @@ describe('workspace', () => { npmScope: 'proj', implicitDependencies: { 'workspace.json': '*', - 'package.json': '*', + 'package.json': { + dependencies: '*', + devDependencies: '*' + }, 'tsconfig.json': '*', 'tslint.json': '*', 'nx.json': '*' diff --git a/packages/workspace/src/utils/ast-utils.ts b/packages/workspace/src/utils/ast-utils.ts index 2346704bb2c42..dd8b1c670734a 100644 --- a/packages/workspace/src/utils/ast-utils.ts +++ b/packages/workspace/src/utils/ast-utils.ts @@ -16,7 +16,11 @@ import * as stripJsonComments from 'strip-json-comments'; import { serializeJson } from './fileutils'; import { NodePackageInstallTask } from '@angular-devkit/schematics/tasks'; import { getWorkspacePath } from './cli-config-utils'; -import { createProjectGraph, ProjectGraph } from '../core/project-graph'; +import { + createProjectGraph, + onlyWorkspaceProjects, + ProjectGraph +} from '../core/project-graph'; import { FileData } from '../core/file-utils'; import { extname, join, normalize, Path } from '@angular-devkit/core'; import { NxJson } from '@nrwl/workspace/src/core/shared-interfaces'; @@ -417,12 +421,8 @@ export function getProjectGraphFromHost(host: Tree): ProjectGraph { ); }); - return createProjectGraph( - workspaceJson, - nxJson, - workspaceFiles, - fileRead, - false + return onlyWorkspaceProjects( + createProjectGraph(workspaceJson, nxJson, workspaceFiles, fileRead, false) ); }