From c2581dcf0b5a605a9a74e7bf6911bc8e93210170 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Fri, 23 Feb 2024 13:20:44 -0500 Subject: [PATCH] feat(core): execute plugins in isolated processes (#21760) --- docs/generated/devkit/logger.md | 17 +- .../src/utils/convert-nx-executor.spec.ts | 7 +- .../devkit/src/utils/convert-nx-executor.ts | 2 + packages/nx/plugins/package-json.ts | 2 +- packages/nx/src/adapter/angular-json.ts | 4 +- packages/nx/src/adapter/ngcli-adapter.ts | 2 +- .../command-line/generate/generator-utils.ts | 2 +- .../nx/src/command-line/run/executor-utils.ts | 2 +- packages/nx/src/config/schema-utils.ts | 2 +- packages/nx/src/config/workspaces.spec.ts | 5 +- packages/nx/src/devkit-exports.ts | 9 +- packages/nx/src/devkit-internals.ts | 1 + .../generators/utils/project-configuration.ts | 6 +- .../update-15-1-0/set-project-names.ts | 2 +- packages/nx/src/plugins/js/index.ts | 2 +- .../nx/src/plugins/js/lock-file/lock-file.ts | 2 +- .../plugins/js/lock-file/npm-parser.spec.ts | 2 +- .../nx/src/plugins/js/lock-file/npm-parser.ts | 2 +- .../plugins/js/lock-file/pnpm-parser.spec.ts | 2 +- .../src/plugins/js/lock-file/pnpm-parser.ts | 2 +- .../plugins/js/lock-file/yarn-parser.spec.ts | 2 +- .../src/plugins/js/lock-file/yarn-parser.ts | 2 +- .../build-dependencies/build-dependencies.ts | 2 +- ...explicit-package-json-dependencies.spec.ts | 2 +- .../explicit-package-json-dependencies.ts | 2 +- .../explicit-project-dependencies.spec.ts | 8 +- .../explicit-project-dependencies.ts | 2 +- .../package-json-workspaces/create-nodes.ts | 64 +-- .../plugins/package-json-workspaces/index.ts | 1 + .../package-json-next-to-project-json.spec.ts | 2 +- .../package-json-next-to-project-json.ts | 4 +- .../build-nodes/project-json.spec.ts | 2 +- .../project-json/build-nodes/project-json.ts | 4 +- .../target-defaults-plugin.spec.ts | 2 +- .../target-defaults/target-defaults-plugin.ts | 4 +- .../locators/project-glob-changes.spec.ts | 2 +- .../affected/locators/project-glob-changes.ts | 11 +- .../src/project-graph/build-project-graph.ts | 19 +- packages/nx/src/project-graph/file-utils.ts | 19 +- .../nx/src/project-graph/plugins/index.ts | 6 + .../src/project-graph/plugins/internal-api.ts | 137 +++++ .../nx/src/project-graph/plugins/messaging.ts | 150 +++++ .../src/project-graph/plugins/plugin-pool.ts | 221 ++++++++ .../project-graph/plugins/plugin-worker.ts | 155 +++++ .../src/project-graph/plugins/public-api.ts | 119 ++++ .../src/project-graph/plugins/worker-api.ts | 264 +++++++++ .../project-graph/project-graph-builder.ts | 2 +- .../nx/src/project-graph/project-graph.ts | 3 - .../utils/normalize-project-nodes.ts | 3 +- .../utils/project-configuration-utils.ts | 95 +--- .../utils/retrieve-workspace-files.spec.ts | 10 +- .../utils/retrieve-workspace-files.ts | 52 +- packages/nx/src/utils/logger.ts | 5 + packages/nx/src/utils/nx-plugin.deprecated.ts | 12 +- packages/nx/src/utils/nx-plugin.ts | 531 ------------------ .../src/utils/plugins/plugin-capabilities.ts | 35 +- 56 files changed, 1253 insertions(+), 776 deletions(-) create mode 100644 packages/nx/src/project-graph/plugins/index.ts create mode 100644 packages/nx/src/project-graph/plugins/internal-api.ts create mode 100644 packages/nx/src/project-graph/plugins/messaging.ts create mode 100644 packages/nx/src/project-graph/plugins/plugin-pool.ts create mode 100644 packages/nx/src/project-graph/plugins/plugin-worker.ts create mode 100644 packages/nx/src/project-graph/plugins/public-api.ts create mode 100644 packages/nx/src/project-graph/plugins/worker-api.ts delete mode 100644 packages/nx/src/utils/nx-plugin.ts diff --git a/docs/generated/devkit/logger.md b/docs/generated/devkit/logger.md index 6ef004c9e6645..c2cf2a082d1eb 100644 --- a/docs/generated/devkit/logger.md +++ b/docs/generated/devkit/logger.md @@ -4,11 +4,12 @@ #### Type declaration -| Name | Type | -| :------ | :-------------------------- | -| `debug` | (...`s`: `any`[]) => `void` | -| `error` | (`s`: `any`) => `void` | -| `fatal` | (...`s`: `any`[]) => `void` | -| `info` | (`s`: `any`) => `void` | -| `log` | (...`s`: `any`[]) => `void` | -| `warn` | (`s`: `any`) => `void` | +| Name | Type | +| :-------- | :-------------------------- | +| `debug` | (...`s`: `any`[]) => `void` | +| `error` | (`s`: `any`) => `void` | +| `fatal` | (...`s`: `any`[]) => `void` | +| `info` | (`s`: `any`) => `void` | +| `log` | (...`s`: `any`[]) => `void` | +| `verbose` | (...`s`: `any`[]) => `void` | +| `warn` | (`s`: `any`) => `void` | diff --git a/packages/devkit/src/utils/convert-nx-executor.spec.ts b/packages/devkit/src/utils/convert-nx-executor.spec.ts index f4240aa0dbd8b..8e89f8d701eff 100644 --- a/packages/devkit/src/utils/convert-nx-executor.spec.ts +++ b/packages/devkit/src/utils/convert-nx-executor.spec.ts @@ -1,17 +1,22 @@ +import { requireNx } from '../../nx'; import { convertNxExecutor } from './convert-nx-executor'; +const { workspaceRoot } = requireNx(); + describe('Convert Nx Executor', () => { it('should convertNxExecutor to builder correctly and produce the same output', async () => { // ARRANGE const { schema } = require('@angular-devkit/core'); const { TestingArchitectHost, - } = require('@angular-devkit/architect/testing'); + // nx-ignore-next-line + } = require('@angular-devkit/architect/testing') as typeof import('@angular-devkit/architect/testing'); const { Architect } = require('@angular-devkit/architect'); const registry = new schema.CoreSchemaRegistry(); registry.addPostTransform(schema.transforms.addUndefinedDefaults); const testArchitectHost = new TestingArchitectHost(); + testArchitectHost.workspaceRoot = workspaceRoot; const architect = new Architect(testArchitectHost, registry); const convertedExecutor = convertNxExecutor(echoExecutor); diff --git a/packages/devkit/src/utils/convert-nx-executor.ts b/packages/devkit/src/utils/convert-nx-executor.ts index 3fcfd6bdc8b48..1e6f8c956e805 100644 --- a/packages/devkit/src/utils/convert-nx-executor.ts +++ b/packages/devkit/src/utils/convert-nx-executor.ts @@ -8,6 +8,7 @@ const { Workspaces, readNxJsonFromDisk, retrieveProjectConfigurationsWithAngularProjects, + shutdownPluginWorkers, } = requireNx(); /** @@ -38,6 +39,7 @@ export function convertNxExecutor(executor: Executor) { (workspaces as any).readProjectsConfigurations({ _includeProjectsFromAngularJson: true, }); + shutdownPluginWorkers?.(); const context: ExecutorContext = { root: builderContext.workspaceRoot, diff --git a/packages/nx/plugins/package-json.ts b/packages/nx/plugins/package-json.ts index 9ba453f652994..936ca3e8970a4 100644 --- a/packages/nx/plugins/package-json.ts +++ b/packages/nx/plugins/package-json.ts @@ -1,4 +1,4 @@ -import type { NxPluginV2 } from '../src/utils/nx-plugin'; +import type { NxPluginV2 } from '../src/project-graph/plugins'; import { workspaceRoot } from '../src/utils/workspace-root'; import { createNodeFromPackageJson } from '../src/plugins/package-json-workspaces'; diff --git a/packages/nx/src/adapter/angular-json.ts b/packages/nx/src/adapter/angular-json.ts index b509554da8874..585cd66c1e623 100644 --- a/packages/nx/src/adapter/angular-json.ts +++ b/packages/nx/src/adapter/angular-json.ts @@ -2,7 +2,7 @@ import { existsSync } from 'fs'; import * as path from 'path'; import { readJsonFile } from '../utils/fileutils'; import { ProjectsConfigurations } from '../config/workspace-json-project-json'; -import { NxPluginV2 } from '../utils/nx-plugin'; +import { NxPluginV2 } from '../project-graph/plugins'; export const NX_ANGULAR_JSON_PLUGIN_NAME = 'nx-angular-json-plugin'; @@ -16,6 +16,8 @@ export const NxAngularJsonPlugin: NxPluginV2 = { ], }; +export default NxAngularJsonPlugin; + export function shouldMergeAngularProjects( root: string, includeProjectsFromAngularJson: boolean diff --git a/packages/nx/src/adapter/ngcli-adapter.ts b/packages/nx/src/adapter/ngcli-adapter.ts index 1f1d137ad3e28..a0445d98c9287 100644 --- a/packages/nx/src/adapter/ngcli-adapter.ts +++ b/packages/nx/src/adapter/ngcli-adapter.ts @@ -59,7 +59,7 @@ import { ExecutorsJson, TaskGraphExecutor, } from '../config/misc-interfaces'; -import { readPluginPackageJson } from '../utils/nx-plugin'; +import { readPluginPackageJson } from '../project-graph/plugins'; import { getImplementationFactory, resolveImplementation, diff --git a/packages/nx/src/command-line/generate/generator-utils.ts b/packages/nx/src/command-line/generate/generator-utils.ts index 1fffe47dcbb00..20d57f4eff259 100644 --- a/packages/nx/src/command-line/generate/generator-utils.ts +++ b/packages/nx/src/command-line/generate/generator-utils.ts @@ -10,7 +10,7 @@ import { resolveSchema, } from '../../config/schema-utils'; import { readJsonFile } from '../../utils/fileutils'; -import { readPluginPackageJson } from '../../utils/nx-plugin'; +import { readPluginPackageJson } from '../../project-graph/plugins'; export function getGeneratorInformation( collectionName: string, diff --git a/packages/nx/src/command-line/run/executor-utils.ts b/packages/nx/src/command-line/run/executor-utils.ts index e12f7e752cd83..ecbac0c0f8bea 100644 --- a/packages/nx/src/command-line/run/executor-utils.ts +++ b/packages/nx/src/command-line/run/executor-utils.ts @@ -1,6 +1,6 @@ import { dirname, join } from 'path'; -import { readPluginPackageJson } from '../../utils/nx-plugin'; +import { readPluginPackageJson } from '../../project-graph/plugins'; import { CustomHasher, Executor, diff --git a/packages/nx/src/config/schema-utils.ts b/packages/nx/src/config/schema-utils.ts index 6c92129a643d7..30d7d083130f0 100644 --- a/packages/nx/src/config/schema-utils.ts +++ b/packages/nx/src/config/schema-utils.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs'; import { extname, join } from 'path'; -import { registerPluginTSTranspiler } from '../utils/nx-plugin'; +import { registerPluginTSTranspiler } from '../project-graph/plugins'; /** * This function is used to get the implementation factory of an executor or generator. diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts index 81d3d1a183171..95b2a2d636750 100644 --- a/packages/nx/src/config/workspaces.spec.ts +++ b/packages/nx/src/config/workspaces.spec.ts @@ -3,6 +3,7 @@ import { TempFs } from '../internal-testing-utils/temp-fs'; import { withEnvironmentVariables } from '../internal-testing-utils/with-environment'; import { retrieveProjectConfigurations } from '../project-graph/utils/retrieve-workspace-files'; import { readNxJson } from './configuration'; +import { shutdownPluginWorkers } from '../project-graph/plugins/plugin-pool'; const libConfig = (root, name?: string) => ({ name: name ?? toProjectName(`${root}/some-file`), @@ -48,10 +49,11 @@ describe('Workspaces', () => { const { projects } = await withEnvironmentVariables( { - NX_WORKSPACE_ROOT: fs.tempDir, + NX_WORKSPACE_ROOT_PATH: fs.tempDir, }, () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir)) ); + await shutdownPluginWorkers(); expect(projects['my-package']).toEqual({ name: 'my-package', root: 'packages/my-package', @@ -65,6 +67,7 @@ describe('Workspaces', () => { }, }, }); + await shutdownPluginWorkers(); }); }); }); diff --git a/packages/nx/src/devkit-exports.ts b/packages/nx/src/devkit-exports.ts index be21b376ce019..cd2b92ec18efe 100644 --- a/packages/nx/src/devkit-exports.ts +++ b/packages/nx/src/devkit-exports.ts @@ -47,16 +47,19 @@ export { workspaceLayout } from './config/configuration'; export type { NxPlugin, - NxPluginV1, NxPluginV2, - ProjectTargetConfigurator, CreateNodes, CreateNodesFunction, CreateNodesResult, CreateNodesContext, CreateDependencies, CreateDependenciesContext, -} from './utils/nx-plugin'; +} from './project-graph/plugins'; + +export type { + NxPluginV1, + ProjectTargetConfigurator, +} from './utils/nx-plugin.deprecated'; /** * @category Workspace diff --git a/packages/nx/src/devkit-internals.ts b/packages/nx/src/devkit-internals.ts index a24b33c98670a..cdb257674d0b4 100644 --- a/packages/nx/src/devkit-internals.ts +++ b/packages/nx/src/devkit-internals.ts @@ -21,3 +21,4 @@ export { findProjectForPath, } from './project-graph/utils/find-project-for-path'; export { registerTsProject } from './plugins/js/utils/register'; +export { shutdownPluginWorkers } from './project-graph/plugins/plugin-pool'; diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 0efb1d3bc0f10..7c595eb0fa72c 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -4,7 +4,7 @@ import { basename, join, relative } from 'path'; import { buildProjectConfigurationFromPackageJson, getGlobPatternsFromPackageManagerWorkspaces, - getNxPackageJsonWorkspacesPlugin, + createNodes as packageJsonWorkspacesCreateNodes, } from '../../plugins/package-json-workspaces'; import { buildProjectFromProjectJson, @@ -196,8 +196,8 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { ), ]; const projectGlobPatterns = configurationGlobs([ - { plugin: ProjectJsonProjectsPlugin }, - { plugin: getNxPackageJsonWorkspacesPlugin(tree.root) }, + ProjectJsonProjectsPlugin, + { createNodes: packageJsonWorkspacesCreateNodes }, ]); const globbedFiles = globWithWorkspaceContext(tree.root, projectGlobPatterns); const createdFiles = findCreatedProjectFiles(tree, patterns); diff --git a/packages/nx/src/migrations/update-15-1-0/set-project-names.ts b/packages/nx/src/migrations/update-15-1-0/set-project-names.ts index 24c9e426ac331..dd2a02f30aa6e 100644 --- a/packages/nx/src/migrations/update-15-1-0/set-project-names.ts +++ b/packages/nx/src/migrations/update-15-1-0/set-project-names.ts @@ -4,7 +4,7 @@ import { dirname } from 'path'; import { readJson, writeJson } from '../../generators/utils/json'; import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available'; import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files'; -import { loadNxPlugins } from '../../utils/nx-plugin'; +import { loadNxPlugins } from '../../project-graph/plugins/internal-api'; export default async function (tree: Tree) { const nxJson = readNxJson(tree); diff --git a/packages/nx/src/plugins/js/index.ts b/packages/nx/src/plugins/js/index.ts index 466340e1f2b28..a815762c523dc 100644 --- a/packages/nx/src/plugins/js/index.ts +++ b/packages/nx/src/plugins/js/index.ts @@ -9,7 +9,7 @@ import { CreateDependencies, CreateDependenciesContext, CreateNodes, -} from '../../utils/nx-plugin'; +} from '../../project-graph/plugins'; import { getLockFileDependencies, getLockFileName, diff --git a/packages/nx/src/plugins/js/lock-file/lock-file.ts b/packages/nx/src/plugins/js/lock-file/lock-file.ts index 017a92b5f3991..00165fe15aca1 100644 --- a/packages/nx/src/plugins/js/lock-file/lock-file.ts +++ b/packages/nx/src/plugins/js/lock-file/lock-file.ts @@ -37,7 +37,7 @@ import { import { pruneProjectGraph } from './project-graph-pruning'; import { normalizePackageJson } from './utils/package-json'; import { readJsonFile } from '../../../utils/fileutils'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; const YARN_LOCK_FILE = 'yarn.lock'; const NPM_LOCK_FILE = 'package-lock.json'; diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts index 2ebb0dbb3a8b7..817e100c023d7 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts @@ -8,7 +8,7 @@ import { pruneProjectGraph } from './project-graph-pruning'; import { vol } from 'memfs'; import { ProjectGraph } from '../../../config/project-graph'; import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; jest.mock('fs', () => { const memFs = require('memfs').fs; diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.ts index c048ce515c44b..00ce0290631a9 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.ts @@ -13,7 +13,7 @@ import { ProjectGraphExternalNode, } from '../../../config/project-graph'; import { hashArray } from '../../../hasher/file-hasher'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; /** * NPM diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts index bdbcea297d715..52ee056631895 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts @@ -11,7 +11,7 @@ import { ProjectGraphBuilder, RawProjectGraphDependency, } from '../../../project-graph/project-graph-builder'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; jest.mock('fs', () => { const memFs = require('memfs').fs; diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts index 8f6a6692ccaf3..068fddaae4e5e 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts @@ -25,7 +25,7 @@ import { ProjectGraphExternalNode, } from '../../../config/project-graph'; import { hashArray } from '../../../hasher/file-hasher'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; // we use key => node map to avoid duplicate work when parsing keys let keyMap = new Map(); diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts index 41e83ac85c542..2e920bb5c12ea 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts @@ -9,7 +9,7 @@ import { vol } from 'memfs'; import { ProjectGraph } from '../../../config/project-graph'; import { PackageJson } from '../../../utils/package-json'; import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; jest.mock('fs', () => { const memFs = require('memfs').fs; diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.ts index 7098220352778..d96f3e5125335 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.ts @@ -14,7 +14,7 @@ import { } from '../../../config/project-graph'; import { hashArray } from '../../../hasher/file-hasher'; import { sortObjectByKeys } from '../../../utils/object-sort'; -import { CreateDependenciesContext } from '../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../project-graph/plugins'; /** * Yarn diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts index 09d8da40396e6..87530bc624ca8 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/build-dependencies.ts @@ -1,6 +1,6 @@ import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies'; import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies'; -import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { RawProjectGraphDependency } from '../../../../project-graph/project-graph-builder'; export function buildExplicitDependencies( diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts index 5cf832fb6f26e..bd253f3c7ce91 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts @@ -6,7 +6,7 @@ import { buildExplicitPackageJsonDependencies } from './explicit-package-json-de import { ProjectGraphProjectNode } from '../../../../config/project-graph'; import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; import { createFileMap } from '../../../../project-graph/file-map-utils'; -import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { getAllFileDataInContext } from '../../../../utils/workspace-context'; describe('explicit package json dependencies', () => { diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts index 58ba45c8843b8..3724c4498a4a3 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.ts @@ -9,7 +9,7 @@ import { } from '../../../../config/workspace-json-project-json'; import { NxJsonConfiguration } from '../../../../config/nx-json'; import { PackageJson } from '../../../../utils/package-json'; -import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { RawProjectGraphDependency, validateDependency, diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts index 0cf2c22dd9d0b..a4b8d1b52dc69 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts @@ -4,12 +4,12 @@ const tempFs = new TempFs('explicit-project-deps'); import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies'; import { - retrieveProjectConfigurationPaths, retrieveProjectConfigurations, retrieveWorkspaceFiles, } from '../../../../project-graph/utils/retrieve-workspace-files'; -import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { setupWorkspaceContext } from '../../../../utils/workspace-context'; +import { shutdownPluginWorkers } from '../../../../project-graph/plugins/plugin-pool'; // projectName => tsconfig import path const dependencyProjectNamesToImportPaths = { @@ -23,6 +23,10 @@ describe('explicit project dependencies', () => { tempFs.reset(); }); + afterEach(async () => { + await shutdownPluginWorkers(); + }); + describe('static imports, dynamic imports, and commonjs requires', () => { it('should build explicit dependencies for static imports, and top-level dynamic imports and commonjs requires', async () => { const source = 'proj'; diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts index 948e917fd09b0..cffc478a3a685 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.ts @@ -6,7 +6,7 @@ import { import { join, relative } from 'path'; import { workspaceRoot } from '../../../../utils/workspace-root'; import { normalizePath } from '../../../../utils/path'; -import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../../../../project-graph/plugins'; import { RawProjectGraphDependency, validateDependency, diff --git a/packages/nx/src/plugins/package-json-workspaces/create-nodes.ts b/packages/nx/src/plugins/package-json-workspaces/create-nodes.ts index d1547b5deb09f..de78d677347a7 100644 --- a/packages/nx/src/plugins/package-json-workspaces/create-nodes.ts +++ b/packages/nx/src/plugins/package-json-workspaces/create-nodes.ts @@ -8,48 +8,42 @@ import { toProjectName } from '../../config/workspaces'; import { readJsonFile, readYamlFile } from '../../utils/fileutils'; import { combineGlobPatterns } from '../../utils/globs'; import { NX_PREFIX } from '../../utils/logger'; -import { NxPluginV2 } from '../../utils/nx-plugin'; import { output } from '../../utils/output'; import { PackageJson, readTargetsFromPackageJson, } from '../../utils/package-json'; import { joinPathFragments } from '../../utils/path'; - -export function getNxPackageJsonWorkspacesPlugin(root: string): NxPluginV2 { - const readJson = (f) => readJsonFile(join(root, f)); - const patterns = getGlobPatternsFromPackageManagerWorkspaces(root, readJson); - - // If the user only specified a negative pattern, we should find all package.json - // files and only return those that don't match a negative pattern. - const negativePatterns = patterns.filter((p) => p.startsWith('!')); - let positivePatterns = patterns.filter((p) => !p.startsWith('!')); - - if ( - // There are some negative patterns - negativePatterns.length > 0 && - // No positive patterns - (positivePatterns.length === 0 || - // Or only a single positive pattern that is the default coming from root package - (positivePatterns.length === 1 && positivePatterns[0] === 'package.json')) - ) { - positivePatterns.push('**/package.json'); - } - - return { - name: 'nx/core/package-json-workspaces', - createNodes: [ - combineGlobPatterns(positivePatterns), - (p) => { - if (!negativePatterns.some((negative) => minimatch(p, negative))) { - return createNodeFromPackageJson(p, root); - } - // A negative pattern matched, so we should not create a node for this package.json - return {}; - }, - ], - }; +import { workspaceRoot } from '../../utils/workspace-root'; +import { CreateNodes } from '../../project-graph/plugins'; + +const readJson = (f) => readJsonFile(join(workspaceRoot, f)); +const patterns = getGlobPatternsFromPackageManagerWorkspaces( + workspaceRoot, + readJson +); +const negativePatterns = patterns.filter((p) => p.startsWith('!')); +const positivePatterns = patterns.filter((p) => !p.startsWith('!')); +if ( + // There are some negative patterns + negativePatterns.length > 0 && + // No positive patterns + (positivePatterns.length === 0 || + // Or only a single positive pattern that is the default coming from root package + (positivePatterns.length === 1 && positivePatterns[0] === 'package.json')) +) { + positivePatterns.push('**/package.json'); } +export const createNodes: CreateNodes = [ + combineGlobPatterns(positivePatterns), + (p, _, { workspaceRoot }) => { + if (!negativePatterns.some((negative) => minimatch(p, negative))) { + return createNodeFromPackageJson(p, workspaceRoot); + } + // A negative pattern matched, so we should not create a node for this package.json + return {}; + }, +]; export function createNodeFromPackageJson(pkgJsonPath: string, root: string) { const json: PackageJson = readJsonFile(join(root, pkgJsonPath)); diff --git a/packages/nx/src/plugins/package-json-workspaces/index.ts b/packages/nx/src/plugins/package-json-workspaces/index.ts index e675dd81f1475..7ac34ae661e87 100644 --- a/packages/nx/src/plugins/package-json-workspaces/index.ts +++ b/packages/nx/src/plugins/package-json-workspaces/index.ts @@ -1 +1,2 @@ export * from './create-nodes'; +export const name = 'nx/core/package-json-workspaces'; diff --git a/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.spec.ts b/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.spec.ts index 688c85af1b493..e4b3dd0d81895 100644 --- a/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.spec.ts +++ b/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.spec.ts @@ -3,7 +3,7 @@ import * as memfs from 'memfs'; import '../../../internal-testing-utils/mock-fs'; import { PackageJsonProjectsNextToProjectJsonPlugin } from './package-json-next-to-project-json'; -import { CreateNodesContext } from '../../../utils/nx-plugin'; +import { CreateNodesContext } from '../../../project-graph/plugins'; const { createNodes } = PackageJsonProjectsNextToProjectJsonPlugin; describe('nx project.json plugin', () => { diff --git a/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.ts b/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.ts index bb241cf307a2b..19421aea54b1f 100644 --- a/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.ts +++ b/packages/nx/src/plugins/project-json/build-nodes/package-json-next-to-project-json.ts @@ -1,6 +1,6 @@ import { dirname, join } from 'path'; import { existsSync } from 'fs'; -import { NxPluginV2 } from '../../../utils/nx-plugin'; +import { NxPluginV2 } from '../../../project-graph/plugins'; import { readJsonFile } from '../../../utils/fileutils'; import { ProjectConfiguration } from '../../../config/workspace-json-project-json'; import { @@ -33,6 +33,8 @@ export const PackageJsonProjectsNextToProjectJsonPlugin: NxPluginV2 = { ], }; +export default PackageJsonProjectsNextToProjectJsonPlugin; + function createProjectFromPackageJsonNextToProjectJson( projectJsonPath: string, workspaceRoot: string diff --git a/packages/nx/src/plugins/project-json/build-nodes/project-json.spec.ts b/packages/nx/src/plugins/project-json/build-nodes/project-json.spec.ts index 138be521698e9..f6baa247e974d 100644 --- a/packages/nx/src/plugins/project-json/build-nodes/project-json.spec.ts +++ b/packages/nx/src/plugins/project-json/build-nodes/project-json.spec.ts @@ -3,7 +3,7 @@ import * as memfs from 'memfs'; import '../../../internal-testing-utils/mock-fs'; import { ProjectJsonProjectsPlugin } from './project-json'; -import { CreateNodesContext } from '../../../utils/nx-plugin'; +import { CreateNodesContext } from '../../../project-graph/plugins'; const { createNodes } = ProjectJsonProjectsPlugin; describe('nx project.json plugin', () => { diff --git a/packages/nx/src/plugins/project-json/build-nodes/project-json.ts b/packages/nx/src/plugins/project-json/build-nodes/project-json.ts index 90048a133f8b6..9dfc44fcbc262 100644 --- a/packages/nx/src/plugins/project-json/build-nodes/project-json.ts +++ b/packages/nx/src/plugins/project-json/build-nodes/project-json.ts @@ -3,7 +3,7 @@ import { dirname, join } from 'node:path'; import { ProjectConfiguration } from '../../../config/workspace-json-project-json'; import { toProjectName } from '../../../config/workspaces'; import { readJsonFile } from '../../../utils/fileutils'; -import { NxPluginV2 } from '../../../utils/nx-plugin'; +import { NxPluginV2 } from '../../../project-graph/plugins'; export const ProjectJsonProjectsPlugin: NxPluginV2 = { name: 'nx/core/project-json', @@ -23,6 +23,8 @@ export const ProjectJsonProjectsPlugin: NxPluginV2 = { ], }; +export default ProjectJsonProjectsPlugin; + export function buildProjectFromProjectJson( json: Partial, path: string diff --git a/packages/nx/src/plugins/target-defaults/target-defaults-plugin.spec.ts b/packages/nx/src/plugins/target-defaults/target-defaults-plugin.spec.ts index 11370e8c934be..1ec89af52f174 100644 --- a/packages/nx/src/plugins/target-defaults/target-defaults-plugin.spec.ts +++ b/packages/nx/src/plugins/target-defaults/target-defaults-plugin.spec.ts @@ -3,7 +3,7 @@ import * as memfs from 'memfs'; import '../../../src/internal-testing-utils/mock-fs'; import { getTargetInfo, TargetDefaultsPlugin } from './target-defaults-plugin'; -import { CreateNodesContext } from '../../utils/nx-plugin'; +import { CreateNodesContext } from '../../project-graph/plugins'; const { createNodes: [, createNodesFn], } = TargetDefaultsPlugin; diff --git a/packages/nx/src/plugins/target-defaults/target-defaults-plugin.ts b/packages/nx/src/plugins/target-defaults/target-defaults-plugin.ts index 1fddbe19f790b..5266bbceee68c 100644 --- a/packages/nx/src/plugins/target-defaults/target-defaults-plugin.ts +++ b/packages/nx/src/plugins/target-defaults/target-defaults-plugin.ts @@ -8,7 +8,7 @@ import { } from '../../config/workspace-json-project-json'; import { readJsonFile } from '../../utils/fileutils'; import { combineGlobPatterns } from '../../utils/globs'; -import { NxPluginV2 } from '../../utils/nx-plugin'; +import { NxPluginV2 } from '../../project-graph/plugins'; import { PackageJson, readTargetsFromPackageJson, @@ -116,6 +116,8 @@ export const TargetDefaultsPlugin: NxPluginV2 = { ], }; +export default TargetDefaultsPlugin; + function getExecutorToTargetMap( packageJsonTargets: Record, projectJsonTargets: Record diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.spec.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.spec.ts index 2c8d3078cc9bc..31578fd73cb10 100644 --- a/packages/nx/src/project-graph/affected/locators/project-glob-changes.spec.ts +++ b/packages/nx/src/project-graph/affected/locators/project-glob-changes.spec.ts @@ -1,7 +1,7 @@ import { ProjectGraphProjectNode } from '../../../config/project-graph'; import { ProjectConfiguration } from '../../../config/workspace-json-project-json'; -import * as nxPlugin from '../../../utils/nx-plugin'; +import * as nxPlugin from '../../../project-graph/plugins'; import { DeletedFileChange } from '../../file-utils'; import { getTouchedProjectsFromProjectGlobChanges } from './project-glob-changes'; diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts index afd7c4f8d7db0..32c126cc8b813 100644 --- a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts +++ b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts @@ -1,23 +1,16 @@ import { TouchedProjectLocator } from '../affected-project-graph-models'; import { minimatch } from 'minimatch'; import { workspaceRoot } from '../../../utils/workspace-root'; -import { getNxRequirePaths } from '../../../utils/installation-directory'; import { join } from 'path'; import { existsSync } from 'fs'; import { configurationGlobs } from '../../utils/retrieve-workspace-files'; -import { loadNxPlugins } from '../../../utils/nx-plugin'; +import { loadNxPlugins } from '../../plugins/internal-api'; import { combineGlobPatterns } from '../../../utils/globs'; export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator = async (touchedFiles, projectGraphNodes, nxJson): Promise => { const globPattern = combineGlobPatterns( - configurationGlobs( - await loadNxPlugins( - nxJson?.plugins, - getNxRequirePaths(workspaceRoot), - workspaceRoot - ) - ) + configurationGlobs(await loadNxPlugins(nxJson?.plugins, workspaceRoot)) ); const touchedProjects = new Set(); diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index c0c57848864a8..e25815ce13e5d 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -14,11 +14,11 @@ import { import { applyImplicitDependencies } from './utils/implicit-project-dependencies'; import { normalizeProjectNodes } from './utils/normalize-project-nodes'; import { - CreateDependenciesContext, isNxPluginV1, isNxPluginV2, loadNxPlugins, -} from '../utils/nx-plugin'; +} from './plugins/internal-api'; +import { CreateDependenciesContext } from './plugins'; import { getRootTsConfigPath } from '../plugins/js/utils/typescript'; import { FileMap, @@ -35,6 +35,7 @@ import { PackageJson } from '../utils/package-json'; import { getNxRequirePaths } from '../utils/installation-directory'; import { output } from '../utils/output'; import { ExternalObject, NxWorkspaceFilesExternals } from '../native'; +import { shutdownPluginWorkers } from './plugins/plugin-pool'; let storedFileMap: FileMap | null = null; let storedAllWorkspaceFiles: FileData[] | null = null; @@ -131,6 +132,7 @@ export async function buildProjectGraphUsingProjectFileMap( if (shouldWriteCache) { writeCache(projectFileMapCache, projectGraph); } + await shutdownPluginWorkers(); return { projectGraph, projectFileMapCache, @@ -240,12 +242,10 @@ async function updateProjectGraphWithPlugins( ) { const plugins = await loadNxPlugins( context.nxJsonConfiguration?.plugins, - getNxRequirePaths(), - context.workspaceRoot, - context.projects + context.workspaceRoot ); let graph = initProjectGraph; - for (const { plugin } of plugins) { + for (const plugin of plugins) { try { if ( isNxPluginV1(plugin) && @@ -297,17 +297,18 @@ async function updateProjectGraphWithPlugins( ); const createDependencyPlugins = plugins.filter( - ({ plugin }) => isNxPluginV2(plugin) && plugin.createDependencies + (plugin) => isNxPluginV2(plugin) && plugin.createDependencies ); await Promise.all( - createDependencyPlugins.map(async ({ plugin, options }) => { + createDependencyPlugins.map(async (plugin) => { performance.mark(`${plugin.name}:createDependencies - start`); // Set this globally to allow plugins to know if they are being called from the project graph creation global.NX_GRAPH_CREATION = true; try { - const dependencies = await plugin.createDependencies(options, { + // TODO: we shouldn't have to pass null here + const dependencies = await plugin.createDependencies(null, { ...context, }); diff --git a/packages/nx/src/project-graph/file-utils.ts b/packages/nx/src/project-graph/file-utils.ts index c07c9edfe8221..86d2aedc35922 100644 --- a/packages/nx/src/project-graph/file-utils.ts +++ b/packages/nx/src/project-graph/file-utils.ts @@ -27,7 +27,6 @@ import { getDefaultPluginsSync } from '../utils/nx-plugin.deprecated'; import { minimatch } from 'minimatch'; import { CreateNodesResult } from '../devkit-exports'; import { PackageJsonProjectsNextToProjectJsonPlugin } from '../plugins/project-json/build-nodes/package-json-next-to-project-json'; -import { LoadedNxPlugin } from '../utils/nx-plugin'; export interface Change { type: string; @@ -184,9 +183,9 @@ export { readNxJson, workspaceLayout } from '../config/configuration'; function getProjectsSyncNoInference(root: string, nxJson: NxJsonConfiguration) { const projectFiles = retrieveProjectConfigurationPaths( root, - getDefaultPluginsSync(root) + getDefaultPluginsSync(root).map((p) => p.plugin) ); - const plugins: LoadedNxPlugin[] = [ + const plugins = [ { plugin: PackageJsonProjectsNextToProjectJsonPlugin }, ...getDefaultPluginsSync(root), ]; @@ -194,17 +193,21 @@ function getProjectsSyncNoInference(root: string, nxJson: NxJsonConfiguration) { const projectRootMap: Map = new Map(); // We iterate over plugins first - this ensures that plugins specified first take precedence. - for (const { plugin, options } of plugins) { + for (const { plugin } of plugins) { const [pattern, createNodes] = plugin.createNodes ?? []; if (!pattern) { continue; } for (const file of projectFiles) { if (minimatch(file, pattern, { dot: true })) { - let r = createNodes(file, options, { - nxJsonConfiguration: nxJson, - workspaceRoot: root, - }) as CreateNodesResult; + let r = createNodes( + file, + {}, + { + nxJsonConfiguration: nxJson, + workspaceRoot: root, + } + ) as CreateNodesResult; for (const node in r.projects) { const project = { root: node, diff --git a/packages/nx/src/project-graph/plugins/index.ts b/packages/nx/src/project-graph/plugins/index.ts new file mode 100644 index 0000000000000..0c939a25f6db0 --- /dev/null +++ b/packages/nx/src/project-graph/plugins/index.ts @@ -0,0 +1,6 @@ +export * from './public-api'; + +export { + readPluginPackageJson, + registerPluginTSTranspiler, +} from './worker-api'; diff --git a/packages/nx/src/project-graph/plugins/internal-api.ts b/packages/nx/src/project-graph/plugins/internal-api.ts new file mode 100644 index 0000000000000..e71f7519db7ab --- /dev/null +++ b/packages/nx/src/project-graph/plugins/internal-api.ts @@ -0,0 +1,137 @@ +// This file contains the bits and bobs of the internal API for loading and interacting with Nx plugins. +// For the public API, used by plugin authors, see `./public-api.ts`. + +import { join, dirname } from 'path'; + +import { toProjectName } from '../../config/workspaces'; +import { combineGlobPatterns } from '../../utils/globs'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { PluginConfiguration } from '../../config/nx-json'; +import { NxPluginV1 } from '../../utils/nx-plugin.deprecated'; +import { shouldMergeAngularProjects } from '../../adapter/angular-json'; + +import { loadRemoteNxPlugin } from './plugin-pool'; +import { + CreateNodesContext, + CreateNodesResult, + NxPlugin, + NxPluginV2, +} from './public-api'; + +export type CreateNodesResultWithContext = CreateNodesResult & { + file: string; + pluginName: string; +}; + +export type NormalizedPlugin = NxPluginV2 & + Pick; + +// This represents a plugin loaded in a plugin-worker. This is not an API for plugin authors, +// rather an internal representation of how to interact with a loaded plugin. +export type RemotePlugin = + // A remote plugin is a v2 plugin, with a slightly different API for create nodes. + Omit & { + createNodes: [ + filePattern: string, + // The create nodes function takes all matched files instead of just one, and includes + // the result's context. + fn: ( + matchedFiles: string[], + context: CreateNodesContext + ) => Promise + ]; + }; + +// Short lived cache (cleared between cmd runs) +// holding resolved nx plugin objects. +// Allows loaded plugins to not be reloaded when +// referenced multiple times. +export const nxPluginCache: Map = new Map(); + +export async function loadNxPlugins( + plugins: PluginConfiguration[], + root = workspaceRoot +): Promise { + const result: Promise[] = []; + + plugins ??= []; + + plugins.unshift( + join( + __dirname, + '../../plugins/project-json/build-nodes/package-json-next-to-project-json' + ) + ); + + // We push the nx core node plugins onto the end, s.t. it overwrites any other plugins + plugins.push(...(await getDefaultPlugins(root))); + + for (const plugin of plugins) { + result.push(loadNxPlugin(plugin, root)); + } + + return Promise.all(result); +} + +export async function loadNxPlugin( + plugin: PluginConfiguration, + root = workspaceRoot +): Promise { + const cacheKey = JSON.stringify(plugin); + + if (nxPluginCache.has(cacheKey)) { + return nxPluginCache.get(cacheKey)!; + } + + const loadedPlugin = await loadRemoteNxPlugin(plugin, root); + nxPluginCache.set(cacheKey, loadedPlugin); + return loadedPlugin; +} + +export function isNxPluginV2(plugin: NxPlugin): plugin is NxPluginV2 { + return 'createNodes' in plugin || 'createDependencies' in plugin; +} + +export function isNxPluginV1( + plugin: NxPlugin | RemotePlugin +): plugin is NxPluginV1 { + return 'processProjectGraph' in plugin || 'projectFilePatterns' in plugin; +} + +export function normalizeNxPlugin(plugin: NxPlugin): NormalizedPlugin { + if (isNxPluginV2(plugin)) { + return plugin; + } + if (isNxPluginV1(plugin) && plugin.projectFilePatterns) { + return { + ...plugin, + createNodes: [ + `*/**/${combineGlobPatterns(plugin.projectFilePatterns)}`, + (configFilePath) => { + const root = dirname(configFilePath); + return { + projects: { + [root]: { + name: toProjectName(configFilePath), + targets: plugin.registerProjectTargets?.(configFilePath), + }, + }, + }; + }, + ], + }; + } + return plugin; +} + +export async function getDefaultPlugins(root: string) { + return [ + join(__dirname, '../../plugins/js'), + join(__dirname, '../../plugins/target-defaults/target-defaults-plugin'), + ...(shouldMergeAngularProjects(root, false) + ? [join(__dirname, '../../adapter/angular-json')] + : []), + join(__dirname, '../../plugins/package-json-workspaces'), + join(__dirname, '../../plugins/project-json/build-nodes/project-json'), + ]; +} diff --git a/packages/nx/src/project-graph/plugins/messaging.ts b/packages/nx/src/project-graph/plugins/messaging.ts new file mode 100644 index 0000000000000..9bd277997a9cf --- /dev/null +++ b/packages/nx/src/project-graph/plugins/messaging.ts @@ -0,0 +1,150 @@ +import { + ProjectGraph, + ProjectGraphProcessorContext, +} from '../../config/project-graph'; +import { PluginConfiguration } from '../../config/nx-json'; +import { CreateDependenciesContext, CreateNodesContext } from './public-api'; +import { RemotePlugin } from './internal-api'; + +export interface PluginWorkerLoadMessage { + type: 'load'; + payload: { + plugin: PluginConfiguration; + root: string; + }; +} + +export interface PluginWorkerLoadResult { + type: 'load-result'; + payload: + | { + name: string; + createNodesPattern: string; + hasCreateDependencies: boolean; + hasProcessProjectGraph: boolean; + success: true; + } + | { + success: false; + error: string; + }; +} + +export interface PluginWorkerShutdownMessage { + type: 'shutdown'; + payload: undefined; +} + +export interface PluginWorkerCreateNodesMessage { + type: 'createNodes'; + payload: { + configFiles: string[]; + context: CreateNodesContext; + }; +} + +export interface PluginWorkerCreateNodesResult { + type: 'createNodesResult'; + payload: + | { + success: true; + result: Awaited>; + } + | { + success: false; + error: string; + }; +} + +export interface PluginCreateDependenciesMessage { + type: 'createDependencies'; + payload: { + context: CreateDependenciesContext; + }; +} + +export interface PluginCreateDependenciesResult { + type: 'createDependenciesResult'; + payload: + | { + dependencies: ReturnType; + success: true; + } + | { + success: false; + error: string; + }; +} + +export interface PluginWorkerProcessProjectGraphMessage { + type: 'processProjectGraph'; + payload: { + graph: ProjectGraph; + ctx: ProjectGraphProcessorContext; + }; +} + +export interface PluginWorkerProcessProjectGraphResult { + type: 'processProjectGraphResult'; + payload: + | { + graph: ProjectGraph; + success: true; + } + | { + success: false; + error: string; + }; +} + +export type PluginWorkerMessage = + | PluginWorkerLoadMessage + | PluginWorkerShutdownMessage + | PluginWorkerCreateNodesMessage + | PluginCreateDependenciesMessage + | PluginWorkerProcessProjectGraphMessage; + +export type PluginWorkerResult = + | PluginWorkerLoadResult + | PluginWorkerCreateNodesResult + | PluginCreateDependenciesResult + | PluginWorkerProcessProjectGraphResult; + +type MaybePromise = T | Promise; + +// The handler can return a message to be sent back to the process from which the message originated +type MessageHandlerReturn = + T extends PluginWorkerResult + ? MaybePromise + : MaybePromise; + +// Takes a message and a map of handlers and calls the appropriate handler +// type safe and requires all handlers to be handled +export async function consumeMessage< + T extends PluginWorkerMessage | PluginWorkerResult +>( + raw: string | T, + handlers: { + [K in T['type']]: ( + // Extract restricts the type of payload to the payload of the message with the type K + payload: Extract['payload'] + ) => MessageHandlerReturn; + } +) { + const message: T = typeof raw === 'string' ? JSON.parse(raw) : raw; + const handler = handlers[message.type]; + if (handler) { + const response = await handler(message.payload); + if (response) { + process.send!(createMessage(response)); + } + } else { + throw new Error(`Unhandled message type: ${message.type}`); + } +} + +export function createMessage( + message: PluginWorkerMessage | PluginWorkerResult +): string { + return JSON.stringify(message); +} diff --git a/packages/nx/src/project-graph/plugins/plugin-pool.ts b/packages/nx/src/project-graph/plugins/plugin-pool.ts new file mode 100644 index 0000000000000..36746634ddcbf --- /dev/null +++ b/packages/nx/src/project-graph/plugins/plugin-pool.ts @@ -0,0 +1,221 @@ +import { ChildProcess, fork } from 'child_process'; +import path = require('path'); + +import { PluginConfiguration } from '../../config/nx-json'; +import { ProjectGraph } from '../../config/project-graph'; +import { logger } from '../../utils/logger'; + +import { RemotePlugin, nxPluginCache } from './internal-api'; +import { PluginWorkerResult, consumeMessage, createMessage } from './messaging'; + +const pool: ChildProcess[] = []; + +const pidMap = new Map(); + +export function loadRemoteNxPlugin(plugin: PluginConfiguration, root: string) { + // this should only really be true when running unit tests within + // the Nx repo. We still need to start the worker in this case, + // but its typescript. + const isWorkerTypescript = path.extname(__filename) === '.ts'; + const workerPath = path.join(__dirname, 'plugin-worker'); + const worker = fork(workerPath, [], { + stdio: ['ignore', 'inherit', 'inherit', 'ipc'], + env: { + ...process.env, + ...(isWorkerTypescript + ? { + // Ensures that the worker uses the same tsconfig as the main process + TS_NODE_PROJECT: path.join(__dirname, '../../../tsconfig.lib.json'), + } + : {}), + }, + execArgv: [ + ...process.execArgv, + // If the worker is typescript, we need to register ts-node + ...(isWorkerTypescript ? ['-r', 'ts-node/register'] : []), + ], + }); + worker.send(createMessage({ type: 'load', payload: { plugin, root } })); + pool.push(worker); + + logger.verbose(`[plugin-worker] started worker: ${worker.pid}`); + + return new Promise((res, rej) => { + worker.on('message', createWorkerHandler(worker, res, rej)); + worker.on('exit', () => workerOnExitHandler(worker)); + }); +} + +let pluginWorkersShutdown = false; + +export async function shutdownPluginWorkers() { + // Clears the plugin cache so no refs to the workers are held + nxPluginCache.clear(); + + // Marks the workers as shutdown so that we don't report unexpected exits + pluginWorkersShutdown = true; + + const promises = []; + + for (const p of pool) { + p.send(createMessage({ type: 'shutdown', payload: undefined }), (error) => { + if (error) { + // This occurs when the worker is already dead, and we can ignore it + } else { + promises.push( + // Create a promise that resolves when the worker exits + new Promise((res, rej) => { + p.once('exit', () => res()); + }) + ); + } + }); + } + return Promise.all(promises); +} + +/** + * Creates a message handler for the given worker. + * @param worker Instance of plugin-worker + * @param onload Resolver for RemotePlugin promise + * @param onloadError Rejecter for RemotePlugin promise + * @returns Function to handle messages from the worker + */ +function createWorkerHandler( + worker: ChildProcess, + onload: (plugin: RemotePlugin) => void, + onloadError: (err?: unknown) => void +) { + // We store resolver and rejecter functions in the outer scope so that we can + // resolve/reject the promise from the message handler. The flow is something like: + // 1. plugin api called + // 2. remote plugin sends message to worker, creates promise and stores resolver/rejecter + // 3. worker performs API request + // 4. worker sends result back to main process + // 5. main process resolves/rejects promise based on result + + let createNodesResolver: ( + result: Awaited> + ) => void | undefined; + let createNodesRejecter: (err: unknown) => void | undefined; + let createDependenciesResolver: ( + result: ReturnType + ) => void | undefined; + let createDependenciesRejecter: (err: unknown) => void | undefined; + let processProjectGraphResolver: (updatedGraph: ProjectGraph) => void; + let processProjectGraphRejecter: (err: unknown) => void | undefined; + + let pluginName: string; + + return function (message: string) { + const parsed = JSON.parse(message); + logger.verbose( + `[plugin-pool] received message: ${parsed.type} from ${ + pluginName ?? worker.pid + }` + ); + consumeMessage(parsed, { + 'load-result': (result) => { + if (result.success) { + const { name, createNodesPattern } = result; + pluginName = name; + pidMap.set(worker.pid, name); + onload({ + name, + createNodes: createNodesPattern + ? [ + createNodesPattern, + (configFiles, ctx) => { + return new Promise((res, rej) => { + worker.send( + createMessage({ + type: 'createNodes', + payload: { configFiles, context: ctx }, + }) + ); + createNodesResolver = res; + createNodesRejecter = rej; + }); + }, + ] + : undefined, + createDependencies: result.hasCreateDependencies + ? (opts, ctx) => { + return new Promise((res, rej) => { + worker.send( + createMessage({ + type: 'createDependencies', + payload: { context: ctx }, + }) + ); + createDependenciesResolver = res; + createDependenciesRejecter = rej; + }); + } + : undefined, + processProjectGraph: result.hasProcessProjectGraph + ? (graph, ctx) => { + return new Promise((res, rej) => { + worker.send( + createMessage({ + type: 'processProjectGraph', + payload: { graph, ctx }, + }) + ); + processProjectGraphResolver = res; + processProjectGraphRejecter = rej; + }); + } + : undefined, + }); + } else if (result.success === false) { + onloadError(result.error); + } + }, + createDependenciesResult: (result) => { + if (result.success) { + createDependenciesResolver(result.dependencies); + createDependenciesResolver = undefined; + } else if (result.success === false) { + createDependenciesRejecter(result.error); + createDependenciesRejecter = undefined; + } + }, + createNodesResult: (payload) => { + if (payload.success) { + createNodesResolver(payload.result); + createNodesResolver = undefined; + } else if (payload.success === false) { + createNodesRejecter(payload.error); + createNodesRejecter = undefined; + } + }, + processProjectGraphResult: (result) => { + if (result.success) { + processProjectGraphResolver(result.graph); + processProjectGraphResolver = undefined; + } else if (result.success === false) { + processProjectGraphRejecter(result.error); + processProjectGraphRejecter = undefined; + } + }, + }); + }; +} + +function workerOnExitHandler(worker: ChildProcess) { + return () => { + if (!pluginWorkersShutdown) { + shutdownPluginWorkers(); + throw new Error( + `[Nx] plugin worker ${ + pidMap.get(worker.pid) ?? worker.pid + } exited unexpectedly` + ); + } + }; +} + +process.on('exit', () => { + shutdownPluginWorkers(); +}); diff --git a/packages/nx/src/project-graph/plugins/plugin-worker.ts b/packages/nx/src/project-graph/plugins/plugin-worker.ts new file mode 100644 index 0000000000000..08dc0cd84de55 --- /dev/null +++ b/packages/nx/src/project-graph/plugins/plugin-worker.ts @@ -0,0 +1,155 @@ +import { getNxRequirePaths } from '../../utils/installation-directory'; +import { loadNxPluginAsync } from './worker-api'; +import { PluginWorkerMessage, consumeMessage } from './messaging'; +import { PluginConfiguration } from '../../config/nx-json'; +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { retrieveProjectConfigurationsWithoutPluginInference } from '../utils/retrieve-workspace-files'; +import { CreateNodesResultWithContext, NormalizedPlugin } from './internal-api'; +import { CreateNodesContext } from './public-api'; + +global.NX_GRAPH_CREATION = true; + +let plugin: NormalizedPlugin; +let pluginOptions: unknown; + +process.on('message', async (message: string) => { + consumeMessage(message, { + load: async ({ plugin: pluginConfiguration, root }) => { + process.chdir(root); + try { + ({ plugin, options: pluginOptions } = await loadPluginFromWorker( + pluginConfiguration, + root + )); + return { + type: 'load-result', + payload: { + name: plugin.name, + createNodesPattern: plugin.createNodes?.[0], + hasCreateDependencies: + 'createDependencies' in plugin && !!plugin.createDependencies, + hasProcessProjectGraph: + 'processProjectGraph' in plugin && !!plugin.processProjectGraph, + success: true, + }, + }; + } catch (e) { + return { + type: 'load-result', + payload: { + success: false, + error: `Could not load plugin ${plugin} \n ${ + e instanceof Error ? e.stack : '' + }`, + }, + }; + } + }, + shutdown: async () => { + process.exit(0); + }, + createNodes: async ({ configFiles, context }) => { + try { + const result = await runCreateNodesInParallel(configFiles, context); + return { + type: 'createNodesResult', + payload: { result, success: true }, + }; + } catch (e) { + return { + type: 'createNodesResult', + payload: { success: false, error: e.stack }, + }; + } + }, + createDependencies: async (payload) => { + try { + const result = await plugin.createDependencies( + pluginOptions, + payload.context + ); + return { + type: 'createDependenciesResult', + payload: { dependencies: result, success: true }, + }; + } catch (e) { + return { + type: 'createDependenciesResult', + payload: { success: false, error: e.stack }, + }; + } + }, + processProjectGraph: async ({ graph, ctx }) => { + try { + const result = await plugin.processProjectGraph(graph, ctx); + return { + type: 'processProjectGraphResult', + payload: { graph: result, success: true }, + }; + } catch (e) { + return { + type: 'processProjectGraphResult', + payload: { success: false, error: e.stack }, + }; + } + }, + }); +}); + +let projectsWithoutInference: Record; + +async function loadPluginFromWorker(plugin: PluginConfiguration, root: string) { + try { + require.resolve(typeof plugin === 'string' ? plugin : plugin.plugin); + } catch { + // If a plugin cannot be resolved, we will need projects to resolve it + projectsWithoutInference ??= + await retrieveProjectConfigurationsWithoutPluginInference(root); + } + return await loadNxPluginAsync( + plugin, + getNxRequirePaths(root), + projectsWithoutInference, + root + ); +} + +function runCreateNodesInParallel( + configFiles: string[], + context: CreateNodesContext +): Promise { + const promises: Array< + CreateNodesResultWithContext | Promise + > = configFiles.map((file) => { + performance.mark(`${plugin.name}:createNodes:${file} - start`); + const value = plugin.createNodes[1](file, pluginOptions, context); + if (value instanceof Promise) { + return value + .catch((e) => { + performance.mark(`${plugin.name}:createNodes:${file} - end`); + throw new Error( + `Unable to create nodes for ${file} using plugin ${plugin.name}.`, + e + ); + }) + .then((r) => { + performance.mark(`${plugin.name}:createNodes:${file} - end`); + performance.measure( + `${plugin.name}:createNodes:${file}`, + `${plugin.name}:createNodes:${file} - start`, + `${plugin.name}:createNodes:${file} - end` + ); + return { ...r, pluginName: plugin.name, file }; + }); + } else { + performance.mark(`${plugin.name}:createNodes:${file} - end`); + performance.measure( + `${plugin.name}:createNodes:${file}`, + `${plugin.name}:createNodes:${file} - start`, + `${plugin.name}:createNodes:${file} - end` + ); + return { ...value, pluginName: plugin.name, file }; + } + }); + return Promise.all(promises); +} diff --git a/packages/nx/src/project-graph/plugins/public-api.ts b/packages/nx/src/project-graph/plugins/public-api.ts new file mode 100644 index 0000000000000..917853ca37de4 --- /dev/null +++ b/packages/nx/src/project-graph/plugins/public-api.ts @@ -0,0 +1,119 @@ +// This file represents the public API for plugins which live in nx.json's plugins array. +// For methods to interact with plugins from within Nx, see `./internal-api.ts`. + +import { NxPluginV1 } from '../../utils/nx-plugin.deprecated'; +import { + FileMap, + ProjectGraph, + ProjectGraphExternalNode, +} from '../../config/project-graph'; + +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; + +import { NxJsonConfiguration } from '../../config/nx-json'; +import { RawProjectGraphDependency } from '../project-graph-builder'; + +/** + * Context for {@link CreateNodesFunction} + */ +export interface CreateNodesContext { + readonly nxJsonConfiguration: NxJsonConfiguration; + readonly workspaceRoot: string; +} + +/** + * A function which parses a configuration file into a set of nodes. + * Used for creating nodes for the {@link ProjectGraph} + */ +export type CreateNodesFunction = ( + projectConfigurationFile: string, + options: T | undefined, + context: CreateNodesContext +) => CreateNodesResult | Promise; + +export type Optional = Omit & Partial>; + +export interface CreateNodesResult { + /** + * A map of project root -> project configuration + */ + projects?: Record>; + + /** + * A map of external node name -> external node. External nodes do not have a root, so the key is their name. + */ + externalNodes?: Record; +} + +/** + * A pair of file patterns and {@link CreateNodesFunction} + */ +export type CreateNodes = readonly [ + projectFilePattern: string, + createNodesFunction: CreateNodesFunction +]; + +/** + * Context for {@link CreateDependencies} + */ +export interface CreateDependenciesContext { + /** + * The external nodes that have been added to the graph. + */ + readonly externalNodes: ProjectGraph['externalNodes']; + + /** + * The configuration of each project in the workspace. + */ + readonly projects: Record; + + /** + * The `nx.json` configuration from the workspace + */ + readonly nxJsonConfiguration: NxJsonConfiguration; + + /** + * All files in the workspace + */ + readonly fileMap: FileMap; + + /** + * Files changes since last invocation + */ + readonly filesToProcess: FileMap; + + readonly workspaceRoot: string; +} + +/** + * A function which parses files in the workspace to create dependencies in the {@link ProjectGraph} + * Use {@link validateDependency} to validate dependencies + */ +export type CreateDependencies = ( + options: T | undefined, + context: CreateDependenciesContext +) => RawProjectGraphDependency[] | Promise; + +/** + * A plugin for Nx which creates nodes and dependencies for the {@link ProjectGraph} + */ +export type NxPluginV2 = { + name: string; + + /** + * Provides a file pattern and function that retrieves configuration info from + * those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFile } + */ + createNodes?: CreateNodes; + + // Todo(@AgentEnder): This shouldn't be a full processor, since its only responsible for defining edges between projects. What do we want the API to be? + /** + * Provides a function to analyze files to create dependencies for the {@link ProjectGraph} + */ + createDependencies?: CreateDependencies; +}; + +/** + * A plugin for Nx + */ +export type NxPlugin = NxPluginV1 | NxPluginV2; diff --git a/packages/nx/src/project-graph/plugins/worker-api.ts b/packages/nx/src/project-graph/plugins/worker-api.ts new file mode 100644 index 0000000000000..dc92c6fa2a52e --- /dev/null +++ b/packages/nx/src/project-graph/plugins/worker-api.ts @@ -0,0 +1,264 @@ +// This file contains methods and utilities that should **only** be used by the plugin worker. + +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { PluginConfiguration } from '../../config/nx-json'; + +import { join } from 'node:path/posix'; +import { getNxRequirePaths } from '../../utils/installation-directory'; +import { + PackageJson, + readModulePackageJsonWithoutFallbacks, +} from '../../utils/package-json'; +import { readJsonFile } from '../../utils/fileutils'; +import path = require('node:path/posix'); +import { workspaceRoot } from '../../utils/workspace-root'; +import { existsSync } from 'node:fs'; +import { readTsConfig } from '../../utils/typescript'; +import { + registerTranspiler, + registerTsConfigPaths, +} from '../../plugins/js/utils/register'; +import { + createProjectRootMappingsFromProjectConfigurations, + findProjectForPath, +} from '../utils/find-project-for-path'; +import { normalizePath } from '../../utils/path'; +import { logger } from '../../utils/logger'; + +import type * as ts from 'typescript'; +import { extname } from 'node:path'; +import { NormalizedPlugin, normalizeNxPlugin } from './internal-api'; +import { NxPlugin } from './public-api'; + +export type LoadedNxPlugin = { + plugin: NxPlugin; + options?: unknown; +}; + +export async function loadNxPluginAsync( + pluginConfiguration: PluginConfiguration, + paths: string[], + projects: Record, + root: string +): Promise { + const { plugin: moduleName, options } = + typeof pluginConfiguration === 'object' + ? pluginConfiguration + : { plugin: pluginConfiguration, options: undefined }; + + performance.mark(`Load Nx Plugin: ${moduleName} - start`); + let { pluginPath, name } = await getPluginPathAndName( + moduleName, + paths, + projects, + root + ); + const plugin = normalizeNxPlugin(await importPluginModule(pluginPath)); + plugin.name ??= name; + performance.mark(`Load Nx Plugin: ${moduleName} - end`); + performance.measure( + `Load Nx Plugin: ${moduleName}`, + `Load Nx Plugin: ${moduleName} - start`, + `Load Nx Plugin: ${moduleName} - end` + ); + return { plugin, options }; +} + +export function readPluginPackageJson( + pluginName: string, + projects: Record, + paths = getNxRequirePaths() +): { + path: string; + json: PackageJson; +} { + try { + const result = readModulePackageJsonWithoutFallbacks(pluginName, paths); + return { + json: result.packageJson, + path: result.path, + }; + } catch (e) { + if (e.code === 'MODULE_NOT_FOUND') { + const localPluginPath = resolveLocalNxPlugin(pluginName, projects); + if (localPluginPath) { + const localPluginPackageJson = path.join( + localPluginPath.path, + 'package.json' + ); + return { + path: localPluginPackageJson, + json: readJsonFile(localPluginPackageJson), + }; + } + } + throw e; + } +} + +export function resolveLocalNxPlugin( + importPath: string, + projects: Record, + root = workspaceRoot +): { path: string; projectConfig: ProjectConfiguration } | null { + return lookupLocalPlugin(importPath, projects, root); +} + +/** + * Register swc-node or ts-node if they are not currently registered + * with some default settings which work well for Nx plugins. + */ +export function registerPluginTSTranspiler() { + // Get the first tsconfig that matches the allowed set + const tsConfigName = [ + join(workspaceRoot, 'tsconfig.base.json'), + join(workspaceRoot, 'tsconfig.json'), + ].find((x) => existsSync(x)); + + const tsConfig: Partial = tsConfigName + ? readTsConfig(tsConfigName) + : {}; + + registerTsConfigPaths(tsConfigName); + registerTranspiler({ + experimentalDecorators: true, + emitDecoratorMetadata: true, + ...tsConfig.options, + }); +} + +function lookupLocalPlugin( + importPath: string, + projects: Record, + root = workspaceRoot +) { + const plugin = findNxProjectForImportPath(importPath, projects, root); + if (!plugin) { + return null; + } + + const projectConfig: ProjectConfiguration = projects[plugin]; + return { path: path.join(root, projectConfig.root), projectConfig }; +} + +function findNxProjectForImportPath( + importPath: string, + projects: Record, + root = workspaceRoot +): string | null { + const tsConfigPaths: Record = readTsConfigPaths(root); + const possiblePaths = tsConfigPaths[importPath]?.map((p) => + normalizePath(path.relative(root, path.join(root, p))) + ); + if (possiblePaths?.length) { + const projectRootMappings = + createProjectRootMappingsFromProjectConfigurations(projects); + for (const tsConfigPath of possiblePaths) { + const nxProject = findProjectForPath(tsConfigPath, projectRootMappings); + if (nxProject) { + return nxProject; + } + } + logger.verbose( + 'Unable to find local plugin', + possiblePaths, + projectRootMappings + ); + throw new Error( + 'Unable to resolve local plugin with import path ' + importPath + ); + } +} + +let tsconfigPaths: Record; + +function readTsConfigPaths(root: string = workspaceRoot) { + if (!tsconfigPaths) { + const tsconfigPath: string | null = ['tsconfig.base.json', 'tsconfig.json'] + .map((x) => path.join(root, x)) + .filter((x) => existsSync(x))[0]; + if (!tsconfigPath) { + throw new Error('unable to find tsconfig.base.json or tsconfig.json'); + } + const { compilerOptions } = readJsonFile(tsconfigPath); + tsconfigPaths = compilerOptions?.paths; + } + return tsconfigPaths ?? {}; +} + +function readPluginMainFromProjectConfiguration( + plugin: ProjectConfiguration +): string | null { + const { main } = + Object.values(plugin.targets).find((x) => + [ + '@nx/js:tsc', + '@nrwl/js:tsc', + '@nx/js:swc', + '@nrwl/js:swc', + '@nx/node:package', + '@nrwl/node:package', + ].includes(x.executor) + )?.options || + plugin.targets?.build?.options || + {}; + return main; +} + +export function getPluginPathAndName( + moduleName: string, + paths: string[], + projects: Record, + root: string +) { + let pluginPath: string; + let registerTSTranspiler = false; + try { + pluginPath = require.resolve(moduleName, { + paths, + }); + const extension = path.extname(pluginPath); + registerTSTranspiler = extension === '.ts'; + } catch (e) { + if (e.code === 'MODULE_NOT_FOUND') { + const plugin = resolveLocalNxPlugin(moduleName, projects, root); + if (plugin) { + registerTSTranspiler = true; + const main = readPluginMainFromProjectConfiguration( + plugin.projectConfig + ); + pluginPath = main ? path.join(root, main) : plugin.path; + } else { + logger.error(`Plugin listed in \`nx.json\` not found: ${moduleName}`); + throw e; + } + } else { + throw e; + } + } + const packageJsonPath = path.join(pluginPath, 'package.json'); + + // Register the ts-transpiler if we are pointing to a + // plain ts file that's not part of a plugin project + if (registerTSTranspiler) { + registerPluginTSTranspiler(); + } + + const { name } = + !['.ts', '.js'].some((x) => extname(moduleName) === x) && // Not trying to point to a ts or js file + existsSync(packageJsonPath) // plugin has a package.json + ? readJsonFile(packageJsonPath) // read name from package.json + : { name: moduleName }; + return { pluginPath, name }; +} + +async function importPluginModule(pluginPath: string): Promise { + const m = await import(pluginPath); + if ( + m.default && + ('createNodes' in m.default || 'createDependencies' in m.default) + ) { + return m.default; + } + return m; +} diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index 39978d05ceeeb..d136c4f1064ba 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -15,7 +15,7 @@ import { ProjectGraphProjectNode, } from '../config/project-graph'; import { ProjectConfiguration } from '../config/workspace-json-project-json'; -import { CreateDependenciesContext } from '../utils/nx-plugin'; +import { CreateDependenciesContext } from './plugins'; import { getFileMap } from './build-project-graph'; /** diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 713ffe0b88672..134c155682fa9 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -17,7 +17,6 @@ import { retrieveWorkspaceFiles, } from './utils/retrieve-workspace-files'; import { readNxJson } from '../config/nx-json'; -import { unregisterPluginTSTranspiler } from '../utils/nx-plugin'; /** * Synchronously reads the latest cached copy of the workspace's ProjectGraph. @@ -105,8 +104,6 @@ export async function buildProjectGraphAndSourceMapsWithoutDaemon() { ).projectGraph; performance.mark('build-project-graph-using-project-file-map:end'); - unregisterPluginTSTranspiler(); - return { projectGraph, sourceMaps }; } diff --git a/packages/nx/src/project-graph/utils/normalize-project-nodes.ts b/packages/nx/src/project-graph/utils/normalize-project-nodes.ts index c7e12a17e085f..4493df96640d5 100644 --- a/packages/nx/src/project-graph/utils/normalize-project-nodes.ts +++ b/packages/nx/src/project-graph/utils/normalize-project-nodes.ts @@ -5,9 +5,8 @@ import { TargetConfiguration, } from '../../config/workspace-json-project-json'; import { findMatchingProjects } from '../../utils/find-matching-projects'; -import { NX_PREFIX } from '../../utils/logger'; import { resolveNxTokensInOptions } from '../utils/project-configuration-utils'; -import { CreateDependenciesContext } from '../../utils/nx-plugin'; +import { CreateDependenciesContext } from '../plugins'; export async function normalizeProjectNodes( ctx: CreateDependenciesContext, diff --git a/packages/nx/src/project-graph/utils/project-configuration-utils.ts b/packages/nx/src/project-graph/utils/project-configuration-utils.ts index 0713b7a5cd2a1..ddae60fe08dd4 100644 --- a/packages/nx/src/project-graph/utils/project-configuration-utils.ts +++ b/packages/nx/src/project-graph/utils/project-configuration-utils.ts @@ -5,13 +5,17 @@ import { TargetConfiguration, } from '../../config/workspace-json-project-json'; import { NX_PREFIX } from '../../utils/logger'; -import { CreateNodesResult, LoadedNxPlugin } from '../../utils/nx-plugin'; import { readJsonFile } from '../../utils/fileutils'; import { workspaceRoot } from '../../utils/workspace-root'; import { ONLY_MODIFIES_EXISTING_TARGET } from '../../plugins/target-defaults/target-defaults-plugin'; import { minimatch } from 'minimatch'; import { join } from 'path'; +import { CreateNodesResult } from '../plugins'; +import { + CreateNodesResultWithContext, + RemotePlugin, +} from '../plugins/internal-api'; export type SourceInformation = [file: string, plugin: string]; export type ConfigurationSourceMaps = Record< @@ -197,94 +201,43 @@ export type ConfigurationResult = { export function buildProjectsConfigurationsFromProjectPathsAndPlugins( nxJson: NxJsonConfiguration, projectFiles: string[], // making this parameter allows devkit to pick up newly created projects - plugins: LoadedNxPlugin[], + plugins: RemotePlugin[], root: string = workspaceRoot ): Promise { - type CreateNodesResultWithContext = CreateNodesResult & { - file: string; - pluginName: string; - }; - const results: Array>> = []; // We iterate over plugins first - this ensures that plugins specified first take precedence. - for (const { plugin, options } of plugins) { + for (const plugin of plugins) { const [pattern, createNodes] = plugin.createNodes ?? []; - const pluginResults: Array< - CreateNodesResultWithContext | Promise - > = []; - performance.mark(`${plugin.name}:createNodes - start`); if (!pattern) { continue; } + const matchedFiles = []; + + performance.mark(`${plugin.name}:createNodes - start`); // Set this globally to allow plugins to know if they are being called from the project graph creation global.NX_GRAPH_CREATION = true; for (const file of projectFiles) { - performance.mark(`${plugin.name}:createNodes:${file} - start`); if (minimatch(file, pattern, { dot: true })) { - try { - let r = createNodes(file, options, { - nxJsonConfiguration: nxJson, - workspaceRoot: root, - }); - - if (r instanceof Promise) { - pluginResults.push( - r - .catch((e) => { - performance.mark(`${plugin.name}:createNodes:${file} - end`); - throw new CreateNodesError( - `Unable to create nodes for ${file} using plugin ${plugin.name}.`, - e - ); - }) - .then((r) => { - performance.mark(`${plugin.name}:createNodes:${file} - end`); - performance.measure( - `${plugin.name}:createNodes:${file}`, - `${plugin.name}:createNodes:${file} - start`, - `${plugin.name}:createNodes:${file} - end` - ); - return { ...r, file, pluginName: plugin.name }; - }) - ); - } else { - performance.mark(`${plugin.name}:createNodes:${file} - end`); - performance.measure( - `${plugin.name}:createNodes:${file}`, - `${plugin.name}:createNodes:${file} - start`, - `${plugin.name}:createNodes:${file} - end` - ); - pluginResults.push({ - ...r, - file, - pluginName: plugin.name, - }); - } - } catch (e) { - throw new CreateNodesError( - `Unable to create nodes for ${file} using plugin ${plugin.name}.`, - e - ); - } + matchedFiles.push(file); } } - // If there are no promises (counter undefined) or all promises have resolved (counter === 0) - results.push( - Promise.all(pluginResults).then((results) => { - delete global.NX_GRAPH_CREATION; - performance.mark(`${plugin.name}:createNodes - end`); - performance.measure( - `${plugin.name}:createNodes`, - `${plugin.name}:createNodes - start`, - `${plugin.name}:createNodes - end` - ); - return results; - }) - ); + try { + let r = createNodes(matchedFiles, { + nxJsonConfiguration: nxJson, + workspaceRoot: root, + }); + + results.push(r); + } catch (e) { + throw new CreateNodesError( + `Unable to create nodes using plugin ${plugin.name}.`, + e + ); + } } return Promise.all(results).then((results) => { diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts index 54eaa75de789a..8ed64564f6f8d 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts @@ -1,4 +1,3 @@ -import { getDefaultPlugins } from '../../utils/nx-plugin'; import { TempFs } from '../../internal-testing-utils/temp-fs'; import { retrieveProjectConfigurationPaths } from './retrieve-workspace-files'; @@ -26,10 +25,11 @@ describe('retrieveProjectConfigurationPaths', () => { }) ); - const configPaths = await retrieveProjectConfigurationPaths( - fs.tempDir, - await getDefaultPlugins(fs.tempDir) - ); + const configPaths = await retrieveProjectConfigurationPaths(fs.tempDir, [ + { + createNodes: ['{project.json,**/project.json}', () => {}], + }, + ]); expect(configPaths).not.toContain('not-projects/project.json'); expect(configPaths).toContain('projects/project.json'); diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 271f4746b58eb..d07fb705d7802 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -1,29 +1,22 @@ import { performance } from 'perf_hooks'; -import { getNxRequirePaths } from '../../utils/installation-directory'; import { ProjectConfiguration } from '../../config/workspace-json-project-json'; import { NX_ANGULAR_JSON_PLUGIN_NAME, - NxAngularJsonPlugin, shouldMergeAngularProjects, } from '../../adapter/angular-json'; import { NxJsonConfiguration, readNxJson } from '../../config/nx-json'; import { ProjectGraphExternalNode } from '../../config/project-graph'; -import { getNxPackageJsonWorkspacesPlugin } from '../../plugins/package-json-workspaces'; import { buildProjectsConfigurationsFromProjectPathsAndPlugins, ConfigurationSourceMaps, } from './project-configuration-utils'; -import { - getDefaultPlugins, - LoadedNxPlugin, - loadNxPlugins, -} from '../../utils/nx-plugin'; -import { ProjectJsonProjectsPlugin } from '../../plugins/project-json/build-nodes/project-json'; +import { RemotePlugin, loadNxPlugins } from '../plugins/internal-api'; import { getNxWorkspaceFilesFromContext, globWithWorkspaceContext, } from '../../utils/workspace-context'; import { buildAllWorkspaceFiles } from './build-all-workspace-files'; +import { join } from 'path'; /** * Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles` @@ -74,11 +67,7 @@ export async function retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration ): Promise { - const plugins = await loadNxPlugins( - nxJson?.plugins ?? [], - getNxRequirePaths(workspaceRoot), - workspaceRoot - ); + const plugins = await loadNxPlugins(nxJson?.plugins ?? [], workspaceRoot); return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins); } @@ -87,19 +76,21 @@ export async function retrieveProjectConfigurationsWithAngularProjects( workspaceRoot: string, nxJson: NxJsonConfiguration ): Promise { - const plugins = await loadNxPlugins( - nxJson?.plugins ?? [], - getNxRequirePaths(workspaceRoot), - workspaceRoot - ); + const pluginsToLoad = nxJson?.plugins ?? []; if ( shouldMergeAngularProjects(workspaceRoot, true) && - !plugins.some((p) => p.plugin.name === NX_ANGULAR_JSON_PLUGIN_NAME) + !pluginsToLoad.some( + (p) => + p === NX_ANGULAR_JSON_PLUGIN_NAME || + (typeof p === 'object' && p.plugin === NX_ANGULAR_JSON_PLUGIN_NAME) + ) ) { - plugins.push({ plugin: NxAngularJsonPlugin }); + pluginsToLoad.push(join(__dirname, '../../adapter/angular-json')); } + const plugins = await loadNxPlugins(nxJson?.plugins ?? [], workspaceRoot); + return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins); } @@ -113,7 +104,7 @@ export type RetrievedGraphNodes = { function _retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration, - plugins: LoadedNxPlugin[] + plugins: RemotePlugin[] ): Promise { const globPatterns = configurationGlobs(plugins); const projectFiles = globWithWorkspaceContext(workspaceRoot, globPatterns); @@ -128,7 +119,7 @@ function _retrieveProjectConfigurations( export function retrieveProjectConfigurationPaths( root: string, - plugins: LoadedNxPlugin[] + plugins: PluginGlobsOnly ): string[] { const projectGlobPatterns = configurationGlobs(plugins); return globWithWorkspaceContext(root, projectGlobPatterns); @@ -144,7 +135,7 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( root: string ): Promise> { const nxJson = readNxJson(root); - const plugins = await getDefaultPlugins(root); + const plugins = await loadNxPlugins([]); // only load default plugins const projectGlobPatterns = retrieveProjectConfigurationPaths(root, plugins); const cacheKey = root + ',' + projectGlobPatterns.join(','); @@ -157,10 +148,7 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( root, nxJson, projectFiles, - [ - { plugin: getNxPackageJsonWorkspacesPlugin(root) }, - { plugin: ProjectJsonProjectsPlugin }, - ] + plugins ); projectsWithoutPluginCache.set(cacheKey, projects); @@ -172,7 +160,7 @@ export async function createProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration, configFiles: string[], - plugins: LoadedNxPlugin[] + plugins: RemotePlugin[] ): Promise { performance.mark('build-project-configs:start'); @@ -199,9 +187,11 @@ export async function createProjectConfigurations( }; } -export function configurationGlobs(plugins: LoadedNxPlugin[]): string[] { +type PluginGlobsOnly = Array<{ createNodes?: readonly [string, ...unknown[]] }>; + +export function configurationGlobs(plugins: PluginGlobsOnly): string[] { const globPatterns = []; - for (const { plugin } of plugins) { + for (const plugin of plugins) { if (plugin.createNodes) { globPatterns.push(plugin.createNodes[0]); } diff --git a/packages/nx/src/utils/logger.ts b/packages/nx/src/utils/logger.ts index 8096d5dc478bf..54b03cd730a23 100644 --- a/packages/nx/src/utils/logger.ts +++ b/packages/nx/src/utils/logger.ts @@ -31,6 +31,11 @@ export const logger = { fatal: (...s) => { console.error(...s); }, + verbose: (...s) => { + if (process.env.NX_VERBOSE_LOGGING) { + console.log(...s); + } + }, }; export function stripIndent(str: string): string { diff --git a/packages/nx/src/utils/nx-plugin.deprecated.ts b/packages/nx/src/utils/nx-plugin.deprecated.ts index c5c24129b964c..f3f370fcc5a24 100644 --- a/packages/nx/src/utils/nx-plugin.deprecated.ts +++ b/packages/nx/src/utils/nx-plugin.deprecated.ts @@ -1,10 +1,10 @@ import { shouldMergeAngularProjects } from '../adapter/angular-json'; import { ProjectGraphProcessor } from '../config/project-graph'; import { TargetConfiguration } from '../config/workspace-json-project-json'; -import { ProjectJsonProjectsPlugin } from '../plugins/project-json/build-nodes/project-json'; -import { TargetDefaultsPlugin } from '../plugins/target-defaults/target-defaults-plugin'; -import { getNxPackageJsonWorkspacesPlugin } from '../plugins/package-json-workspaces'; -import { LoadedNxPlugin, NxPluginV2 } from './nx-plugin'; +import ProjectJsonProjectsPlugin from '../plugins/project-json/build-nodes/project-json'; +import TargetDefaultsPlugin from '../plugins/target-defaults/target-defaults-plugin'; +import * as PackageJsonWorkspacesPlugin from '../plugins/package-json-workspaces'; +import { NxPluginV2 } from '../project-graph/plugins'; /** * @deprecated Add targets to the projects in a {@link CreateNodes} function instead. This will be removed in Nx 19 @@ -39,14 +39,14 @@ export type NxPluginV1 = { /** * @todo(@agentender) v19: Remove this fn when we remove readWorkspaceConfig */ -export function getDefaultPluginsSync(root: string): LoadedNxPlugin[] { +export function getDefaultPluginsSync(root: string) { const plugins: NxPluginV2[] = [ require('../plugins/js'), ...(shouldMergeAngularProjects(root, false) ? [require('../adapter/angular-json').NxAngularJsonPlugin] : []), TargetDefaultsPlugin, - getNxPackageJsonWorkspacesPlugin(root), + PackageJsonWorkspacesPlugin, ProjectJsonProjectsPlugin, ]; diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts deleted file mode 100644 index 836dfe23337fb..0000000000000 --- a/packages/nx/src/utils/nx-plugin.ts +++ /dev/null @@ -1,531 +0,0 @@ -import { existsSync } from 'fs'; -import * as path from 'path'; -import { - FileMap, - ProjectGraph, - ProjectGraphExternalNode, -} from '../config/project-graph'; -import { toProjectName } from '../config/workspaces'; - -import { workspaceRoot } from './workspace-root'; -import { readJsonFile } from '../utils/fileutils'; -import { - PackageJson, - readModulePackageJsonWithoutFallbacks, -} from './package-json'; -import { - registerTranspiler, - registerTsConfigPaths, -} from '../plugins/js/utils/register'; -import { ProjectConfiguration } from '../config/workspace-json-project-json'; -import { logger } from './logger'; -import { - createProjectRootMappingsFromProjectConfigurations, - findProjectForPath, -} from '../project-graph/utils/find-project-for-path'; -import { normalizePath } from './path'; -import { dirname, join } from 'path'; -import { getNxRequirePaths } from './installation-directory'; -import { readTsConfig } from '../plugins/js/utils/typescript'; -import { - NxJsonConfiguration, - PluginConfiguration, - readNxJson, -} from '../config/nx-json'; - -import type * as ts from 'typescript'; -import { NxPluginV1 } from './nx-plugin.deprecated'; -import { RawProjectGraphDependency } from '../project-graph/project-graph-builder'; -import { combineGlobPatterns } from './globs'; -import { shouldMergeAngularProjects } from '../adapter/angular-json'; -import { getNxPackageJsonWorkspacesPlugin } from '../plugins/package-json-workspaces'; -import { ProjectJsonProjectsPlugin } from '../plugins/project-json/build-nodes/project-json'; -import { PackageJsonProjectsNextToProjectJsonPlugin } from '../plugins/project-json/build-nodes/package-json-next-to-project-json'; -import { retrieveProjectConfigurationsWithoutPluginInference } from '../project-graph/utils/retrieve-workspace-files'; -import { TargetDefaultsPlugin } from '../plugins/target-defaults/target-defaults-plugin'; - -/** - * Context for {@link CreateNodesFunction} - */ -export interface CreateNodesContext { - readonly nxJsonConfiguration: NxJsonConfiguration; - readonly workspaceRoot: string; -} - -/** - * A function which parses a configuration file into a set of nodes. - * Used for creating nodes for the {@link ProjectGraph} - */ -export type CreateNodesFunction = ( - projectConfigurationFile: string, - options: T | undefined, - context: CreateNodesContext -) => CreateNodesResult | Promise; - -export interface CreateNodesResult { - /** - * A map of project root -> project configuration - */ - projects?: Record>; - - /** - * A map of external node name -> external node. External nodes do not have a root, so the key is their name. - */ - externalNodes?: Record; -} - -/** - * A pair of file patterns and {@link CreateNodesFunction} - */ -export type CreateNodes = readonly [ - projectFilePattern: string, - createNodesFunction: CreateNodesFunction -]; - -/** - * Context for {@link CreateDependencies} - */ -export interface CreateDependenciesContext { - /** - * The external nodes that have been added to the graph. - */ - readonly externalNodes: ProjectGraph['externalNodes']; - - /** - * The configuration of each project in the workspace. - */ - readonly projects: Record; - - /** - * The `nx.json` configuration from the workspace - */ - readonly nxJsonConfiguration: NxJsonConfiguration; - - /** - * All files in the workspace - */ - readonly fileMap: FileMap; - - /** - * Files changes since last invocation - */ - readonly filesToProcess: FileMap; - - readonly workspaceRoot: string; -} - -/** - * A function which parses files in the workspace to create dependencies in the {@link ProjectGraph} - * Use {@link validateDependency} to validate dependencies - */ -export type CreateDependencies = ( - options: T | undefined, - context: CreateDependenciesContext -) => RawProjectGraphDependency[] | Promise; - -/** - * A plugin for Nx which creates nodes and dependencies for the {@link ProjectGraph} - */ -export type NxPluginV2 = { - name: string; - - /** - * Provides a file pattern and function that retrieves configuration info from - * those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFile } - */ - createNodes?: CreateNodes; - - // Todo(@AgentEnder): This shouldn't be a full processor, since its only responsible for defining edges between projects. What do we want the API to be? - /** - * Provides a function to analyze files to create dependencies for the {@link ProjectGraph} - */ - createDependencies?: CreateDependencies; -}; - -export * from './nx-plugin.deprecated'; - -/** - * A plugin for Nx - */ -export type NxPlugin = NxPluginV1 | NxPluginV2; - -export type LoadedNxPlugin = { - plugin: NxPluginV2 & Pick; - options?: unknown; -}; - -// Short lived cache (cleared between cmd runs) -// holding resolved nx plugin objects. -// Allows loadNxPlugins to be called multiple times w/o -// executing resolution mulitple times. -export const nxPluginCache: Map = new Map(); - -export function getPluginPathAndName( - moduleName: string, - paths: string[], - projects: Record, - root: string -) { - let pluginPath: string; - try { - pluginPath = require.resolve(moduleName, { - paths, - }); - } catch (e) { - if (e.code === 'MODULE_NOT_FOUND') { - const plugin = resolveLocalNxPlugin( - moduleName, - readNxJson(root), - projects, - root - ); - if (plugin) { - const main = readPluginMainFromProjectConfiguration( - plugin.projectConfig - ); - pluginPath = main ? path.join(root, main) : plugin.path; - } else { - logger.error(`Plugin listed in \`nx.json\` not found: ${moduleName}`); - throw e; - } - } else { - throw e; - } - } - const packageJsonPath = path.join(pluginPath, 'package.json'); - - const extension = path.extname(pluginPath); - - // Register the ts-transpiler if we are pointing to a - // plain ts file that's not part of a plugin project - if (extension === '.ts' && !tsNodeAndPathsUnregisterCallback) { - registerPluginTSTranspiler(); - } - - const { name } = - !['.ts', '.js'].some((x) => x === extension) && // Not trying to point to a ts or js file - existsSync(packageJsonPath) // plugin has a package.json - ? readJsonFile(packageJsonPath) // read name from package.json - : { name: moduleName }; - return { pluginPath, name }; -} - -export async function loadNxPluginAsync( - pluginConfiguration: PluginConfiguration, - paths: string[], - projects: Record, - root: string -): Promise { - const { plugin: moduleName, options } = - typeof pluginConfiguration === 'object' - ? pluginConfiguration - : { plugin: pluginConfiguration, options: undefined }; - let pluginModule = nxPluginCache.get(moduleName); - if (pluginModule) { - return { plugin: pluginModule, options }; - } - performance.mark(`Load Nx Plugin: ${moduleName} - start`); - let { pluginPath, name } = await getPluginPathAndName( - moduleName, - paths, - projects, - root - ); - const plugin = ensurePluginIsV2( - (await import(pluginPath)) as LoadedNxPlugin['plugin'] - ); - plugin.name ??= name; - nxPluginCache.set(moduleName, plugin); - performance.mark(`Load Nx Plugin: ${moduleName} - end`); - performance.measure( - `Load Nx Plugin: ${moduleName}`, - `Load Nx Plugin: ${moduleName} - start`, - `Load Nx Plugin: ${moduleName} - end` - ); - return { plugin, options }; -} - -export async function loadNxPlugins( - plugins: PluginConfiguration[], - paths = getNxRequirePaths(), - root = workspaceRoot, - projects?: Record -): Promise { - const result: LoadedNxPlugin[] = [ - { plugin: PackageJsonProjectsNextToProjectJsonPlugin }, - ]; - - plugins ??= []; - - // When loading plugins for `createNodes`, we don't know what projects exist yet. - // Try resolving plugins - for (const plugin of plugins) { - try { - require.resolve(typeof plugin === 'string' ? plugin : plugin.plugin); - } catch { - // If a plugin cannot be resolved, we will need projects to resolve it - projects ??= await retrieveProjectConfigurationsWithoutPluginInference( - root - ); - break; - } - } - for (const plugin of plugins) { - result.push(await loadNxPluginAsync(plugin, paths, projects, root)); - } - - // We push the nx core node plugins onto the end, s.t. it overwrites any other plugins - result.push(...(await getDefaultPlugins(root))); - - return result; -} - -export function ensurePluginIsV2(plugin: NxPlugin): NxPluginV2 { - if (isNxPluginV2(plugin)) { - return plugin; - } - if (isNxPluginV1(plugin) && plugin.projectFilePatterns) { - return { - ...plugin, - createNodes: [ - `*/**/${combineGlobPatterns(plugin.projectFilePatterns)}`, - (configFilePath) => { - const root = dirname(configFilePath); - return { - projects: { - [root]: { - name: toProjectName(configFilePath), - root, - targets: plugin.registerProjectTargets?.(configFilePath), - }, - }, - }; - }, - ], - }; - } - return plugin; -} - -export function isNxPluginV2(plugin: NxPlugin): plugin is NxPluginV2 { - return 'createNodes' in plugin || 'createDependencies' in plugin; -} - -export function isNxPluginV1(plugin: NxPlugin): plugin is NxPluginV1 { - return 'processProjectGraph' in plugin || 'projectFilePatterns' in plugin; -} - -export function readPluginPackageJson( - pluginName: string, - projects: Record, - paths = getNxRequirePaths() -): { - path: string; - json: PackageJson; -} { - try { - const result = readModulePackageJsonWithoutFallbacks(pluginName, paths); - return { - json: result.packageJson, - path: result.path, - }; - } catch (e) { - if (e.code === 'MODULE_NOT_FOUND') { - const nxJson = readNxJson(); - const localPluginPath = resolveLocalNxPlugin( - pluginName, - nxJson, - projects - ); - if (localPluginPath) { - const localPluginPackageJson = path.join( - localPluginPath.path, - 'package.json' - ); - return { - path: localPluginPackageJson, - json: readJsonFile(localPluginPackageJson), - }; - } - } - throw e; - } -} - -/** - * Builds a plugin package and returns the path to output - * @param importPath What is the import path that refers to a potential plugin? - * @returns The path to the built plugin, or null if it doesn't exist - */ -const localPluginCache: Record< - string, - { path: string; projectConfig: ProjectConfiguration } -> = {}; - -export function resolveLocalNxPlugin( - importPath: string, - nxJsonConfiguration: NxJsonConfiguration, - projects: Record, - root = workspaceRoot -): { path: string; projectConfig: ProjectConfiguration } | null { - localPluginCache[importPath] ??= lookupLocalPlugin( - importPath, - nxJsonConfiguration, - projects, - root - ); - return localPluginCache[importPath]; -} - -let tsNodeAndPathsUnregisterCallback: (() => void) | undefined = undefined; - -/** - * Register swc-node or ts-node if they are not currently registered - * with some default settings which work well for Nx plugins. - */ -export function registerPluginTSTranspiler() { - if (!tsNodeAndPathsUnregisterCallback) { - // nx-ignore-next-line - const ts: typeof import('typescript') = require('typescript'); - - // Get the first tsconfig that matches the allowed set - const tsConfigName = [ - join(workspaceRoot, 'tsconfig.base.json'), - join(workspaceRoot, 'tsconfig.json'), - ].find((x) => existsSync(x)); - - const tsConfig: Partial = tsConfigName - ? readTsConfig(tsConfigName) - : {}; - - const unregisterTsConfigPaths = registerTsConfigPaths(tsConfigName); - const unregisterTranspiler = registerTranspiler({ - experimentalDecorators: true, - emitDecoratorMetadata: true, - ...tsConfig.options, - }); - tsNodeAndPathsUnregisterCallback = () => { - unregisterTsConfigPaths(); - unregisterTranspiler(); - }; - } -} - -/** - * Unregister the ts-node transpiler if it is registered - */ -export function unregisterPluginTSTranspiler() { - if (tsNodeAndPathsUnregisterCallback) { - tsNodeAndPathsUnregisterCallback(); - tsNodeAndPathsUnregisterCallback = undefined; - } -} - -function lookupLocalPlugin( - importPath: string, - nxJsonConfiguration: NxJsonConfiguration, - projects: Record, - root = workspaceRoot -) { - const plugin = findNxProjectForImportPath(importPath, projects, root); - if (!plugin) { - return null; - } - - if (!tsNodeAndPathsUnregisterCallback) { - registerPluginTSTranspiler(); - } - - const projectConfig: ProjectConfiguration = projects[plugin]; - return { path: path.join(root, projectConfig.root), projectConfig }; -} - -function findNxProjectForImportPath( - importPath: string, - projects: Record, - root = workspaceRoot -): string | null { - const tsConfigPaths: Record = readTsConfigPaths(root); - const possiblePaths = tsConfigPaths[importPath]?.map((p) => - normalizePath(path.relative(root, path.join(root, p))) - ); - if (possiblePaths?.length) { - const projectRootMappings = - createProjectRootMappingsFromProjectConfigurations(projects); - for (const tsConfigPath of possiblePaths) { - const nxProject = findProjectForPath(tsConfigPath, projectRootMappings); - if (nxProject) { - return nxProject; - } - } - if (process.env.NX_VERBOSE_LOGGING) { - console.log( - 'Unable to find local plugin', - possiblePaths, - projectRootMappings - ); - } - throw new Error( - 'Unable to resolve local plugin with import path ' + importPath - ); - } -} - -let tsconfigPaths: Record; - -function readTsConfigPaths(root: string = workspaceRoot) { - if (!tsconfigPaths) { - const tsconfigPath: string | null = ['tsconfig.base.json', 'tsconfig.json'] - .map((x) => path.join(root, x)) - .filter((x) => existsSync(x))[0]; - if (!tsconfigPath) { - throw new Error('unable to find tsconfig.base.json or tsconfig.json'); - } - const { compilerOptions } = readJsonFile(tsconfigPath); - tsconfigPaths = compilerOptions?.paths; - } - return tsconfigPaths ?? {}; -} - -function readPluginMainFromProjectConfiguration( - plugin: ProjectConfiguration -): string | null { - const { main } = - Object.values(plugin.targets).find((x) => - [ - '@nx/js:tsc', - '@nrwl/js:tsc', - '@nx/js:swc', - '@nrwl/js:swc', - '@nx/node:package', - '@nrwl/node:package', - ].includes(x.executor) - )?.options || - plugin.targets?.build?.options || - {}; - return main; -} - -export async function getDefaultPlugins( - root: string -): Promise { - const plugins: NxPluginV2[] = [ - await import('../plugins/js'), - TargetDefaultsPlugin, - ...(shouldMergeAngularProjects(root, false) - ? [ - await import('../adapter/angular-json').then( - (m) => m.NxAngularJsonPlugin - ), - ] - : []), - getNxPackageJsonWorkspacesPlugin(root), - ProjectJsonProjectsPlugin, - ]; - - return plugins.map((p) => ({ - plugin: p, - })); -} - -type Optional = Omit & Partial>; diff --git a/packages/nx/src/utils/plugins/plugin-capabilities.ts b/packages/nx/src/utils/plugins/plugin-capabilities.ts index 156cc111415b8..6b105847768b8 100644 --- a/packages/nx/src/utils/plugins/plugin-capabilities.ts +++ b/packages/nx/src/utils/plugins/plugin-capabilities.ts @@ -1,19 +1,19 @@ -import { workspaceRoot } from '../workspace-root'; import * as chalk from 'chalk'; import { dirname, join } from 'path'; -import { output } from '../output'; -import type { PluginCapabilities } from './models'; -import { hasElements } from './shared'; + +import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { readPluginPackageJson } from '../../project-graph/plugins'; +import { RemotePlugin } from '../../project-graph/plugins/internal-api'; +import { loadRemoteNxPlugin } from '../../project-graph/plugins/plugin-pool'; import { readJsonFile } from '../fileutils'; -import { getPackageManagerCommand } from '../package-manager'; -import { - loadNxPluginAsync, - NxPlugin, - readPluginPackageJson, -} from '../nx-plugin'; import { getNxRequirePaths } from '../installation-directory'; +import { output } from '../output'; import { PackageJson } from '../package-json'; -import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { getPackageManagerCommand } from '../package-manager'; +import { workspaceRoot } from '../workspace-root'; +import { hasElements } from './shared'; + +import type { PluginCapabilities } from './models'; function tryGetCollection( packageJsonPath: string, @@ -101,24 +101,17 @@ async function tryGetModule( packageJson: PackageJson, workspaceRoot: string, projects: Record -): Promise { +): Promise { try { return packageJson.generators ?? packageJson.executors ?? packageJson['nx-migrations'] ?? packageJson['schematics'] ?? packageJson['builders'] - ? ( - await loadNxPluginAsync( - packageJson.name, - getNxRequirePaths(workspaceRoot), - projects, - workspaceRoot - ) - ).plugin + ? await loadRemoteNxPlugin(packageJson.name, workspaceRoot) : ({ name: packageJson.name, - } as NxPlugin); + } as RemotePlugin); } catch { return null; }