fix(@angular/cli): redirect Angular schematic dependency requests to …

…known versions

This change adds logic to redirect module resolution requests for Angular schematics to ensure that the correct versions of core schematic related packages are used.  This also ensures that the runtime version of the schematics package matches the version used inside the schematic and that object instances passed into the schematic are compatible.
The current set of core schematic related packages are `@angular-devkit/*` and `@schematics/angular`.  Only first-party Angular schematics are currently affected by this change.

packages/angular/cli/commands/update-impl.ts

@@ -14,6 +14,7 @@ import * as semver from 'semver';
 import { PackageManager } from '../lib/config/schema';
 import { Command } from '../models/command';
 import { Arguments } from '../models/interface';
+import { SchematicEngineHost } from '../models/schematic-engine-host';
 import { colors } from '../utilities/color';
 import { runTempPackageBin } from '../utilities/install-package';
 import { writeErrorToLogFile } from '../utilities/log-file';
@@ -71,6 +72,7 @@ export class UpdateCommand extends Command<UpdateCommandSchema> {
         // Otherwise, use packages from the active workspace (migrations)
         resolvePaths: [__dirname, this.context.root],
         schemaValidation: true,
+        engineHostCreator: (options) => new SchematicEngineHost(options.resolvePaths),
       },
     );
   }
@@ -265,28 +267,6 @@ export class UpdateCommand extends Command<UpdateCommandSchema> {
 
   // tslint:disable-next-line:no-big-function
   async run(options: UpdateCommandSchema & Arguments) {
-    // Check if the @angular-devkit/schematics package can be resolved from the workspace root
-    // This works around issues with packages containing migrations that cannot directly depend on the package
-    // This check can be removed once the schematic runtime handles this situation
-    try {
-      require.resolve('@angular-devkit/schematics', { paths: [this.context.root] });
-    } catch (e) {
-      if (e.code === 'MODULE_NOT_FOUND') {
-        this.logger.fatal(
-          'The "@angular-devkit/schematics" package cannot be resolved from the workspace root directory. ' +
-            'This may be due to an unsupported node modules structure.\n' +
-            'Please remove both the "node_modules" directory and the package lock file; and then reinstall.\n' +
-            'If this does not correct the problem, ' +
-            'please temporarily install the "@angular-devkit/schematics" package within the workspace. ' +
-            'It can be removed once the update is complete.',
-        );
-
-        return 1;
-      }
-
-      throw e;
-    }
-
     // Check if the current installed CLI version is older than the latest version.
     if (!disableVersionCheck && await this.checkCLILatestVersion(options.verbose, options.next)) {
       this.logger.warn(

packages/angular/cli/models/schematic-command.ts

@@ -23,7 +23,6 @@ import {
   FileSystemCollection,
   FileSystemEngine,
   FileSystemSchematic,
-  FileSystemSchematicDescription,
   NodeWorkflow,
 } from '@angular-devkit/schematics/tools';
 import * as inquirer from 'inquirer';
@@ -37,6 +36,7 @@ import { isPackageNameSafeForAnalytics } from './analytics';
 import { BaseCommandOptions, Command } from './command';
 import { Arguments, CommandContext, CommandDescription, Option } from './interface';
 import { parseArguments, parseFreeFormArguments } from './parser';
+import { SchematicEngineHost } from './schematic-engine-host';
 
 export interface BaseSchematicSchema {
   debug?: boolean;
@@ -258,6 +258,7 @@ export abstract class SchematicCommand<
           ...current,
         }),
       ],
+      engineHostCreator: (options) => new SchematicEngineHost(options.resolvePaths),
     });
 
     const getProjectName = () => {

packages/angular/cli/models/schematic-engine-host.ts

@@ -0,0 +1,189 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { RuleFactory, SchematicsException, Tree } from '@angular-devkit/schematics';
+import { NodeModulesEngineHost } from '@angular-devkit/schematics/tools';
+import { readFileSync } from 'fs';
+import { parse as parseJson } from 'jsonc-parser';
+import { dirname, resolve } from 'path';
+import { Script } from 'vm';
+
+/**
+ * Environment variable to control schematic package redirection
+ * Default: Angular schematics only
+ */
+const schematicRedirectVariable = process.env['NG_SCHEMATIC_REDIRECT']?.toLowerCase();
+
+function shouldWrapSchematic(schematicFile: string): boolean {
+  // Check environment variable if present
+  if (schematicRedirectVariable !== undefined) {
+    switch (schematicRedirectVariable) {
+      case '0':
+      case 'false':
+      case 'off':
+      case 'none':
+        return false;
+      case 'all':
+        return true;
+    }
+  }
+
+  // Never wrap `@schematics/update` when executed directly
+  // It communicates with the update command via `global`
+  if (/[\/\\]node_modules[\/\\]@schematics[\/\\]update[\/\\]/.test(schematicFile)) {
+    return false;
+  }
+
+  // Default is only first-party Angular schematic packages
+  // Angular schematics are safe to use in the wrapped VM context
+  return /[\/\\]node_modules[\/\\]@(?:angular|schematics|nguniversal)[\/\\]/.test(schematicFile);
+}
+
+export class SchematicEngineHost extends NodeModulesEngineHost {
+  protected _resolveReferenceString(refString: string, parentPath: string) {
+    const [path, name] = refString.split('#', 2);
+    // Mimic behavior of ExportStringRef class used in default behavior
+    const fullPath = path[0] === '.' ? resolve(parentPath ?? process.cwd(), path) : path;
+
+    const schematicFile = require.resolve(fullPath, { paths: [parentPath] });
+
+    if (shouldWrapSchematic(schematicFile)) {
+      const schematicPath = dirname(schematicFile);
+
+      const moduleCache = new Map<string, unknown>();
+      const factoryInitializer = wrap(
+        schematicFile,
+        schematicPath,
+        moduleCache,
+        name || 'default',
+      ) as () => RuleFactory<{}>;
+
+      const factory = factoryInitializer();
+      if (!factory || typeof factory !== 'function') {
+        return null;
+      }
+
+      return { ref: factory, path: schematicPath };
+    }
+
+    // All other schematics use default behavior
+    return super._resolveReferenceString(refString, parentPath);
+  }
+}
+
+/**
+ * Minimal shim modules for legacy deep imports of `@schematics/angular`
+ */
+const legacyModules: Record<string, unknown> = {
+  '@schematics/angular/utility/config': {
+    getWorkspace(host: Tree) {
+      const path = '/.angular.json';
+      const data = host.read(path);
+      if (!data) {
+        throw new SchematicsException(`Could not find (${path})`);
+      }
+
+      return parseJson(data.toString(), [], { allowTrailingComma: true });
+    },
+  },
+  '@schematics/angular/utility/project': {
+    buildDefaultPath(project: { sourceRoot?: string; root: string; projectType: string }): string {
+      const root = project.sourceRoot ? `/${project.sourceRoot}/` : `/${project.root}/src/`;
+
+      return `${root}${project.projectType === 'application' ? 'app' : 'lib'}`;
+    },
+  },
+};
+
+/**
+ * Wrap a JavaScript file in a VM context to allow specific Angular dependencies to be redirected.
+ * This VM setup is ONLY intended to redirect dependencies.
+ *
+ * @param schematicFile A JavaScript schematic file path that should be wrapped.
+ * @param schematicDirectory A directory that will be used as the location of the JavaScript file.
+ * @param moduleCache A map to use for caching repeat module usage and proper `instanceof` support.
+ * @param exportName An optional name of a specific export to return. Otherwise, return all exports.
+ */
+function wrap(
+  schematicFile: string,
+  schematicDirectory: string,
+  moduleCache: Map<string, unknown>,
+  exportName?: string,
+): () => unknown {
+  const { createRequire, createRequireFromPath } = require('module');
+  // Node.js 10.x does not support `createRequire` so fallback to `createRequireFromPath`
+  // `createRequireFromPath` is deprecated in 12+ and can be removed once 10.x support is removed
+  const scopedRequire = createRequire?.(schematicFile) || createRequireFromPath(schematicFile);
+
+  const customRequire = function (id: string) {
+    if (legacyModules[id]) {
+      // Provide compatibility modules for older versions of @angular/cdk
+      return legacyModules[id];
+    } else if (id.startsWith('@angular-devkit/') || id.startsWith('@schematics/')) {
+      // Resolve from inside the `@angular/cli` project
+      const packagePath = require.resolve(id);
+
+      return require(packagePath);
+    } else if (id.startsWith('.') || id.startsWith('@angular/cdk')) {
+      // Wrap relative files inside the schematic collection
+      // Also wrap `@angular/cdk`, it contains helper utilities that import core schematic packages
+
+      // Resolve from the original file
+      const modulePath = scopedRequire.resolve(id);
+
+      // Use cached module if available
+      const cachedModule = moduleCache.get(modulePath);
+      if (cachedModule) {
+        return cachedModule;
+      }
+
+      // Do not wrap vendored third-party packages or JSON files
+      if (
+        !/[\/\\]node_modules[\/\\]@schematics[\/\\]angular[\/\\]third_party[\/\\]/.test(
+          modulePath,
+        ) &&
+        !modulePath.endsWith('.json')
+      ) {
+        // Wrap module and save in cache
+        const wrappedModule = wrap(modulePath, dirname(modulePath), moduleCache)();
+        moduleCache.set(modulePath, wrappedModule);
+
+        return wrappedModule;
+      }
+    }
+
+    // All others are required directly from the original file
+    return scopedRequire(id);
+  };
+
+  // Setup a wrapper function to capture the module's exports
+  const schematicCode = readFileSync(schematicFile, 'utf8');
+  // `module` is required due to @angular/localize ng-add being in UMD format
+  const headerCode = '(function() {\nvar exports = {};\nvar module = { exports };\n';
+  const footerCode = exportName ? `\nreturn exports['${exportName}'];});` : '\nreturn exports;});';
+
+  const script = new Script(headerCode + schematicCode + footerCode, {
+    filename: schematicFile,
+    lineOffset: 3,
+  });
+
+  const context = {
+    __dirname: schematicDirectory,
+    __filename: schematicFile,
+    Buffer,
+    console,
+    process,
+    get global() {
+      return this;
+    },
+    require: customRequire,
+  };
+
+  const exportsFactory = script.runInNewContext(context);
+
+  return exportsFactory;
+}

tests/legacy-cli/e2e/tests/misc/invalid-schematic-dependencies.ts

@@ -0,0 +1,21 @@
+import { join } from 'path';
+import { expectFileToMatch } from '../../utils/fs';
+import { ng } from '../../utils/process';
+import { installPackage, uninstallPackage } from '../../utils/packages';
+import { isPrereleaseCli } from '../../utils/project';
+
+export default async function () {
+  const componentDir = join('src', 'app', 'test-component');
+
+  // Install old and incompatible version
+  // Must directly use npm registry since old versions are not hosted locally
+  await installPackage('@schematics/angular@7', 'https://registry.npmjs.org')
+
+  const tag = await isPrereleaseCli() ?  '@next' : '';
+  await ng('add', `@angular/material${tag}`);
+  await expectFileToMatch('package.json', /@angular\/material/);
+
+  // Clean up existing cdk package
+  // Not doing so can cause adding material to fail if an incompatible cdk is present
+  await uninstallPackage('@angular/cdk');
+}

tests/legacy-cli/e2e/utils/packages.ts

@@ -27,12 +27,13 @@ export async function installWorkspacePackages(updateWebdriver = true): Promise<
   }
 }
 
-export async function installPackage(specifier: string): Promise<ProcessOutput> {
+export async function installPackage(specifier: string, registry?: string): Promise<ProcessOutput> {
+  const registryOption = registry ? [`--registry=${registry}`] : [];
   switch (getActivePackageManager()) {
     case 'npm':
-      return silentNpm('install', specifier);
+      return silentNpm('install', specifier, ...registryOption);
     case 'yarn':
-      return silentYarn('add', specifier);
+      return silentYarn('add', specifier, ...registryOption);
   }
 }