perf(ngcc): process tasks in parallel in async mode (#32427)

`ngcc` supports both synchronous and asynchronous execution. The default
mode when using `ngcc` programmatically (which is how `@angular/cli` is
using it) is synchronous. When running `ngcc` from the command line
(i.e. via the `ivy-ngcc` script), it runs in async mode.

Previously, the work would be executed in the same way in both modes.

This commit improves the performance of `ngcc` in async mode by
processing tasks in parallel on multiple processes. It uses the Node.js
built-in [`cluster` module](https://nodejs.org/api/cluster.html) to
launch a cluster of Node.js processes and take advantage of multi-core
systems.

Preliminary comparisons indicate a 1.8x to 2.6x speed improvement when
processing the angular.io app (apparently depending on the OS, number of
available cores, system load, etc.). Further investigation is needed to
better understand these numbers and identify potential areas of
improvement.

Inspired by/Based on @alxhub's prototype: alxhub/angular@cb631bdb1
Original design doc: https://hackmd.io/uYG9CJrFQZ-6FtKqpnYJAA?view

Jira issue: [FW-1460](https://angular-team.atlassian.net/browse/FW-1460)

PR Close #32427

Author: gkalpak

integration/ngcc/test.sh

@@ -82,6 +82,10 @@ grep "FocusMonitor.decorators =" node_modules/@angular/cdk/bundles/cdk-a11y.umd.
 ivy-ngcc -l debug | grep 'Skipping'
 if [[ $? != 0 ]]; then exit 1; fi
 
+# Does it process the tasks in parallel?
+ivy-ngcc -l debug | grep 'Running ngcc on ClusterExecutor'
+if [[ $? != 0 ]]; then exit 1; fi
+
 # Check that running it with logging level error outputs nothing
 ivy-ngcc -l error | grep '.' && exit 1
 

packages/compiler-cli/ngcc/src/dependencies/dependency_resolver.ts

@@ -63,16 +63,17 @@ export interface DependencyDiagnostics {
 export type PartiallyOrderedEntryPoints = PartiallyOrderedList<EntryPoint>;
 
 /**
- * A list of entry-points, sorted by their dependencies.
+ * A list of entry-points, sorted by their dependencies, and the dependency graph.
  *
  * The `entryPoints` array will be ordered so that no entry point depends upon an entry point that
  * appears later in the array.
  *
- * Some entry points or their dependencies may be have been ignored. These are captured for
+ * Some entry points or their dependencies may have been ignored. These are captured for
  * diagnostic purposes in `invalidEntryPoints` and `ignoredDependencies` respectively.
  */
 export interface SortedEntryPointsInfo extends DependencyDiagnostics {
   entryPoints: PartiallyOrderedEntryPoints;
+  graph: DepGraph<EntryPoint>;
 }
 
 /**
@@ -109,6 +110,7 @@ export class DependencyResolver {
     return {
       entryPoints: (sortedEntryPointNodes as PartiallyOrderedList<string>)
                        .map(path => graph.getNodeData(path)),
+      graph,
       invalidEntryPoints,
       ignoredDependencies,
     };

packages/compiler-cli/ngcc/src/execution/api.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {EntryPoint, EntryPointJsonProperty} from '../packages/entry_point';
+import {EntryPoint, EntryPointJsonProperty, JsonObject} from '../packages/entry_point';
 import {PartiallyOrderedList} from '../utils';
 
 
@@ -49,7 +49,7 @@ export interface Executor {
 export type PartiallyOrderedTasks = PartiallyOrderedList<Task>;
 
 /** Represents a unit of work: processing a specific format property of an entry-point. */
-export interface Task {
+export interface Task extends JsonObject {
   /** The `EntryPoint` which needs to be processed as part of the task. */
   entryPoint: EntryPoint;
 

packages/compiler-cli/ngcc/src/execution/cluster/api.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {AbsoluteFsPath} from '../../../../src/ngtsc/file_system';
+import {JsonObject} from '../../packages/entry_point';
+import {PackageJsonChange} from '../../writing/package_json_updater';
+import {Task, TaskProcessingOutcome} from '../api';
+
+
+/** A message reporting that an unrecoverable error occurred. */
+export interface ErrorMessage extends JsonObject {
+  type: 'error';
+  error: string;
+}
+
+/** A message requesting the processing of a task. */
+export interface ProcessTaskMessage extends JsonObject {
+  type: 'process-task';
+  task: Task;
+}
+
+/**
+ * A message reporting the result of processing the currently assigned task.
+ *
+ * NOTE: To avoid the communication overhead, the task is not included in the message. Instead, the
+ *       master is responsible for keeping a mapping of workers to their currently assigned tasks.
+ */
+export interface TaskCompletedMessage extends JsonObject {
+  type: 'task-completed';
+  outcome: TaskProcessingOutcome;
+}
+
+/** A message requesting the update of a `package.json` file. */
+export interface UpdatePackageJsonMessage extends JsonObject {
+  type: 'update-package-json';
+  packageJsonPath: AbsoluteFsPath;
+  changes: PackageJsonChange[];
+}
+
+/** The type of messages sent from cluster workers to the cluster master. */
+export type MessageFromWorker = ErrorMessage | TaskCompletedMessage | UpdatePackageJsonMessage;
+
+/** The type of messages sent from the cluster master to cluster workers. */
+export type MessageToWorker = ProcessTaskMessage;

packages/compiler-cli/ngcc/src/execution/cluster/executor.ts

@@ -0,0 +1,46 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/// <reference types="node" />
+
+import * as cluster from 'cluster';
+
+import {Logger} from '../../logging/logger';
+import {PackageJsonUpdater} from '../../writing/package_json_updater';
+import {AnalyzeEntryPointsFn, CreateCompileFn, Executor} from '../api';
+
+import {ClusterMaster} from './master';
+import {ClusterWorker} from './worker';
+
+
+/**
+ * An `Executor` that processes tasks in parallel (on multiple processes) and completes
+ * asynchronously.
+ */
+export class ClusterExecutor implements Executor {
+  constructor(
+      private workerCount: number, private logger: Logger,
+      private pkgJsonUpdater: PackageJsonUpdater) {}
+
+  async execute(analyzeEntryPoints: AnalyzeEntryPointsFn, createCompileFn: CreateCompileFn):
+      Promise<void> {
+    if (cluster.isMaster) {
+      this.logger.debug(
+          `Running ngcc on ${this.constructor.name} (using ${this.workerCount} worker processes).`);
+
+      // This process is the cluster master.
+      const master =
+          new ClusterMaster(this.workerCount, this.logger, this.pkgJsonUpdater, analyzeEntryPoints);
+      return master.run();
+    } else {
+      // This process is a cluster worker.
+      const worker = new ClusterWorker(createCompileFn);
+      return worker.run();
+    }
+  }
+}