Perf logging & Incremental Compilation

packages/compiler-cli/BUILD.bazel

@@ -28,8 +28,10 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/entry_point",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/path",
+        "//packages/compiler-cli/src/ngtsc/perf",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/routing",
         "//packages/compiler-cli/src/ngtsc/scope",

packages/compiler-cli/ngcc/BUILD.bazel

@@ -16,6 +16,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/imports",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/path",
+        "//packages/compiler-cli/src/ngtsc/perf",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/scope",
         "//packages/compiler-cli/src/ngtsc/transform",

packages/compiler-cli/ngcc/src/analysis/decoration_analyzer.ts

@@ -6,6 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import {ConstantPool} from '@angular/compiler';
+import {NOOP_PERF_RECORDER} from '@angular/compiler-cli/src/ngtsc/perf';
 import * as path from 'canonical-path';
 import * as fs from 'fs';
 import * as ts from 'typescript';
@@ -90,7 +91,8 @@ export class DecorationAnalyzer {
         this.reflectionHost, this.evaluator, this.scopeRegistry, NOOP_DEFAULT_IMPORT_RECORDER,
         this.isCore),
     new InjectableDecoratorHandler(
-        this.reflectionHost, NOOP_DEFAULT_IMPORT_RECORDER, this.isCore, /* strictCtorDeps */ false),
+        this.reflectionHost, NOOP_DEFAULT_IMPORT_RECORDER, this.isCore,
+        /* strictCtorDeps */ false),
     new NgModuleDecoratorHandler(
         this.reflectionHost, this.evaluator, this.scopeRegistry, this.referencesRegistry,
         this.isCore, /* routeAnalyzer */ null, this.refEmitter, NOOP_DEFAULT_IMPORT_RECORDER),

packages/compiler-cli/src/main.ts

@@ -22,7 +22,9 @@ import {performWatchCompilation, createPerformWatchHost} from './perform_watch'
 
 export function main(
     args: string[], consoleError: (s: string) => void = console.error,
-    config?: NgcParsedConfiguration, customTransformers?: api.CustomTransformers): number {
+    config?: NgcParsedConfiguration, customTransformers?: api.CustomTransformers, programReuse?: {
+      program: api.Program | undefined,
+    }): number {
   let {project, rootNames, options, errors: configErrors, watch, emitFlags} =
       config || readNgcCommandLineAndConfiguration(args);
   if (configErrors.length) {
@@ -32,12 +34,22 @@ export function main(
     const result = watchMode(project, options, consoleError);
     return reportErrorsAndExit(result.firstCompileResult, options, consoleError);
   }
-  const {diagnostics: compileDiags} = performCompilation({
+
+  let oldProgram: api.Program|undefined;
+  if (programReuse !== undefined) {
+    oldProgram = programReuse.program;
+  }
+
+  const {diagnostics: compileDiags, program} = performCompilation({
     rootNames,
     options,
     emitFlags,
+    oldProgram,
     emitCallback: createEmitCallback(options), customTransformers
   });
+  if (programReuse !== undefined) {
+    programReuse.program = program;
+  }
   return reportErrorsAndExit(compileDiags, options, consoleError);
 }
 

packages/compiler-cli/src/ngtsc/annotations/src/injectable.ts

@@ -50,6 +50,8 @@ export class InjectableDecoratorHandler implements
 
   analyze(node: ClassDeclaration, decorator: Decorator): AnalysisOutput<InjectableHandlerData> {
     return {
+      // @Injectable()s cannot depend on other files for their compilation output.
+      allowSkipAnalysisAndEmit: true,
       analysis: {
         meta: extractInjectableMetadata(
             node, decorator, this.reflector, this.defaultImportRecorder, this.isCore,

packages/compiler-cli/src/ngtsc/incremental/BUILD.bazel

@@ -0,0 +1,13 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "incremental",
+    srcs = ["index.ts"] + glob([
+        "src/**/*.ts",
+    ]),
+    deps = [
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/src/ngtsc/incremental/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export {IncrementalState} from './src/state';

packages/compiler-cli/src/ngtsc/incremental/src/README.md

@@ -0,0 +1,43 @@
+# What is the `incremental` package?
+
+This package contains logic related to incremental compilation in ngtsc.
+
+In particular, it tracks metadata for `ts.SourceFile`s in between compilations, so the compiler can make intelligent decisions about when to skip certain operations and rely on previously analyzed data.
+
+# How does incremental compilation work?
+
+The initial compilation is no different from a standalone compilation; the compiler is unaware that incremental compilation will be utilized.
+
+When an `NgtscProgram` is created for a _subsequent_ compilation, it is initialized with the `NgtscProgram` from the previous compilation. It is therefore able to take advantage of any information present in the previous compilation to optimize the next one.
+
+This information is leveraged in two major ways:
+
+1) The previous `ts.Program` itself is used to create the next `ts.Program`, allowing TypeScript internally to leverage information from the previous compile in much the same way.
+
+2) An `IncrementalState` instance is constructed from the previous compilation's `IncrementalState` and its `ts.Program`.
+
+After this initialization, the `IncrementalState` contains the knowledge from the previous compilation which will be used to optimize the next one.
+
+# What optimizations can be made?
+
+Currently, ngtsc makes a decision to skip the emit of a file if it can prove that the contents of the file will not have changed. To prove this, two conditions must be true.
+
+* The input file itself must not have changed since the previous compilation.
+
+* As a result of analyzing the file, no dependencies must exist where the output of compilation could vary depending on the contents of any other file.
+
+The second condition is challenging, as Angular allows statically evaluated expressions in lots of contexts that could result in changes from file to file. For example, the `name` of an `@Pipe` could be a reference to a constant in a different file.
+
+Therefore, only two types of files meet these conditions and can be optimized today:
+
+* Files with no Angular decorated classes at all.
+
+* Files with only `@Injectable`s.
+
+# What optimizations are possible in the future?
+
+There is plenty of room for improvement here, with diminishing returns for the work involved.
+
+* The compiler could track the dependencies of each file being compiled, and know whether an `@Pipe` gets its name from a second file, for example. This is sufficient to skip the analysis and emit of more files when none of the dependencies have changed.
+
+* The compiler could also perform analysis on files which _have_ changed dependencies, and skip emit if the analysis indicates nothing has changed which would affect the file being emitted.

packages/compiler-cli/src/ngtsc/incremental/src/state.ts

@@ -0,0 +1,86 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+/**
+ * Accumulates state between compilations.
+ */
+export class IncrementalState {
+  private constructor(
+      private unchangedFiles: Set<ts.SourceFile>,
+      private metadata: Map<ts.SourceFile, FileMetadata>) {}
+
+  static reconcile(previousState: IncrementalState, oldProgram: ts.Program, newProgram: ts.Program):
+      IncrementalState {
+    const unchangedFiles = new Set<ts.SourceFile>();
+    const metadata = new Map<ts.SourceFile, FileMetadata>();
+
+    // Compute the set of files that's unchanged.
+    const oldFiles = new Set<ts.SourceFile>();
+    for (const oldFile of oldProgram.getSourceFiles()) {
+      if (!oldFile.isDeclarationFile) {
+        oldFiles.add(oldFile);
+      }
+    }
+
+    // Look for files in the new program which haven't changed.
+    for (const newFile of newProgram.getSourceFiles()) {
+      if (oldFiles.has(newFile)) {
+        unchangedFiles.add(newFile);
+
+        // Copy over metadata for the unchanged file if available.
+        if (previousState.metadata.has(newFile)) {
+          metadata.set(newFile, previousState.metadata.get(newFile) !);
+        }
+      }
+    }
+
+    return new IncrementalState(unchangedFiles, metadata);
+  }
+
+  static fresh(): IncrementalState {
+    return new IncrementalState(new Set<ts.SourceFile>(), new Map<ts.SourceFile, FileMetadata>());
+  }
+
+  safeToSkipEmit(sf: ts.SourceFile): boolean {
+    if (!this.unchangedFiles.has(sf)) {
+      // The file has changed since the last run, and must be re-emitted.
+      return false;
+    }
+
+    // The file hasn't changed since the last emit. Whether or not it's safe to emit depends on
+    // what metadata was gathered about the file.
+
+    if (!this.metadata.has(sf)) {
+      // The file has no metadata from the previous or current compilations, so it must be emitted.
+      return false;
+    }
+
+    const meta = this.metadata.get(sf) !;
+
+    // Check if this file was explicitly marked as safe. This would only be done if every
+    // `DecoratorHandler` agreed that the file didn't depend on any other file's contents.
+    if (meta.safeToSkipEmitIfUnchanged) {
+      return true;
+    }
+
+    // The file wasn't explicitly marked as safe to skip emitting, so require an emit.
+    return false;
+  }
+
+  markFileAsSafeToSkipEmitIfUnchanged(sf: ts.SourceFile): void {
+    this.metadata.set(sf, {
+      safeToSkipEmitIfUnchanged: true,
+    });
+  }
+}
+
+interface FileMetadata {
+  safeToSkipEmitIfUnchanged: boolean;
+}

packages/compiler-cli/src/ngtsc/perf/BUILD.bazel

@@ -0,0 +1,15 @@
+package(default_visibility = ["//visibility:public"])
+
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "perf",
+    srcs = ["index.ts"] + glob([
+        "src/*.ts",
+    ]),
+    deps = [
+        "//packages:types",
+        "@npm//@types/node",
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/src/ngtsc/perf/README.md

@@ -0,0 +1,21 @@
+# What is the `perf` package?
+
+This package contains utilities for collecting performance information from around the compiler and for producing ngtsc performance traces.
+
+This feature is currently undocumented and not exposed to users as the trace file format is still unstable.
+
+# What is a performance trace?
+
+A performance trace is a JSON file which logs events within ngtsc with microsecond precision. It tracks the phase of compilation, the file and possibly symbol being compiled, and other metadata explaining what the compiler was doing during the event.
+
+Traces track two specific kinds of events: marks and spans. A mark is a single event with a timestamp, while a span is two events (start and end) that cover a section of work in the compiler. Analyzing a file is a span event, while a decision (such as deciding not to emit a particular file) is a mark event.
+
+# Enabling performance traces
+
+Performance traces are enabled via the undocumented `tracePerformance` option in `angularCompilerOptions` inside the tsconfig.json file. This option takes a string path relative to the current directory, which will be used to write the trace JSON file.
+
+## In-Memory TS Host Tracing
+
+By default, the trace file will be written with the `fs` package directly. However, ngtsc supports in-memory compilation using a `ts.CompilerHost` for all operations. In the event that tracing is required when using an in-memory filesystem, a `ts:` prefix can be added to the value of `tracePerformance`, which will cause the trace JSON file to be written with the TS host's `writeFile` method instead.
+
+This is not done by default as `@angular/cli` does not allow writing arbitrary JSON files via its host.

packages/compiler-cli/src/ngtsc/perf/index.ts

@@ -0,0 +1,11 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export {PerfRecorder} from './src/api';
+export {NOOP_PERF_RECORDER} from './src/noop';
+export {PerfTracker} from './src/tracking';

packages/compiler-cli/src/ngtsc/perf/src/api.ts

@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+export interface PerfRecorder {
+  readonly enabled: boolean;
+
+  mark(name: string, node?: ts.SourceFile|ts.Declaration, category?: string, detail?: string): void;
+  start(name: string, node?: ts.SourceFile|ts.Declaration, category?: string, detail?: string):
+      number;
+  stop(span: number): void;
+}

packages/compiler-cli/src/ngtsc/perf/src/clock.ts

@@ -0,0 +1,21 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+// This file uses 'process'
+/// <reference types="node" />
+
+export type HrTime = [number, number];
+
+export function mark(): HrTime {
+  return process.hrtime();
+}
+
+export function timeSinceInMicros(mark: HrTime): number {
+  const delta = process.hrtime(mark);
+  return (delta[0] * 1000000) + Math.floor(delta[1] / 1000);
+}

packages/compiler-cli/src/ngtsc/perf/src/noop.ts

@@ -0,0 +1,20 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {PerfRecorder} from './api';
+
+export const NOOP_PERF_RECORDER: PerfRecorder = {
+  enabled: false,
+  mark: (name: string, node: ts.SourceFile | ts.Declaration, category?: string, detail?: string):
+            void => {},
+  start: (name: string, node: ts.SourceFile | ts.Declaration, category?: string, detail?: string):
+             number => { return 0;},
+  stop: (span: number | false): void => {},
+};