Skip to content
Permalink
Browse files

perf(ivy): reuse prior analysis work during incremental builds (#34288)

Previously, the compiler performed an incremental build by analyzing and
resolving all classes in the program (even unchanged ones) and then using
the dependency graph information to determine which .js files were stale and
needed to be re-emitted. This algorithm produced "correct" rebuilds, but the
cost of re-analyzing the entire program turned out to be higher than
anticipated, especially for component-heavy compilations.

To achieve performant rebuilds, it is necessary to reuse previous analysis
results if possible. Doing this safely requires knowing when prior work is
viable and when it is stale and needs to be re-done.

The new algorithm implemented by this commit is such:

1) Each incremental build starts with knowledge of the last known good
   dependency graph and analysis results from the last successful build,
   plus of course information about the set of files changed.

2) The previous dependency graph's information is used to determine the
   set of source files which have "logically" changed. A source file is
   considered logically changed if it or any of its dependencies have
   physically changed (on disk) since the last successful compilation. Any
   logically unchanged dependencies have their dependency information copied
   over to the new dependency graph.

3) During the `TraitCompiler`'s loop to consider all source files in the
   program, if a source file is logically unchanged then its previous
   analyses are "adopted" (and their 'register' steps are run). If the file
   is logically changed, then it is re-analyzed as usual.

4) Then, incremental build proceeds as before, with the new dependency graph
   being used to determine the set of files which require re-emitting.

This analysis reuse avoids template parsing operations in many circumstances
and significantly reduces the time it takes ngtsc to rebuild a large
application.

Future work will increase performance even more, by tackling a variety of
other opportunities to reuse or avoid work.

PR Close #34288
  • Loading branch information
alxhub authored and kara committed Dec 6, 2019
1 parent e6ead8a commit c387952739f69fe7d5c7cf530e418a4f3d125126
Showing with 582 additions and 224 deletions.
  1. +1 −0 packages/compiler-cli/ngcc/BUILD.bazel
  2. +5 −4 packages/compiler-cli/ngcc/src/analysis/decoration_analyzer.ts
  3. +11 −1 packages/compiler-cli/ngcc/src/analysis/util.ts
  4. +1 −1 packages/compiler-cli/ngcc/test/analysis/migration_host_spec.ts
  5. +1 −1 packages/compiler-cli/ngcc/test/analysis/references_registry_spec.ts
  6. +1 −0 packages/compiler-cli/src/ngtsc/annotations/BUILD.bazel
  7. +9 −6 packages/compiler-cli/src/ngtsc/annotations/src/component.ts
  8. +1 −0 packages/compiler-cli/src/ngtsc/annotations/src/directive.ts
  9. +1 −0 packages/compiler-cli/src/ngtsc/annotations/src/injectable.ts
  10. +21 −6 packages/compiler-cli/src/ngtsc/annotations/src/ng_module.ts
  11. +1 −0 packages/compiler-cli/src/ngtsc/annotations/src/pipe.ts
  12. +3 −2 packages/compiler-cli/src/ngtsc/annotations/test/component_spec.ts
  13. +1 −1 packages/compiler-cli/src/ngtsc/annotations/test/directive_spec.ts
  14. +1 −1 packages/compiler-cli/src/ngtsc/annotations/test/ng_module_spec.ts
  15. +10 −0 packages/compiler-cli/src/ngtsc/incremental/BUILD.bazel
  16. +53 −0 packages/compiler-cli/src/ngtsc/incremental/api.ts
  17. +57 −10 packages/compiler-cli/src/ngtsc/incremental/src/README.md
  18. +142 −0 packages/compiler-cli/src/ngtsc/incremental/src/dependency_tracking.ts
  19. +75 −69 packages/compiler-cli/src/ngtsc/incremental/src/state.ts
  20. +1 −0 packages/compiler-cli/src/ngtsc/partial_evaluator/BUILD.bazel
  21. +1 −1 packages/compiler-cli/src/ngtsc/partial_evaluator/index.ts
  22. +2 −9 packages/compiler-cli/src/ngtsc/partial_evaluator/src/interface.ts
  23. +5 −4 packages/compiler-cli/src/ngtsc/partial_evaluator/src/interpreter.ts
  24. +1 −0 packages/compiler-cli/src/ngtsc/partial_evaluator/test/BUILD.bazel
  25. +27 −20 packages/compiler-cli/src/ngtsc/partial_evaluator/test/evaluator_spec.ts
  26. +4 −2 packages/compiler-cli/src/ngtsc/partial_evaluator/test/utils.ts
  27. +40 −38 packages/compiler-cli/src/ngtsc/program.ts
  28. +1 −0 packages/compiler-cli/src/ngtsc/transform/BUILD.bazel
  29. +1 −1 packages/compiler-cli/src/ngtsc/transform/index.ts
  30. +2 −0 packages/compiler-cli/src/ngtsc/transform/src/api.ts
  31. +82 −12 packages/compiler-cli/src/ngtsc/transform/src/compilation.ts
  32. +1 −0 packages/compiler-cli/src/ngtsc/util/BUILD.bazel
  33. +0 −20 packages/compiler-cli/src/ngtsc/util/src/resource_recorder.ts
  34. +1 −1 packages/compiler-cli/test/metadata/evaluator_spec.ts
  35. +6 −5 packages/compiler-cli/test/ngtsc/incremental_error_spec.ts
  36. +5 −3 packages/compiler-cli/test/ngtsc/incremental_spec.ts
  37. +2 −2 packages/core/schematics/migrations/missing-injectable/transform.ts
  38. +3 −2 packages/core/schematics/migrations/module-with-providers/transform.ts
  39. +2 −2 packages/core/schematics/migrations/undecorated-classes-with-di/index.ts
@@ -17,6 +17,7 @@ ts_library(
"//packages/compiler-cli/src/ngtsc/diagnostics",
"//packages/compiler-cli/src/ngtsc/file_system",
"//packages/compiler-cli/src/ngtsc/imports",
"//packages/compiler-cli/src/ngtsc/incremental:api",
"//packages/compiler-cli/src/ngtsc/metadata",
"//packages/compiler-cli/src/ngtsc/partial_evaluator",
"//packages/compiler-cli/src/ngtsc/perf",
@@ -27,7 +27,7 @@ import {isDefined} from '../utils';

import {DefaultMigrationHost} from './migration_host';
import {AnalyzedClass, AnalyzedFile, CompiledClass, CompiledFile, DecorationAnalyses} from './types';
import {analyzeDecorators, isWithinPackage} from './util';
import {NOOP_DEPENDENCY_TRACKER, analyzeDecorators, isWithinPackage} from './util';


/**
@@ -79,7 +79,8 @@ export class DecorationAnalyzer {
scopeRegistry = new LocalModuleScopeRegistry(
this.metaRegistry, this.dtsModuleScopeResolver, this.refEmitter, this.aliasingHost);
fullRegistry = new CompoundMetadataRegistry([this.metaRegistry, this.scopeRegistry]);
evaluator = new PartialEvaluator(this.reflectionHost, this.typeChecker);
evaluator =
new PartialEvaluator(this.reflectionHost, this.typeChecker, /* dependencyTracker */ null);
moduleResolver = new ModuleResolver(this.program, this.options, this.host);
importGraph = new ImportGraph(this.moduleResolver);
cycleAnalyzer = new CycleAnalyzer(this.importGraph);
@@ -90,9 +91,9 @@ export class DecorationAnalyzer {
/* defaultPreserveWhitespaces */ false,
/* i18nUseExternalIds */ true, this.bundle.enableI18nLegacyMessageIdFormat,
this.moduleResolver, this.cycleAnalyzer, this.refEmitter, NOOP_DEFAULT_IMPORT_RECORDER,
/* annotateForClosureCompiler */ false),
NOOP_DEPENDENCY_TRACKER, /* annotateForClosureCompiler */ false),
// clang-format off
// See the note in ngtsc about why this cast is needed.
// See the note in ngtsc about why this cast is needed.
new DirectiveDecoratorHandler(
this.reflectionHost, this.evaluator, this.fullRegistry, NOOP_DEFAULT_IMPORT_RECORDER,
this.isCore, /* annotateForClosureCompiler */ false) as DecoratorHandler<unknown, unknown, unknown>,
@@ -9,8 +9,9 @@ import * as ts from 'typescript';

import {isFatalDiagnosticError} from '../../../src/ngtsc/diagnostics';
import {AbsoluteFsPath, absoluteFromSourceFile, relative} from '../../../src/ngtsc/file_system';
import {DependencyTracker} from '../../../src/ngtsc/incremental/api';
import {Decorator} from '../../../src/ngtsc/reflection';
import {DecoratorHandler, DetectResult, HandlerFlags, HandlerPrecedence} from '../../../src/ngtsc/transform';
import {DecoratorHandler, HandlerFlags, HandlerPrecedence} from '../../../src/ngtsc/transform';
import {NgccClassSymbol} from '../host/ngcc_host';

import {AnalyzedClass, MatchingHandler} from './types';
@@ -103,3 +104,12 @@ export function analyzeDecorators(
diagnostics: allDiagnostics.length > 0 ? allDiagnostics : undefined,
};
}

class NoopDependencyTracker implements DependencyTracker {
addDependency(): void {}
addResourceDependency(): void {}
addTransitiveDependency(): void {}
addTransitiveResources(): void {}
}

export const NOOP_DEPENDENCY_TRACKER: DependencyTracker = new NoopDependencyTracker();
@@ -298,7 +298,7 @@ runInEachFileSystem(() => {
});

class TestHandler implements DecoratorHandler<unknown, unknown, unknown> {
constructor(protected name: string, protected log: string[]) {}
constructor(readonly name: string, protected log: string[]) {}

precedence = HandlerPrecedence.PRIMARY;
detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<unknown>|undefined {
@@ -47,7 +47,7 @@ runInEachFileSystem(() => {
const testArrayExpression = testArrayDeclaration.initializer !;

const reflectionHost = new TypeScriptReflectionHost(checker);
const evaluator = new PartialEvaluator(reflectionHost, checker);
const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
const registry = new NgccReferencesRegistry(reflectionHost);

const references = (evaluator.evaluate(testArrayExpression) as any[]).filter(isReference);
@@ -13,6 +13,7 @@ ts_library(
"//packages/compiler-cli/src/ngtsc/diagnostics",
"//packages/compiler-cli/src/ngtsc/file_system",
"//packages/compiler-cli/src/ngtsc/imports",
"//packages/compiler-cli/src/ngtsc/incremental:api",
"//packages/compiler-cli/src/ngtsc/indexer",
"//packages/compiler-cli/src/ngtsc/metadata",
"//packages/compiler-cli/src/ngtsc/partial_evaluator",
@@ -13,6 +13,7 @@ import {CycleAnalyzer} from '../../cycles';
import {ErrorCode, FatalDiagnosticError} from '../../diagnostics';
import {absoluteFrom, relative} from '../../file_system';
import {DefaultImportRecorder, ModuleResolver, Reference, ReferenceEmitter} from '../../imports';
import {DependencyTracker} from '../../incremental/api';
import {IndexingContext} from '../../indexer';
import {DirectiveMeta, MetadataReader, MetadataRegistry, extractDirectiveGuards} from '../../metadata';
import {flattenInheritedDirectiveMetadata} from '../../metadata/src/inheritance';
@@ -21,7 +22,6 @@ import {ClassDeclaration, Decorator, ReflectionHost, reflectObjectLiteral} from
import {ComponentScopeReader, LocalModuleScopeRegistry} from '../../scope';
import {AnalysisOutput, CompileResult, DecoratorHandler, DetectResult, HandlerFlags, HandlerPrecedence, ResolveResult} from '../../transform';
import {TemplateSourceMapping, TypeCheckContext} from '../../typecheck';
import {NoopResourceDependencyRecorder, ResourceDependencyRecorder} from '../../util/src/resource_recorder';
import {tsSourceMapBug29300Fixed} from '../../util/src/ts_source_map_bug_29300';
import {SubsetOfKeys} from '../../util/src/typescript';

@@ -73,9 +73,7 @@ export class ComponentDecoratorHandler implements
private enableI18nLegacyMessageIdFormat: boolean, private moduleResolver: ModuleResolver,
private cycleAnalyzer: CycleAnalyzer, private refEmitter: ReferenceEmitter,
private defaultImportRecorder: DefaultImportRecorder,
private annotateForClosureCompiler: boolean,
private resourceDependencies:
ResourceDependencyRecorder = new NoopResourceDependencyRecorder()) {}
private depTracker: DependencyTracker|null, private annotateForClosureCompiler: boolean) {}

private literalCache = new Map<Decorator, ts.ObjectLiteralExpression>();
private elementSchemaRegistry = new DomElementSchemaRegistry();
@@ -88,6 +86,7 @@ export class ComponentDecoratorHandler implements
private preanalyzeTemplateCache = new Map<ts.Declaration, PreanalyzedTemplate>();

readonly precedence = HandlerPrecedence.PRIMARY;
readonly name = ComponentDecoratorHandler.name;

detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
if (!decorators) {
@@ -255,7 +254,9 @@ export class ComponentDecoratorHandler implements
const resourceUrl = this.resourceLoader.resolve(styleUrl, containingFile);
const resourceStr = this.resourceLoader.load(resourceUrl);
styles.push(resourceStr);
this.resourceDependencies.recordResourceDependency(node.getSourceFile(), resourceUrl);
if (this.depTracker !== null) {
this.depTracker.addResourceDependency(node.getSourceFile(), resourceUrl);
}
}
}
if (component.has('styles')) {
@@ -638,7 +639,9 @@ export class ComponentDecoratorHandler implements
templateSourceMapping: TemplateSourceMapping
} {
const templateStr = this.resourceLoader.load(resourceUrl);
this.resourceDependencies.recordResourceDependency(node.getSourceFile(), resourceUrl);
if (this.depTracker !== null) {
this.depTracker.addResourceDependency(node.getSourceFile(), resourceUrl);
}
const parseTemplate = (options?: ParseTemplateOptions) => this._parseTemplate(
component, templateStr, sourceMapUrl(resourceUrl),
/* templateRange */ undefined,
@@ -45,6 +45,7 @@ export class DirectiveDecoratorHandler implements
private isCore: boolean, private annotateForClosureCompiler: boolean) {}

readonly precedence = HandlerPrecedence.PRIMARY;
readonly name = DirectiveDecoratorHandler.name;

detect(node: ClassDeclaration, decorators: Decorator[]|null):
DetectResult<Decorator|null>|undefined {
@@ -42,6 +42,7 @@ export class InjectableDecoratorHandler implements
private errorOnDuplicateProv = true) {}

readonly precedence = HandlerPrecedence.SHARED;
readonly name = InjectableDecoratorHandler.name;

detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
if (!decorators) {
@@ -36,13 +36,15 @@ export interface NgModuleAnalysis {
factorySymbolName: string;
}

export interface NgModuleResolution { injectorImports: Expression[]; }

/**
* Compiles @NgModule annotations to ngModuleDef fields.
*
* TODO(alxhub): handle injector side of things as well.
*/
export class NgModuleDecoratorHandler implements
DecoratorHandler<Decorator, NgModuleAnalysis, unknown> {
DecoratorHandler<Decorator, NgModuleAnalysis, NgModuleResolution> {
constructor(
private reflector: ReflectionHost, private evaluator: PartialEvaluator,
private metaReader: MetadataReader, private metaRegistry: MetadataRegistry,
@@ -54,6 +56,7 @@ export class NgModuleDecoratorHandler implements
private annotateForClosureCompiler: boolean, private localeId?: string) {}

readonly precedence = HandlerPrecedence.PRIMARY;
readonly name = NgModuleDecoratorHandler.name;

detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
if (!decorators) {
@@ -270,17 +273,21 @@ export class NgModuleDecoratorHandler implements
}
}

resolve(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>): ResolveResult<unknown> {
resolve(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>):
ResolveResult<NgModuleResolution> {
const scope = this.scopeRegistry.getScopeOfModule(node);
const diagnostics = this.scopeRegistry.getDiagnosticsOfModule(node) || undefined;
const data: NgModuleResolution = {
injectorImports: [],
};

if (scope !== null) {
// Using the scope information, extend the injector's imports using the modules that are
// specified as module exports.
const context = getSourceFile(node);
for (const exportRef of analysis.exports) {
if (isNgModule(exportRef.node, scope.compilation)) {
analysis.inj.imports.push(this.refEmitter.emit(exportRef, context));
data.injectorImports.push(this.refEmitter.emit(exportRef, context));
}
}

@@ -296,17 +303,25 @@ export class NgModuleDecoratorHandler implements
}

if (scope === null || scope.reexports === null) {
return {diagnostics};
return {data, diagnostics};
} else {
return {
data,
diagnostics,
reexports: scope.reexports,
};
}
}

compile(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>): CompileResult[] {
const ngInjectorDef = compileInjector(analysis.inj);
compile(
node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>,
resolution: Readonly<NgModuleResolution>): CompileResult[] {
// Merge the injector imports (which are 'exports' that were later found to be NgModules)
// computed during resolution with the ones from analysis.
const ngInjectorDef = compileInjector({
...analysis.inj,
imports: [...analysis.inj.imports, ...resolution.injectorImports],
});
const ngModuleDef = compileNgModule(analysis.mod);
const ngModuleStatements = ngModuleDef.additionalStatements;
if (analysis.metadataStmt !== null) {
@@ -32,6 +32,7 @@ export class PipeDecoratorHandler implements DecoratorHandler<Decorator, PipeHan
private isCore: boolean) {}

readonly precedence = HandlerPrecedence.PRIMARY;
readonly name = PipeDecoratorHandler.name;

detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
if (!decorators) {
@@ -47,7 +47,7 @@ runInEachFileSystem(() => {
]);
const checker = program.getTypeChecker();
const reflectionHost = new TypeScriptReflectionHost(checker);
const evaluator = new PartialEvaluator(reflectionHost, checker);
const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
const moduleResolver = new ModuleResolver(program, options, host);
const importGraph = new ImportGraph(moduleResolver);
const cycleAnalyzer = new CycleAnalyzer(importGraph);
@@ -64,7 +64,8 @@ runInEachFileSystem(() => {
/* isCore */ false, new NoopResourceLoader(), /* rootDirs */[''],
/* defaultPreserveWhitespaces */ false, /* i18nUseExternalIds */ true,
/* enableI18nLegacyMessageIdFormat */ false, moduleResolver, cycleAnalyzer, refEmitter,
NOOP_DEFAULT_IMPORT_RECORDER, /* annotateForClosureCompiler */ false);
NOOP_DEFAULT_IMPORT_RECORDER, /* depTracker */ null,
/* annotateForClosureCompiler */ false);
const TestCmp = getDeclaration(program, _('/entry.ts'), 'TestCmp', isNamedClassDeclaration);
const detected = handler.detect(TestCmp, reflectionHost.getDecoratorsOfDeclaration(TestCmp));
if (detected === undefined) {
@@ -42,7 +42,7 @@ runInEachFileSystem(() => {

const checker = program.getTypeChecker();
const reflectionHost = new TestReflectionHost(checker);
const evaluator = new PartialEvaluator(reflectionHost, checker);
const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
const metaReader = new LocalMetadataRegistry();
const dtsReader = new DtsMetadataReader(checker, reflectionHost);
const scopeRegistry = new LocalModuleScopeRegistry(
@@ -57,7 +57,7 @@ runInEachFileSystem(() => {
]);
const checker = program.getTypeChecker();
const reflectionHost = new TypeScriptReflectionHost(checker);
const evaluator = new PartialEvaluator(reflectionHost, checker);
const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
const referencesRegistry = new NoopReferencesRegistry();
const metaRegistry = new LocalMetadataRegistry();
const metaReader = new CompoundMetadataReader([metaRegistry]);
@@ -8,12 +8,22 @@ ts_library(
"src/**/*.ts",
]),
deps = [
":api",
"//packages/compiler-cli/src/ngtsc/imports",
"//packages/compiler-cli/src/ngtsc/metadata",
"//packages/compiler-cli/src/ngtsc/partial_evaluator",
"//packages/compiler-cli/src/ngtsc/reflection",
"//packages/compiler-cli/src/ngtsc/scope",
"//packages/compiler-cli/src/ngtsc/transform",
"//packages/compiler-cli/src/ngtsc/util",
"@npm//typescript",
],
)

ts_library(
name = "api",
srcs = ["api.ts"],
deps = [
"@npm//typescript",
],
)
@@ -0,0 +1,53 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/

import * as ts from 'typescript';

/**
* Interface of the incremental build engine.
*
* `W` is a generic type representing a unit of work. This is generic to avoid a cyclic dependency
* between the incremental engine API definition and its consumer(s).
*/
export interface IncrementalBuild<W> {
/**
* Retrieve the prior analysis work, if any, done for the given source file.
*/
priorWorkFor(sf: ts.SourceFile): W[]|null;
}

/**
* Tracks dependencies between source files or resources in the application.
*/
export interface DependencyTracker<T extends{fileName: string} = ts.SourceFile> {
/**
* Record that the file `from` depends on the file `on`.
*/
addDependency(from: T, on: T): void;

/**
* Record that the file `from` depends on the resource file `on`.
*/
addResourceDependency(from: T, on: string): void;

/**
* Record that the file `from` depends on the file `on` as well as `on`'s direct dependencies.
*
* This operation is reified immediately, so if future dependencies are added to `on` they will
* not automatically be added to `from`.
*/
addTransitiveDependency(from: T, on: T): void;

/**
* Record that the file `from` depends on the resource dependencies of `resourcesOf`.
*
* This operation is reified immediately, so if future resource dependencies are added to
* `resourcesOf` they will not automatically be added to `from`.
*/
addTransitiveResources(from: T, resourcesOf: T): void;
}

0 comments on commit c387952

Please sign in to comment.
You can’t perform that action at this time.