perf(compiler): make the creation of ts.Program faster. (#19275)

We now create 2 programs with exactly the same fileNames and
exactly the same `import` / `export` declarations,
allowing TS to reuse the structure of first program
completely. When passing in an oldProgram and the files didn’t change,
TS can also reuse the old program completely.

This is possible buy adding generated files to TS
in `host.geSourceFile` via `ts.SourceFile.referencedFiles`.

This commit also:
- has a minor side effect on how we generate shared stylesheets:
  - previously every import in a stylesheet would generate a new
    `.ngstyles.ts` file.
  - now, we only generate 1 `.ngstyles.ts` file per entry in `@Component.styleUrls`.
  This was required as we need to be able to determine the program files
  without loading the resources (which can be async).
- makes all angular related methods in `CompilerHost`
  optional, allowing to just use a regular `ts.CompilerHost` as `CompilerHost`.
- simplifies the logic around `Compiler.analyzeNgModules` by introducing `NgAnalyzedFile`.

Perf impact: 1.5s improvement in compiling angular io
PR Close #19275

integration/ng-cli-create.sh

@@ -38,8 +38,7 @@ else
 
     sed -i -E 's/ng build/ng build --prod --build-optimizer/g' package.json
     sed -i -E 's/ng test/ng test --single-run/g' package.json
-    # workaround for https://github.com/angular/angular-cli/issues/7401
-    sed -i -E 's/"@angular\/cli\"\: \".*\"/"@angular\/cli":  "https:\/\/github.com\/angular\/cli-builds"/g' package.json
+    sed -i -E 's/"typescript\"\: \".*\"/"typescript":  "2.4.2"/g' package.json
 
     yarn add \
       file:../../dist/packages-dist/compiler-cli \

packages/compiler-cli/src/compiler_host.ts

@@ -22,8 +22,6 @@ const GENERATED_FILES = /\.ngfactory\.ts$|\.ngstyle\.ts$|\.ngsummary\.ts$/;
 const GENERATED_OR_DTS_FILES = /\.d\.ts$|\.ngfactory\.ts$|\.ngstyle\.ts$|\.ngsummary\.ts$/;
 const SHALLOW_IMPORT = /^((\w|-)+|(@(\w|-)+(\/(\w|-)+)+))$/;
 
-export interface MetadataProvider { getMetadata(source: ts.SourceFile): ModuleMetadata|undefined; }
-
 export interface BaseAotCompilerHostContext extends ts.ModuleResolutionHost {
   readResource?(fileName: string): Promise<string>|string;
 }
@@ -35,9 +33,7 @@ export abstract class BaseAotCompilerHost<C extends BaseAotCompilerHostContext>
   private flatModuleIndexNames = new Set<string>();
   private flatModuleIndexRedirectNames = new Set<string>();
 
-  constructor(
-      protected program: ts.Program, protected options: CompilerOptions, protected context: C,
-      protected metadataProvider: MetadataProvider = new MetadataCollector()) {}
+  constructor(protected options: CompilerOptions, protected context: C) {}
 
   abstract moduleNameToFileName(m: string, containingFile: string): string|null;
 
@@ -49,17 +45,7 @@ export abstract class BaseAotCompilerHost<C extends BaseAotCompilerHostContext>
 
   abstract fromSummaryFileName(fileName: string, referringLibFileName: string): string;
 
-  protected getSourceFile(filePath: string): ts.SourceFile {
-    const sf = this.program.getSourceFile(filePath);
-    if (!sf) {
-      if (this.context.fileExists(filePath)) {
-        const sourceText = this.context.readFile(filePath);
-        return ts.createSourceFile(filePath, sourceText, ts.ScriptTarget.Latest, true);
-      }
-      throw new Error(`Source file ${filePath} not present in program.`);
-    }
-    return sf;
-  }
+  abstract getMetadataForSourceFile(filePath: string): ModuleMetadata|undefined;
 
   getMetadataFor(filePath: string): ModuleMetadata[]|undefined {
     if (!this.context.fileExists(filePath)) {
@@ -82,8 +68,7 @@ export abstract class BaseAotCompilerHost<C extends BaseAotCompilerHostContext>
       }
     }
 
-    const sf = this.getSourceFile(filePath);
-    const metadata = this.metadataProvider.getMetadata(sf);
+    const metadata = this.getMetadataForSourceFile(filePath);
     return metadata ? [metadata] : [];
   }
 
@@ -122,7 +107,7 @@ export abstract class BaseAotCompilerHost<C extends BaseAotCompilerHostContext>
       v3Metadata.metadata[prop] = v1Metadata.metadata[prop];
     }
 
-    const exports = this.metadataProvider.getMetadata(this.getSourceFile(dtsFilePath));
+    const exports = this.getMetadataForSourceFile(dtsFilePath);
     if (exports) {
       for (let prop in exports.metadata) {
         if (!v3Metadata.metadata[prop]) {
@@ -233,6 +218,7 @@ export interface CompilerHostContext extends ts.ModuleResolutionHost {
 
 // TODO(tbosch): remove this once G3 uses the transformer compiler!
 export class CompilerHost extends BaseAotCompilerHost<CompilerHostContext> {
+  protected metadataProvider: MetadataCollector;
   protected basePath: string;
   private moduleFileNames = new Map<string, string|null>();
   private isGenDirChildOfRootDir: boolean;
@@ -241,10 +227,10 @@ export class CompilerHost extends BaseAotCompilerHost<CompilerHostContext> {
   private urlResolver: UrlResolver;
 
   constructor(
-      program: ts.Program, options: CompilerOptions, context: CompilerHostContext,
-      collectorOptions?: CollectorOptions,
-      metadataProvider: MetadataProvider = new MetadataCollector(collectorOptions)) {
-    super(program, options, context, metadataProvider);
+      protected program: ts.Program, options: CompilerOptions, context: CompilerHostContext,
+      collectorOptions?: CollectorOptions) {
+    super(options, context);
+    this.metadataProvider = new MetadataCollector(collectorOptions);
     // normalize the path so that it never ends with '/'.
     this.basePath = path.normalize(path.join(this.options.basePath !, '.')).replace(/\\/g, '/');
     this.genDir = path.normalize(path.join(this.options.genDir !, '.')).replace(/\\/g, '/');
@@ -274,6 +260,19 @@ export class CompilerHost extends BaseAotCompilerHost<CompilerHostContext> {
     this.urlResolver = createOfflineCompileUrlResolver();
   }
 
+  getMetadataForSourceFile(filePath: string): ModuleMetadata|undefined {
+    let sf = this.program.getSourceFile(filePath);
+    if (!sf) {
+      if (this.context.fileExists(filePath)) {
+        const sourceText = this.context.readFile(filePath);
+        sf = ts.createSourceFile(filePath, sourceText, ts.ScriptTarget.Latest, true);
+      } else {
+        throw new Error(`Source file ${filePath} not present in program.`);
+      }
+    }
+    return this.metadataProvider.getMetadata(sf);
+  }
+
   toSummaryFileName(fileName: string, referringSrcFileName: string): string {
     return fileName.replace(EXT, '') + '.d.ts';
   }

packages/compiler-cli/src/diagnostics/translate_diagnostics.ts

@@ -13,7 +13,7 @@ import {DEFAULT_ERROR_CODE, Diagnostic, SOURCE} from '../transformers/api';
 import {GENERATED_FILES} from '../transformers/util';
 
 export interface TypeCheckHost {
-  ngSpanOf(fileName: string, line: number, character: number): ParseSourceSpan|null;
+  parseSourceSpanOf(fileName: string, line: number, character: number): ParseSourceSpan|null;
 }
 
 export function translateDiagnostics(host: TypeCheckHost, untranslatedDiagnostics: ts.Diagnostic[]):
@@ -49,7 +49,7 @@ export function translateDiagnostics(host: TypeCheckHost, untranslatedDiagnostic
 function sourceSpanOf(host: TypeCheckHost, source: ts.SourceFile, start: number): ParseSourceSpan|
     null {
   const {line, character} = ts.getLineAndCharacterOfPosition(source, start);
-  return host.ngSpanOf(source.fileName, line, character);
+  return host.parseSourceSpanOf(source.fileName, line, character);
 }
 
 function diagnosticMessageToString(message: ts.DiagnosticMessageChain | string): string {

packages/compiler-cli/src/diagnostics/typescript_symbols.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {AotSummaryResolver, CompileMetadataResolver, CompilePipeSummary, CompilerConfig, DEFAULT_INTERPOLATION_CONFIG, DirectiveNormalizer, DirectiveResolver, DomElementSchemaRegistry, HtmlParser, InterpolationConfig, NgAnalyzedModules, NgModuleResolver, ParseTreeResult, PipeResolver, ResourceLoader, StaticReflector, StaticSymbol, StaticSymbolCache, StaticSymbolResolver, SummaryResolver, analyzeNgModules, extractProgramSymbols} from '@angular/compiler';
+import {AotSummaryResolver, CompileMetadataResolver, CompilePipeSummary, CompilerConfig, DEFAULT_INTERPOLATION_CONFIG, DirectiveNormalizer, DirectiveResolver, DomElementSchemaRegistry, HtmlParser, InterpolationConfig, NgAnalyzedModules, NgModuleResolver, ParseTreeResult, PipeResolver, ResourceLoader, StaticReflector, StaticSymbol, StaticSymbolCache, StaticSymbolResolver, SummaryResolver} from '@angular/compiler';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as ts from 'typescript';

packages/compiler-cli/src/language_services.ts

@@ -14,7 +14,7 @@ Angular modules and Typescript as this will indirectly add a dependency
 to the language service.
 
 */
-export {CompilerHost, CompilerHostContext, MetadataProvider, ModuleResolutionHostAdapter, NodeCompilerHostContext} from './compiler_host';
+export {CompilerHost, CompilerHostContext, ModuleResolutionHostAdapter, NodeCompilerHostContext} from './compiler_host';
 export {DiagnosticTemplateInfo, ExpressionDiagnostic, getExpressionDiagnostics, getExpressionScope, getTemplateExpressionDiagnostics} from './diagnostics/expression_diagnostics';
 export {AstType, DiagnosticKind, ExpressionDiagnosticsContext, TypeDiagnostic} from './diagnostics/expression_type';
 export {BuiltinType, DeclarationKind, Definition, Location, PipeInfo, Pipes, Signature, Span, Symbol, SymbolDeclaration, SymbolQuery, SymbolTable} from './diagnostics/symbols';

packages/compiler-cli/src/ngtools_api2.ts

@@ -55,11 +55,11 @@ export interface CompilerOptions extends ts.CompilerOptions {
 }
 
 export interface CompilerHost extends ts.CompilerHost {
-  moduleNameToFileName(moduleName: string, containingFile?: string): string|null;
-  fileNameToModuleName(importedFilePath: string, containingFilePath: string): string|null;
-  resourceNameToFileName(resourceName: string, containingFilePath: string): string|null;
-  toSummaryFileName(fileName: string, referringSrcFileName: string): string;
-  fromSummaryFileName(fileName: string, referringLibFileName: string): string;
+  moduleNameToFileName?(moduleName: string, containingFile?: string): string|null;
+  fileNameToModuleName?(importedFilePath: string, containingFilePath: string): string;
+  resourceNameToFileName?(resourceName: string, containingFilePath: string): string|null;
+  toSummaryFileName?(fileName: string, referringSrcFileName: string): string;
+  fromSummaryFileName?(fileName: string, referringLibFileName: string): string;
   readResource?(fileName: string): Promise<string>|string;
 }
 

packages/compiler-cli/src/path_mapped_compiler_host.ts

@@ -131,9 +131,7 @@ export class PathMappedCompilerHost extends CompilerHost {
           return this.readMetadata(metadataPath, rootedPath);
         }
       } else {
-        const sf = this.getSourceFile(rootedPath);
-        sf.fileName = sf.fileName;
-        const metadata = this.metadataProvider.getMetadata(sf);
+        const metadata = this.getMetadataForSourceFile(rootedPath);
         return metadata ? [metadata] : [];
       }
     }

packages/compiler-cli/src/perform_compile.ts

@@ -141,12 +141,6 @@ export function performCompilation({rootNames, options, host, oldProgram, emitCa
   customTransformers?: api.CustomTransformers,
   emitFlags?: api.EmitFlags
 }): PerformCompilationResult {
-  const [major, minor] = ts.version.split('.');
-
-  if (Number(major) < 2 || (Number(major) === 2 && Number(minor) < 4)) {
-    throw new Error('The Angular Compiler requires TypeScript >= 2.4.');
-  }
-
   let program: api.Program|undefined;
   let emitResult: ts.EmitResult|undefined;
   let allDiagnostics: Diagnostics = [];

packages/compiler-cli/src/perform_watch.ts

@@ -53,8 +53,8 @@ export interface PerformWatchHost {
 
 export function createPerformWatchHost(
     configFileName: string, reportDiagnostics: (diagnostics: Diagnostics) => void,
-    existingOptions?: ts.CompilerOptions,
-    createEmitCallback?: (options: api.CompilerOptions) => api.TsEmitCallback): PerformWatchHost {
+    existingOptions?: ts.CompilerOptions, createEmitCallback?: (options: api.CompilerOptions) =>
+                                              api.TsEmitCallback | undefined): PerformWatchHost {
   return {
     reportDiagnostics: reportDiagnostics,
     createCompilerHost: options => createCompilerHost({options}),

packages/compiler-cli/src/transformers/api.ts

@@ -135,17 +135,17 @@ export interface CompilerHost extends ts.CompilerHost {
    * Converts a module name that is used in an `import` to a file path.
    * I.e. `path/to/containingFile.ts` containing `import {...} from 'module-name'`.
    */
-  moduleNameToFileName(moduleName: string, containingFile?: string): string|null;
+  moduleNameToFileName?(moduleName: string, containingFile: string): string|null;
   /**
    * Converts a file path to a module name that can be used as an `import ...`
    * I.e. `path/to/importedFile.ts` should be imported by `path/to/containingFile.ts`.
    */
-  fileNameToModuleName(importedFilePath: string, containingFilePath: string): string|null;
+  fileNameToModuleName?(importedFilePath: string, containingFilePath: string): string;
   /**
    * Converts a file path for a resource that is used in a source file or another resource
    * into a filepath.
    */
-  resourceNameToFileName(resourceName: string, containingFilePath: string): string|null;
+  resourceNameToFileName?(resourceName: string, containingFilePath: string): string|null;
   /**
    * Converts a file name into a representation that should be stored in a summary file.
    * This has to include changing the suffix as well.
@@ -154,12 +154,12 @@ export interface CompilerHost extends ts.CompilerHost {
    *
    * @param referringSrcFileName the soure file that refers to fileName
    */
-  toSummaryFileName(fileName: string, referringSrcFileName: string): string;
+  toSummaryFileName?(fileName: string, referringSrcFileName: string): string;
   /**
    * Converts a fileName that was processed by `toSummaryFileName` back into a real fileName
    * given the fileName of the library that is referrig to it.
    */
-  fromSummaryFileName(fileName: string, referringLibFileName: string): string;
+  fromSummaryFileName?(fileName: string, referringLibFileName: string): string;
   /**
    * Load a referenced resource either statically or asynchronously. If the host returns a
    * `Promise<string>` it is assumed the user of the corresponding `Program` will call
@@ -267,7 +267,7 @@ export interface Program {
    *
    * Angular structural information is required to emit files.
    */
-  emit({emitFlags, cancellationToken, customTransformers, emitCallback}: {
+  emit({emitFlags, cancellationToken, customTransformers, emitCallback}?: {
     emitFlags?: EmitFlags,
     cancellationToken?: ts.CancellationToken,
     customTransformers?: CustomTransformers,