refactor(ivy): update the compiler to emit $localize tags (angular#31609)

This commit changes the Angular compiler (ivy-only) to generate `$localize`
tagged strings for component templates that use `i18n` attributes.

BREAKING CHANGE

Since `$localize` is a global function, it must be included in any applications
that use i18n. This is achieved by importing the `@angular/localize` package
into an appropriate bundle, where it will be executed before the renderer
needs to call `$localize`. For CLI based projects, this is best done in
the `polyfills.ts` file.

```ts
import '@angular/localize';
```

For non-CLI applications this could be added as a script to the index.html
file or another suitable script file.

PR Close angular#31609

Author: petebacondarwin

integration/_payload-limits.json

@@ -21,7 +21,7 @@
     "master": {
       "uncompressed": {
         "runtime": 1440,
-        "main": 125448,
+        "main": 125882,
         "polyfills": 45340
       }
     }

integration/side-effects/snapshots/core/esm2015.js

@@ -1,3 +1,17 @@
 import "rxjs";
 
 import "rxjs/operators";
+
+const __globalThis = "undefined" !== typeof globalThis && globalThis;
+
+const __window = "undefined" !== typeof window && window;
+
+const __self = "undefined" !== typeof self && "undefined" !== typeof WorkerGlobalScope && self instanceof WorkerGlobalScope && self;
+
+const __global = "undefined" !== typeof global && global;
+
+const _global = __globalThis || __global || __window || __self;
+
+if (ngDevMode) _global.$localize = _global.$localize || function() {
+    throw new Error("The global function `$localize` is missing. Please add `import '@angular/localize';` to your polyfills.ts file.");
+};

integration/side-effects/snapshots/core/esm5.js

@@ -3,3 +3,17 @@ import "tslib";
 import "rxjs";
 
 import "rxjs/operators";
+
+var __globalThis = "undefined" !== typeof globalThis && globalThis;
+
+var __window = "undefined" !== typeof window && window;
+
+var __self = "undefined" !== typeof self && "undefined" !== typeof WorkerGlobalScope && self instanceof WorkerGlobalScope && self;
+
+var __global = "undefined" !== typeof global && global;
+
+var _global = __globalThis || __global || __window || __self;
+
+if (ngDevMode) _global.$localize = _global.$localize || function() {
+    throw new Error("The global function `$localize` is missing. Please add `import '@angular/localize';` to your polyfills.ts file.");
+};

karma-js.conf.js

@@ -82,6 +82,7 @@ module.exports = function(config) {
       'dist/all/@angular/elements/schematics/**',
       'dist/all/@angular/examples/**/e2e_test/*',
       'dist/all/@angular/language-service/**',
+      'dist/all/@angular/localize/**/test/**',
       'dist/all/@angular/router/**/test/**',
       'dist/all/@angular/platform-browser/testing/e2e_util.js',
       'dist/all/angular1_router.js',

modules/benchmarks/src/expanding_rows/BUILD.bazel

@@ -14,6 +14,7 @@ ng_module(
         "//packages:types",
         "//packages/common",
         "//packages/core",
+        "//packages/localize",
         "//packages/platform-browser",
         "@npm//rxjs",
     ],

modules/benchmarks/src/expanding_rows/index.ts

@@ -5,6 +5,8 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+// This benchmark uses i18n in its `ExpandingRowSummary` component so `$localize` must be loaded.
+import '@angular/localize';
 import {enableProdMode} from '@angular/core';
 import {platformBrowser} from '@angular/platform-browser';
 

packages/compiler-cli/src/ngtsc/translator/src/translator.ts

@@ -7,6 +7,7 @@
  */
 
 import {ArrayType, AssertNotNull, BinaryOperator, BinaryOperatorExpr, BuiltinType, BuiltinTypeName, CastExpr, ClassStmt, CommaExpr, CommentStmt, ConditionalExpr, DeclareFunctionStmt, DeclareVarStmt, Expression, ExpressionStatement, ExpressionType, ExpressionVisitor, ExternalExpr, ExternalReference, FunctionExpr, IfStmt, InstantiateExpr, InvokeFunctionExpr, InvokeMethodExpr, JSDocCommentStmt, LiteralArrayExpr, LiteralExpr, LiteralMapExpr, MapType, NotExpr, ReadKeyExpr, ReadPropExpr, ReadVarExpr, ReturnStatement, Statement, StatementVisitor, StmtModifier, ThrowStmt, TryCatchStmt, Type, TypeVisitor, TypeofExpr, WrappedNodeExpr, WriteKeyExpr, WritePropExpr, WriteVarExpr} from '@angular/compiler';
+import {LocalizedString} from '@angular/compiler/src/output/output_ast';
 import * as ts from 'typescript';
 
 import {DefaultImportRecorder, ImportRewriter, NOOP_DEFAULT_IMPORT_RECORDER, NoopImportRewriter} from '../../imports';
@@ -249,6 +250,10 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     return expr;
   }
 
+  visitLocalizedString(ast: LocalizedString, context: Context): ts.Expression {
+    return visitLocalizedString(ast, context, this);
+  }
+
   visitExternalExpr(ast: ExternalExpr, context: Context): ts.PropertyAccessExpression
       |ts.Identifier {
     if (ast.value.moduleName === null || ast.value.name === null) {
@@ -435,6 +440,10 @@ export class TypeTranslatorVisitor implements ExpressionVisitor, TypeVisitor {
     return ts.createLiteral(ast.value as string);
   }
 
+  visitLocalizedString(ast: LocalizedString, context: Context): ts.Expression {
+    return visitLocalizedString(ast, context, this);
+  }
+
   visitExternalExpr(ast: ExternalExpr, context: Context): ts.TypeNode {
     if (ast.value.moduleName === null || ast.value.name === null) {
       throw new Error(`Import unknown module or symbol`);
@@ -512,3 +521,44 @@ export class TypeTranslatorVisitor implements ExpressionVisitor, TypeVisitor {
     return ts.createTypeQueryNode(expr as ts.Identifier);
   }
 }
+
+/**
+ * A helper to reduce duplication, since this functionality is required in both
+ * `ExpressionTranslatorVisitor` and `TypeTranslatorVisitor`.
+ */
+function visitLocalizedString(ast: LocalizedString, context: Context, visitor: ExpressionVisitor) {
+  let template: ts.TemplateLiteral;
+  if (ast.messageParts.length === 1) {
+    template = ts.createNoSubstitutionTemplateLiteral(ast.messageParts[0]);
+  } else {
+    const head = ts.createTemplateHead(ast.messageParts[0]);
+    const spans: ts.TemplateSpan[] = [];
+    for (let i = 1; i < ast.messageParts.length; i++) {
+      const resolvedExpression = ast.expressions[i - 1].visitExpression(visitor, context);
+      spans.push(ts.createTemplateSpan(
+          resolvedExpression, ts.createTemplateMiddle(prefixWithPlaceholderMarker(
+                                  ast.messageParts[i], ast.placeHolderNames[i - 1]))));
+    }
+    if (spans.length > 0) {
+      // The last span is supposed to have a tail rather than a middle
+      spans[spans.length - 1].literal.kind = ts.SyntaxKind.TemplateTail;
+    }
+    template = ts.createTemplateExpression(head, spans);
+  }
+  return ts.createTaggedTemplate(ts.createIdentifier('$localize'), template);
+}
+
+/**
+ * We want our tagged literals to include placeholder name information to aid runtime translation.
+ *
+ * The expressions are marked with placeholder names by postfixing the expression with
+ * `:placeHolderName:`. To achieve this, we actually "prefix" the message part that follows the
+ * expression.
+ *
+ * @param messagePart the message part that follows the current expression.
+ * @param placeHolderName the name of the placeholder for the current expression.
+ * @returns the prefixed message part.
+ */
+function prefixWithPlaceholderMarker(messagePart: string, placeHolderName: string) {
+  return `:${placeHolderName}:${messagePart}`;
+}

packages/compiler-cli/src/transformers/node_emitter.ts

@@ -7,6 +7,7 @@
  */
 
 import {AssertNotNull, BinaryOperator, BinaryOperatorExpr, BuiltinMethod, BuiltinVar, CastExpr, ClassStmt, CommaExpr, CommentStmt, ConditionalExpr, DeclareFunctionStmt, DeclareVarStmt, ExpressionStatement, ExpressionVisitor, ExternalExpr, ExternalReference, FunctionExpr, IfStmt, InstantiateExpr, InvokeFunctionExpr, InvokeMethodExpr, JSDocCommentStmt, LiteralArrayExpr, LiteralExpr, LiteralMapExpr, NotExpr, ParseSourceFile, ParseSourceSpan, PartialModule, ReadKeyExpr, ReadPropExpr, ReadVarExpr, ReturnStatement, Statement, StatementVisitor, StmtModifier, ThrowStmt, TryCatchStmt, TypeofExpr, WrappedNodeExpr, WriteKeyExpr, WritePropExpr, WriteVarExpr} from '@angular/compiler';
+import {LocalizedString} from '@angular/compiler/src/output/output_ast';
 import * as ts from 'typescript';
 
 import {error} from './util';
@@ -535,6 +536,10 @@ export class NodeEmitterVisitor implements StatementVisitor, ExpressionVisitor {
 
   visitLiteralExpr(expr: LiteralExpr) { return this.record(expr, createLiteral(expr.value)); }
 
+  visitLocalizedString(expr: LocalizedString, context: any) {
+    throw new Error('localized strings are not supported in pre-ivy mode.');
+  }
+
   visitExternalExpr(expr: ExternalExpr) {
     return this.record(expr, this._visitIdentifier(expr.value));
   }

packages/compiler-cli/test/compliance/mock_compile.ts

@@ -15,12 +15,14 @@ import {NgtscProgram} from '../../src/ngtsc/program';
 const IDENTIFIER = /[A-Za-z_$ɵ][A-Za-z0-9_$]*/;
 const OPERATOR =
     /!|\?|%|\*|\/|\^|&&?|\|\|?|\(|\)|\{|\}|\[|\]|:|;|<=?|>=?|={1,3}|!==?|=>|\+\+?|--?|@|,|\.|\.\.\./;
-const STRING = /'(\\'|[^'])*'|"(\\"|[^"])*"|`(\\`[\s\S])*?`/;
+const STRING = /'(\\'|[^'])*'|"(\\"|[^"])*"/;
+const BACKTICK_STRING = /\\`(([\s\S]*?)(\$\{[^}]*?\})?)*?\\`/;
+const BACKTICK_INTERPOLATION = /(\$\{[^}]*\})/;
 const NUMBER = /\d+/;
 
 const ELLIPSIS = '…';
 const TOKEN = new RegExp(
-    `\\s*((${IDENTIFIER.source})|(${OPERATOR.source})|(${STRING.source})|${NUMBER.source}|${ELLIPSIS})\\s*`,
+    `\\s*((${IDENTIFIER.source})|(${OPERATOR.source})|(${STRING.source})|(${BACKTICK_STRING.source})|${NUMBER.source}|${ELLIPSIS})\\s*`,
     'y');
 
 type Piece = string | RegExp;
@@ -30,6 +32,8 @@ const SKIP = /(?:.|\n|\r)*/;
 const ERROR_CONTEXT_WIDTH = 30;
 // Transform the expected output to set of tokens
 function tokenize(text: string): Piece[] {
+  // TOKEN.lastIndex is stateful so we cache the `lastIndex` and restore it at the end of the call.
+  const lastIndex = TOKEN.lastIndex;
   TOKEN.lastIndex = 0;
 
   let match: RegExpMatchArray|null;
@@ -42,6 +46,8 @@ function tokenize(text: string): Piece[] {
       pieces.push(IDENTIFIER);
     } else if (token === ELLIPSIS) {
       pieces.push(SKIP);
+    } else if (match = BACKTICK_STRING.exec(token)) {
+      pieces.push(...tokenizeBackTickString(token));
     } else {
       pieces.push(token);
     }
@@ -57,10 +63,33 @@ function tokenize(text: string): Piece[] {
         `Invalid test, no token found for "${text[tokenizedTextEnd]}" ` +
         `(context = '${text.substr(from, to)}...'`);
   }
+  // Reset the lastIndex in case we are in a recursive `tokenize()` call.
+  TOKEN.lastIndex = lastIndex;
 
   return pieces;
 }
 
+/**
+ * Back-ticks are escaped as "\`" so we must strip the backslashes.
+ * Also the string will likely contain interpolations and if an interpolation holds an
+ * identifier we will need to match that later. So tokenize the interpolation too!
+ */
+function tokenizeBackTickString(str: string): Piece[] {
+  const pieces: Piece[] = ['`'];
+  const backTickPieces = str.slice(2, -2).split(BACKTICK_INTERPOLATION);
+  backTickPieces.forEach((backTickPiece) => {
+    if (BACKTICK_INTERPOLATION.test(backTickPiece)) {
+      // An interpolation so tokenize this expression
+      pieces.push(...tokenize(backTickPiece));
+    } else {
+      // Not an interpolation so just add it as a piece
+      pieces.push(backTickPiece);
+    }
+  });
+  pieces.push('`');
+  return pieces;
+}
+
 export function expectEmit(
     source: string, expected: string, description: string,
     assertIdentifiers?: {[name: string]: RegExp}) {