I18n message extractor

modules/angular2/src/compiler/html_ast.ts

@@ -23,10 +23,16 @@ export class HtmlElementAst implements HtmlAst {
   visit(visitor: HtmlAstVisitor, context: any): any { return visitor.visitElement(this, context); }
 }
 
+export class HtmlCommentAst implements HtmlAst {
+  constructor(public value: string, public sourceSpan: ParseSourceSpan) {}
+  visit(visitor: HtmlAstVisitor, context: any): any { return visitor.visitComment(this, context); }
+}
+
 export interface HtmlAstVisitor {
   visitElement(ast: HtmlElementAst, context: any): any;
   visitAttr(ast: HtmlAttrAst, context: any): any;
   visitText(ast: HtmlTextAst, context: any): any;
+  visitComment(ast: HtmlCommentAst, context: any): any;
 }
 
 export function htmlVisitAll(visitor: HtmlAstVisitor, asts: HtmlAst[], context: any = null): any[] {

modules/angular2/src/compiler/html_parser.ts

@@ -11,7 +11,7 @@ import {
 
 import {ListWrapper} from 'angular2/src/facade/collection';
 
-import {HtmlAst, HtmlAttrAst, HtmlTextAst, HtmlElementAst} from './html_ast';
+import {HtmlAst, HtmlAttrAst, HtmlTextAst, HtmlCommentAst, HtmlElementAst} from './html_ast';
 
 import {Injectable} from 'angular2/src/core/di';
 import {HtmlToken, HtmlTokenType, tokenizeHtml} from './html_lexer';
@@ -98,9 +98,11 @@ class TreeBuilder {
     this._advanceIf(HtmlTokenType.CDATA_END);
   }
 
-  private _consumeComment(startToken: HtmlToken) {
-    this._advanceIf(HtmlTokenType.RAW_TEXT);
+  private _consumeComment(token: HtmlToken) {
+    var text = this._advanceIf(HtmlTokenType.RAW_TEXT);
     this._advanceIf(HtmlTokenType.COMMENT_END);
+    var value = isPresent(text) ? text.parts[0].trim() : null;
+    this._addToParent(new HtmlCommentAst(value, token.sourceSpan));
   }
 
   private _consumeText(token: HtmlToken) {

modules/angular2/src/compiler/legacy_template.ts

@@ -8,7 +8,14 @@ import {
   isPresent
 } from 'angular2/src/facade/lang';
 
-import {HtmlAstVisitor, HtmlAttrAst, HtmlElementAst, HtmlTextAst, HtmlAst} from './html_ast';
+import {
+  HtmlAstVisitor,
+  HtmlAttrAst,
+  HtmlElementAst,
+  HtmlTextAst,
+  HtmlCommentAst,
+  HtmlAst
+} from './html_ast';
 import {HtmlParser, HtmlParseTreeResult} from './html_parser';
 
 import {dashCaseToCamelCase, camelCaseToDashCase} from './util';
@@ -37,6 +44,8 @@ export class LegacyHtmlAstTransformer implements HtmlAstVisitor {
 
   constructor(private dashCaseSelectors?: string[]) {}
 
+  visitComment(ast: HtmlCommentAst, context: any): any { return ast; }
+
   visitElement(ast: HtmlElementAst, context: any): HtmlElementAst {
     this.visitingTemplateEl = ast.name.toLowerCase() == 'template';
     let attrs = ast.attrs.map(attr => attr.visit(this, null));

modules/angular2/src/compiler/template_normalizer.ts

@@ -20,6 +20,7 @@ import {
   HtmlTextAst,
   HtmlAttrAst,
   HtmlAst,
+  HtmlCommentAst,
   htmlVisitAll
 } from './html_ast';
 import {HtmlParser} from './html_parser';
@@ -126,6 +127,7 @@ class TemplatePreparseVisitor implements HtmlAstVisitor {
     }
     return null;
   }
+  visitComment(ast: HtmlCommentAst, context: any): any { return null; }
   visitAttr(ast: HtmlAttrAst, context: any): any { return null; }
   visitText(ast: HtmlTextAst, context: any): any { return null; }
 }

modules/angular2/src/compiler/template_parser.ts

@@ -41,6 +41,7 @@ import {
   HtmlElementAst,
   HtmlAttrAst,
   HtmlTextAst,
+  HtmlCommentAst,
   htmlVisitAll
 } from './html_ast';
 
@@ -209,6 +210,8 @@ class TemplateParseVisitor implements HtmlAstVisitor {
     return new AttrAst(ast.name, ast.value, ast.sourceSpan);
   }
 
+  visitComment(ast: HtmlCommentAst, context: any): any { return null; }
+
   visitElement(element: HtmlElementAst, component: Component): any {
     var nodeName = element.name;
     var preparsedElement = preparseElement(element);
@@ -676,6 +679,7 @@ class NonBindableVisitor implements HtmlAstVisitor {
     return new ElementAst(ast.name, htmlVisitAll(this, ast.attrs), [], [], [], [], children,
                           ngContentIndex, ast.sourceSpan);
   }
+  visitComment(ast: HtmlCommentAst, context: any): any { return null; }
   visitAttr(ast: HtmlAttrAst, context: any): AttrAst {
     return new AttrAst(ast.name, ast.value, ast.sourceSpan);
   }

modules/angular2/src/facade/collection.dart

@@ -85,6 +85,10 @@ class StringMapWrapper {
     return a.keys.toList();
   }
 
+  static List values(Map<String, dynamic> a) {
+    return a.values.toList();
+  }
+
   static bool isEmpty(Map m) => m.isEmpty;
   static bool equals/*<V>*/(Map/*<String,V>*/ m1, Map/*<String,V>*/ m2) {
     if (m1.length != m2.length) {

modules/angular2/src/facade/collection.ts

@@ -116,6 +116,12 @@ export class StringMapWrapper {
   }
   static set<V>(map: {[key: string]: V}, key: string, value: V) { map[key] = value; }
   static keys(map: {[key: string]: any}): string[] { return Object.keys(map); }
+  static values<T>(map: {[key: string]: T}): T[] {
+    return Object.keys(map).reduce((r, a) => {
+      r.push(map[a]);
+      return r;
+    }, []);
+  }
   static isEmpty(map: {[key: string]: any}): boolean {
     for (var prop in map) {
       return false;

modules/angular2/src/i18n/message_extractor.ts

@@ -0,0 +1,261 @@
+import {HtmlParser} from 'angular2/src/compiler/html_parser';
+import {ParseSourceSpan, ParseError} from 'angular2/src/compiler/parse_util';
+import {
+  HtmlAst,
+  HtmlAstVisitor,
+  HtmlElementAst,
+  HtmlAttrAst,
+  HtmlTextAst,
+  HtmlCommentAst,
+  htmlVisitAll
+} from 'angular2/src/compiler/html_ast';
+import {isPresent, isBlank} from 'angular2/src/facade/lang';
+import {StringMapWrapper} from 'angular2/src/facade/collection';
+import {Parser} from 'angular2/src/core/change_detection/parser/parser';
+import {Interpolation} from 'angular2/src/core/change_detection/parser/ast';
+
+const I18N_ATTR = "i18n";
+const I18N_ATTR_PREFIX = "i18n-";
+
+/**
+ * A message extracted from a template.
+ *
+ * The identity of a message is comprised of `content` and `meaning`.
+ *
+ * `description` is additional information provided to the translator.
+ */
+export class Message {
+  constructor(public content: string, public meaning: string, public description: string) {}
+}
+
+/**
+ * All messages extracted from a template.
+ */
+export class ExtractionResult {
+  constructor(public messages: Message[], public errors: ParseError[]) {}
+}
+
+/**
+ * An extraction error.
+ */
+export class I18nExtractionError extends ParseError {
+  constructor(span: ParseSourceSpan, msg: string) { super(span, msg); }
+}
+
+/**
+ * Removes duplicate messages.
+ *
+ * E.g.
+ *
+ * ```
+ *  var m = [new Message("message", "meaning", "desc1"), new Message("message", "meaning",
+ * "desc2")];
+ *  expect(removeDuplicates(m)).toEqual([new Message("message", "meaning", "desc1")]);
+ * ```
+ */
+export function removeDuplicates(messages: Message[]): Message[] {
+  let uniq: {[key: string]: Message} = {};
+  messages.forEach(m => {
+    let key = `$ng__${m.meaning}__|${m.content}`;
+    if (!StringMapWrapper.contains(uniq, key)) {
+      uniq[key] = m;
+    }
+  });
+  return StringMapWrapper.values(uniq);
+}
+
+/**
+ * Extracts all messages from a template.
+ *
+ * It works like this. First, the extractor uses the provided html parser to get
+ * the html AST of the template. Then it partitions the root nodes into parts.
+ * Everything between two i18n comments becomes a single part. Every other nodes becomes
+ * a part too.
+ *
+ * We process every part as follows. Say we have a part A.
+ *
+ * If the part has the i18n attribute, it gets converted into a message.
+ * And we do not recurse into that part, except to extract messages from the attributes.
+ *
+ * If the part doesn't have the i18n attribute, we recurse into that part and
+ * partition its children.
+ *
+ * While walking the AST we also remove i18n attributes from messages.
+ */
+export class MessageExtractor {
+  messages: Message[];
+  errors: ParseError[];
+
+  constructor(private _htmlParser: HtmlParser, private _parser: Parser) {}
+
+  extract(template: string, sourceUrl: string): ExtractionResult {
+    this.messages = [];
+    this.errors = [];
+
+    let res = this._htmlParser.parse(template, sourceUrl);
+    if (res.errors.length > 0) {
+      return new ExtractionResult([], res.errors);
+    } else {
+      let ps = this._partition(res.rootNodes);
+      ps.forEach(p => this._extractMessagesFromPart(p));
+      return new ExtractionResult(this.messages, this.errors);
+    }
+  }
+
+  private _extractMessagesFromPart(p: _Part): void {
+    if (p.hasI18n) {
+      this.messages.push(new Message(_stringifyNodes(p.children, this._parser), _meaning(p.i18n),
+                                     _description(p.i18n)));
+      this._recurseToExtractMessagesFromAttributes(p.children);
+    } else {
+      this._recurse(p.children);
+    }
+
+    if (isPresent(p.rootElement)) {
+      this._extractMessagesFromAttributes(p.rootElement);
+    }
+  }
+
+  private _recurse(nodes: HtmlAst[]): void {
+    let ps = this._partition(nodes);
+    ps.forEach(p => this._extractMessagesFromPart(p));
+  }
+
+  private _recurseToExtractMessagesFromAttributes(nodes: HtmlAst[]): void {
+    nodes.forEach(n => {
+      if (n instanceof HtmlElementAst) {
+        this._extractMessagesFromAttributes(n);
+        this._recurseToExtractMessagesFromAttributes(n.children);
+      }
+    });
+  }
+
+  private _extractMessagesFromAttributes(p: HtmlElementAst): void {
+    p.attrs.forEach(attr => {
+      if (attr.name.startsWith(I18N_ATTR_PREFIX)) {
+        let expectedName = attr.name.substring(5);
+        let matching = p.attrs.filter(a => a.name == expectedName);
+
+        if (matching.length > 0) {
+          let value = _removeInterpolation(matching[0].value, p.sourceSpan, this._parser);
+          this.messages.push(new Message(value, _meaning(attr.value), _description(attr.value)));
+        } else {
+          this.errors.push(
+              new I18nExtractionError(p.sourceSpan, `Missing attribute '${expectedName}'.`));
+        }
+      }
+    });
+  }
+
+  // Man, this is so ugly!
+  private _partition(nodes: HtmlAst[]): _Part[] {
+    let res = [];
+
+    for (let i = 0; i < nodes.length; ++i) {
+      let n = nodes[i];
+      let temp = [];
+      if (_isOpeningComment(n)) {
+        let i18n = (<HtmlCommentAst>n).value.substring(5).trim();
+        i++;
+        while (!_isClosingComment(nodes[i])) {
+          temp.push(nodes[i++]);
+          if (i === nodes.length) {
+            this.errors.push(
+                new I18nExtractionError(n.sourceSpan, "Missing closing 'i18n' comment."));
+            break;
+          }
+        }
+        res.push(new _Part(null, temp, i18n, true));
+
+      } else if (n instanceof HtmlElementAst) {
+        let i18n = _findI18nAttr(n);
+        res.push(new _Part(n, n.children, isPresent(i18n) ? i18n.value : null, isPresent(i18n)));
+      }
+    }
+
+    return res;
+  }
+}
+
+class _Part {
+  constructor(public rootElement: HtmlElementAst, public children: HtmlAst[], public i18n: string,
+              public hasI18n: boolean) {}
+}
+
+function _isOpeningComment(n: HtmlAst): boolean {
+  return n instanceof HtmlCommentAst && isPresent(n.value) && n.value.startsWith("i18n:");
+}
+
+function _isClosingComment(n: HtmlAst): boolean {
+  return n instanceof HtmlCommentAst && isPresent(n.value) && n.value == "/i18n";
+}
+
+function _stringifyNodes(nodes: HtmlAst[], parser: Parser) {
+  let visitor = new _StringifyVisitor(parser);
+  return htmlVisitAll(visitor, nodes).join("");
+}
+
+class _StringifyVisitor implements HtmlAstVisitor {
+  constructor(private _parser: Parser) {}
+
+  visitElement(ast: HtmlElementAst, context: any): any {
+    let attrs = this._join(htmlVisitAll(this, ast.attrs), " ");
+    let children = this._join(htmlVisitAll(this, ast.children), "");
+    return `<${ast.name} ${attrs}>${children}</${ast.name}>`;
+  }
+
+  visitAttr(ast: HtmlAttrAst, context: any): any {
+    if (ast.name.startsWith(I18N_ATTR_PREFIX)) {
+      return "";
+    } else {
+      return `${ast.name}="${ast.value}"`;
+    }
+  }
+
+  visitText(ast: HtmlTextAst, context: any): any {
+    return _removeInterpolation(ast.value, ast.sourceSpan, this._parser);
+  }
+
+  visitComment(ast: HtmlCommentAst, context: any): any { return ""; }
+
+  private _join(strs: string[], str: string): string {
+    return strs.filter(s => s.length > 0).join(str);
+  }
+}
+
+function _removeInterpolation(value: string, source: ParseSourceSpan, parser: Parser): string {
+  try {
+    let parsed = parser.parseInterpolation(value, source.toString());
+    if (isPresent(parsed)) {
+      let ast: Interpolation = <any>parsed.ast;
+      let res = "";
+      for (let i = 0; i < ast.strings.length; ++i) {
+        res += ast.strings[i];
+        if (i != ast.strings.length - 1) {
+          res += `{{I${i}}}`;
+        }
+      }
+      return res;
+    } else {
+      return value;
+    }
+  } catch (e) {
+    return value;
+  }
+}
+
+function _findI18nAttr(p: HtmlElementAst): HtmlAttrAst {
+  let i18n = p.attrs.filter(a => a.name == I18N_ATTR);
+  return i18n.length == 0 ? null : i18n[0];
+}
+
+function _meaning(i18n: string): string {
+  if (isBlank(i18n) || i18n == "") return null;
+  return i18n.split("|")[0];
+}
+
+function _description(i18n: string): string {
+  if (isBlank(i18n) || i18n == "") return null;
+  let parts = i18n.split("|");
+  return parts.length > 1 ? parts[1] : null;
+}