Parser syntax sparse bl (#76)

* Handle sparse messages in fluent-syntax * Fix tests affected by the sparse message parsing change * Handle sparse messages in fluent runtime * Fix linting errors * Apply reviewers feedback * One more indent
projectfluent · Oct 4, 2017 · 7e52517 · 7e52517
1 parent 64ef212
commit 7e52517
Show file tree

Hide file tree

Showing 10 changed files with 613 additions and 26 deletions.
diff --git a/fluent-syntax/src/ftlstream.js b/fluent-syntax/src/ftlstream.js
@@ -31,6 +31,21 @@ export class FTLParserStream extends ParserStream {
     }
   }
 
+  peekBlankLines() {
+    while (true) {
+      const lineStart = this.getPeekIndex();
+
+      this.peekInlineWS();
+
+      if (this.currentPeekIs('\n')) {
+        this.peek();
+      } else {
+        this.resetPeek(lineStart);
+        break;
+      }
+    }
+  }
+
   skipInlineWS() {
     while (this.ch) {
       if (!includes(INLINE_WS, this.ch)) {
@@ -54,6 +69,13 @@ export class FTLParserStream extends ParserStream {
     throw new ParseError('E0003', ch);
   }
 
+  expectIndent() {
+    this.expectChar('\n');
+    this.skipBlankLines();
+    this.expectChar(' ');
+    this.skipInlineWS();
+  }
+
   takeCharIf(ch) {
     if (this.ch === ch) {
       this.next();
@@ -94,6 +116,8 @@ export class FTLParserStream extends ParserStream {
 
     this.peek();
 
+    this.peekBlankLines();
+
     const ptr = this.getPeekIndex();
 
     this.peekInlineWS();
@@ -122,6 +146,8 @@ export class FTLParserStream extends ParserStream {
 
     this.peek();
 
+    this.peekBlankLines();
+
     const ptr = this.getPeekIndex();
 
     this.peekInlineWS();
@@ -140,13 +166,15 @@ export class FTLParserStream extends ParserStream {
     return false;
   }
 
-  isPeekNextLinePattern() {
+  isPeekNextNonBlankLinePattern() {
     if (!this.currentPeekIs('\n')) {
       return false;
     }
 
     this.peek();
 
+    this.peekBlankLines();
+
     const ptr = this.getPeekIndex();
 
     this.peekInlineWS();
@@ -176,6 +204,8 @@ export class FTLParserStream extends ParserStream {
 
     this.peek();
 
+    this.peekBlankLines();
+
     const ptr = this.getPeekIndex();
 
     this.peekInlineWS();

diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js
@@ -197,6 +197,7 @@ export default class FluentParser {
     if (ps.currentIs('=')) {
       ps.next();
       ps.skipInlineWS();
+      ps.skipBlankLines();
 
       pattern = this.getPattern(ps);
     }
@@ -241,8 +242,7 @@ export default class FluentParser {
     const attrs = [];
 
     while (true) {
-      ps.expectChar('\n');
-      ps.skipInlineWS();
+      ps.expectIndent();
 
       const attr = this.getAttribute(ps);
       attrs.push(attr);
@@ -264,8 +264,7 @@ export default class FluentParser {
     const tags = [];
 
     while (true) {
-      ps.expectChar('\n');
-      ps.skipInlineWS();
+      ps.expectIndent();
 
       const tag = this.getTag(ps);
       tags.push(tag);
@@ -340,8 +339,7 @@ export default class FluentParser {
     let hasDefault = false;
 
     while (true) {
-      ps.expectChar('\n');
-      ps.skipInlineWS();
+      ps.expectIndent();
 
       const variant = this.getVariant(ps, hasDefault);
 
@@ -419,7 +417,7 @@ export default class FluentParser {
     ps.skipInlineWS();
 
     // Special-case: trim leading whitespace and newlines.
-    if (ps.isPeekNextLinePattern()) {
+    if (ps.isPeekNextNonBlankLinePattern()) {
       ps.skipBlankLines();
       ps.skipInlineWS();
     }
@@ -429,7 +427,7 @@ export default class FluentParser {
 
       // The end condition for getPattern's while loop is a newline
       // which is not followed by a valid pattern continuation.
-      if (ch === '\n' && !ps.isPeekNextLinePattern()) {
+      if (ch === '\n' && !ps.isPeekNextNonBlankLinePattern()) {
         break;
       }
 
@@ -456,7 +454,7 @@ export default class FluentParser {
       }
 
       if (ch === '\n') {
-        if (!ps.isPeekNextLinePattern()) {
+        if (!ps.isPeekNextNonBlankLinePattern()) {
           return new AST.TextElement(buffer);
         }
 
@@ -498,9 +496,7 @@ export default class FluentParser {
     if (ps.isPeekNextLineVariantStart()) {
       const variants = this.getVariants(ps);
 
-      ps.expectChar('\n');
-      ps.expectChar(' ');
-      ps.skipInlineWS();
+      ps.expectIndent();
 
       return new AST.SelectExpression(null, variants);
     }
@@ -523,13 +519,12 @@ export default class FluentParser {
 
         const variants = this.getVariants(ps);
 
+
         if (variants.length === 0) {
           throw new ParseError('E0011');
         }
 
-        ps.expectChar('\n');
-        ps.expectChar(' ');
-        ps.skipInlineWS();
+        ps.expectIndent();
 
         return new AST.SelectExpression(selector, variants);
       }

diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js
@@ -102,9 +102,16 @@ export class ParserStream {
     return ret === ch;
   }
 
-  resetPeek() {
-    this.peekIndex = this.index;
-    this.peekEnd = this.iterEnd;
+  resetPeek(pos) {
+    if (pos) {
+      if (pos < this.peekIndex) {
+        this.peekEnd = false;
+      }
+      this.peekIndex = pos;
+    } else {
+      this.peekIndex = this.index;
+      this.peekEnd = this.iterEnd;
+    }
   }
 
   skipToPeek() {

diff --git a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl
@@ -1,2 +1,2 @@
 key = { $foo ->
-//~ ERROR E0003, pos 16, args "["
+//~ ERROR E0003, pos 16, args " "
diff --git a/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json
@@ -18,8 +18,8 @@
         "elements": [],
         "span": {
           "type": "Span",
-          "start": 6,
-          "end": 6
+          "start": 7,
+          "end": 7
         }
       },
       "attributes": [],
@@ -28,7 +28,7 @@
       "span": {
         "type": "Span",
         "start": 0,
-        "end": 6
+        "end": 7
       }
     }
   ],

diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json
@@ -62,7 +62,7 @@
         ],
         "span": {
           "type": "Span",
-          "start": 6,
+          "start": 7,
           "end": 131
         }
       },
@@ -127,7 +127,7 @@
         ],
         "span": {
           "type": "Span",
-          "start": 139,
+          "start": 140,
           "end": 184
         }
       },

diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.ftl b/fluent-syntax/test/fixtures_structure/sparse-messages.ftl
@@ -0,0 +1,46 @@
+key =
+
+
+    Value
+
+key2
+
+
+    .attr = Attribute
+
+
+key3 =
+    Value
+    Value2
+
+
+    Value 4
+    Value3
+
+
+
+    .attr2 = Attr 2
+
+
+key4 = Value
+
+
+    #tag1
+
+
+    #tag2
+key5 = Value 5
+
+key6 = {
+
+
+         [one] One
+
+
+
+
+        *[two] Two
+
+
+
+    }