Syntax parser tests

fluent-syntax/src/errors.js

@@ -45,6 +45,8 @@ function getErrorMessage(code, args) {
       return 'Expected variant key';
     case 'E0014':
       return 'Expected literal';
+    case 'E0015':
+      return 'Only one variant can be marked as default (*)';
     default:
       return code;
   }

fluent-syntax/src/ftlstream.js

@@ -43,6 +43,11 @@ export class FTLParserStream extends ParserStream {
       return true;
     }
 
+    if (ch === '\n') {
+      // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
+      throw new ParseError('E0003', '\u2424');
+    }
+
     throw new ParseError('E0003', ch);
   }
 
@@ -56,14 +61,18 @@ export class FTLParserStream extends ParserStream {
 
   takeChar(f) {
     const ch = this.ch;
-    if (f(ch)) {
+    if (ch !== undefined && f(ch)) {
       this.next();
       return ch;
     }
     return undefined;
   }
 
   isIDStart() {
+    if (this.ch === undefined) {
+      return false;
+    }
+
     const cc = this.ch.charCodeAt(0);
     return ((cc >= 97 && cc <= 122) || // a-z
             (cc >= 65 && cc <= 90) ||  // A-Z
@@ -98,8 +107,15 @@ export class FTLParserStream extends ParserStream {
 
     this.peek();
 
+    const ptr = this.getPeekIndex();
+
     this.peekLineWS();
 
+    if (this.getPeekIndex() - ptr === 0) {
+      this.resetPeek();
+      return false;
+    }
+
     if (this.currentPeekIs('*')) {
       this.peek();
     }
@@ -119,8 +135,15 @@ export class FTLParserStream extends ParserStream {
 
     this.peek();
 
+    const ptr = this.getPeekIndex();
+
     this.peekLineWS();
 
+    if (this.getPeekIndex() - ptr === 0) {
+      this.resetPeek();
+      return false;
+    }
+
     if (this.currentPeekIs('.')) {
       this.resetPeek();
       return true;
@@ -130,15 +153,52 @@ export class FTLParserStream extends ParserStream {
     return false;
   }
 
+  isPeekNextLinePattern() {
+    if (!this.currentPeekIs('\n')) {
+      return false;
+    }
+
+    this.peek();
+
+    const ptr = this.getPeekIndex();
+
+    this.peekLineWS();
+
+    if (this.getPeekIndex() - ptr === 0) {
+      this.resetPeek();
+      return false;
+    }
+
+    if (this.currentPeekIs('}') ||
+        this.currentPeekIs('.') ||
+        this.currentPeekIs('#') ||
+        this.currentPeekIs('[') ||
+        this.currentPeekIs('*') ||
+        this.currentPeekIs('{')) {
+      this.resetPeek();
+      return false;
+    }
+
+    this.resetPeek();
+    return true;
+  }
+
   isPeekNextLineTagStart() {
     if (!this.currentPeekIs('\n')) {
       return false;
     }
 
     this.peek();
 
+    const ptr = this.getPeekIndex();
+
     this.peekLineWS();
 
+    if (this.getPeekIndex() - ptr === 0) {
+      this.resetPeek();
+      return false;
+    }
+
     if (this.currentPeekIs('#')) {
       this.resetPeek();
       return true;

fluent-syntax/src/parser.js

@@ -156,7 +156,7 @@ function getMessage(ps, comment) {
   }
 
   if (pattern === undefined && attrs === undefined && tags === undefined) {
-    throw new ParseError('E0005', id);
+    throw new ParseError('E0005', id.name);
   }
 
   return new AST.Message(id, pattern, attrs, tags, comment);
@@ -254,6 +254,9 @@ function getVariants(ps) {
     ps.skipLineWS();
 
     if (ps.currentIs('*')) {
+      if (hasDefault) {
+        throw new ParseError('E0015');
+      }
       ps.next();
       defaultIndex = true;
       hasDefault = true;
@@ -350,15 +353,12 @@ function getPattern(ps) {
         break;
       }
 
-      ps.peek();
-
-      if (!ps.currentPeekIs(' ')) {
-        ps.resetPeek();
+      if (!ps.isPeekNextLinePattern()) {
         break;
       }
 
-      ps.peekLineWS();
-      ps.skipToPeek();
+      ps.next();
+      ps.skipLineWS();
 
       firstLine = false;
 

fluent-syntax/test/fixtures_behavior/attribute_expression_with_wrong_attr.ftl

@@ -0,0 +1,7 @@
+key = { foo.23 }
+
+//~ ERROR E0004, pos 12, args "a-zA-Z"
+
+key = { foo. }
+
+//~ ERROR E0004, pos 31, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/attribute_starts_from_nl.ftl

@@ -0,0 +1,3 @@
+foo = Value
+.attr = Value 2
+//~ ERROR E0002, pos 12

fluent-syntax/test/fixtures_behavior/attribute_with_empty_pattern.ftl

@@ -0,0 +1,2 @@
+key = Value
+    .label =

fluent-syntax/test/fixtures_behavior/attribute_without_equal_sign.ftl

@@ -0,0 +1,3 @@
+key = Value
+    .label
+//~ ERROR E0003, pos 22, args "="

fluent-syntax/test/fixtures_behavior/broken_number.ftl

@@ -0,0 +1,14 @@
+key = { -2.4.5 }
+//~ ERROR E0003, pos 12, args "}"
+
+key = { -2.4. }
+//~ ERROR E0003, pos 30, args "}"
+
+key = { -.4 }
+//~ ERROR E0004, pos 44, args "0-9"
+
+key = { -2..4 }
+//~ ERROR E0004, pos 61, args "0-9"
+
+key = { 24d }
+//~ ERROR E0003, pos 77, args "}"

fluent-syntax/test/fixtures_behavior/call_expression_with_trailing_comma.ftl

@@ -0,0 +1 @@
+key = { BUILTIN(23, ) }

fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl

@@ -0,0 +1,2 @@
+key = { BUILTIN(2: "foo") }
+//~ ERROR E0009, pos 17

fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl

@@ -0,0 +1,2 @@
+key = { BUILTIN(key: foo) }
+//~ ERROR E0006, pos 21, args "value"

fluent-syntax/test/fixtures_behavior/comment_continues_with_one_slash.ftl

@@ -0,0 +1,3 @@
+// This is a normal comment
+/ but this is not
+//~ ERROR E0003, pos 29, args "/"

fluent-syntax/test/fixtures_behavior/comment_with_eof.ftl

@@ -0,0 +1 @@
+// This is a comment with no new line

fluent-syntax/test/fixtures_behavior/empty_resource_with_ws.ftl

@@ -0,0 +1,2 @@
+
+

fluent-syntax/test/fixtures_behavior/entry_start_with_one_slash.ftl

@@ -0,0 +1,3 @@
+
+/ Test
+//~ ERROR E0003, pos 2, args "/"

fluent-syntax/test/fixtures_behavior/leading_empty_lines.ftl

@@ -0,0 +1,3 @@
+
+
+foo = Value

fluent-syntax/test/fixtures_behavior/leading_empty_lines_with_ws.ftl

@@ -0,0 +1,5 @@
+
+
+
+
+foo = Value

fluent-syntax/test/fixtures_behavior/multiline_string.ftl

@@ -0,0 +1,3 @@
+key = { BUILTIN(key: "
+ text
+    ") }

fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl

@@ -0,0 +1,3 @@
+key = Value
+    Value 2
+//~ ERROR E0002, pos 12

fluent-syntax/test/fixtures_behavior/non_id_attribute_name.ftl

@@ -0,0 +1,3 @@
+key = Value
+    .2 = Foo
+//~ ERROR E0004, pos 17, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/placeable_without_close_bracket.ftl

@@ -0,0 +1,3 @@
+key = { $num
+
+//~ ERROR E0003, pos 12, args "}"

fluent-syntax/test/fixtures_behavior/second_attribute_starts_from_nl.ftl

@@ -0,0 +1,4 @@
+key = Value
+    .label = Value
+.accesskey = K
+//~ ERROR E0002, pos 31

fluent-syntax/test/fixtures_behavior/second_tag_starts_from_nl.ftl

@@ -0,0 +1,4 @@
+key = Value
+    #tag
+#tag2
+//~ ERROR E0002, pos 21

fluent-syntax/test/fixtures_behavior/section_starts_with_one_bracket.ftl

@@ -0,0 +1,2 @@
+[ This is a broken section ]]
+//~ ERROR E0003, pos 1, args "["

fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl

@@ -0,0 +1,3 @@
+[[ This is
+a broken section]]
+//~ ERROR E0003, pos 10, args "]"

fluent-syntax/test/fixtures_behavior/section_with_no_nl_after_it.ftl

@@ -0,0 +1 @@
+[[ This is a correct section ]]

fluent-syntax/test/fixtures_behavior/section_with_one_bracket_at_the_end.ftl

@@ -0,0 +1,2 @@
+[[ This is a broken section ]
+//~ ERROR E0003, pos 29, args "]"

fluent-syntax/test/fixtures_behavior/select_expression_with_two_selectors.ftl

@@ -0,0 +1,2 @@
+key = { $foo $faa }
+//~ ERROR E0003, pos 13, args "}"

fluent-syntax/test/fixtures_behavior/select_expression_without_arrow.ftl

@@ -0,0 +1,2 @@
+key = { $foo - }
+//~ ERROR E0003, pos 13, args "}"

fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl

@@ -0,0 +1,6 @@
+key = { $foo -> }
+//~ ERROR E0003, pos 16, args "␤"
+
+key = { $foo ->
+    }
+//~ ERROR E0003, pos 39, args "["

fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl

@@ -0,0 +1,2 @@
+key = { $foo ->
+//~ ERROR E0003, pos 16, args "["

fluent-syntax/test/fixtures_behavior/single_char_id.ftl

@@ -0,0 +1,2 @@
+k = Value
+    .l = Foo

fluent-syntax/test/fixtures_behavior/standalone_identifier.ftl

@@ -0,0 +1,2 @@
+foo
+//~ ERROR E0005, pos 3, args "foo"

fluent-syntax/test/fixtures_behavior/tag_and_attribute_together.ftl

@@ -0,0 +1,4 @@
+key = Value
+    .label = Foo
+    #masculine
+//~ ERROR E0012, pos 28

fluent-syntax/test/fixtures_behavior/tag_starts_from_nl.ftl

@@ -0,0 +1,3 @@
+key = Value
+#tag
+//~ ERROR E0002, pos 12

fluent-syntax/test/fixtures_behavior/unknown_entry_start.ftl

@@ -0,0 +1,3 @@
+
+8Foo = Foo
+//~ ERROR E0002, pos 1

fluent-syntax/test/fixtures_behavior/variant_ends_abruptly.ftl

@@ -0,0 +1,3 @@
+key = { $foo ->
+    *[
+//~ ERROR E0004, pos 22, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/variant_expression_empty_key.ftl

@@ -0,0 +1,2 @@
+key = { foo[] }
+//~ ERROR E0004, pos 12, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/variant_starts_from_nl.ftl

@@ -0,0 +1,4 @@
+key = {
+*[one] Value
+    }
+//~ ERROR E0004, pos 7, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/variant_with_digit_key.ftl

@@ -0,0 +1,3 @@
+key = {
+        *[-2] Foo
+    }

fluent-syntax/test/fixtures_behavior/variant_with_empty_pattern.ftl

@@ -0,0 +1,3 @@
+key = {
+    *[one]
+    }

fluent-syntax/test/fixtures_behavior/variant_with_leading_space_in_name.ftl

@@ -0,0 +1,4 @@
+key = {
+        *[  one] Foo
+    }
+//~ ERROR E0004, pos 18, args "a-zA-Z"

fluent-syntax/test/fixtures_behavior/variant_with_symbol_with_space.ftl

@@ -0,0 +1,3 @@
+key = {
+        *[New York] Nowy Jork
+    }

fluent-syntax/test/fixtures_behavior/variants_with_two_defaults.ftl

@@ -0,0 +1,5 @@
+key = {
+    *[one] Foo
+    *[two] Two
+    }
+//~ ERROR E0015, pos 27

fluent-syntax/test/fixtures_structure/message_with_empty_pattern.ftl

@@ -0,0 +1 @@
+foo = 

fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json

@@ -0,0 +1,26 @@
+{
+  "type": "Resource",
+  "body": [
+    {
+      "type": "Message",
+      "span": {
+        "type": "Span",
+        "start": 0,
+        "end": 6
+      },
+      "annotations": [],
+      "id": {
+        "type": "Identifier",
+        "name": "foo"
+      },
+      "value": {
+        "type": "Pattern",
+        "elements": []
+      },
+      "attributes": null,
+      "tags": null,
+      "comment": null
+    }
+  ],
+  "comment": null
+}

fluent-syntax/test/fixtures_structure/resource_comment.ftl

@@ -0,0 +1,2 @@
+// This is a resource wide comment
+// It's multiline

fluent-syntax/test/fixtures_structure/resource_comment.json

@@ -0,0 +1,14 @@
+{
+  "type": "Resource",
+  "body": [],
+  "comment": {
+    "type": "Comment",
+    "span": {
+      "type": "Span",
+      "start": 0,
+      "end": 53
+    },
+    "annotations": [],
+    "content": "This is a resource wide comment\nIt's multiline"
+  }
+}

fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.ftl

@@ -0,0 +1,3 @@
+// This is a comment
+// This comment is multiline
+//