diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 43c358987..0d124ae01 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -45,6 +45,8 @@ function getErrorMessage(code, args) { return 'Expected variant key'; case 'E0014': return 'Expected literal'; + case 'E0015': + return 'Only one variant can be marked as default (*)'; default: return code; } diff --git a/fluent-syntax/src/ftlstream.js b/fluent-syntax/src/ftlstream.js index 705d74a8c..82aef23ea 100644 --- a/fluent-syntax/src/ftlstream.js +++ b/fluent-syntax/src/ftlstream.js @@ -43,6 +43,11 @@ export class FTLParserStream extends ParserStream { return true; } + if (ch === '\n') { + // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) + throw new ParseError('E0003', '\u2424'); + } + throw new ParseError('E0003', ch); } @@ -56,7 +61,7 @@ export class FTLParserStream extends ParserStream { takeChar(f) { const ch = this.ch; - if (f(ch)) { + if (ch !== undefined && f(ch)) { this.next(); return ch; } @@ -64,6 +69,10 @@ export class FTLParserStream extends ParserStream { } isIDStart() { + if (this.ch === undefined) { + return false; + } + const cc = this.ch.charCodeAt(0); return ((cc >= 97 && cc <= 122) || // a-z (cc >= 65 && cc <= 90) || // A-Z @@ -98,8 +107,15 @@ export class FTLParserStream extends ParserStream { this.peek(); + const ptr = this.getPeekIndex(); + this.peekLineWS(); + if (this.getPeekIndex() - ptr === 0) { + this.resetPeek(); + return false; + } + if (this.currentPeekIs('*')) { this.peek(); } @@ -119,8 +135,15 @@ export class FTLParserStream extends ParserStream { this.peek(); + const ptr = this.getPeekIndex(); + this.peekLineWS(); + if (this.getPeekIndex() - ptr === 0) { + this.resetPeek(); + return false; + } + if (this.currentPeekIs('.')) { this.resetPeek(); return true; @@ -130,6 +153,36 @@ export class FTLParserStream extends ParserStream { return false; } + isPeekNextLinePattern() { + if (!this.currentPeekIs('\n')) { + return false; + } + + this.peek(); + + const ptr = this.getPeekIndex(); + + this.peekLineWS(); + + if (this.getPeekIndex() - ptr === 0) { + this.resetPeek(); + return false; + } + + if (this.currentPeekIs('}') || + this.currentPeekIs('.') || + this.currentPeekIs('#') || + this.currentPeekIs('[') || + this.currentPeekIs('*') || + this.currentPeekIs('{')) { + this.resetPeek(); + return false; + } + + this.resetPeek(); + return true; + } + isPeekNextLineTagStart() { if (!this.currentPeekIs('\n')) { return false; @@ -137,8 +190,15 @@ export class FTLParserStream extends ParserStream { this.peek(); + const ptr = this.getPeekIndex(); + this.peekLineWS(); + if (this.getPeekIndex() - ptr === 0) { + this.resetPeek(); + return false; + } + if (this.currentPeekIs('#')) { this.resetPeek(); return true; diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 3ca8d7996..57f5e7f89 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -156,7 +156,7 @@ function getMessage(ps, comment) { } if (pattern === undefined && attrs === undefined && tags === undefined) { - throw new ParseError('E0005', id); + throw new ParseError('E0005', id.name); } return new AST.Message(id, pattern, attrs, tags, comment); @@ -254,6 +254,9 @@ function getVariants(ps) { ps.skipLineWS(); if (ps.currentIs('*')) { + if (hasDefault) { + throw new ParseError('E0015'); + } ps.next(); defaultIndex = true; hasDefault = true; @@ -350,15 +353,12 @@ function getPattern(ps) { break; } - ps.peek(); - - if (!ps.currentPeekIs(' ')) { - ps.resetPeek(); + if (!ps.isPeekNextLinePattern()) { break; } - ps.peekLineWS(); - ps.skipToPeek(); + ps.next(); + ps.skipLineWS(); firstLine = false; diff --git a/fluent-syntax/test/fixtures_behavior/attribute_expression_with_wrong_attr.ftl b/fluent-syntax/test/fixtures_behavior/attribute_expression_with_wrong_attr.ftl new file mode 100644 index 000000000..5fbf7b777 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/attribute_expression_with_wrong_attr.ftl @@ -0,0 +1,7 @@ +key = { foo.23 } + +//~ ERROR E0004, pos 12, args "a-zA-Z" + +key = { foo. } + +//~ ERROR E0004, pos 31, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/attribute_starts_from_nl.ftl b/fluent-syntax/test/fixtures_behavior/attribute_starts_from_nl.ftl new file mode 100644 index 000000000..380b08cc5 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/attribute_starts_from_nl.ftl @@ -0,0 +1,3 @@ +foo = Value +.attr = Value 2 +//~ ERROR E0002, pos 12 diff --git a/fluent-syntax/test/fixtures_behavior/attribute_with_empty_pattern.ftl b/fluent-syntax/test/fixtures_behavior/attribute_with_empty_pattern.ftl new file mode 100644 index 000000000..66e4df3e2 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/attribute_with_empty_pattern.ftl @@ -0,0 +1,2 @@ +key = Value + .label = diff --git a/fluent-syntax/test/fixtures_behavior/attribute_without_equal_sign.ftl b/fluent-syntax/test/fixtures_behavior/attribute_without_equal_sign.ftl new file mode 100644 index 000000000..f412f83c7 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/attribute_without_equal_sign.ftl @@ -0,0 +1,3 @@ +key = Value + .label +//~ ERROR E0003, pos 22, args "=" diff --git a/fluent-syntax/test/fixtures_behavior/broken_number.ftl b/fluent-syntax/test/fixtures_behavior/broken_number.ftl new file mode 100644 index 000000000..46cce00c0 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/broken_number.ftl @@ -0,0 +1,14 @@ +key = { -2.4.5 } +//~ ERROR E0003, pos 12, args "}" + +key = { -2.4. } +//~ ERROR E0003, pos 30, args "}" + +key = { -.4 } +//~ ERROR E0004, pos 44, args "0-9" + +key = { -2..4 } +//~ ERROR E0004, pos 61, args "0-9" + +key = { 24d } +//~ ERROR E0003, pos 77, args "}" diff --git a/fluent-syntax/test/fixtures_behavior/call_expression_with_trailing_comma.ftl b/fluent-syntax/test/fixtures_behavior/call_expression_with_trailing_comma.ftl new file mode 100644 index 000000000..974cf99c9 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/call_expression_with_trailing_comma.ftl @@ -0,0 +1 @@ +key = { BUILTIN(23, ) } diff --git a/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl new file mode 100644 index 000000000..b0bd8500c --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl @@ -0,0 +1,2 @@ +key = { BUILTIN(2: "foo") } +//~ ERROR E0009, pos 17 diff --git a/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl new file mode 100644 index 000000000..26ac13981 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl @@ -0,0 +1,2 @@ +key = { BUILTIN(key: foo) } +//~ ERROR E0006, pos 21, args "value" diff --git a/fluent-syntax/test/fixtures_behavior/comment_continues_with_one_slash.ftl b/fluent-syntax/test/fixtures_behavior/comment_continues_with_one_slash.ftl new file mode 100644 index 000000000..5dd830cd2 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/comment_continues_with_one_slash.ftl @@ -0,0 +1,3 @@ +// This is a normal comment +/ but this is not +//~ ERROR E0003, pos 29, args "/" diff --git a/fluent-syntax/test/fixtures_behavior/comment_with_eof.ftl b/fluent-syntax/test/fixtures_behavior/comment_with_eof.ftl new file mode 100644 index 000000000..0ae104701 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/comment_with_eof.ftl @@ -0,0 +1 @@ +// This is a comment with no new line diff --git a/fluent-syntax/test/fixtures_behavior/empty_resource.ftl b/fluent-syntax/test/fixtures_behavior/empty_resource.ftl new file mode 100644 index 000000000..e69de29bb diff --git a/fluent-syntax/test/fixtures_behavior/empty_resource_with_ws.ftl b/fluent-syntax/test/fixtures_behavior/empty_resource_with_ws.ftl new file mode 100644 index 000000000..4e4354436 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/empty_resource_with_ws.ftl @@ -0,0 +1,2 @@ + + diff --git a/fluent-syntax/test/fixtures_behavior/entry_start_with_one_slash.ftl b/fluent-syntax/test/fixtures_behavior/entry_start_with_one_slash.ftl new file mode 100644 index 000000000..440b3ffcd --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/entry_start_with_one_slash.ftl @@ -0,0 +1,3 @@ + +/ Test +//~ ERROR E0003, pos 2, args "/" diff --git a/fluent-syntax/test/fixtures_behavior/leading_empty_lines.ftl b/fluent-syntax/test/fixtures_behavior/leading_empty_lines.ftl new file mode 100644 index 000000000..60e34ee63 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/leading_empty_lines.ftl @@ -0,0 +1,3 @@ + + +foo = Value diff --git a/fluent-syntax/test/fixtures_behavior/leading_empty_lines_with_ws.ftl b/fluent-syntax/test/fixtures_behavior/leading_empty_lines_with_ws.ftl new file mode 100644 index 000000000..cd626c8d3 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/leading_empty_lines_with_ws.ftl @@ -0,0 +1,5 @@ + + + + +foo = Value diff --git a/fluent-syntax/test/fixtures_behavior/multiline_string.ftl b/fluent-syntax/test/fixtures_behavior/multiline_string.ftl new file mode 100644 index 000000000..9f3cdba06 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/multiline_string.ftl @@ -0,0 +1,3 @@ +key = { BUILTIN(key: " + text + ") } diff --git a/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl b/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl new file mode 100644 index 000000000..70417cc57 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl @@ -0,0 +1,3 @@ +key = Value + Value 2 +//~ ERROR E0002, pos 12 diff --git a/fluent-syntax/test/fixtures_behavior/non_id_attribute_name.ftl b/fluent-syntax/test/fixtures_behavior/non_id_attribute_name.ftl new file mode 100644 index 000000000..1cf580b29 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/non_id_attribute_name.ftl @@ -0,0 +1,3 @@ +key = Value + .2 = Foo +//~ ERROR E0004, pos 17, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/placeable_without_close_bracket.ftl b/fluent-syntax/test/fixtures_behavior/placeable_without_close_bracket.ftl new file mode 100644 index 000000000..282cf2b61 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/placeable_without_close_bracket.ftl @@ -0,0 +1,3 @@ +key = { $num + +//~ ERROR E0003, pos 12, args "}" diff --git a/fluent-syntax/test/fixtures_behavior/second_attribute_starts_from_nl.ftl b/fluent-syntax/test/fixtures_behavior/second_attribute_starts_from_nl.ftl new file mode 100644 index 000000000..e87c2b98f --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/second_attribute_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = Value + .label = Value +.accesskey = K +//~ ERROR E0002, pos 31 diff --git a/fluent-syntax/test/fixtures_behavior/second_tag_starts_from_nl.ftl b/fluent-syntax/test/fixtures_behavior/second_tag_starts_from_nl.ftl new file mode 100644 index 000000000..00c9dcd96 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/second_tag_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = Value + #tag +#tag2 +//~ ERROR E0002, pos 21 diff --git a/fluent-syntax/test/fixtures_behavior/section_starts_with_one_bracket.ftl b/fluent-syntax/test/fixtures_behavior/section_starts_with_one_bracket.ftl new file mode 100644 index 000000000..eb4235618 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/section_starts_with_one_bracket.ftl @@ -0,0 +1,2 @@ +[ This is a broken section ]] +//~ ERROR E0003, pos 1, args "[" diff --git a/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl b/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl new file mode 100644 index 000000000..d4731b4a9 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl @@ -0,0 +1,3 @@ +[[ This is +a broken section]] +//~ ERROR E0003, pos 10, args "]" diff --git a/fluent-syntax/test/fixtures_behavior/section_with_no_nl_after_it.ftl b/fluent-syntax/test/fixtures_behavior/section_with_no_nl_after_it.ftl new file mode 100644 index 000000000..f11f4c062 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/section_with_no_nl_after_it.ftl @@ -0,0 +1 @@ +[[ This is a correct section ]] diff --git a/fluent-syntax/test/fixtures_behavior/section_with_one_bracket_at_the_end.ftl b/fluent-syntax/test/fixtures_behavior/section_with_one_bracket_at_the_end.ftl new file mode 100644 index 000000000..0464afea0 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/section_with_one_bracket_at_the_end.ftl @@ -0,0 +1,2 @@ +[[ This is a broken section ] +//~ ERROR E0003, pos 29, args "]" diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_with_two_selectors.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_with_two_selectors.ftl new file mode 100644 index 000000000..0a05e57de --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/select_expression_with_two_selectors.ftl @@ -0,0 +1,2 @@ +key = { $foo $faa } +//~ ERROR E0003, pos 13, args "}" diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_without_arrow.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_without_arrow.ftl new file mode 100644 index 000000000..b89618dd8 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/select_expression_without_arrow.ftl @@ -0,0 +1,2 @@ +key = { $foo - } +//~ ERROR E0003, pos 13, args "}" diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl new file mode 100644 index 000000000..198654bbc --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl @@ -0,0 +1,6 @@ +key = { $foo -> } +//~ ERROR E0003, pos 16, args "␤" + +key = { $foo -> + } +//~ ERROR E0003, pos 39, args "[" diff --git a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl new file mode 100644 index 000000000..a2ad26bec --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -0,0 +1,2 @@ +key = { $foo -> +//~ ERROR E0003, pos 16, args "[" diff --git a/fluent-syntax/test/fixtures_behavior/foo.ftl b/fluent-syntax/test/fixtures_behavior/simple_message.ftl similarity index 100% rename from fluent-syntax/test/fixtures_behavior/foo.ftl rename to fluent-syntax/test/fixtures_behavior/simple_message.ftl diff --git a/fluent-syntax/test/fixtures_behavior/single_char_id.ftl b/fluent-syntax/test/fixtures_behavior/single_char_id.ftl new file mode 100644 index 000000000..b4d3b479c --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/single_char_id.ftl @@ -0,0 +1,2 @@ +k = Value + .l = Foo diff --git a/fluent-syntax/test/fixtures_behavior/standalone_identifier.ftl b/fluent-syntax/test/fixtures_behavior/standalone_identifier.ftl new file mode 100644 index 000000000..d3ee59c70 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/standalone_identifier.ftl @@ -0,0 +1,2 @@ +foo +//~ ERROR E0005, pos 3, args "foo" diff --git a/fluent-syntax/test/fixtures_behavior/tag_and_attribute_together.ftl b/fluent-syntax/test/fixtures_behavior/tag_and_attribute_together.ftl new file mode 100644 index 000000000..2adfb21a5 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/tag_and_attribute_together.ftl @@ -0,0 +1,4 @@ +key = Value + .label = Foo + #masculine +//~ ERROR E0012, pos 28 diff --git a/fluent-syntax/test/fixtures_behavior/tag_starts_from_nl.ftl b/fluent-syntax/test/fixtures_behavior/tag_starts_from_nl.ftl new file mode 100644 index 000000000..797ab57a9 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/tag_starts_from_nl.ftl @@ -0,0 +1,3 @@ +key = Value +#tag +//~ ERROR E0002, pos 12 diff --git a/fluent-syntax/test/fixtures_behavior/bar.ftl b/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl similarity index 100% rename from fluent-syntax/test/fixtures_behavior/bar.ftl rename to fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl diff --git a/fluent-syntax/test/fixtures_behavior/unknown_entry_start.ftl b/fluent-syntax/test/fixtures_behavior/unknown_entry_start.ftl new file mode 100644 index 000000000..357d3b46d --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/unknown_entry_start.ftl @@ -0,0 +1,3 @@ + +8Foo = Foo +//~ ERROR E0002, pos 1 diff --git a/fluent-syntax/test/fixtures_behavior/variant_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/variant_ends_abruptly.ftl new file mode 100644 index 000000000..94dfcd4b5 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_ends_abruptly.ftl @@ -0,0 +1,3 @@ +key = { $foo -> + *[ +//~ ERROR E0004, pos 22, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/variant_expression_empty_key.ftl b/fluent-syntax/test/fixtures_behavior/variant_expression_empty_key.ftl new file mode 100644 index 000000000..c6fd0b8d1 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_expression_empty_key.ftl @@ -0,0 +1,2 @@ +key = { foo[] } +//~ ERROR E0004, pos 12, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/variant_starts_from_nl.ftl b/fluent-syntax/test/fixtures_behavior/variant_starts_from_nl.ftl new file mode 100644 index 000000000..139f4f108 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = { +*[one] Value + } +//~ ERROR E0004, pos 7, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/variant_with_digit_key.ftl b/fluent-syntax/test/fixtures_behavior/variant_with_digit_key.ftl new file mode 100644 index 000000000..3ab304d69 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_with_digit_key.ftl @@ -0,0 +1,3 @@ +key = { + *[-2] Foo + } diff --git a/fluent-syntax/test/fixtures_behavior/variant_with_empty_pattern.ftl b/fluent-syntax/test/fixtures_behavior/variant_with_empty_pattern.ftl new file mode 100644 index 000000000..8dfb5bc14 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_with_empty_pattern.ftl @@ -0,0 +1,3 @@ +key = { + *[one] + } diff --git a/fluent-syntax/test/fixtures_behavior/variant_with_leading_space_in_name.ftl b/fluent-syntax/test/fixtures_behavior/variant_with_leading_space_in_name.ftl new file mode 100644 index 000000000..0b8f9c956 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_with_leading_space_in_name.ftl @@ -0,0 +1,4 @@ +key = { + *[ one] Foo + } +//~ ERROR E0004, pos 18, args "a-zA-Z" diff --git a/fluent-syntax/test/fixtures_behavior/variant_with_symbol_with_space.ftl b/fluent-syntax/test/fixtures_behavior/variant_with_symbol_with_space.ftl new file mode 100644 index 000000000..68a60c6e7 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variant_with_symbol_with_space.ftl @@ -0,0 +1,3 @@ +key = { + *[New York] Nowy Jork + } diff --git a/fluent-syntax/test/fixtures_behavior/variants_with_two_defaults.ftl b/fluent-syntax/test/fixtures_behavior/variants_with_two_defaults.ftl new file mode 100644 index 000000000..f49c2998c --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/variants_with_two_defaults.ftl @@ -0,0 +1,5 @@ +key = { + *[one] Foo + *[two] Two + } +//~ ERROR E0015, pos 27 diff --git a/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.ftl b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.ftl new file mode 100644 index 000000000..179dbacc6 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.ftl @@ -0,0 +1 @@ +foo = diff --git a/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json new file mode 100644 index 000000000..e4ee002ef --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json @@ -0,0 +1,26 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 6 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [] + }, + "attributes": null, + "tags": null, + "comment": null + } + ], + "comment": null +} diff --git a/fluent-syntax/test/fixtures_structure/resource_comment.ftl b/fluent-syntax/test/fixtures_structure/resource_comment.ftl new file mode 100644 index 000000000..4c7a33c9d --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/resource_comment.ftl @@ -0,0 +1,2 @@ +// This is a resource wide comment +// It's multiline diff --git a/fluent-syntax/test/fixtures_structure/resource_comment.json b/fluent-syntax/test/fixtures_structure/resource_comment.json new file mode 100644 index 000000000..d8a529e09 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/resource_comment.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [], + "comment": { + "type": "Comment", + "span": { + "type": "Span", + "start": 0, + "end": 53 + }, + "annotations": [], + "content": "This is a resource wide comment\nIt's multiline" + } +} diff --git a/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.ftl b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.ftl new file mode 100644 index 000000000..14b5cced0 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.ftl @@ -0,0 +1,3 @@ +// This is a comment +// This comment is multiline +// diff --git a/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json new file mode 100644 index 000000000..49c8c1677 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [], + "comment": { + "type": "Comment", + "span": { + "type": "Span", + "start": 0, + "end": 53 + }, + "annotations": [], + "content": "This is a comment\nThis comment is multiline\n" + } +} diff --git a/fluent-syntax/test/fixtures_structure/section.ftl b/fluent-syntax/test/fixtures_structure/section.ftl new file mode 100644 index 000000000..60a26d326 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/section.ftl @@ -0,0 +1,2 @@ + +[[ This is a section ]] diff --git a/fluent-syntax/test/fixtures_structure/section.json b/fluent-syntax/test/fixtures_structure/section.json new file mode 100644 index 000000000..657f491bc --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/section.json @@ -0,0 +1,20 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Section", + "span": { + "type": "Span", + "start": 1, + "end": 25 + }, + "annotations": [], + "name": { + "type": "Symbol", + "name": "This is a section" + }, + "comment": null + } + ], + "comment": null +} diff --git a/fluent-syntax/test/fixtures_structure/foo.ftl b/fluent-syntax/test/fixtures_structure/simple_message.ftl similarity index 100% rename from fluent-syntax/test/fixtures_structure/foo.ftl rename to fluent-syntax/test/fixtures_structure/simple_message.ftl diff --git a/fluent-syntax/test/fixtures_structure/foo.json b/fluent-syntax/test/fixtures_structure/simple_message.json similarity index 100% rename from fluent-syntax/test/fixtures_structure/foo.json rename to fluent-syntax/test/fixtures_structure/simple_message.json diff --git a/fluent-syntax/test/fixtures_structure/standalone_comment.ftl b/fluent-syntax/test/fixtures_structure/standalone_comment.ftl new file mode 100644 index 000000000..49fce24e8 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/standalone_comment.ftl @@ -0,0 +1,3 @@ +foo = Value + +// This is a standalone comment diff --git a/fluent-syntax/test/fixtures_structure/standalone_comment.json b/fluent-syntax/test/fixtures_structure/standalone_comment.json new file mode 100644 index 000000000..0ae862574 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/standalone_comment.json @@ -0,0 +1,41 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 11 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": null, + "tags": null, + "comment": null + }, + { + "type": "Comment", + "span": { + "type": "Span", + "start": 13, + "end": 45 + }, + "annotations": [], + "content": "This is a standalone comment" + } + ], + "comment": null +} diff --git a/package.json b/package.json index 3408e82a3..d873679c7 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "commander": "^2.9", "documentation": "^4.0.0-beta10", "eslint": "^3.9.1", + "fuzzer": "^0.2.1", "gh-pages": "^0.12.0", "jsdoc": "^3.4.3", "mocha": "^3.0.2", diff --git a/tools/README.md b/tools/README.md index 166eb0a98..08579eaee 100644 --- a/tools/README.md +++ b/tools/README.md @@ -49,6 +49,26 @@ Examples: ./format.js -e data.json file.ftl +fuzz.js +-------- + +`fuzz.js` is a fuzzer for fluent.js's parsers. + + Usage: fuzz [options] [file] + + Options: + + -h, --help output usage information + -V, --version output the version number + -r, --runtime Use the runtime parser + -i, --repetitions Number of repetitions [100000] + +Examples: + + echo "foo = Foo" | ./fuzz.js + ./fuzz.js -i 1000 file.ftl + + perf ---- diff --git a/tools/fuzz.js b/tools/fuzz.js new file mode 100755 index 000000000..3c8f0433f --- /dev/null +++ b/tools/fuzz.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +'use strict'; + +const fs = require('fs'); +const readline = require('readline'); +const program = require('commander'); +const fuzzer = require('fuzzer'); + +require('babel-register')({ + plugins: ['transform-es2015-modules-commonjs'] +}); + +fuzzer.seed(Math.random() * 1000000000); + +program + .version('0.0.1') + .usage('[options] [file]') + .option('-r, --runtime', 'Use the runtime parser') + .option('-i, --repetitions ', 'Number of repetitions [100000]', 100000) + .parse(process.argv); + +if (program.args.length) { + fs.readFile(program.args[0], fuzz); +} else { + process.stdin.resume(); + process.stdin.on('data', data => fuzz(null, data)); +} + +function fuzz(err, data) { + if (err) { + return console.error('File not found: ' + err.path); + } + + const parse = program.runtime + ? require('../fluent/src/parser').default + : require('../fluent-syntax/src').parse; + + const source = data.toString(); + const mutations = new Set(); + + let i = 1; + + while (i <= program.repetitions) { + const mutated = fuzzer.mutate.string(source); + + if (mutations.has(mutated)) { + continue; + } + + mutations.add(mutated); + + const progress = Math.round(i / program.repetitions * 100); + readline.cursorTo(process.stdout, 0); + process.stdout.write(`mutation ${i} ... ${progress}%`); + + try { + parse(mutated); + } catch (e) { + console.log(`! mutation ${i}`); + console.log(e); + console.log(mutated); + break; + } + + i++; + } +} diff --git a/tools/parse.js b/tools/parse.js index d76e79f61..edcb9fdb7 100755 --- a/tools/parse.js +++ b/tools/parse.js @@ -64,10 +64,10 @@ function printAnnotations(source, entry) { } function printAnnotation(source, span, annot) { - const { name, message, pos } = annot; + const { name, message, span: { start } } = annot; const slice = source.substring(span.start, span.end).trimRight(); - const lineNumber = FluentSyntax.lineOffset(source, pos) + 1; - const columnOffset = FluentSyntax.columnOffset(source, pos); + const lineNumber = FluentSyntax.lineOffset(source, start) + 1; + const columnOffset = FluentSyntax.columnOffset(source, start); const showLines = lineNumber - FluentSyntax.lineOffset(source, span.start); const lines = slice.split('\n'); const head = lines.slice(0, showLines);