From f21097394ac653d72eab9d47927b70fb6a22709c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 27 Nov 2018 13:31:56 +0100 Subject: [PATCH 1/2] Update reference fixtures wrt. Junk content Port fixture changes from: https://github.com/projectfluent/fluent/commit/bf8fff4333c8b3a3182fb4658368c8b756d9c29e --- .../test/fixtures_reference/astral.json | 4 +- .../fixtures_reference/call_expressions.json | 20 +++++-- .../test/fixtures_reference/crlf.json | 2 +- .../escaped_characters.json | 2 +- .../test/fixtures_reference/junk.ftl | 19 ++++++- .../test/fixtures_reference/junk.json | 57 ++++++++++++++++++- .../test/fixtures_reference/leading_dots.json | 12 ++-- .../member_expressions.json | 7 ++- .../test/fixtures_reference/messages.json | 2 +- .../fixtures_reference/mixed_entries.json | 4 +- .../test/fixtures_reference/placeables.json | 6 +- .../select_expressions.json | 8 +-- .../fixtures_reference/select_indent.json | 7 ++- .../test/fixtures_reference/tab.json | 4 +- .../test/fixtures_reference/terms.json | 8 +-- .../test/fixtures_reference/variant_keys.json | 4 +- .../fixtures_reference/variant_lists.json | 8 +-- fluent-syntax/test/reference_test.js | 9 +-- 18 files changed, 137 insertions(+), 46 deletions(-) diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json index 6f748c9c0..b69743cad 100644 --- a/fluent-syntax/test/fixtures_reference/astral.json +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -154,7 +154,7 @@ { "type": "Junk", "annotations": [], - "content": "err-😂 = Value\n" + "content": "err-😂 = Value\n\n" }, { "type": "Comment", @@ -163,7 +163,7 @@ { "type": "Junk", "annotations": [], - "content": "err-invalid-expression = { 😂 }\n" + "content": "err-invalid-expression = { 😂 }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index 4cc36d404..286ea4f39 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -70,7 +70,7 @@ { "type": "Junk", "annotations": [], - "content": "mixed-case-callee = {Function()}\n" + "content": "mixed-case-callee = {Function()}\n\n" }, { "type": "Comment", @@ -88,7 +88,7 @@ { "type": "Junk", "annotations": [], - "content": "variable-callee = {$variable()}\n" + "content": "variable-callee = {$variable()}\n\n" }, { "type": "GroupComment", @@ -323,7 +323,7 @@ { "type": "Junk", "annotations": [], - "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n" + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n\n" }, { "type": "Comment", @@ -332,7 +332,7 @@ { "type": "Junk", "annotations": [], - "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n" + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n\n\n" }, { "type": "GroupComment", @@ -1063,7 +1063,17 @@ { "type": "Junk", "annotations": [], - "content": "one-argument = {FUN(1,,)}\nmissing-arg = {FUN(,)}\nmissing-sparse-arg = {FUN( , )}\n" + "content": "one-argument = {FUN(1,,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-arg = {FUN(,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-sparse-arg = {FUN( , )}\n\n\n" }, { "type": "GroupComment", diff --git a/fluent-syntax/test/fixtures_reference/crlf.json b/fluent-syntax/test/fixtures_reference/crlf.json index 6b324cd4e..58d26a777 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.json +++ b/fluent-syntax/test/fixtures_reference/crlf.json @@ -61,7 +61,7 @@ { "type": "Junk", "annotations": [], - "content": "err03 = { \"str\r\n" + "content": "err03 = { \"str\r\n\r\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 91de49944..e05c0efe6 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -169,7 +169,7 @@ { "type": "Junk", "annotations": [], - "content": "unknown-escape = {\"\\x\"}\n" + "content": "unknown-escape = {\"\\x\"}\n\n" }, { "type": "GroupComment", diff --git a/fluent-syntax/test/fixtures_reference/junk.ftl b/fluent-syntax/test/fixtures_reference/junk.ftl index 62fc2ea1b..b0b0c5f3b 100644 --- a/fluent-syntax/test/fixtures_reference/junk.ftl +++ b/fluent-syntax/test/fixtures_reference/junk.ftl @@ -1,4 +1,21 @@ +## Two adjacent Junks. +err01 = {1x} +err02 = {2x} + +# A single Junk. +err03 = {1x +2 + +# A single Junk. ą=Invalid identifier ć=Another one -key01 = { +# The COMMENT ends this junk. +err04 = { +# COMMENT + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. +err04 = { +# COMMENT +} diff --git a/fluent-syntax/test/fixtures_reference/junk.json b/fluent-syntax/test/fixtures_reference/junk.json index 530ff3d5a..15e62a4db 100644 --- a/fluent-syntax/test/fixtures_reference/junk.json +++ b/fluent-syntax/test/fixtures_reference/junk.json @@ -1,15 +1,68 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Two adjacent Junks." + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {1x}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {2x}\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {1x\n2\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, { "type": "Junk", "annotations": [], - "content": "ą=Invalid identifier\nć=Another one\n" + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" }, { "type": "Junk", "annotations": [], - "content": "key01 = {\n" + "content": "}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/leading_dots.json b/fluent-syntax/test/fixtures_reference/leading_dots.json index 1787d0ee0..a185d4896 100644 --- a/fluent-syntax/test/fixtures_reference/leading_dots.json +++ b/fluent-syntax/test/fixtures_reference/leading_dots.json @@ -173,7 +173,7 @@ { "type": "Junk", "annotations": [], - "content": " .Continued\n" + "content": " .Continued\n\n" }, { "type": "Comment", @@ -182,7 +182,7 @@ { "type": "Junk", "annotations": [], - "content": "key08 =\n .Value\n" + "content": "key08 =\n .Value\n\n" }, { "type": "Comment", @@ -191,7 +191,7 @@ { "type": "Junk", "annotations": [], - "content": "key09 =\n .Value\n Continued\n" + "content": "key09 =\n .Value\n Continued\n\n" }, { "type": "Message", @@ -410,7 +410,7 @@ { "type": "Junk", "annotations": [], - "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n" + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n\n" }, { "type": "Comment", @@ -419,7 +419,7 @@ { "type": "Junk", "annotations": [], - "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n" + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n\n" }, { "type": "Comment", @@ -428,7 +428,7 @@ { "type": "Junk", "annotations": [], - "content": "key18 =\n.Value\n" + "content": "key18 =\n.Value\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.json b/fluent-syntax/test/fixtures_reference/member_expressions.json index a95879011..541bd795a 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.json +++ b/fluent-syntax/test/fixtures_reference/member_expressions.json @@ -70,7 +70,12 @@ { "type": "Junk", "annotations": [], - "content": "variant-expression = {msg[case]}\nattribute-expression = {-term.attr}\n" + "content": "variant-expression = {msg[case]}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "attribute-expression = {-term.attr}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/messages.json b/fluent-syntax/test/fixtures_reference/messages.json index fa8dcbd7b..7d779d3e2 100644 --- a/fluent-syntax/test/fixtures_reference/messages.json +++ b/fluent-syntax/test/fixtures_reference/messages.json @@ -234,7 +234,7 @@ { "type": "Junk", "annotations": [], - "content": "key07 =\n" + "content": "key07 =\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/mixed_entries.json b/fluent-syntax/test/fixtures_reference/mixed_entries.json index 8f334f4fa..a9dc501f6 100644 --- a/fluent-syntax/test/fixtures_reference/mixed_entries.json +++ b/fluent-syntax/test/fixtures_reference/mixed_entries.json @@ -61,7 +61,7 @@ { "type": "Junk", "annotations": [], - "content": "ą=Invalid identifier\nć=Another one\n" + "content": "ą=Invalid identifier\nć=Another one\n\n" }, { "type": "Message", @@ -91,7 +91,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr = Dangling attribute\n" + "content": " .attr = Dangling attribute\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/placeables.json b/fluent-syntax/test/fixtures_reference/placeables.json index ff7fac8a7..7d67d940b 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.json +++ b/fluent-syntax/test/fixtures_reference/placeables.json @@ -80,7 +80,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-open1 = { 1\n" + "content": "unmatched-open1 = { 1\n\n" }, { "type": "Comment", @@ -89,7 +89,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-open2 = {{ 1 }\n" + "content": "unmatched-open2 = {{ 1 }\n\n" }, { "type": "Comment", @@ -98,7 +98,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-close1 = 1 }\n" + "content": "unmatched-close1 = 1 }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 7cce4070d..4e84c0c2a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -137,7 +137,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n" + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n\n" }, { "type": "Comment", @@ -146,7 +146,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n" + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n\n" }, { "type": "Comment", @@ -155,7 +155,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n" + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n\n" }, { "type": "Message", @@ -279,7 +279,7 @@ { "type": "Junk", "annotations": [], - "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/select_indent.json b/fluent-syntax/test/fixtures_reference/select_indent.json index 03109cb0f..d8c0fa880 100644 --- a/fluent-syntax/test/fixtures_reference/select_indent.json +++ b/fluent-syntax/test/fixtures_reference/select_indent.json @@ -548,7 +548,12 @@ { "type": "Junk", "annotations": [], - "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\nContinued without indent.\n}\n" + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "Continued without indent.\n}\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/tab.json b/fluent-syntax/test/fixtures_reference/tab.json index fe438ab43..714eb947e 100644 --- a/fluent-syntax/test/fixtures_reference/tab.json +++ b/fluent-syntax/test/fixtures_reference/tab.json @@ -29,7 +29,7 @@ { "type": "Junk", "annotations": [], - "content": "key02\t= Value 02\n" + "content": "key02\t= Value 02\n\n" }, { "type": "Comment", @@ -38,7 +38,7 @@ { "type": "Junk", "annotations": [], - "content": "key03 =\n\tThis line isn't properly indented.\n" + "content": "key03 =\n\tThis line isn't properly indented.\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index d3eff8065..2321283f0 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -65,7 +65,7 @@ { "type": "Junk", "annotations": [], - "content": "-term03 =\n .attr = Attribute\n" + "content": "-term03 =\n .attr = Attribute\n\n" }, { "type": "Comment", @@ -74,7 +74,7 @@ { "type": "Junk", "annotations": [], - "content": "-term04 = \n .attr1 = Attribute 1\n" + "content": "-term04 = \n .attr1 = Attribute 1\n\n" }, { "type": "Comment", @@ -83,7 +83,7 @@ { "type": "Junk", "annotations": [], - "content": "-term05 =\n" + "content": "-term05 =\n\n" }, { "type": "Comment", @@ -92,7 +92,7 @@ { "type": "Junk", "annotations": [], - "content": "-term06 = \n" + "content": "-term06 = \n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/variant_keys.json b/fluent-syntax/test/fixtures_reference/variant_keys.json index e8c60b002..cf752e433 100644 --- a/fluent-syntax/test/fixtures_reference/variant_keys.json +++ b/fluent-syntax/test/fixtures_reference/variant_keys.json @@ -132,7 +132,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-identifier =\n {\n *[two words] value\n }\n" + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n\n" }, { "type": "Comment", @@ -141,7 +141,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-int =\n {\n *[1 apple] value\n }\n" + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.json b/fluent-syntax/test/fixtures_reference/variant_lists.json index d10484046..aaac9f7b0 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.json +++ b/fluent-syntax/test/fixtures_reference/variant_lists.json @@ -56,7 +56,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { "type": "Comment", @@ -65,7 +65,7 @@ { "type": "Junk", "annotations": [], - "content": "variant-list-in-message =\n {\n *[key] Value\n }\n" + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n\n" }, { "type": "Message", @@ -91,7 +91,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { "type": "Term", @@ -200,7 +200,7 @@ { "type": "Junk", "annotations": [], - "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n" + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/reference_test.js b/fluent-syntax/test/reference_test.js index a6cd8404e..39cd7ad32 100644 --- a/fluent-syntax/test/reference_test.js +++ b/fluent-syntax/test/reference_test.js @@ -46,10 +46,11 @@ readdir(fixtures, function(err, filenames) { const ref = JSON.parse(expected) const ast = parser.parse(ftl); - // Ignore Junk which is parsed differently by the tooling parser, and - // which doesn't carry spans nor annotations in the reference parser. - ref.body = ref.body.filter(entry => entry.type !== "Junk"); - ast.body = ast.body.filter(entry => entry.type !== "Junk"); + // Only compare Junk content and ignore annotations, which carry error + // messages and positions. The reference parser doesn't produce + // annotations at the moment. + ast.body = ast.body.map(entry => entry.type === "Junk" ? + {...entry, annotations: []} : entry); assert.deepEqual( ast, ref, From 20444375ea332585396fd1f08bc589b2151d99d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 27 Nov 2018 13:34:31 +0100 Subject: [PATCH 2/2] Update reference fixtures wrt. allowing all Unicode characters Port reference fixtures from: https://github.com/projectfluent/fluent/commit/741be047bb6bd55c3e761c0d2fcdd8f5dc360b6b --- .../test/fixtures_reference/any_char.ftl | 8 +++ .../test/fixtures_reference/any_char.json | 68 +++++++++++++++++++ fluent/test/fixtures_reference/any_char.json | 5 ++ 3 files changed, 81 insertions(+) create mode 100644 fluent-syntax/test/fixtures_reference/any_char.ftl create mode 100644 fluent-syntax/test/fixtures_reference/any_char.json create mode 100644 fluent/test/fixtures_reference/any_char.json diff --git a/fluent-syntax/test/fixtures_reference/any_char.ftl b/fluent-syntax/test/fixtures_reference/any_char.ftl new file mode 100644 index 000000000..6966a0dae --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/any_char.ftl @@ -0,0 +1,8 @@ +# ↓ BEL, U+0007 +control0 = abcdef + +# ↓ DEL, U+007F +delete = abcdef + +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/fluent-syntax/test/fixtures_reference/any_char.json b/fluent-syntax/test/fixtures_reference/any_char.json new file mode 100644 index 000000000..07e7dc4b2 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/any_char.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc\u0007def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BEL, U+0007" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "delete" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcdef" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ DEL, U+007F" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc‚def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BPM, U+0082" + } + } + ] +} diff --git a/fluent/test/fixtures_reference/any_char.json b/fluent/test/fixtures_reference/any_char.json new file mode 100644 index 000000000..21eb2e4c3 --- /dev/null +++ b/fluent/test/fixtures_reference/any_char.json @@ -0,0 +1,5 @@ +{ + "control0": "abc\u0007def", + "delete": "abcdef", + "control1": "abc‚def" +}