From d009ea21c6d58d3d824a20060c3ee62fe16aa53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 14:42:23 +0100 Subject: [PATCH 1/5] Support astral Unicode characters in TextElements and StringLiterals --- eslint_src.json | 11 -- .../test/fixtures_reference/astral.ftl | 20 ++ .../test/fixtures_reference/astral.json | 174 ++++++++++++++++++ fluent/src/resource.js | 8 +- fluent/test/fixtures_reference/astral.json | 19 ++ fluent/test/parser_behavior_test.js | 4 +- fluent/test/parser_reference_test.js | 2 +- fluent/test/parser_structure_test.js | 2 +- 8 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/astral.ftl create mode 100644 fluent-syntax/test/fixtures_reference/astral.json create mode 100644 fluent/test/fixtures_reference/astral.json diff --git a/eslint_src.json b/eslint_src.json index 252b3cd95..328febcee 100644 --- a/eslint_src.json +++ b/eslint_src.json @@ -32,17 +32,6 @@ ], "no-implied-eval": 2, "no-loop-func": 2, - "no-magic-numbers": [ - 1, - { - "ignore": [ - -1, - 0, - 1, - 2 - ] - } - ], "no-useless-call": 2, "no-useless-concat": 2, "no-delete-var": 2, diff --git a/fluent-syntax/test/fixtures_reference/astral.ftl b/fluent-syntax/test/fixtures_reference/astral.ftl new file mode 100644 index 000000000..b77e32e38 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/astral.ftl @@ -0,0 +1,20 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 + +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = {"\uD83D\uDE02"} +surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} + +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = {"A face 😂 with tears of joy."} + +# ERROR Invalid identifier +err-😂 = Value + +# ERROR Invalid expression +err-invalid-expression = { 😂 } + +# ERROR Invalid variant key +err-invalid-variant-key = { $sel -> + *[😂] Value +} diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json new file mode 100644 index 000000000..6056fb727 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -0,0 +1,174 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "face-with-tears-of-joy" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "😂" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "tetragram-for-centre" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "𝌆" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\uD83D\\uDE02" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-adjacent-strings" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A face 😂 with tears of joy." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "A face 😂 with tears of joy." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Invalid identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-😂 = Value\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid expression" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-expression = { 😂 }\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid variant key" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" + } + ] +} diff --git a/fluent/src/resource.js b/fluent/src/resource.js index a7c6b5675..62025ad8c 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -420,7 +420,13 @@ export default class FluentResource extends Map { function parseEscapeSequence(reSpecialized) { if (test(RE_UNICODE_ESCAPE)) { let sequence = match(RE_UNICODE_ESCAPE); - return String.fromCodePoint(parseInt(sequence, 16)); + let codepoint = parseInt(sequence, 16); + return codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Lonely surrogates can cause trouble when the parsing result is + // saved using UTF-8. Use U+FFFD REPLACEMENT CHARACTER instead. + : "�"; } if (test(reSpecialized)) { diff --git a/fluent/test/fixtures_reference/astral.json b/fluent/test/fixtures_reference/astral.json new file mode 100644 index 000000000..9a9469f0c --- /dev/null +++ b/fluent/test/fixtures_reference/astral.json @@ -0,0 +1,19 @@ +{ + "face-with-tears-of-joy": "😂", + "tetragram-for-centre": "𝌆", + "surrogates-in-text": [ + "�", + "�" + ], + "surrogates-in-string": [ + "��" + ], + "surrogates-in-adjacent-strings": [ + "�", + "�" + ], + "emoji-in-text": "A face 😂 with tears of joy.", + "emoji-in-string": [ + "A face 😂 with tears of joy." + ] +} diff --git a/fluent/test/parser_behavior_test.js b/fluent/test/parser_behavior_test.js index 239a4b47c..53a169ed2 100644 --- a/fluent/test/parser_behavior_test.js +++ b/fluent/test/parser_behavior_test.js @@ -47,8 +47,8 @@ readdir(ftlFixtures, function(err, filenames) { if (expected.attributes) { assert(hasAttrs(entry), `Expected ${id} to have attributes`); assert.deepEqual( - Object.keys(expected.attributes), - Object.keys(entry.attrs) + Object.keys(entry.attrs), + Object.keys(expected.attributes) ); } else { assert(!hasAttrs(entry), `Expected ${id} to have zero attributes`); diff --git a/fluent/test/parser_reference_test.js b/fluent/test/parser_reference_test.js index 1058fb85d..de69b757b 100644 --- a/fluent/test/parser_reference_test.js +++ b/fluent/test/parser_reference_test.js @@ -30,7 +30,7 @@ readdir(ftlFixtures, function(err, filenames) { ); const resource = FluentResource.fromString(ftl); assert.deepEqual( - JSON.parse(expected), toObject(resource), + toObject(resource), JSON.parse(expected), 'Actual Annotations don\'t match the expected ones' ); }); diff --git a/fluent/test/parser_structure_test.js b/fluent/test/parser_structure_test.js index 8c406c5b5..8462192e7 100644 --- a/fluent/test/parser_structure_test.js +++ b/fluent/test/parser_structure_test.js @@ -30,7 +30,7 @@ readdir(ftlFixtures, function(err, filenames) { ); const resource = FluentResource.fromString(ftl); assert.deepEqual( - JSON.parse(expected), toObject(resource), + toObject(resource), JSON.parse(expected), 'Actual Annotations don\'t match the expected ones' ); }); From 5941f1bbf98300a193b61d0e69e2918dc6ec5e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 13:51:49 +0100 Subject: [PATCH 2/5] Don't store the term sigil in the identifier name --- fluent-syntax/src/errors.js | 2 +- fluent-syntax/src/parser.js | 20 +-- fluent-syntax/src/serializer.js | 33 ++++- fluent-syntax/test/fixtures_behavior/term.ftl | 4 +- .../fixtures_reference/call_expressions.json | 2 +- .../test/fixtures_reference/comments.json | 2 +- .../member_expressions.json | 2 +- .../fixtures_reference/mixed_entries.json | 2 +- .../reference_expressions.json | 2 +- .../select_expressions.json | 2 +- .../test/fixtures_reference/terms.json | 4 +- .../test/fixtures_reference/variables.ftl | 17 +++ .../test/fixtures_reference/variables.json | 132 ++++++++++++++++++ .../test/fixtures_reference/variant_keys.json | 8 +- .../fixtures_reference/variant_lists.json | 8 +- .../fixtures_reference/variants_indent.json | 8 +- .../fixtures_structure/sparse-messages.json | 8 +- .../test/fixtures_structure/term.json | 24 ++-- .../term_with_empty_pattern.json | 4 +- .../test/fixtures_behavior/make_fixtures.js | 20 +-- fluent/test/fixtures_reference/variables.json | 32 +++++ 21 files changed, 268 insertions(+), 68 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/variables.ftl create mode 100644 fluent-syntax/test/fixtures_reference/variables.json create mode 100644 fluent/test/fixtures_reference/variables.json diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index df0c71db9..81caf7c73 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -28,7 +28,7 @@ function getErrorMessage(code, args) { } case "E0006": { const [id] = args; - return `Expected term "${id}" to have a value`; + return `Expected term "-${id}" to have a value`; } case "E0007": return "Keyword cannot end with a whitespace"; diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index c0b0e4449..58877f01b 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -39,10 +39,9 @@ export default class FluentParser { // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getTermIdentifier", "getVariant", "getNumber", - "getValue", "getPattern", "getVariantList", "getTextElement", - "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg", - "getString", "getLiteral" + "getVariant", "getNumber", "getValue", "getPattern", "getVariantList", + "getTextElement", "getPlaceable", "getExpression", + "getSelectorExpression", "getCallArg", "getString", "getLiteral" ]; for (const name of methodNames) { this[name] = withSpan(this[name]); @@ -242,7 +241,8 @@ export default class FluentParser { } getTerm(ps) { - const id = this.getTermIdentifier(ps); + ps.expectChar("-"); + const id = this.getIdentifier(ps); ps.skipBlankInline(); ps.expectChar("="); @@ -301,13 +301,6 @@ export default class FluentParser { return new AST.Identifier(name); } - getTermIdentifier(ps) { - ps.expectChar("-"); - const id = this.getIdentifier(ps); - return new AST.Identifier(`-${id.name}`); - - } - getVariantKey(ps) { const ch = ps.currentChar; @@ -777,7 +770,8 @@ export default class FluentParser { } if (ch === "-") { - const id = this.getTermIdentifier(ps); + ps.next(); + const id = this.getIdentifier(ps); return new AST.TermReference(id); } diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 91cc8d635..618496b7f 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -44,8 +44,9 @@ export default class FluentSerializer { serializeEntry(entry, state = 0) { switch (entry.type) { case "Message": - case "Term": return serializeMessage(entry); + case "Term": + return serializeTerm(entry); case "Comment": if (state & HAS_ENTRIES) { return `\n${serializeComment(entry, "#")}\n`; @@ -95,8 +96,7 @@ function serializeMessage(message) { parts.push(serializeComment(message.comment)); } - parts.push(serializeIdentifier(message.id)); - parts.push(" ="); + parts.push(`${serializeIdentifier(message.id)} =`); if (message.value) { parts.push(serializeValue(message.value)); @@ -111,6 +111,25 @@ function serializeMessage(message) { } +function serializeTerm(term) { + const parts = []; + + if (term.comment) { + parts.push(serializeComment(term.comment)); + } + + parts.push(`-${serializeIdentifier(term.id)} =`); + parts.push(serializeValue(term.value)); + + for (const attribute of term.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + function serializeAttribute(attribute) { const id = serializeIdentifier(attribute.id); const value = indent(serializeValue(attribute.value)); @@ -202,8 +221,9 @@ function serializeExpression(expr) { case "NumberLiteral": return serializeNumberLiteral(expr); case "MessageReference": - case "TermReference": return serializeMessageReference(expr); + case "TermReference": + return serializeTermReference(expr); case "VariableReference": return serializeVariableReference(expr); case "AttributeExpression": @@ -237,6 +257,11 @@ function serializeMessageReference(expr) { } +function serializeTermReference(expr) { + return `-${serializeIdentifier(expr.id)}`; +} + + function serializeVariableReference(expr) { return `$${serializeIdentifier(expr.id)}`; } diff --git a/fluent-syntax/test/fixtures_behavior/term.ftl b/fluent-syntax/test/fixtures_behavior/term.ftl index 272e30397..58426d638 100644 --- a/fluent-syntax/test/fixtures_behavior/term.ftl +++ b/fluent-syntax/test/fixtures_behavior/term.ftl @@ -25,8 +25,8 @@ err4 = { -brand() } # ~ERROR E0008, pos 339 -err5 = -# ~ERROR E0006, pos 351, args "-err5" +# ~ERROR E0006, pos 351, args "err5" -err6 = .attr = Attribute -# ~ERROR E0006, pos 360, args "-err6" +# ~ERROR E0006, pos 360, args "err6" diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index ae9029a05..a860260fe 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -548,7 +548,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-msg" + "name": "msg" } } ], diff --git a/fluent-syntax/test/fixtures_reference/comments.json b/fluent-syntax/test/fixtures_reference/comments.json index 484f0b155..c28115ac7 100644 --- a/fluent-syntax/test/fixtures_reference/comments.json +++ b/fluent-syntax/test/fixtures_reference/comments.json @@ -30,7 +30,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term" + "name": "term" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.json b/fluent-syntax/test/fixtures_reference/member_expressions.json index 939ff5e80..a95879011 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.json +++ b/fluent-syntax/test/fixtures_reference/member_expressions.json @@ -18,7 +18,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } }, "key": { diff --git a/fluent-syntax/test/fixtures_reference/mixed_entries.json b/fluent-syntax/test/fixtures_reference/mixed_entries.json index 4e349a5ab..8f334f4fa 100644 --- a/fluent-syntax/test/fixtures_reference/mixed_entries.json +++ b/fluent-syntax/test/fixtures_reference/mixed_entries.json @@ -13,7 +13,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-brand-name" + "name": "brand-name" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/reference_expressions.json b/fluent-syntax/test/fixtures_reference/reference_expressions.json index 080e54f1e..47e4dfab0 100644 --- a/fluent-syntax/test/fixtures_reference/reference_expressions.json +++ b/fluent-syntax/test/fixtures_reference/reference_expressions.json @@ -40,7 +40,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } } } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 3dac7c79d..13f5b71bc 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -92,7 +92,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } }, "name": { diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index 790e6073c..f63122e57 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term01" + "name": "term01" }, "value": { "type": "Pattern", @@ -40,7 +40,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term02" + "name": "term02" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/variables.ftl b/fluent-syntax/test/fixtures_reference/variables.ftl new file mode 100644 index 000000000..6c3436929 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/variables.ftl @@ -0,0 +1,17 @@ +key01 = {$var} +key02 = { $var } +key03 = { + $var +} +key04 = { +$var} + + +## Errors + +# ERROR Missing variable identifier +err01 = {$} +# ERROR Double $$ +err02 = {$$var} +# ERROR Invalid first char of the identifier +err03 = {$-var} diff --git a/fluent-syntax/test/fixtures_reference/variables.json b/fluent-syntax/test/fixtures_reference/variables.json new file mode 100644 index 000000000..58682e5ba --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/variables.json @@ -0,0 +1,132 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Errors" + }, + { + "type": "Comment", + "content": "ERROR Missing variable identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {$}\n" + }, + { + "type": "Comment", + "content": "ERROR Double $$" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {$$var}\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid first char of the identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {$-var}\n" + } + ] +} diff --git a/fluent-syntax/test/fixtures_reference/variant_keys.json b/fluent-syntax/test/fixtures_reference/variant_keys.json index f85abe669..e8c60b002 100644 --- a/fluent-syntax/test/fixtures_reference/variant_keys.json +++ b/fluent-syntax/test/fixtures_reference/variant_keys.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-simple-identifier" + "name": "simple-identifier" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-identifier-surrounded-by-whitespace" + "name": "identifier-surrounded-by-whitespace" }, "value": { "type": "VariantList", @@ -67,7 +67,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-int-number" + "name": "int-number" }, "value": { "type": "VariantList", @@ -98,7 +98,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-float-number" + "name": "float-number" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.json b/fluent-syntax/test/fixtures_reference/variant_lists.json index 9713e098e..d10484046 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.json +++ b/fluent-syntax/test/fixtures_reference/variant_lists.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variant-list-in-term" + "name": "variant-list-in-term" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variant-list-in-term-attr" + "name": "variant-list-in-term-attr" }, "value": { "type": "Pattern", @@ -97,7 +97,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-nested-variant-list-in-term" + "name": "nested-variant-list-in-term" }, "value": { "type": "VariantList", @@ -141,7 +141,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-nested-select" + "name": "nested-select" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_reference/variants_indent.json b/fluent-syntax/test/fixtures_reference/variants_indent.json index ad5c1723a..7b987d487 100644 --- a/fluent-syntax/test/fixtures_reference/variants_indent.json +++ b/fluent-syntax/test/fixtures_reference/variants_indent.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-1tbs" + "name": "variants-1tbs" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-allman" + "name": "variants-allman" }, "value": { "type": "VariantList", @@ -67,7 +67,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-gnu" + "name": "variants-gnu" }, "value": { "type": "VariantList", @@ -98,7 +98,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-no-indent" + "name": "variants-no-indent" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index 90f8fdb31..bdeb917b5 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -351,10 +351,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key7", + "name": "key7", "span": { "type": "Span", - "start": 211, + "start": 212, "end": 216 } }, @@ -417,10 +417,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key8", + "name": "key8", "span": { "type": "Span", - "start": 254, + "start": 255, "end": 259 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index f9cf94d4f..6da90f3bb 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -5,10 +5,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 0, + "start": 1, "end": 11 } }, @@ -172,10 +172,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 145, + "start": 146, "end": 156 } }, @@ -254,10 +254,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 199, + "start": 200, "end": 210 } }, @@ -303,10 +303,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 243, + "start": 244, "end": 254 } }, @@ -365,10 +365,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 311, + "start": 312, "end": 322 } }, @@ -436,10 +436,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 385, + "start": 386, "end": 396 } }, diff --git a/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json index 66b7e06b7..13e018ec8 100644 --- a/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json @@ -8,7 +8,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-foo" + "foo" ], "message": "Expected term \"-foo\" to have a value", "span": { @@ -32,7 +32,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-bar" + "bar" ], "message": "Expected term \"-bar\" to have a value", "span": { diff --git a/fluent/test/fixtures_behavior/make_fixtures.js b/fluent/test/fixtures_behavior/make_fixtures.js index 10e05f35e..591fdab90 100644 --- a/fluent/test/fixtures_behavior/make_fixtures.js +++ b/fluent/test/fixtures_behavior/make_fixtures.js @@ -36,22 +36,22 @@ function isLocalizable(entry) { function printEntries(source) { const {body} = parser.parse(source); - const messages = body.filter(isLocalizable); - const entries = {}; - for (const msg of messages) { - const entry = entries[msg.id.name] = {}; + for (const entry of body.filter(isLocalizable)) { + const id = entry.type === 'Term' + ? `-${entry.id.name}` : entry.id.name; + const expected = entries[id] = {}; - if (msg.value !== null) { - entry.value = true; + if (entry.value !== null) { + expected.value = true; } - if (msg.attributes.length > 0) { - entry.attributes = {}; + if (entry.attributes.length > 0) { + expected.attributes = {}; - for (const attr of msg.attributes) { - entry.attributes[attr.id.name] = true + for (const attr of entry.attributes) { + expected.attributes[attr.id.name] = true } } diff --git a/fluent/test/fixtures_reference/variables.json b/fluent/test/fixtures_reference/variables.json new file mode 100644 index 000000000..6f0de9904 --- /dev/null +++ b/fluent/test/fixtures_reference/variables.json @@ -0,0 +1,32 @@ +{ + "key01": [ + { + "type": "var", + "name": "var" + } + ], + "key02": [ + { + "type": "var", + "name": "var" + } + ], + "key03": [ + { + "type": "var", + "name": "var" + } + ], + "key04": [ + { + "type": "var", + "name": "var" + } + ], + "err03": [ + { + "type": "var", + "name": "-var" + } + ] +} From bf3f6c8caaba9f86bb377f3f3d8eff217f214036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 15:59:03 +0100 Subject: [PATCH 3/5] Treat backslash as normal char in TextElements --- fluent-syntax/src/parser.js | 12 +- .../fixtures_behavior/escape_sequences.ftl | 22 +- .../fixtures_reference/escaped_characters.ftl | 23 +- .../escaped_characters.json | 156 +++++- .../fixtures_structure/escape_sequences.ftl | 23 +- .../fixtures_structure/escape_sequences.json | 458 +++++++++++++++--- fluent-syntax/test/serializer_test.js | 6 +- fluent/src/resource.js | 43 +- .../fixtures_behavior/escape_sequences.json | 20 +- fluent/test/fixtures_reference/astral.json | 5 +- .../escaped_characters.json | 40 +- .../fixtures_structure/escape_sequences.json | 40 +- 12 files changed, 676 insertions(+), 172 deletions(-) diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 58877f01b..14066cbda 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -487,12 +487,6 @@ export default class FluentParser { continue; } - if (ch === "\\") { - ps.next(); - buffer += this.getEscapeSequence(ps); - continue; - } - buffer += ch; ps.next(); } @@ -500,10 +494,10 @@ export default class FluentParser { return new AST.TextElement(buffer); } - getEscapeSequence(ps, specials = ["{", "\\"]) { + getEscapeSequence(ps) { const next = ps.currentChar; - if (specials.includes(next)) { + if (next === "\\" || next === "\"") { ps.next(); return `\\${next}`; } @@ -731,7 +725,7 @@ export default class FluentParser { let ch; while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { if (ch === "\\") { - val += this.getEscapeSequence(ps, ["{", "\\", "\""]); + val += this.getEscapeSequence(ps); } else { val += ch; } diff --git a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl index d0bbb85ae..801923be2 100644 --- a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl @@ -1,10 +1,16 @@ -# ~ERROR E0025, pos 8, args "A" -key1 = \A +## Backslash is a regular character in text elements. +key01 = \A +key02 = \u0041 +key03 = \\u0041 +key04 = \u000z +key05 = \{Value} -# ~ERROR E0026, pos 23, args "000z" -key2 = \u000z +key06 = {"Escaped \" quote"} +key07 = {"Escaped \\ backslash"} +key08 = {"Escaped \u0041 A"} -key3 = \{Escaped} -key4 = {"Escaped \" quote"} -key5 = \u0041 -key6 = \\u0041 +# ~ERROR E0025, pos 232, args "A" +key09 = {"\A"} + +# ~ERROR E0026, pos 252, args "000z" +key10 = {"\u000z"} diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl index d3a5a078f..3c64fcef7 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl @@ -1,9 +1,22 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (a closing brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-sequence = {"\u0041"} +string-escaped-unicode = {"\\u0041"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing } brace. diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 560277517..6c26f82ee 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -1,18 +1,22 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text" + }, { "type": "Message", "id": { "type": "Identifier", - "name": "backslash" + "name": "text-backslash-one" }, "value": { "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)" + "value": "Value with \\ a backslash" } ] }, @@ -23,14 +27,14 @@ "type": "Message", "id": { "type": "Identifier", - "name": "closing-brace" + "name": "text-backslash-two" }, "value": { "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\{ (a closing brace)" + "value": "Value with \\\\ two backslashes" } ] }, @@ -41,7 +45,35 @@ "type": "Message", "id": { "type": "Identifier", - "name": "unicode-escape" + "name": "text-backslash-brace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\" + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u" }, "value": { "type": "Pattern", @@ -59,7 +91,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "escaped-unicode" + "name": "text-backslash-backslash-u" }, "value": { "type": "Pattern", @@ -75,7 +107,7 @@ }, { "type": "GroupComment", - "content": "String Expressions" + "content": "String literals" }, { "type": "Message", @@ -119,10 +151,120 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Mismatched quote" + }, { "type": "Junk", "annotations": [], "content": "mismatched-quote = {\"\\\\\"\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Unknown escape" + }, + { + "type": "Junk", + "annotations": [], + "content": "unknown-escape = {\"\\x\"}\n" + }, + { + "type": "GroupComment", + "content": "Unicode escapes" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-sequence" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-escaped-unicode" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Literal braces" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "{" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing } brace." + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl index e91de86bd..3c64fcef7 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl @@ -1,9 +1,22 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (an opening brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-sequence = {"\u0041"} +string-escaped-unicode = {"\\u0041"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing } brace. diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index f49683f67..3259d024f 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -1,15 +1,24 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text", + "span": { + "type": "Span", + "start": 0, + "end": 15 + } + }, { "type": "Message", "id": { "type": "Identifier", - "name": "backslash", + "name": "text-backslash-one", "span": { "type": "Span", - "start": 0, - "end": 9 + "start": 16, + "end": 34 } }, "value": { @@ -17,37 +26,37 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)", + "value": "Value with \\ a backslash", "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } } ], "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 0, - "end": 48 + "start": 16, + "end": 61 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "closing-brace", + "name": "text-backslash-two", "span": { "type": "Span", - "start": 49, - "end": 62 + "start": 62, + "end": 80 } }, "value": { @@ -55,37 +64,100 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\{ (an opening brace)", + "value": "Value with \\\\ two backslashes", "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } } ], "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 49, - "end": 97 + "start": 62, + "end": 112 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "unicode-escape", + "name": "text-backslash-brace", "span": { "type": "Span", - "start": 98, - "end": 112 + "start": 113, + "end": 133 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\", + "span": { + "type": "Span", + "start": 136, + "end": 148 + } + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 148, + "end": 159 + } + } + ], + "span": { + "type": "Span", + "start": 136, + "end": 159 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 113, + "end": 159 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u", + "span": { + "type": "Span", + "start": 160, + "end": 176 } }, "value": { @@ -96,34 +168,34 @@ "value": "\\u0041", "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } } ], "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 98, - "end": 121 + "start": 160, + "end": 185 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "escaped-unicode", + "name": "text-backslash-backslash-u", "span": { "type": "Span", - "start": 122, - "end": 137 + "start": 186, + "end": 212 } }, "value": { @@ -134,32 +206,32 @@ "value": "\\\\u0041", "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } } ], "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 122, - "end": 147 + "start": 186, + "end": 222 } }, { "type": "GroupComment", - "content": "String Expressions", + "content": "String literals", "span": { "type": "Span", - "start": 149, - "end": 170 + "start": 224, + "end": 242 } }, { @@ -169,8 +241,8 @@ "name": "quote-in-string", "span": { "type": "Span", - "start": 171, - "end": 186 + "start": 243, + "end": 258 } }, "value": { @@ -183,29 +255,29 @@ "value": "\\\"", "span": { "type": "Span", - "start": 190, - "end": 194 + "start": 262, + "end": 266 } }, "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } } ], "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 171, - "end": 195 + "start": 243, + "end": 267 } }, { @@ -215,8 +287,8 @@ "name": "backslash-in-string", "span": { "type": "Span", - "start": 196, - "end": 215 + "start": 268, + "end": 287 } }, "value": { @@ -229,29 +301,38 @@ "value": "\\\\", "span": { "type": "Span", - "start": 219, - "end": 223 + "start": 291, + "end": 295 } }, "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } } ], "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 196, - "end": 224 + "start": 268, + "end": 296 + } + }, + { + "type": "Comment", + "content": "ERROR Mismatched quote", + "span": { + "type": "Span", + "start": 297, + "end": 321 } }, { @@ -266,22 +347,267 @@ "message": "Expected token: \"}\"", "span": { "type": "Span", - "start": 249, - "end": 249 + "start": 346, + "end": 346 } } ], "content": "mismatched-quote = {\"\\\\\"\"}\n", "span": { "type": "Span", - "start": 225, - "end": 252 + "start": 322, + "end": 349 + } + }, + { + "type": "Comment", + "content": "ERROR Unknown escape", + "span": { + "type": "Span", + "start": 349, + "end": 371 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0025", + "args": [ + "x" + ], + "message": "Unknown escape sequence: \\x.", + "span": { + "type": "Span", + "start": 392, + "end": 392 + } + } + ], + "content": "unknown-escape = {\"\\x\"}\n\n", + "span": { + "type": "Span", + "start": 372, + "end": 397 + } + }, + { + "type": "GroupComment", + "content": "Unicode escapes", + "span": { + "type": "Span", + "start": 397, + "end": 415 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-sequence", + "span": { + "type": "Span", + "start": 416, + "end": 439 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u0041", + "span": { + "type": "Span", + "start": 443, + "end": 451 + } + }, + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + } + ], + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 416, + "end": 452 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-escaped-unicode", + "span": { + "type": "Span", + "start": 453, + "end": 475 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\\u0041", + "span": { + "type": "Span", + "start": 479, + "end": 488 + } + }, + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + } + ], + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 453, + "end": 489 + } + }, + { + "type": "GroupComment", + "content": "Literal braces", + "span": { + "type": "Span", + "start": 491, + "end": 508 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open", + "span": { + "type": "Span", + "start": 509, + "end": 519 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening ", + "span": { + "type": "Span", + "start": 522, + "end": 533 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "{", + "span": { + "type": "Span", + "start": 534, + "end": 537 + } + }, + "span": { + "type": "Span", + "start": 533, + "end": 538 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 538, + "end": 545 + } + } + ], + "span": { + "type": "Span", + "start": 522, + "end": 545 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 509, + "end": 545 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close", + "span": { + "type": "Span", + "start": 546, + "end": 557 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing } brace.", + "span": { + "type": "Span", + "start": 560, + "end": 578 + } + } + ], + "span": { + "type": "Span", + "start": 560, + "end": 578 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 546, + "end": 578 } } ], "span": { "type": "Span", "start": 0, - "end": 252 + "end": 579 } } diff --git a/fluent-syntax/test/serializer_test.js b/fluent-syntax/test/serializer_test.js index d4b775db2..cb32b9caa 100644 --- a/fluent-syntax/test/serializer_test.js +++ b/fluent-syntax/test/serializer_test.js @@ -457,9 +457,9 @@ suite("Serialize resource", function() { assert.equal(pretty(input), input); }); - test("Escaped special char in TextElement", function() { + test("Backslash in TextElement", function() { const input = ftl` - foo = \\{Escaped} + foo = \\{ placeable } `; assert.equal(pretty(input), input); }); @@ -473,7 +473,7 @@ suite("Serialize resource", function() { test("Unicode escape sequence", function() { const input = ftl` - foo = \\u0065 + foo = { "\\u0065" } `; assert.equal(pretty(input), input); }); diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 62025ad8c..19ea001cc 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -15,19 +15,17 @@ const RE_IDENTIFIER = /(-?[a-zA-Z][a-zA-Z0-9_-]*)/y; const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; // A "run" is a sequence of text or string literal characters which don't -// require any special handling. For TextElements such special characters are: -// { (starts a placeable), \ (starts an escape sequence), and line breaks which -// require additional logic to check if the next line is indented. For -// StringLiterals they are: \ (starts an escape sequence), " (ends the -// literal), and line breaks which are not allowed in StringLiterals. Also note -// that string runs may be empty, but text runs may not. -const RE_TEXT_RUN = /([^\\{\n\r]+)/y; +// require any special handling. For TextElements such special characters are: { +// (starts a placeable), and line breaks which require additional logic to check +// if the next line is indented. For StringLiterals they are: \ (starts an +// escape sequence), " (ends the literal), and line breaks which are not allowed +// in StringLiterals. Note that string runs may be empty; text runs may not. +const RE_TEXT_RUN = /([^{\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -const RE_TEXT_ESCAPE = /\\([\\{])/y; // Used for trimming TextElements and indents. With the /m flag, the $ matches // the end of every line. @@ -180,15 +178,12 @@ export default class FluentResource extends Map { var first = match(RE_TEXT_RUN); } - // If there's a backslash escape or a placeable on the first line, fall - // back to parsing a complex pattern. - switch (source[cursor]) { - case "{": - case "\\": - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // If there's a placeable on the first line, parse a complex pattern. + if (source[cursor] === "{") { + return first + // Re-use the text parsed above, if possible. + ? parsePatternElements(first) + : parsePatternElements(); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if @@ -240,12 +235,6 @@ export default class FluentResource extends Map { continue; } - if (source[cursor] === "\\") { - elements.push(parseEscapeSequence(RE_TEXT_ESCAPE)); - needsTrimming = false; - continue; - } - break; } @@ -403,7 +392,7 @@ export default class FluentResource extends Map { value += match(RE_STRING_RUN); if (source[cursor] === "\\") { - value += parseEscapeSequence(RE_STRING_ESCAPE); + value += parseEscapeSequence(); continue; } @@ -417,7 +406,7 @@ export default class FluentResource extends Map { } // Unescape known escape sequences. - function parseEscapeSequence(reSpecialized) { + function parseEscapeSequence() { if (test(RE_UNICODE_ESCAPE)) { let sequence = match(RE_UNICODE_ESCAPE); let codepoint = parseInt(sequence, 16); @@ -429,8 +418,8 @@ export default class FluentResource extends Map { : "�"; } - if (test(reSpecialized)) { - return match(reSpecialized); + if (test(RE_STRING_ESCAPE)) { + return match(RE_STRING_ESCAPE); } throw new FluentError("Unknown escape sequence"); diff --git a/fluent/test/fixtures_behavior/escape_sequences.json b/fluent/test/fixtures_behavior/escape_sequences.json index 638f1bd08..95c0834db 100644 --- a/fluent/test/fixtures_behavior/escape_sequences.json +++ b/fluent/test/fixtures_behavior/escape_sequences.json @@ -1,14 +1,26 @@ { - "key3": { + "key01": { "value": true }, - "key4": { + "key02": { "value": true }, - "key5": { + "key03": { "value": true }, - "key6": { + "key04": { + "value": true + }, + "key05": { + "value": true + }, + "key06": { + "value": true + }, + "key07": { + "value": true + }, + "key08": { "value": true } } diff --git a/fluent/test/fixtures_reference/astral.json b/fluent/test/fixtures_reference/astral.json index 9a9469f0c..5c4850b60 100644 --- a/fluent/test/fixtures_reference/astral.json +++ b/fluent/test/fixtures_reference/astral.json @@ -1,10 +1,7 @@ { "face-with-tears-of-joy": "😂", "tetragram-for-centre": "𝌆", - "surrogates-in-text": [ - "�", - "�" - ], + "surrogates-in-text": "\\uD83D\\uDE02", "surrogates-in-string": [ "��" ], diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index 1b358bc60..c371cab9f 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -1,25 +1,31 @@ { - "backslash": [ - "Value with ", - "\\", - " (an escaped backslash)" - ], - "closing-brace": [ - "Value with ", - "{", - " (a closing brace)" - ], - "unicode-escape": [ - "A" - ], - "escaped-unicode": [ - "\\", - "u0041" + "text-backslash-one": "Value with \\ a backslash", + "text-backslash-two": "Value with \\\\ two backslashes", + "text-backslash-brace": [ + "Value with \\", + { + "type": "ref", + "name": "placeable" + } ], + "text-backslash-u": "\\u0041", + "text-backslash-backslash-u": "\\\\u0041", "quote-in-string": [ "\"" ], "backslash-in-string": [ "\\" - ] + ], + "string-unicode-sequence": [ + "A" + ], + "string-escaped-unicode": [ + "\\u0041" + ], + "brace-open": [ + "An opening ", + "{", + " brace." + ], + "brace-close": "A closing } brace." } diff --git a/fluent/test/fixtures_structure/escape_sequences.json b/fluent/test/fixtures_structure/escape_sequences.json index 22d8f1cd4..c371cab9f 100644 --- a/fluent/test/fixtures_structure/escape_sequences.json +++ b/fluent/test/fixtures_structure/escape_sequences.json @@ -1,25 +1,31 @@ { - "backslash": [ - "Value with ", - "\\", - " (an escaped backslash)" - ], - "closing-brace": [ - "Value with ", - "{", - " (an opening brace)" - ], - "unicode-escape": [ - "A" - ], - "escaped-unicode": [ - "\\", - "u0041" + "text-backslash-one": "Value with \\ a backslash", + "text-backslash-two": "Value with \\\\ two backslashes", + "text-backslash-brace": [ + "Value with \\", + { + "type": "ref", + "name": "placeable" + } ], + "text-backslash-u": "\\u0041", + "text-backslash-backslash-u": "\\\\u0041", "quote-in-string": [ "\"" ], "backslash-in-string": [ "\\" - ] + ], + "string-unicode-sequence": [ + "A" + ], + "string-escaped-unicode": [ + "\\u0041" + ], + "brace-open": [ + "An opening ", + "{", + " brace." + ], + "brace-close": "A closing } brace." } From 1758b959e167ba307fdb95332118a73d9e7255fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 16 Nov 2018 13:45:36 +0100 Subject: [PATCH 4/5] End comment lines on EOLs recognized by JS RegExp --- fluent-syntax/.gitattributes | 1 + fluent-syntax/src/parser.js | 8 +++-- fluent-syntax/test/fixtures_reference/cr.ftl | 1 + fluent-syntax/test/fixtures_reference/cr.json | 10 ++++++ .../test/fixtures_reference/crlf.ftl | 11 +++++-- .../test/fixtures_reference/crlf.json | 32 +++++++++++++++++-- fluent/test/fixtures_reference/cr.json | 23 +++++++++++++ fluent/test/fixtures_reference/crlf.json | 22 ++++++++++--- 8 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/cr.ftl create mode 100644 fluent-syntax/test/fixtures_reference/cr.json create mode 100644 fluent/test/fixtures_reference/cr.json diff --git a/fluent-syntax/.gitattributes b/fluent-syntax/.gitattributes index 9e0125713..dc3c32a74 100644 --- a/fluent-syntax/.gitattributes +++ b/fluent-syntax/.gitattributes @@ -1,2 +1,3 @@ test/fixtures_reference/crlf.ftl eol=crlf +test/fixtures_reference/cr.ftl eol=cr test/fixtures_structure/crlf.ftl eol=crlf diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 14066cbda..bb4f60311 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -6,6 +6,10 @@ import { ParseError } from "./errors"; const trailingWSRe = /[ \t\n\r]+$/; +// The Fluent Syntax spec uses /.*/ to parse comment lines. It matches all +// characters except the following ones, which are considered line endings by +// the regex engine. +const COMMENT_EOL = ["\n", "\r", "\u2028", "\u2029"]; function withSpan(fn) { @@ -188,10 +192,10 @@ export default class FluentParser { level = i; } - if (ps.currentChar !== EOL) { + if (!COMMENT_EOL.includes(ps.currentChar)) { ps.expectChar(" "); let ch; - while ((ch = ps.takeChar(x => x !== EOL))) { + while ((ch = ps.takeChar(x => !COMMENT_EOL.includes(x)))) { content += ch; } } diff --git a/fluent-syntax/test/fixtures_reference/cr.ftl b/fluent-syntax/test/fixtures_reference/cr.ftl new file mode 100644 index 000000000..549c662ae --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/cr.ftl @@ -0,0 +1 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } \ No newline at end of file diff --git a/fluent-syntax/test/fixtures_reference/cr.json b/fluent-syntax/test/fixtures_reference/cr.json new file mode 100644 index 000000000..afec8119a --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/cr.json @@ -0,0 +1,10 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Junk", + "annotations": [], + "content": "### This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + } + ] +} diff --git a/fluent-syntax/test/fixtures_reference/crlf.ftl b/fluent-syntax/test/fixtures_reference/crlf.ftl index 7349195be..df3a02c5f 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.ftl +++ b/fluent-syntax/test/fixtures_reference/crlf.ftl @@ -1,7 +1,14 @@ + key01 = Value 01 key02 = + Value 02 Continued -# ERROR (Missing value or attributes) -key03 + .title = Title + +# ERROR Unclosed StringLiteral +err03 = { "str + +# ERROR Missing newline after ->. +err04 = { $sel -> } diff --git a/fluent-syntax/test/fixtures_reference/crlf.json b/fluent-syntax/test/fixtures_reference/crlf.json index a36838a0b..6b324cd4e 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.json +++ b/fluent-syntax/test/fixtures_reference/crlf.json @@ -34,17 +34,43 @@ } ] }, - "attributes": [], + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "title" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Title" + } + ] + } + } + ], "comment": null }, { "type": "Comment", - "content": "ERROR (Missing value or attributes)" + "content": "ERROR Unclosed StringLiteral" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = { \"str\r\n" + }, + { + "type": "Comment", + "content": "ERROR Missing newline after ->." }, { "type": "Junk", "annotations": [], - "content": "key03\n" + "content": "err04 = { $sel -> }\r\n" } ] } diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json new file mode 100644 index 000000000..6c6f39082 --- /dev/null +++ b/fluent/test/fixtures_reference/cr.json @@ -0,0 +1,23 @@ +{ + "err01": "Value 01", + "err02": "Value 02", + "err03": { + "value": [ + "Value 03", + "\r", + "Continued" + ], + "attrs": { + "title": "Title" + } + }, + "err05": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel" + } + } + ] +} diff --git a/fluent/test/fixtures_reference/crlf.json b/fluent/test/fixtures_reference/crlf.json index 65d3a327e..aa9ccd77c 100644 --- a/fluent/test/fixtures_reference/crlf.json +++ b/fluent/test/fixtures_reference/crlf.json @@ -1,8 +1,22 @@ { "key01": "Value 01", - "key02": [ - "Value 02", - "\n", - "Continued" + "key02": { + "value": [ + "Value 02", + "\n", + "Continued" + ], + "attrs": { + "title": "Title" + } + }, + "err04": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel" + } + } ] } From eb0493b3293b58452c18bb28a8571b51f67beec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 16 Nov 2018 14:13:14 +0100 Subject: [PATCH 5/5] Forbid the closing brace in text --- fluent-syntax/src/errors.js | 2 ++ fluent-syntax/src/parser.js | 4 ++- .../placeable_in_placeable.ftl | 3 +- .../fixtures_reference/escaped_characters.ftl | 2 +- .../escaped_characters.json | 13 ++++++- .../test/fixtures_reference/placeables.ftl | 12 +++++++ .../test/fixtures_reference/placeables.json | 36 +++++++++++++++++++ .../fixtures_reference/select_expressions.ftl | 5 +++ .../select_expressions.json | 9 +++++ .../fixtures_structure/escape_sequences.ftl | 2 +- .../fixtures_structure/escape_sequences.json | 36 ++++++++++++++++--- fluent/src/resource.js | 8 +++-- .../placeable_in_placeable.json | 3 -- .../escaped_characters.json | 6 +++- .../fixtures_structure/escape_sequences.json | 6 +++- 15 files changed, 130 insertions(+), 17 deletions(-) diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 81caf7c73..f721d25a4 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -74,6 +74,8 @@ function getErrorMessage(code, args) { const [char] = args; return `Invalid Unicode escape sequence: \\u${char}.`; } + case "E0027": + return "Unbalanced closing brace in TextElement."; default: return code; } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index bb4f60311..1f8651a66 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -452,6 +452,8 @@ export default class FluentParser { if (ch === "{") { const element = this.getPlaceable(ps); elements.push(element); + } else if (ch === "}") { + throw new ParseError("E0027"); } else { const element = this.getTextElement(ps); elements.push(element); @@ -475,7 +477,7 @@ export default class FluentParser { let ch; while ((ch = ps.currentChar)) { - if (ch === "{") { + if (ch === "{" || ch === "}") { return new AST.TextElement(buffer); } diff --git a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl index 7ece456be..827c93e72 100644 --- a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl +++ b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl @@ -7,8 +7,9 @@ key2 = { { foo } } # { foo } # } -key4 = { { foo } # ~ERROR E0003, pos 96, args "}" +key4 = { { foo } +# ~ERROR E0027, pos 111 key5 = { foo } } diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl index 3c64fcef7..5242a4bcb 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl @@ -19,4 +19,4 @@ string-escaped-unicode = {"\\u0041"} ## Literal braces brace-open = An opening {"{"} brace. -brace-close = A closing } brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 6c26f82ee..26b1974b0 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -259,7 +259,18 @@ "elements": [ { "type": "TextElement", - "value": "A closing } brace." + "value": "A closing " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "}" + } + }, + { + "type": "TextElement", + "value": " brace." } ] }, diff --git a/fluent-syntax/test/fixtures_reference/placeables.ftl b/fluent-syntax/test/fixtures_reference/placeables.ftl index c0e515b67..7a1b280f4 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.ftl +++ b/fluent-syntax/test/fixtures_reference/placeables.ftl @@ -1,3 +1,15 @@ nested-placeable = {{{1}}} padded-placeable = { 1 } sparse-placeable = { { 1 } } + +# ERROR Unmatched opening brace +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/test/fixtures_reference/placeables.json b/fluent-syntax/test/fixtures_reference/placeables.json index c0e7cc9e1..ff7fac8a7 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.json +++ b/fluent-syntax/test/fixtures_reference/placeables.json @@ -72,6 +72,42 @@ }, "attributes": [], "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open1 = { 1\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open2 = {{ 1 }\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close1 = 1 }\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close2 = { 1 }}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.ftl b/fluent-syntax/test/fixtures_reference/select_expressions.ftl index 3c54f57c2..859c01a04 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/select_expressions.ftl @@ -34,3 +34,8 @@ nested-variant-list = *[two] Value } } + +# ERROR Missing line end after variant list +missing-line-end = + { 1 -> + *[one] One} diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 13f5b71bc..4c782006a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -257,6 +257,15 @@ "type": "Junk", "annotations": [], "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + }, + { + "type": "Comment", + "content": "ERROR Missing line end after variant list" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-line-end =\n { 1 ->\n *[one] One}\n" } ] } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl index 3c64fcef7..5242a4bcb 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl @@ -19,4 +19,4 @@ string-escaped-unicode = {"\\u0041"} ## Literal braces brace-open = An opening {"{"} brace. -brace-close = A closing } brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index 3259d024f..e9e851c7b 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -582,18 +582,44 @@ "elements": [ { "type": "TextElement", - "value": "A closing } brace.", + "value": "A closing ", "span": { "type": "Span", "start": 560, - "end": 578 + "end": 570 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "}", + "span": { + "type": "Span", + "start": 571, + "end": 574 + } + }, + "span": { + "type": "Span", + "start": 570, + "end": 575 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 575, + "end": 582 } } ], "span": { "type": "Span", "start": 560, - "end": 578 + "end": 582 } }, "attributes": [], @@ -601,13 +627,13 @@ "span": { "type": "Span", "start": 546, - "end": 578 + "end": 582 } } ], "span": { "type": "Span", "start": 0, - "end": 579 + "end": 583 } } diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 19ea001cc..b457a38ec 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -20,7 +20,7 @@ const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; // if the next line is indented. For StringLiterals they are: \ (starts an // escape sequence), " (ends the literal), and line breaks which are not allowed // in StringLiterals. Note that string runs may be empty; text runs may not. -const RE_TEXT_RUN = /([^{\n\r]+)/y; +const RE_TEXT_RUN = /([^{}\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. @@ -179,7 +179,7 @@ export default class FluentResource extends Map { } // If there's a placeable on the first line, parse a complex pattern. - if (source[cursor] === "{") { + if (source[cursor] === "{" || source[cursor] === "}") { return first // Re-use the text parsed above, if possible. ? parsePatternElements(first) @@ -228,6 +228,10 @@ export default class FluentResource extends Map { continue; } + if (source[cursor] === "}") { + throw new FluentError("Unbalanced closing brace"); + } + let indent = parseIndent(); if (indent) { elements.push(trim(indent)); diff --git a/fluent/test/fixtures_behavior/placeable_in_placeable.json b/fluent/test/fixtures_behavior/placeable_in_placeable.json index 824346581..0e54ab88d 100644 --- a/fluent/test/fixtures_behavior/placeable_in_placeable.json +++ b/fluent/test/fixtures_behavior/placeable_in_placeable.json @@ -4,8 +4,5 @@ }, "key2": { "value": true - }, - "key5": { - "value": true } } diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index c371cab9f..cc1619930 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -27,5 +27,9 @@ "{", " brace." ], - "brace-close": "A closing } brace." + "brace-close": [ + "A closing ", + "}", + " brace." + ] } diff --git a/fluent/test/fixtures_structure/escape_sequences.json b/fluent/test/fixtures_structure/escape_sequences.json index c371cab9f..cc1619930 100644 --- a/fluent/test/fixtures_structure/escape_sequences.json +++ b/fluent/test/fixtures_structure/escape_sequences.json @@ -27,5 +27,9 @@ "{", " brace." ], - "brace-close": "A closing } brace." + "brace-close": [ + "A closing ", + "}", + " brace." + ] }