From 3d69f326084345a43ce2dd320316110abad4c933 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Sun, 29 Jul 2018 00:59:35 -0700 Subject: [PATCH 01/36] Initial attempt to write a zero-copy parser Add a simple binary for testing Support menubar.ftl except of expressions Support multiline patterns Add EOF tests support Support basic JSON fixtures Support first set of comparison tests Support most of the comments Use serializer instead of deserializer Uncomment some fixture tests with slight modifications Support Junk Use ps.take_if in more places Support all reference fixtures Fix benches Support TermReference Support variants and select expression Update to rust 2018 preview 2 Use 2018 use extern macros Clean up whitespace collecting Remove try Update bench to Rust 2018 Support parsing of 0.7 reference tests Support CallExpressions --- fluent-syntax/Cargo.toml | 5 + fluent-syntax/benches/lib.rs | 6 +- fluent-syntax/src/ast.rs | 167 ++-- fluent-syntax/src/bin/parser.rs | 58 ++ fluent-syntax/src/lib.rs | 2 + fluent-syntax/src/parser/errors/display.rs | 41 - fluent-syntax/src/parser/errors/list.rs | 122 --- fluent-syntax/src/parser/errors/mod.rs | 124 +-- fluent-syntax/src/parser/ftlstream.rs | 438 +++------ fluent-syntax/src/parser/mod.rs | 786 ++++++++++++++- fluent-syntax/src/parser/parser.rs | 685 ------------- fluent-syntax/src/parser/stream.rs | 159 --- fluent-syntax/tests/ast/mod.rs | 400 ++++++++ fluent-syntax/tests/errors.rs | 291 ------ fluent-syntax/tests/fixtures.rs | 67 -- fluent-syntax/tests/fixtures/Makefile | 11 + fluent-syntax/tests/fixtures/astral.ftl | 20 + fluent-syntax/tests/fixtures/astral.json | 159 +++ .../tests/fixtures/call_expressions.ftl | 102 ++ .../tests/fixtures/call_expressions.json | 922 ++++++++++++++++++ fluent-syntax/tests/fixtures/comments.ftl | 15 + fluent-syntax/tests/fixtures/comments.json | 58 ++ fluent-syntax/tests/fixtures/convert.js | 94 ++ fluent-syntax/tests/fixtures/crlf.ftl | 7 + fluent-syntax/tests/fixtures/crlf.json | 49 + fluent-syntax/tests/fixtures/eof_comment.ftl | 3 + fluent-syntax/tests/fixtures/eof_comment.json | 12 + fluent-syntax/tests/fixtures/eof_empty.ftl | 0 fluent-syntax/tests/fixtures/eof_empty.json | 3 + fluent-syntax/tests/fixtures/eof_id.ftl | 3 + fluent-syntax/tests/fixtures/eof_id.json | 13 + .../tests/fixtures/eof_id_equals.ftl | 3 + .../tests/fixtures/eof_id_equals.json | 13 + fluent-syntax/tests/fixtures/eof_junk.ftl | 3 + fluent-syntax/tests/fixtures/eof_junk.json | 13 + fluent-syntax/tests/fixtures/eof_value.ftl | 3 + fluent-syntax/tests/fixtures/eof_value.json | 24 + .../tests/fixtures/escaped_characters.ftl | 9 + .../tests/fixtures/escaped_characters.json | 115 +++ fluent-syntax/tests/fixtures/junk.ftl | 4 + fluent-syntax/tests/fixtures/junk.json | 14 + fluent-syntax/tests/fixtures/leading_dots.ftl | 76 ++ .../tests/fixtures/leading_dots.json | 443 +++++++++ .../tests/fixtures/literal_expressions.ftl | 3 + .../tests/fixtures/literal_expressions.json | 61 ++ .../tests/fixtures/member_expressions.ftl | 6 + .../tests/fixtures/member_expressions.json | 67 ++ fluent-syntax/tests/fixtures/messages.ftl | 27 + fluent-syntax/tests/fixtures/messages.json | 215 ++++ .../tests/fixtures/mixed_entries.ftl | 24 + .../tests/fixtures/mixed_entries.json | 123 +++ .../tests/fixtures/multiline_values.ftl | 35 + .../tests/fixtures/multiline_values.json | 236 +++++ .../fixtures/parser/ftl/01-basic-errors01.ftl | 23 - .../tests/fixtures/parser/ftl/01-basic01.ftl | 1 - .../tests/fixtures/parser/ftl/01-basic02.ftl | 10 - .../tests/fixtures/parser/ftl/01-basic03.ftl | 9 - .../fixtures/parser/ftl/02-multiline01.ftl | 14 - .../tests/fixtures/parser/ftl/03-comments.ftl | 22 - .../parser/ftl/04-sections-errors.ftl | 25 - .../parser/ftl/05-variants-errors.ftl | 40 - .../tests/fixtures/parser/ftl/05-variants.ftl | 37 - .../parser/ftl/06-placeables-errors.ftl | 87 -- .../fixtures/parser/ftl/06-placeables01.ftl | 44 - .../tests/fixtures/parser/ftl/07-private.ftl | 11 - .../fixtures/parser/ftl/errors/01-empty.ftl | 1 - .../parser/ftl/errors/02-bad-id-start.ftl | 1 - .../fixtures/parser/ftl/errors/03-just-id.ftl | 1 - .../parser/ftl/errors/04-no-equal-sign.ftl | 1 - .../ftl/errors/05-bad-char-in-keyword.ftl | 2 - .../parser/ftl/errors/06-trait-value.ftl | 2 - .../ftl/errors/07-message-missing-fields.ftl | 3 - .../fixtures/parser/ftl/errors/08-private.ftl | 12 - .../fixtures/parser/ftl/junk/01-basic.ftl | 12 - .../fixtures/parser/ftl/junk/02-start.ftl | 2 - .../tests/fixtures/parser/ftl/junk/03-end.ftl | 5 - .../fixtures/parser/ftl/junk/04-multiline.ftl | 11 - .../fixtures/parser/ftl/junk/05-comment.ftl | 7 - fluent-syntax/tests/fixtures/placeables.ftl | 3 + fluent-syntax/tests/fixtures/placeables.json | 70 ++ .../tests/fixtures/reference_expressions.ftl | 5 + .../tests/fixtures/reference_expressions.json | 88 ++ .../tests/fixtures/select_expressions.ftl | 36 + .../tests/fixtures/select_expressions.json | 227 +++++ .../tests/fixtures/select_indent.ftl | 95 ++ .../tests/fixtures/select_indent.json | 587 +++++++++++ .../tests/fixtures/sparse_entries.ftl | 39 + .../tests/fixtures/sparse_entries.json | 160 +++ fluent-syntax/tests/fixtures/tab.ftl | 14 + fluent-syntax/tests/fixtures/tab.json | 65 ++ fluent-syntax/tests/fixtures/terms.ftl | 23 + fluent-syntax/tests/fixtures/terms.json | 98 ++ fluent-syntax/tests/fixtures/variables.ftl | 17 + fluent-syntax/tests/fixtures/variables.json | 119 +++ fluent-syntax/tests/fixtures/variant_keys.ftl | 37 + .../tests/fixtures/variant_keys.json | 135 +++ .../tests/fixtures/variant_lists.ftl | 54 + .../tests/fixtures/variant_lists.json | 188 ++++ .../tests/fixtures/variants_indent.ftl | 19 + .../tests/fixtures/variants_indent.json | 122 +++ .../tests/fixtures/whitespace_in_value.ftl | 10 + .../tests/fixtures/whitespace_in_value.json | 51 + fluent-syntax/tests/junk.rs | 79 -- fluent-syntax/tests/parser_fixtures.rs | 82 ++ fluent-syntax/tests/stream.rs | 185 ---- 105 files changed, 6829 insertions(+), 2501 deletions(-) create mode 100644 fluent-syntax/src/bin/parser.rs delete mode 100644 fluent-syntax/src/parser/errors/display.rs delete mode 100644 fluent-syntax/src/parser/errors/list.rs delete mode 100644 fluent-syntax/src/parser/parser.rs delete mode 100644 fluent-syntax/src/parser/stream.rs create mode 100644 fluent-syntax/tests/ast/mod.rs delete mode 100644 fluent-syntax/tests/errors.rs delete mode 100644 fluent-syntax/tests/fixtures.rs create mode 100644 fluent-syntax/tests/fixtures/Makefile create mode 100644 fluent-syntax/tests/fixtures/astral.ftl create mode 100644 fluent-syntax/tests/fixtures/astral.json create mode 100644 fluent-syntax/tests/fixtures/call_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/call_expressions.json create mode 100644 fluent-syntax/tests/fixtures/comments.ftl create mode 100644 fluent-syntax/tests/fixtures/comments.json create mode 100755 fluent-syntax/tests/fixtures/convert.js create mode 100644 fluent-syntax/tests/fixtures/crlf.ftl create mode 100644 fluent-syntax/tests/fixtures/crlf.json create mode 100644 fluent-syntax/tests/fixtures/eof_comment.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_comment.json create mode 100644 fluent-syntax/tests/fixtures/eof_empty.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_empty.json create mode 100644 fluent-syntax/tests/fixtures/eof_id.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_id.json create mode 100644 fluent-syntax/tests/fixtures/eof_id_equals.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_id_equals.json create mode 100644 fluent-syntax/tests/fixtures/eof_junk.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_junk.json create mode 100644 fluent-syntax/tests/fixtures/eof_value.ftl create mode 100644 fluent-syntax/tests/fixtures/eof_value.json create mode 100644 fluent-syntax/tests/fixtures/escaped_characters.ftl create mode 100644 fluent-syntax/tests/fixtures/escaped_characters.json create mode 100644 fluent-syntax/tests/fixtures/junk.ftl create mode 100644 fluent-syntax/tests/fixtures/junk.json create mode 100644 fluent-syntax/tests/fixtures/leading_dots.ftl create mode 100644 fluent-syntax/tests/fixtures/leading_dots.json create mode 100644 fluent-syntax/tests/fixtures/literal_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/literal_expressions.json create mode 100644 fluent-syntax/tests/fixtures/member_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/member_expressions.json create mode 100644 fluent-syntax/tests/fixtures/messages.ftl create mode 100644 fluent-syntax/tests/fixtures/messages.json create mode 100644 fluent-syntax/tests/fixtures/mixed_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/mixed_entries.json create mode 100644 fluent-syntax/tests/fixtures/multiline_values.ftl create mode 100644 fluent-syntax/tests/fixtures/multiline_values.json delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl delete mode 100644 fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl create mode 100644 fluent-syntax/tests/fixtures/placeables.ftl create mode 100644 fluent-syntax/tests/fixtures/placeables.json create mode 100644 fluent-syntax/tests/fixtures/reference_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/reference_expressions.json create mode 100644 fluent-syntax/tests/fixtures/select_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/select_expressions.json create mode 100644 fluent-syntax/tests/fixtures/select_indent.ftl create mode 100644 fluent-syntax/tests/fixtures/select_indent.json create mode 100644 fluent-syntax/tests/fixtures/sparse_entries.ftl create mode 100644 fluent-syntax/tests/fixtures/sparse_entries.json create mode 100644 fluent-syntax/tests/fixtures/tab.ftl create mode 100644 fluent-syntax/tests/fixtures/tab.json create mode 100644 fluent-syntax/tests/fixtures/terms.ftl create mode 100644 fluent-syntax/tests/fixtures/terms.json create mode 100644 fluent-syntax/tests/fixtures/variables.ftl create mode 100644 fluent-syntax/tests/fixtures/variables.json create mode 100644 fluent-syntax/tests/fixtures/variant_keys.ftl create mode 100644 fluent-syntax/tests/fixtures/variant_keys.json create mode 100644 fluent-syntax/tests/fixtures/variant_lists.ftl create mode 100644 fluent-syntax/tests/fixtures/variant_lists.json create mode 100644 fluent-syntax/tests/fixtures/variants_indent.ftl create mode 100644 fluent-syntax/tests/fixtures/variants_indent.json create mode 100644 fluent-syntax/tests/fixtures/whitespace_in_value.ftl create mode 100644 fluent-syntax/tests/fixtures/whitespace_in_value.json delete mode 100644 fluent-syntax/tests/junk.rs create mode 100644 fluent-syntax/tests/parser_fixtures.rs delete mode 100644 fluent-syntax/tests/stream.rs diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index 56dcbee1..ea86cda4 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -4,6 +4,7 @@ description = """ Parser/Serializer tools for Fluent Syntax. """ version = "0.1.1" +edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " @@ -16,7 +17,11 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] +clap = "2.32" annotate-snippets = {version = "0.1", features = ["color"]} [dev-dependencies] +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" glob = "0.2" diff --git a/fluent-syntax/benches/lib.rs b/fluent-syntax/benches/lib.rs index e0d7e4fd..f1c2f656 100644 --- a/fluent-syntax/benches/lib.rs +++ b/fluent-syntax/benches/lib.rs @@ -3,16 +3,16 @@ extern crate fluent_syntax; extern crate test; +use self::test::Bencher; use fluent_syntax::parser::parse; use std::fs::File; use std::io; use std::io::Read; -use test::Bencher; fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); + let mut f = File::open(path)?; let mut s = String::new(); - try!(f.read_to_string(&mut s)); + f.read_to_string(&mut s)?; Ok(s) } diff --git a/fluent-syntax/src/ast.rs b/fluent-syntax/src/ast.rs index 77a22188..dedc0bbe 100644 --- a/fluent-syntax/src/ast.rs +++ b/fluent-syntax/src/ast.rs @@ -1,129 +1,136 @@ #[derive(Debug, PartialEq)] -pub struct Resource { - pub body: Vec, +pub struct Resource<'ast> { + pub body: Vec>, } #[derive(Debug, PartialEq)] -pub enum Entry { - Message(Message), - Term(Term), - Comment(Comment), - Junk { content: String }, +pub enum ResourceEntry<'ast> { + Entry(Entry<'ast>), + Junk(&'ast str), } #[derive(Debug, PartialEq)] -pub struct Message { - pub id: Identifier, - pub value: Option, - pub attributes: Option>, - pub comment: Option, +pub enum Entry<'ast> { + Message(Message<'ast>), + Term(Term<'ast>), + Comment(Comment<'ast>), } #[derive(Debug, PartialEq)] -pub struct Term { - pub id: Identifier, - pub value: Pattern, - pub attributes: Option>, - pub comment: Option, +pub struct Message<'ast> { + pub id: Identifier<'ast>, + pub value: Option>, + pub attributes: Vec>, + pub comment: Option>, } #[derive(Debug, PartialEq)] -pub struct Pattern { - pub elements: Vec, +pub struct Term<'ast> { + pub id: Identifier<'ast>, + pub value: Value<'ast>, + pub attributes: Vec>, + pub comment: Option>, } #[derive(Debug, PartialEq)] -pub enum PatternElement { - TextElement(String), - Placeable(Expression), +pub enum Value<'ast> { + Pattern(Pattern<'ast>), + VariantList { variants: Vec> }, } #[derive(Debug, PartialEq)] -pub enum Expression { - StringExpression { - value: String, - }, - NumberExpression { - value: Number, - }, - MessageReference { - id: Identifier, - }, - ExternalArgument { - id: Identifier, - }, - SelectExpression { - expression: Option>, - variants: Vec, - }, - AttributeExpression { - id: Identifier, - name: Identifier, - }, - VariantExpression { - id: Identifier, - key: VarKey, - }, - CallExpression { - callee: Function, - args: Vec, - }, +pub struct Pattern<'ast> { + pub elements: Vec>, } #[derive(Debug, PartialEq)] -pub struct Attribute { - pub id: Identifier, - pub value: Pattern, +pub enum PatternElement<'ast> { + TextElement(&'ast str), + Placeable(Expression<'ast>), } #[derive(Debug, PartialEq)] -pub struct Variant { - pub key: VarKey, - pub value: Pattern, - pub default: bool, +pub struct Attribute<'ast> { + pub id: Identifier<'ast>, + pub value: Pattern<'ast>, } #[derive(Debug, PartialEq)] -pub enum VarKey { - VariantName(VariantName), - Number(Number), +pub struct Identifier<'ast> { + pub name: &'ast str, } #[derive(Debug, PartialEq)] -pub enum Argument { - Expression(Expression), - NamedArgument { name: Identifier, val: ArgValue }, +pub struct Function<'ast> { + pub name: &'ast str, } #[derive(Debug, PartialEq)] -pub enum ArgValue { - Number(Number), - String(String), +pub struct Variant<'ast> { + pub key: VariantKey<'ast>, + pub value: Value<'ast>, + pub default: bool, } -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Identifier { - pub name: String, +#[derive(Debug, PartialEq)] +pub enum VariantKey<'ast> { + Identifier { name: &'ast str }, + NumberLiteral { value: &'ast str }, } #[derive(Debug, PartialEq)] -pub struct Number { - pub value: String, +pub enum Comment<'ast> { + Comment { content: Vec<&'ast str> }, + GroupComment { content: Vec<&'ast str> }, + ResourceComment { content: Vec<&'ast str> }, } #[derive(Debug, PartialEq)] -pub struct VariantName { - pub name: String, +pub enum InlineExpression<'ast> { + StringLiteral { + value: &'ast str, + }, + NumberLiteral { + value: &'ast str, + }, + VariableReference { + id: Identifier<'ast>, + }, + CallExpression { + callee: Function<'ast>, + positional: Vec>, + named: Vec>, + }, + AttributeExpression { + reference: Box>, + name: Identifier<'ast>, + }, + VariantExpression { + reference: Box>, + key: VariantKey<'ast>, + }, + MessageReference { + id: Identifier<'ast>, + }, + TermReference { + id: Identifier<'ast>, + }, + Placeable { + expression: Box>, + }, } #[derive(Debug, PartialEq)] -pub enum Comment { - Comment { content: String }, - GroupComment { content: String }, - ResourceComment { content: String }, +pub struct NamedArgument<'ast> { + pub name: Identifier<'ast>, + pub value: InlineExpression<'ast>, } #[derive(Debug, PartialEq)] -pub struct Function { - pub name: String, +pub enum Expression<'ast> { + InlineExpression(InlineExpression<'ast>), + SelectExpression { + selector: InlineExpression<'ast>, + variants: Vec>, + }, } diff --git a/fluent-syntax/src/bin/parser.rs b/fluent-syntax/src/bin/parser.rs new file mode 100644 index 00000000..0cb35bed --- /dev/null +++ b/fluent-syntax/src/bin/parser.rs @@ -0,0 +1,58 @@ +extern crate clap; +extern crate fluent_syntax; + +use std::fs::File; +use std::io; +use std::io::Read; + +use clap::App; + +use fluent_syntax::ast::Resource; +use fluent_syntax::parser::parse; + +fn read_file(path: &str) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn print_entries_resource(res: &Resource) { + println!("{:#?}", res); +} + +fn main() { + let matches = App::new("Fluent Parser") + .version("1.0") + .about("Parses FTL file into an AST") + .args_from_usage( + "-s, --silence 'disable output' + 'Sets the input file to use'", + ) + .get_matches(); + + let input = matches.value_of("INPUT").unwrap(); + + let source = read_file(&input).expect("Read file failed"); + + let res = parse(&source); + + if matches.is_present("silence") { + return; + }; + + match res { + Ok(res) => print_entries_resource(&res), + Err((res, errors)) => { + print_entries_resource(&res); + println!("==============================\n"); + if errors.len() == 1 { + println!("Parser encountered one error:"); + } else { + println!("Parser encountered {} errors:", errors.len()); + } + println!("-----------------------------"); + println!("{:#?}", errors); + } + }; +} diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs index a310c76d..724c7446 100644 --- a/fluent-syntax/src/lib.rs +++ b/fluent-syntax/src/lib.rs @@ -1,2 +1,4 @@ +#![feature(box_syntax, box_patterns)] + pub mod ast; pub mod parser; diff --git a/fluent-syntax/src/parser/errors/display.rs b/fluent-syntax/src/parser/errors/display.rs deleted file mode 100644 index 401a142c..00000000 --- a/fluent-syntax/src/parser/errors/display.rs +++ /dev/null @@ -1,41 +0,0 @@ -extern crate annotate_snippets; - -use super::list::get_error_desc; -use super::ParserError; - -use self::annotate_snippets::display_list::DisplayList; -use self::annotate_snippets::formatter::DisplayListFormatter; -use self::annotate_snippets::snippet; - -pub fn annotate_error(err: &ParserError, file_name: &Option, color: bool) -> String { - let desc = get_error_desc(&err.kind); - - let (source, line_start, pos) = if let Some(ref info) = err.info { - (info.slice.clone(), info.line, info.pos) - } else { - panic!() - }; - - let snippet = snippet::Snippet { - slices: vec![snippet::Slice { - source, - line_start, - origin: file_name.clone(), - fold: false, - annotations: vec![snippet::SourceAnnotation { - label: desc.2.to_string(), - annotation_type: snippet::AnnotationType::Error, - range: (pos, pos + 1), - }], - }], - title: Some(snippet::Annotation { - label: Some(desc.1), - id: Some(desc.0.to_string()), - annotation_type: snippet::AnnotationType::Error, - }), - footer: vec![], - }; - let dl = DisplayList::from(snippet); - let dlf = DisplayListFormatter::new(color); - dlf.format(&dl) -} diff --git a/fluent-syntax/src/parser/errors/list.rs b/fluent-syntax/src/parser/errors/list.rs deleted file mode 100644 index bcf6d5a1..00000000 --- a/fluent-syntax/src/parser/errors/list.rs +++ /dev/null @@ -1,122 +0,0 @@ -#[derive(Debug, PartialEq)] -pub struct ParserError { - pub info: Option, - pub kind: ErrorKind, -} - -#[derive(Debug, PartialEq)] -pub struct ErrorInfo { - pub slice: String, - pub line: usize, - pub pos: usize, -} - -#[derive(Debug, PartialEq)] -pub enum ErrorKind { - Generic, - ExpectedEntry, - ExpectedToken { token: char }, - ExpectedCharRange { range: String }, - ExpectedMessageField { entry_id: String }, - ExpectedTermField { entry_id: String }, - ForbiddenWhitespace, - ForbiddenCallee, - ForbiddenKey, - MissingDefaultVariant, - MissingVariants, - MissingValue, - MissingVariantKey, - MissingLiteral, - MultipleDefaultVariants, - MessageReferenceAsSelector, - VariantAsSelector, - MessageAttributeAsSelector, - TermAttributeAsSelector, - UnterminatedStringExpression, -} - -pub fn get_error_desc(err: &ErrorKind) -> (&'static str, String, &'static str) { - match err { - ErrorKind::Generic => ("E0001", "generic error".to_owned(), ""), - ErrorKind::ExpectedEntry => ( - "E0002", - "Expected an entry start".to_owned(), - "Expected one of ('a'...'Z' | '_' | #') here", - ), - ErrorKind::ExpectedToken { token } => ("E0003", format!("expected token `{}`", token), ""), - ErrorKind::ExpectedCharRange { range } => ( - "E0004", - format!("Expected a character from range ({})", range), - "", - ), - ErrorKind::ExpectedMessageField { entry_id } => ( - "E0005", - format!( - "Expected message `{}` to have a value or attributes", - entry_id - ), - "", - ), - ErrorKind::ExpectedTermField { entry_id } => ( - "E0006", - format!("Expected term `{}` to have a value", entry_id), - "", - ), - ErrorKind::ForbiddenWhitespace => ( - "E0007", - "Keyword cannot end with a whitespace".to_owned(), - "", - ), - ErrorKind::ForbiddenCallee => ( - "E0008", - "The callee has to be a simple, upper-case, identifier".to_owned(), - "", - ), - ErrorKind::ForbiddenKey => ( - "E0009", - "The key has to be a simple identifier".to_owned(), - "", - ), - ErrorKind::MissingDefaultVariant => ( - "E0010", - "Expected one of the variants to be marked as default (*).".to_owned(), - "", - ), - ErrorKind::MissingVariants => ( - "E0011", - "Expected at least one variant after \"->\".".to_owned(), - "", - ), - ErrorKind::MissingValue => ("E0012", "Expected value".to_owned(), ""), - ErrorKind::MissingVariantKey => ("E0013", "Expected variant key".to_owned(), ""), - ErrorKind::MissingLiteral => ("E0014", "Expected literal".to_owned(), ""), - ErrorKind::MultipleDefaultVariants => ( - "E0015", - "Only one variant can be marked as default (*)".to_owned(), - "", - ), - ErrorKind::MessageReferenceAsSelector => ( - "E0016", - "Message references cannot be used as selectors".to_owned(), - "", - ), - ErrorKind::VariantAsSelector => ( - "E0017", - "Variants cannot be used as selectors".to_owned(), - "", - ), - ErrorKind::MessageAttributeAsSelector => ( - "E0018", - "Attributes of messages cannot be used as selectors.".to_owned(), - "", - ), - ErrorKind::TermAttributeAsSelector => ( - "E0019", - "Attributes of terms cannot be used as selectors.".to_owned(), - "", - ), - ErrorKind::UnterminatedStringExpression => { - ("E0020", "Underminated string expression".to_owned(), "") - } - } -} diff --git a/fluent-syntax/src/parser/errors/mod.rs b/fluent-syntax/src/parser/errors/mod.rs index 133dc3b0..0607d912 100644 --- a/fluent-syntax/src/parser/errors/mod.rs +++ b/fluent-syntax/src/parser/errors/mod.rs @@ -1,96 +1,46 @@ -pub mod display; -mod list; - -pub use self::list::get_error_desc; -pub use self::list::ErrorInfo; -pub use self::list::ErrorKind; -pub use self::list::ParserError; +#[derive(Debug, PartialEq)] +pub struct ParserError { + pub pos: usize, + pub slice: Option<(usize, usize)>, + pub kind: ErrorKind, +} macro_rules! error { - ($kind:expr) => {{ + ($ps:ident, $kind:expr) => {{ Err(ParserError { - info: None, + pos: $ps.ptr, + slice: None, kind: $kind, }) }}; } -fn get_line_num(source: &str, pos: usize) -> usize { - let mut ptr = 0; - let mut i = 0; - - let lines = source.lines(); - - for line in lines { - let lnlen = line.chars().count(); - ptr += lnlen + 1; - - if ptr > pos { - break; - } - i += 1; - } - - i -} - -pub fn get_error_lines(source: &str, start: usize, end: usize) -> String { - let l = if start < end { end - start } else { 1 }; - - let lines = source.lines().skip(start).take(l); - - let mut s = String::new(); - - for line in lines { - s.push_str(line); - s.push('\n'); - } - - String::from(s.trim_right()) -} - -pub fn get_error_slice(source: &str, start: usize, end: usize) -> &str { - let len = source.chars().count(); - - let start_pos; - let mut slice_len = end - start; - - if len <= slice_len { - start_pos = 0; - slice_len = len; - } else if start + slice_len >= len { - start_pos = len - slice_len - 1; - } else { - start_pos = start; - } - - let mut iter = source.chars(); - if start_pos > 0 { - iter.by_ref().nth(start_pos - 1); - } - let slice = iter.as_str(); - let endp = slice - .char_indices() - .nth(slice_len) - .map(|(n, _)| n) - .unwrap_or(len); - &slice[..endp] -} - -pub fn get_error_info( - source: &str, - pos: usize, - entry_start: usize, - next_entry_start: usize, -) -> Option { - let first_line_num = get_line_num(source, entry_start); - let next_entry_line = get_line_num(source, next_entry_start); - - let slice = get_error_lines(source, first_line_num, next_entry_line); - - Some(ErrorInfo { - slice, - line: first_line_num, - pos: pos - entry_start, - }) +#[derive(Debug, PartialEq)] +pub enum ErrorKind { + Generic, + ExpectedEntry, + ExpectedToken(char), + ExpectedCharRange { range: String }, + ExpectedMessageField { entry_id: String }, + ExpectedTermField { entry_id: String }, + ForbiddenWhitespace, + ForbiddenCallee, + ForbiddenKey, + MissingDefaultVariant, + MissingVariants, + MissingValue, + MissingVariantKey, + MissingLiteral, + MultipleDefaultVariants, + MessageReferenceAsSelector, + VariantAsSelector, + MessageAttributeAsSelector, + TermAttributeAsPlaceable, + UnterminatedStringExpression, + PositionalArgumentFollowsNamed, + DuplicatedNamedArgument(String), + VariantListInExpression, + ForbiddenVariantAccessor, + UnknownEscapeSequence(String), + InvalidUnicodeEscapeSequence(String), } diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index cdee2c44..0e3d8fd8 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -1,394 +1,216 @@ -use super::errors::ErrorKind; -use super::errors::ParserError; -use super::parser::Result; -use super::stream::ParserStream; - -pub trait FTLParserStream { - fn skip_inline_ws(&mut self); - fn peek_inline_ws(&mut self); - fn skip_blank_lines(&mut self); - fn peek_blank_lines(&mut self); - fn skip_indent(&mut self); - fn expect_char(&mut self, ch: char) -> Result<()>; - fn expect_indent(&mut self) -> Result<()>; - fn take_char_if(&mut self, ch: char) -> bool; - - fn take_char(&mut self, f: F) -> Option - where - F: Fn(char) -> bool; - - fn is_char_id_start(&mut self, ch: Option) -> bool; - fn is_entry_id_start(&mut self) -> bool; - fn is_number_start(&mut self) -> bool; - fn is_char_pattern_continuation(&self, ch: Option) -> bool; - fn is_peek_pattern_start(&mut self) -> bool; - fn is_peek_next_line_zero_four_style_comment(&mut self) -> bool; - fn is_peek_next_line_comment(&mut self, level: i8) -> bool; - fn is_peek_next_line_variant_start(&mut self) -> bool; - fn is_peek_next_line_attribute_start(&mut self) -> bool; - fn is_peek_next_line_pattern_start(&mut self) -> bool; - fn skip_to_next_entry_start(&mut self); - fn take_id_start(&mut self, allow_private: bool) -> Result; - fn take_id_char(&mut self) -> Option; - fn take_variant_name_char(&mut self) -> Option; - fn take_digit(&mut self) -> Option; +use super::errors::{ErrorKind, ParserError}; +use super::Result; +use std::str; + +pub struct ParserStream<'p> { + pub source: &'p [u8], + pub length: usize, + pub ptr: usize, } -static INLINE_WS: [char; 2] = [' ', '\t']; -static SPECIAL_LINE_START_CHARS: [char; 4] = ['}', '.', '[', '*']; - -impl FTLParserStream for ParserStream -where - I: Iterator, -{ - fn skip_inline_ws(&mut self) { - while let Some(ch) = self.ch { - if !INLINE_WS.contains(&ch) { - break; - } - self.next(); +impl<'p> ParserStream<'p> { + pub fn new(stream: &'p str) -> Self { + ParserStream { + source: stream.as_bytes(), + length: stream.len(), + ptr: 0, } } - fn peek_inline_ws(&mut self) { - while let Some(ch) = self.current_peek() { - if !INLINE_WS.contains(&ch) { - break; - } - self.peek(); + pub fn is_current_byte(&self, b: u8) -> bool { + if self.ptr >= self.length { + return false; } + self.source[self.ptr] == b } - fn skip_blank_lines(&mut self) { - loop { - self.peek_inline_ws(); - - if self.current_peek() == Some('\n') { - self.skip_to_peek(); - self.next(); - } else { - self.reset_peek(None); - break; - } - } + pub fn _get_current_byte(&self) -> String { + str::from_utf8(&[self.source[self.ptr]]).unwrap().to_owned() } - fn peek_blank_lines(&mut self) { - loop { - let line_start = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.current_peek_is('\n') { - self.peek(); - } else { - self.reset_peek(Some(line_start)); - break; - } + pub fn is_byte_at(&self, b: u8, pos: usize) -> bool { + if pos >= self.length { + return false; } + self.source[pos] == b } - fn skip_indent(&mut self) { - self.skip_blank_lines(); - self.skip_inline_ws(); - } - - fn expect_char(&mut self, ch: char) -> Result<()> { - if self.ch == Some(ch) { - self.next(); - return Ok(()); - } - - if self.ch == Some('\n') { - // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) - return error!(ErrorKind::ExpectedToken { token: '\u{2424}' }); + pub fn expect_byte(&mut self, b: u8) -> Result<()> { + if !self.is_current_byte(b) { + return error!(self, ErrorKind::ExpectedToken(b as char)); } - - error!(ErrorKind::ExpectedToken { token: ch }) - } - - fn expect_indent(&mut self) -> Result<()> { - self.expect_char('\n')?; - self.skip_blank_lines(); - self.expect_char(' ')?; - self.skip_inline_ws(); + self.ptr += 1; Ok(()) } - fn take_char_if(&mut self, ch: char) -> bool { - if self.ch == Some(ch) { - self.next(); - return true; + pub fn take_if(&mut self, b: u8) -> bool { + if self.is_current_byte(b) { + self.ptr += 1; + true + } else { + false } - - false } - fn take_char(&mut self, f: F) -> Option - where - F: Fn(char) -> bool, - { - if let Some(ch) = self.ch { - if f(ch) { - self.next(); - return Some(ch); + pub fn skip_blank_block(&mut self) -> usize { + let mut count = 0; + loop { + let start = self.ptr; + self.skip_blank_inline(); + if !self.skip_eol() { + self.ptr = start; + break; } + count += 1; } - None - } - - fn is_char_id_start(&mut self, ch: Option) -> bool { - match ch { - Some('a'...'z') | Some('A'...'Z') => true, - _ => false, - } + count } - fn is_entry_id_start(&mut self) -> bool { - if let Some('-') = self.ch { - self.peek(); - } - let ch = self.current_peek(); - let is_id = self.is_char_id_start(ch); - self.reset_peek(None); - is_id - } - - fn is_number_start(&mut self) -> bool { - if let Some('-') = self.ch { - self.peek(); - } - let ch = self.current_peek(); - let is_digit = match ch { - Some('0'...'9') => true, - _ => false, - }; - self.reset_peek(None); - is_digit - } - - fn is_char_pattern_continuation(&self, ch: Option) -> bool { - match ch { - Some(ch) => !SPECIAL_LINE_START_CHARS.contains(&ch), - _ => false, - } - } - - fn is_peek_pattern_start(&mut self) -> bool { - self.peek_inline_ws(); - - if let Some(ch) = self.current_peek() { - if ch != '\n' { - return true; + pub fn skip_blank(&mut self) { + while self.ptr < self.length { + let b = self.source[self.ptr]; + if b == b' ' || b == b'\n' { + self.ptr += 1; + } else { + break; } } - - self.is_peek_next_line_pattern_start() } - fn is_peek_next_line_zero_four_style_comment(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; - } - self.peek(); - - if self.current_peek_is('/') { - self.peek(); - if self.current_peek_is('/') { - self.reset_peek(None); - return true; + pub fn skip_blank_inline(&mut self) -> bool { + let start = self.ptr; + while self.ptr < self.length { + let b = self.source[self.ptr]; + if b == b' ' { + self.ptr += 1; + } else { + break; } } - self.reset_peek(None); - false + start != self.ptr } - fn is_peek_next_line_comment(&mut self, level: i8) -> bool { - if !self.current_peek_is('\n') { - return false; - } - - let mut i = 0; - - while i <= level && (level == -1 && i < 3) { - self.peek(); - if !self.current_peek_is('#') { - if i != level && level != -1 { - self.reset_peek(None); - return false; - } + pub fn skip_to_next_entry_start(&mut self) { + while self.ptr < self.length { + if (self.ptr == 0 || self.is_byte_at(b'\n', self.ptr - 1)) + && (self.is_identifier_start() + || self.is_current_byte(b'-') + || self.is_current_byte(b'#')) + { break; } - i += 1; - } - self.peek(); + self.ptr += 1; - if let Some(ch) = self.current_peek() { - if [' ', '\n'].contains(&ch) { - self.reset_peek(None); - return true; + while self.ptr < self.length && !self.is_byte_at(b'\n', self.ptr - 1) { + self.ptr += 1; } } - self.reset_peek(None); - false } - fn is_peek_next_line_variant_start(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; - } - self.peek(); - - self.peek_blank_lines(); - - let ptr = self.get_peek_index(); - - self.peek_inline_ws(); - - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); + pub fn skip_eol(&mut self) -> bool { + if self.ptr >= self.length { return false; } - if self.current_peek_is('*') { - self.peek(); + if self.is_current_byte(b'\n') { + self.ptr += 1; + return true; } - if self.current_peek_is('[') && !self.peek_char_is('[') { - self.reset_peek(None); + if self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) { + self.ptr += 2; return true; } - self.reset_peek(None); false } - fn is_peek_next_line_attribute_start(&mut self) -> bool { - if !self.current_peek_is('\n') { + pub fn _is_entry_start(&self) -> bool { + if self.ptr >= self.length { return false; } - self.peek(); - - self.peek_blank_lines(); - - let ptr = self.get_peek_index(); + let b = self.source[self.ptr]; + (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') || b == b'-' + } - self.peek_inline_ws(); + pub fn skip_to_value_start(&mut self) -> bool { + let start = self.ptr; + self.skip_blank_inline(); - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); + if self.ptr >= self.length { return false; } - - if self.current_peek_is('.') { - self.reset_peek(None); + if !self.is_eol() { return true; } - - self.reset_peek(None); - false + self.skip_to_next_line_value(start) } - fn is_peek_next_line_pattern_start(&mut self) -> bool { - if !self.current_peek_is('\n') { - return false; - } - self.peek(); - - self.peek_blank_lines(); - - let ptr = self.get_peek_index(); - - self.peek_inline_ws(); + pub fn skip_to_next_line_value(&mut self, start: usize) -> bool { + self.skip_blank_block(); + let inline = self.skip_blank_inline(); - if self.get_peek_index() - ptr == 0 { - self.reset_peek(None); + if self.is_current_byte(b'{') { + return true; + } + if !inline { + self.ptr = start; return false; } - if !self.is_char_pattern_continuation(self.current_peek()) { - self.reset_peek(None); + if !self.is_char_pattern_continuation() { + self.ptr = start; return false; } - - self.reset_peek(None); true } - fn skip_to_next_entry_start(&mut self) { - while let Some(_) = self.next() { - if self.current_is('\n') && !self.peek_char_is('\n') { - self.next(); - - if self.ch.is_none() - || self.is_entry_id_start() - || self.current_is('#') - || (self.current_is('/') && self.peek_char_is('/')) - || (self.current_is('[') && self.peek_char_is('[')) - { - break; - } - } + pub fn is_char_pattern_continuation(&self) -> bool { + if self.ptr >= self.length { + return false; } + + let b = self.source[self.ptr]; + b != b'}' && b != b'.' && b != b'[' && b != b'*' } - fn take_id_start(&mut self, allow_term: bool) -> Result { - if allow_term && self.current_is('-') { - self.next(); - return Ok('-'); + pub fn is_pattern_start(&self) -> bool { + if self.ptr >= self.length { + return false; } + let b = self.source[self.ptr]; + b != b'.' && b != b'[' && b != b'*' && b != b'}' + } - if let Some(ch) = self.ch { - if self.is_char_id_start(Some(ch)) { - let ret = self.ch.unwrap(); - self.next(); - return Ok(ret); - } + pub fn is_identifier_start(&self) -> bool { + if self.ptr >= self.length { + return false; } - - let allowed_range = if allow_term { - "'a'...'z' | 'A'...'Z' | '-'" - } else { - "'a'...'z' | 'A'...'Z'" - }; - error!(ErrorKind::ExpectedCharRange { - range: String::from(allowed_range), - }) + let b = self.source[self.ptr]; + (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') } - fn take_id_char(&mut self) -> Option { - let closure = |x| match x { - 'a'...'z' | 'A'...'Z' | '0'...'9' | '_' | '-' => true, - _ => false, - }; - - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, + pub fn is_number_start(&self) -> bool { + if self.ptr >= self.length { + return false; } + let b = self.source[self.ptr]; + b == b'-' || (b >= b'0' && b <= b'9') } - fn take_variant_name_char(&mut self) -> Option { - let closure = |x| match x { - 'a'...'z' | 'A'...'Z' | '0'...'9' | '_' | '-' | ' ' => true, - _ => false, - }; + pub fn is_eol(&self) -> bool { + if self.ptr >= self.length { + return false; + } - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, + if self.is_current_byte(b'\n') { + return true; } - } - fn take_digit(&mut self) -> Option { - let closure = |x| match x { - '0'...'9' => true, - _ => false, - }; + self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) + } - match self.take_char(closure) { - Some(ch) => Some(ch), - None => None, - } + pub fn get_slice(&self, start: usize, end: usize) -> &'p str { + unsafe { str::from_utf8_unchecked(&self.source[start..end]) } } } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index ac133c9f..79db6404 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -1,14 +1,778 @@ -//! AST, parser and serializer operations -//! -//! This is an internal API used by `FluentBundle` for parsing an FTL syntax -//! into an AST that can be then resolved by the `Resolver`. -//! -//! This module may be useful for tooling that operates on FTL syntax. - #[macro_use] pub mod errors; -pub mod ftlstream; -pub mod parser; -pub mod stream; +mod ftlstream; + +use std::result; +use std::str; + +use self::errors::ErrorKind; +pub use self::errors::ParserError; +use self::ftlstream::ParserStream; +use super::ast; + +pub type Result = result::Result; + +pub fn parse(source: &str) -> result::Result)> { + let mut errors = vec![]; + + let mut ps = ParserStream::new(source); + + let mut body = vec![]; + + ps.skip_blank_block(); + + let mut last_entry_end = ps.ptr; + let mut last_comment = None; + + while ps.ptr < ps.length { + let entry_start = ps.ptr; + match get_entry(&mut ps) { + Ok(entry) => { + if last_entry_end != entry_start { + let mut te = 0; + while ps.is_byte_at(b'\n', entry_start - te - 1) { + te += 1; + } + let te = if te > 1 { te - 1 } else { 0 }; + let slice = ps.get_slice(last_entry_end, entry_start - te); + body.push(ast::ResourceEntry::Junk(slice)); + } + if let Some(content) = last_comment { + match entry { + ast::Entry::Message(mut msg) => { + msg.comment = Some(ast::Comment::Comment { content }); + body.push(ast::ResourceEntry::Entry(ast::Entry::Message(msg))); + last_comment = None; + } + ast::Entry::Term(mut term) => { + term.comment = Some(ast::Comment::Comment { content }); + body.push(ast::ResourceEntry::Entry(ast::Entry::Term(term))); + last_comment = None; + } + ast::Entry::Comment(new_comment) => { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( + ast::Comment::Comment { content }, + ))); + if let ast::Comment::Comment { content } = new_comment { + last_comment = Some(content); + } else { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( + new_comment, + ))); + last_comment = None; + } + } + } + } else { + match entry { + ast::Entry::Comment(ast::Comment::Comment { content }) => { + last_comment = Some(content); + } + _ => { + body.push(ast::ResourceEntry::Entry(entry)); + } + } + } + ps.skip_eol(); + if ps.skip_blank_block() > 0 { + if let Some(content) = last_comment { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( + ast::Comment::Comment { content }, + ))); + last_comment = None; + } + } + last_entry_end = ps.ptr; + } + Err(mut err) => { + if let Some(content) = last_comment { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( + ast::Comment::Comment { content }, + ))); + last_comment = None; + } + ps.skip_to_next_entry_start(); + let mut te = 0; + while ps.is_byte_at(b'\n', ps.ptr - te - 1) { + te += 1; + } + err.slice = Some((last_entry_end, ps.ptr - te)); + errors.push(err); + if te > 1 { + let slice = ps.get_slice(last_entry_end, ps.ptr - te + 1); + body.push(ast::ResourceEntry::Junk(slice)); + last_entry_end = ps.ptr; + } + } + } + } + if let Some(content) = last_comment { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( + ast::Comment::Comment { content }, + ))); + } + if last_entry_end != ps.ptr { + let mut te = 0; + while ps.is_byte_at(b'\n', ps.ptr - te - 1) { + te += 1; + } + let te = if te > 1 { te - 1 } else { 0 }; + let slice = ps.get_slice(last_entry_end, ps.ptr - te); + body.push(ast::ResourceEntry::Junk(slice)); + } + + if errors.is_empty() { + Ok(ast::Resource { body }) + } else { + Err((ast::Resource { body }, errors)) + } +} + +fn get_entry<'p>(ps: &mut ParserStream<'p>) -> Result> { + let entry = match ps.source[ps.ptr] { + b'#' => ast::Entry::Comment(get_comment(ps)?), + b'-' => ast::Entry::Term(get_term(ps)?), + _ => ast::Entry::Message(get_message(ps)?), + }; + Ok(entry) +} + +fn get_message<'p>(ps: &mut ParserStream<'p>) -> Result> { + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + ps.skip_blank_inline(); + + let pattern = if ps.skip_to_value_start() { + get_pattern(ps)? + } else { + None + }; + + ps.skip_blank_block(); + + let ptr = ps.ptr; + let attributes = match get_attributes(ps) { + Ok(attrs) => attrs, + Err(_err) => { + ps.ptr = ptr; + vec![] + } + }; + + if pattern.is_none() && attributes.is_empty() { + return error!( + ps, + ErrorKind::ExpectedMessageField { + entry_id: id.name.to_string() + } + ); + } + + Ok(ast::Message { + id, + value: pattern, + attributes, + comment: None, + }) +} + +fn get_term<'p>(ps: &mut ParserStream<'p>) -> Result> { + ps.expect_byte(b'-')?; + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + ps.skip_blank_inline(); + + let value = get_value(ps)?; + + ps.skip_blank_block(); + + let ptr = ps.ptr; + let attributes = match get_attributes(ps) { + Ok(attrs) => attrs, + Err(_err) => { + ps.ptr = ptr; + vec![] + } + }; + + if let Some(value) = value { + Ok(ast::Term { + id, + value, + attributes, + comment: None, + }) + } else { + error!( + ps, + ErrorKind::ExpectedTermField { + entry_id: id.name.to_string() + } + ) + } +} + +fn get_value<'p>(ps: &mut ParserStream<'p>) -> Result>> { + if !ps.skip_to_value_start() { + return Ok(None); + } + + if ps.is_current_byte(b'{') { + let start = ps.ptr; + ps.ptr += 1; + ps.skip_blank(); + if ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { + let variants = get_variants(ps, true)?; + ps.expect_byte(b'}')?; + return Ok(Some(ast::Value::VariantList { variants })); + } + ps.ptr = start; + } + + let pattern = get_pattern(ps)?; + + Ok(pattern.map(ast::Value::Pattern)) +} + +fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Result>> { + let mut attributes = vec![]; + + loop { + let line_start = ps.ptr; + + ps.skip_blank_inline(); + + if !ps.is_current_byte(b'.') { + ps.ptr = line_start; + break; + } + ps.ptr += 1; // . + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + ps.skip_blank_inline(); + let pattern = get_pattern(ps)?; + + match pattern { + Some(pattern) => attributes.push(ast::Attribute { id, value: pattern }), + None => panic!("Expected Value!"), + }; + ps.skip_eol(); + } + Ok(attributes) +} + +fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> { + let start_pos = ps.ptr; + + while ps.ptr < ps.length { + let b = ps.source[ps.ptr]; + if start_pos == ps.ptr { + if ps.is_identifier_start() { + ps.ptr += 1; + } else { + return error!( + ps, + ErrorKind::ExpectedCharRange { + range: "a-zA-Z".to_string() + } + ); + } + } else if (b >= b'a' && b <= b'z') + || (b >= b'A' && b <= b'Z') + || (b >= b'0' && b <= b'9') + || b == b'_' + || b == b'-' + { + ps.ptr += 1; + } else { + break; + } + } + let name = ps.get_slice(start_pos, ps.ptr); + + Ok(ast::Identifier { name }) +} + +fn get_variant_key<'p>(ps: &mut ParserStream<'p>) -> Result> { + if !ps.take_if(b'[') { + return error!(ps, ErrorKind::ExpectedToken('[')); + } + ps.skip_blank(); + + let key = if ps.is_number_start() { + ast::VariantKey::NumberLiteral { + value: get_number_literal(ps)?, + } + } else { + ast::VariantKey::Identifier { + name: get_identifier(ps)?.name, + } + }; + + ps.skip_blank(); + + ps.expect_byte(b']')?; + + Ok(key) +} + +fn get_variants<'p>( + ps: &mut ParserStream<'p>, + variant_lists: bool, +) -> Result>> { + let mut variants = vec![]; + let mut has_default = false; + + while ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { + let default = ps.take_if(b'*'); + + if default { + if has_default { + return error!(ps, ErrorKind::MultipleDefaultVariants); + } else { + has_default = true; + } + } + + let key = get_variant_key(ps)?; + + ps.skip_blank_inline(); + + let value = if variant_lists { + get_value(ps)? + } else { + get_pattern(ps)?.map(ast::Value::Pattern) + }; + + if let Some(value) = value { + variants.push(ast::Variant { + key, + value, + default, + }); + ps.skip_blank(); + } else { + return error!(ps, ErrorKind::MissingValue); + } + } + + if !has_default { + error!(ps, ErrorKind::MissingDefaultVariant) + } else { + Ok(variants) + } +} + +fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result>> { + let start = ps.ptr; + if ps.skip_eol() { + ps.skip_blank_block(); + if !ps.skip_blank_inline() || !ps.is_pattern_start() { + ps.ptr = start; + return Ok(None); + } + } + + let mut elements = vec![]; + + loop { + let mut start_pos = ps.ptr; + + while ps.ptr < ps.length { + if ps.skip_eol() { + break; + } + let b = ps.source[ps.ptr]; + match b { + b'\\' => { + ps.ptr += 1; + let b = ps.source[ps.ptr]; + match b { + b'{' => ps.ptr += 1, + b'\\' => ps.ptr += 1, + b'u' => { + ps.ptr += 2; + let start = ps.ptr; + for _ in 0..4 { + match ps.source[ps.ptr] { + b'0'...b'9' => ps.ptr += 1, + b'a'...b'f' => ps.ptr += 1, + b'A'...b'F' => ps.ptr += 1, + _ => break, + } + } + if start == ps.ptr { + return error!( + ps, + ErrorKind::InvalidUnicodeEscapeSequence( + ps.get_slice(start, ps.ptr + 1).to_owned() + ) + ); + } + } + _ => panic!(), + } + } + b'{' => { + if start_pos != ps.ptr { + let value = ps.get_slice(start_pos, ps.ptr); + elements.push(ast::PatternElement::TextElement(value)); + } + ps.ptr += 1; // { + ps.skip_blank(); + let exp = get_expression(ps)?; + elements.push(ast::PatternElement::Placeable(exp)); + ps.skip_blank_inline(); + ps.expect_byte(b'}')?; + start_pos = ps.ptr; + } + _ => ps.ptr += 1, + } + } + + if start_pos != ps.ptr { + let value = ps.get_slice(start_pos, ps.ptr); + elements.push(ast::PatternElement::TextElement(value)); + } + + let end_of_line = ps.ptr; + + let bl = ps.skip_blank_block(); + + if !ps.skip_blank_inline() || !ps.is_pattern_start() { + ps.ptr = end_of_line; + break; + } else { + for _ in 0..bl { + elements.push(ast::PatternElement::TextElement("\n")); + } + } + } + + if !elements.is_empty() { + let last_pos = elements.len() - 1; + let val = &elements[last_pos]; + let mut new_val = ""; + let mut modified = false; + if let ast::PatternElement::TextElement(te) = val { + new_val = te.trim_right(); + modified = &new_val != te; + } + if modified { + elements.pop(); + if !new_val.is_empty() { + elements.insert(last_pos, ast::PatternElement::TextElement(new_val)); + ps.ptr -= 1; // move before last \n + } + } + } + + if elements.is_empty() { + Ok(None) + } else { + Ok(Some(ast::Pattern { elements })) + } +} + +fn get_comment<'p>(ps: &mut ParserStream<'p>) -> Result> { + let mut level = None; + let mut content = vec![]; + + while ps.ptr < ps.length { + let line_level = get_comment_level(ps); + if line_level == 0 { + ps.ptr -= 1; + break; + } + if level.is_some() && Some(line_level) != level { + ps.ptr -= line_level; + break; + } + + level = Some(line_level); + + if ps.is_current_byte(b'\n') { + content.push(get_comment_line(ps)?); + } else { + ps.expect_byte(b' ')?; + content.push(get_comment_line(ps)?); + } + ps.skip_eol(); + } + + let comment = if level == Some(3) { + ast::Comment::ResourceComment { content } + } else if level == Some(2) { + ast::Comment::GroupComment { content } + } else { + ast::Comment::Comment { content } + }; + Ok(comment) +} + +fn get_comment_level<'p>(ps: &mut ParserStream<'p>) -> usize { + let mut chars = 0; + + while ps.take_if(b'#') { + chars += 1; + } + + chars +} + +fn get_comment_line<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { + let start_pos = ps.ptr; + + while ps.ptr < ps.length && !ps.is_eol() { + ps.ptr += 1; + } + + Ok(str::from_utf8(&ps.source[start_pos..ps.ptr]).unwrap()) +} + +fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let exp = get_inline_expression(ps)?; + + ps.skip_blank(); + + if !ps.is_current_byte(b'-') || !ps.is_byte_at(b'>', ps.ptr + 1) { + if let ast::InlineExpression::AttributeExpression { ref reference, .. } = exp { + if let box ast::InlineExpression::TermReference { .. } = reference { + return error!(ps, ErrorKind::TermAttributeAsPlaceable); + } + } + return Ok(ast::Expression::InlineExpression(exp)); + } + + match exp { + ast::InlineExpression::MessageReference { .. } => { + return error!(ps, ErrorKind::MessageReferenceAsSelector); + } + ast::InlineExpression::AttributeExpression { ref reference, .. } => { + if let box ast::InlineExpression::MessageReference { .. } = reference { + return error!(ps, ErrorKind::MessageAttributeAsSelector); + } + } + ast::InlineExpression::VariantExpression { .. } => { + return error!(ps, ErrorKind::VariantAsSelector); + } + _ => {} + } + + ps.ptr += 2; // -> + + ps.skip_blank_inline(); + ps.expect_byte(b'\n')?; + ps.skip_blank(); + + let variants = get_variants(ps, false)?; + + Ok(ast::Expression::SelectExpression { + selector: exp, + variants, + }) +} + +fn get_inline_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + match ps.source.get(ps.ptr) { + Some(b'"') => { + ps.ptr += 1; // " + let start = ps.ptr; + while ps.ptr < ps.length { + match ps.source[ps.ptr] { + b'\\' => match ps.source[ps.ptr + 1] { + b'\\' => ps.ptr += 2, + b'{' => ps.ptr += 2, + b'"' => ps.ptr += 2, + b'u' => { + ps.ptr += 2; + let start = ps.ptr; + for _ in 0..4 { + match ps.source[ps.ptr] { + b'0'...b'9' => ps.ptr += 1, + b'a'...b'f' => ps.ptr += 1, + b'A'...b'F' => ps.ptr += 1, + _ => break, + } + } + if start == ps.ptr { + return error!( + ps, + ErrorKind::InvalidUnicodeEscapeSequence( + ps.get_slice(start, ps.ptr + 1).to_owned() + ) + ); + } + } + _ => panic!(), + }, + b'"' => { + break; + } + _ => ps.ptr += 1, + } + } + + ps.expect_byte(b'"')?; + Ok(ast::InlineExpression::StringLiteral { + value: ps.get_slice(start, ps.ptr - 1), + }) + } + Some(b'0'...b'9') => { + let num = get_number_literal(ps)?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + Some(b'-') => { + ps.ptr += 1; // - + if ps.is_identifier_start() { + let id = get_identifier(ps)?; + match ps.source[ps.ptr] { + b'.' => { + ps.ptr += 1; // . + let attr = get_identifier(ps)?; + Ok(ast::InlineExpression::AttributeExpression { + reference: Box::new(ast::InlineExpression::TermReference { id }), + name: attr, + }) + } + b'[' => { + let key = get_variant_key(ps)?; + Ok(ast::InlineExpression::VariantExpression { + reference: Box::new(ast::InlineExpression::TermReference { id }), + key, + }) + } + _ => Ok(ast::InlineExpression::TermReference { id }), + } + } else { + ps.ptr -= 1; + let num = get_number_literal(ps)?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + } + Some(b'$') => { + ps.ptr += 1; // - + let id = get_identifier(ps)?; + Ok(ast::InlineExpression::VariableReference { id }) + } + Some(b'a'...b'z') | Some(b'A'...b'Z') => { + let id = get_identifier(ps)?; + + match ps.source[ps.ptr] { + b'(' => get_call_expression(ps, Some(id)), + b'.' => { + ps.ptr += 1; // . + let attr = get_identifier(ps)?; + Ok(ast::InlineExpression::AttributeExpression { + reference: Box::new(ast::InlineExpression::MessageReference { id }), + name: attr, + }) + } + _ => Ok(ast::InlineExpression::MessageReference { id }), + } + } + Some(b'{') => { + ps.ptr += 1; // { + ps.skip_blank(); + let exp = get_expression(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'}')?; + Ok(ast::InlineExpression::Placeable { + expression: Box::new(exp), + }) + } + _ => error!(ps, ErrorKind::MissingLiteral), + } +} + +fn get_call_expression<'p>( + ps: &mut ParserStream<'p>, + id: Option>, +) -> Result> { + let id = match id { + Some(id) => id, + None => get_identifier(ps)?, + }; + let (positional, named) = get_call_args(ps)?; + Ok(ast::InlineExpression::CallExpression { + callee: ast::Function { name: id.name }, + positional, + named, + }) +} + +fn get_call_args<'p>( + ps: &mut ParserStream<'p>, +) -> Result<(Vec>, Vec>)> { + let mut positional = vec![]; + let mut named = vec![]; + let mut argument_names = vec![]; + + ps.expect_byte(b'(')?; + ps.skip_blank(); + + while ps.ptr < ps.length { + let b = ps.source[ps.ptr]; + if b == b')' { + break; + } + let id = if ps.is_identifier_start() { + Some(get_identifier(ps)?) + } else { + None + }; + + if let Some(id) = id { + ps.skip_blank(); + if ps.is_current_byte(b':') { + if argument_names.contains(&id.name.to_owned()) { + return error!(ps, ErrorKind::DuplicatedNamedArgument(id.name.to_owned())); + } + ps.ptr += 1; + ps.skip_blank(); + let val = get_inline_expression(ps)?; + argument_names.push(id.name.to_owned()); + named.push(ast::NamedArgument { + name: id, + value: val, + }); + } else if ps.is_current_byte(b'(') { + positional.push(get_call_expression(ps, Some(id))?); + } else { + if !argument_names.is_empty() { + return error!(ps, ErrorKind::PositionalArgumentFollowsNamed); + } + positional.push(ast::InlineExpression::MessageReference { id }); + } + } else { + if !argument_names.is_empty() { + return error!(ps, ErrorKind::PositionalArgumentFollowsNamed); + } + positional.push(get_inline_expression(ps)?); + } + + ps.skip_blank(); + ps.take_if(b','); + ps.skip_blank(); + } + + ps.expect_byte(b')')?; + Ok((positional, named)) +} + +fn get_number_literal<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { + let start = ps.ptr; + ps.take_if(b'-'); + while ps.source[ps.ptr] >= b'0' && ps.source[ps.ptr] <= b'9' { + ps.ptr += 1; + } + ps.take_if(b'.'); + while ps.source[ps.ptr] >= b'0' && ps.source[ps.ptr] <= b'9' { + ps.ptr += 1; + } -pub use self::parser::parse; + Ok(ps.get_slice(start, ps.ptr)) +} diff --git a/fluent-syntax/src/parser/parser.rs b/fluent-syntax/src/parser/parser.rs deleted file mode 100644 index 9286a407..00000000 --- a/fluent-syntax/src/parser/parser.rs +++ /dev/null @@ -1,685 +0,0 @@ -pub use super::errors::get_error_info; -pub use super::errors::get_error_slice; -pub use super::errors::ErrorKind; -pub use super::errors::ParserError; - -use super::ftlstream::FTLParserStream; -use super::stream::ParserStream; - -use std::result; - -use super::super::ast; - -pub type Result = result::Result; - -pub fn parse(source: &str) -> result::Result)> { - let mut errors = vec![]; - - let mut ps = ParserStream::new(source.chars()); - - ps.skip_blank_lines(); - - let mut entries = vec![]; - - while ps.current().is_some() { - let entry_start_pos = ps.get_index(); - - match get_entry(&mut ps) { - Ok(entry) => { - entries.push(entry); - } - Err(mut e) => { - let error_pos = ps.get_index(); - entries.push(get_junk_entry(&mut ps, source, entry_start_pos)); - - e.info = get_error_info(source, error_pos, entry_start_pos, ps.get_index()); - errors.push(e); - } - } - - ps.skip_blank_lines(); - } - - if errors.is_empty() { - Ok(ast::Resource { body: entries }) - } else { - Err((ast::Resource { body: entries }, errors)) - } -} - -fn get_entry(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let comment = if ps.current_is('#') { - Some(get_comment(ps)?) - } else { - None - }; - - if ps.is_entry_id_start() { - match comment { - None | Some(ast::Comment::Comment { .. }) => { - return Ok(get_message(ps, comment)?); - } - _ => {} - }; - } - - match comment { - Some(comment) => Ok(ast::Entry::Comment(comment)), - None => error!(ErrorKind::ExpectedEntry), - } -} - -fn get_comment(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut level = -1; - let mut content = String::new(); - - loop { - let mut i = -1; - while ps.current_is('#') && ((level == -1 && i < 2) || (level != -1 && i < level)) { - ps.next(); - i += 1; - } - - if level == -1 { - level = i; - } - - if !ps.current_is('\n') { - ps.expect_char(' ')?; - while let Some(ch) = ps.take_char(|x| x != '\n') { - content.push(ch); - } - } - - if ps.is_peek_next_line_comment(level) { - content.push('\n'); - ps.next(); - } else { - break; - } - } - - match level { - 0 => Ok(ast::Comment::Comment { content }), - 1 => Ok(ast::Comment::GroupComment { content }), - 2 => Ok(ast::Comment::ResourceComment { content }), - _ => panic!("Unknown comment level!"), - } -} - -fn get_message(ps: &mut ParserStream, comment: Option) -> Result -where - I: Iterator, -{ - let id = get_entry_identifier(ps)?; - - ps.skip_inline_ws(); - - ps.expect_char('=')?; - - let pattern = if ps.is_peek_pattern_start() { - ps.skip_indent(); - get_pattern(ps)? - } else { - None - }; - - let attributes = if ps.is_peek_next_line_attribute_start() { - Some(get_attributes(ps)?) - } else { - None - }; - - if id.name.starts_with('-') { - match pattern { - Some(pattern) => { - return Ok(ast::Entry::Term(ast::Term { - id, - value: pattern, - attributes, - comment, - })); - } - None => { - return error!(ErrorKind::ExpectedTermField { entry_id: id.name }); - } - } - } - - if pattern.is_none() && attributes.is_none() { - return error!(ErrorKind::ExpectedMessageField { entry_id: id.name }); - } - - Ok(ast::Entry::Message(ast::Message { - id, - value: pattern, - attributes, - comment, - })) -} - -fn get_attribute(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - ps.expect_char('.')?; - - let key = get_identifier(ps, false)?; - - ps.skip_inline_ws(); - ps.expect_char('=')?; - - if ps.is_peek_pattern_start() { - ps.skip_indent(); - let value = get_pattern(ps)?; - if let Some(value) = value { - return Ok(ast::Attribute { id: key, value }); - } - } - error!(ErrorKind::MissingValue) -} - -fn get_attributes(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut attributes = vec![]; - loop { - ps.expect_indent()?; - let attr = get_attribute(ps)?; - attributes.push(attr); - - if !ps.is_peek_next_line_attribute_start() { - break; - } - } - Ok(attributes) -} - -fn get_entry_identifier(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - get_identifier(ps, true) -} - -fn get_identifier(ps: &mut ParserStream, allow_term: bool) -> Result -where - I: Iterator, -{ - let mut name = String::new(); - - name.push(ps.take_id_start(allow_term)?); - - while let Some(ch) = ps.take_id_char() { - name.push(ch); - } - - Ok(ast::Identifier { name }) -} - -fn get_variant_key(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if let Some(ch) = ps.current() { - match ch { - '0'...'9' | '-' => { - return Ok(ast::VarKey::Number(get_number(ps)?)); - } - _ => { - return Ok(ast::VarKey::VariantName(get_variant_name(ps)?)); - } - } - } else { - return error!(ErrorKind::MissingVariantKey); - } -} - -fn get_variant(ps: &mut ParserStream, has_default: bool) -> Result -where - I: Iterator, -{ - let default_index = if ps.current_is('*') { - if has_default { - return error!(ErrorKind::MultipleDefaultVariants); - } - ps.next(); - true - } else { - false - }; - - ps.expect_char('[')?; - - let key = get_variant_key(ps)?; - - ps.expect_char(']')?; - - if ps.is_peek_pattern_start() { - ps.skip_indent(); - if let Some(value) = get_pattern(ps)? { - return Ok(ast::Variant { - key, - value, - default: default_index, - }); - } - } - return error!(ErrorKind::MissingValue); -} - -fn get_variants(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut variants = vec![]; - let mut has_default = false; - - loop { - ps.expect_indent()?; - let variant = get_variant(ps, has_default)?; - - if variant.default { - has_default = true; - } - - variants.push(variant); - - if !ps.is_peek_next_line_variant_start() { - break; - } - } - if !has_default { - return error!(ErrorKind::MissingDefaultVariant); - } - Ok(variants) -} - -fn get_variant_name(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut name = String::new(); - - name.push(ps.take_id_start(false)?); - - while let Some(ch) = ps.take_variant_name_char() { - name.push(ch); - } - - while name.ends_with(' ') { - name.pop(); - } - - Ok(ast::VariantName { name }) -} - -fn get_digits(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut num = String::new(); - - while let Some(ch) = ps.take_digit() { - num.push(ch); - } - - if num.is_empty() { - return error!(ErrorKind::ExpectedCharRange { - range: "0...9".to_owned(), - }); - } - - Ok(num) -} - -fn get_number(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut num = String::new(); - - if ps.current_is('-') { - num.push('-'); - ps.next(); - } - - num.push_str(&get_digits(ps)?); - - if ps.current_is('.') { - num.push('.'); - ps.next(); - num.push_str(&get_digits(ps)?); - } - Ok(ast::Number { value: num }) -} - -fn get_pattern(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut elements = vec![]; - - ps.skip_inline_ws(); - - while let Some(ch) = ps.current() { - if ch == '\n' && !ps.is_peek_next_line_pattern_start() { - break; - } - - match ch { - '{' => { - elements.push(get_placeable(ps)?); - } - _ => { - elements.push(get_text_element(ps)?); - } - } - } - - Ok(Some(ast::Pattern { elements })) -} - -fn get_text_element(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut buf = String::new(); - - while let Some(ch) = ps.current() { - match ch { - '{' => return Ok(ast::PatternElement::TextElement(buf)), - '\n' => { - if !ps.is_peek_next_line_pattern_start() { - return Ok(ast::PatternElement::TextElement(buf)); - } - ps.next(); - ps.skip_inline_ws(); - - // Add the new line to the buffer - buf.push(ch); - continue; - } - '\\' => { - if let Some(ch2) = ps.next() { - if ch2 == '{' || ch2 == '"' { - buf.push(ch2); - } else { - buf.push(ch); - buf.push(ch2); - } - } - } - _ => buf.push(ch), - } - ps.next(); - } - - Ok(ast::PatternElement::TextElement(buf)) -} - -fn get_placeable(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - ps.expect_char('{')?; - let expression = get_expression(ps)?; - ps.expect_char('}')?; - Ok(ast::PatternElement::Placeable(expression)) -} - -fn get_expression(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if ps.is_peek_next_line_variant_start() { - let variants = get_variants(ps)?; - - ps.expect_indent()?; - - return Ok(ast::Expression::SelectExpression { - expression: None, - variants, - }); - } - - ps.skip_inline_ws(); - - let selector = get_selector_expression(ps)?; - - ps.skip_inline_ws(); - - if ps.current_is('-') { - ps.peek(); - - if !ps.current_peek_is('>') { - ps.reset_peek(None); - return Ok(selector); - } - - match selector { - ast::Expression::MessageReference { .. } => { - return error!(ErrorKind::MessageReferenceAsSelector) - } - ast::Expression::AttributeExpression { ref id, .. } => { - if !id.name.starts_with('-') { - return error!(ErrorKind::MessageAttributeAsSelector); - } - } - ast::Expression::VariantExpression { .. } => { - return error!(ErrorKind::VariantAsSelector) - } - _ => {} - }; - - ps.next(); - ps.next(); - - ps.skip_inline_ws(); - - let variants = get_variants(ps)?; - - if variants.is_empty() { - return error!(ErrorKind::MissingVariants); - } - - ps.expect_indent()?; - - return Ok(ast::Expression::SelectExpression { - expression: Some(Box::new(selector)), - variants, - }); - } else if let ast::Expression::AttributeExpression { ref id, .. } = selector { - if id.name.starts_with('-') { - return error!(ErrorKind::TermAttributeAsSelector); - } - } - - Ok(selector) -} - -fn get_selector_expression(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let literal = get_literal(ps)?; - - match literal { - ast::Expression::MessageReference { id } => match ps.ch { - Some('.') => { - ps.next(); - let attr = get_identifier(ps, false)?; - Ok(ast::Expression::AttributeExpression { id, name: attr }) - } - Some('[') => { - ps.next(); - let key = get_variant_key(ps)?; - ps.expect_char(']')?; - - Ok(ast::Expression::VariantExpression { id, key }) - } - Some('(') => { - if id.name.starts_with('-') || id.name.chars().any(|c| c.is_lowercase()) { - return error!(ErrorKind::ForbiddenCallee); - } - ps.next(); - let args = get_call_args(ps)?; - ps.expect_char(')')?; - - // XXX Make sure that id.name is [A-Z][A-Z_?-]* - Ok(ast::Expression::CallExpression { - callee: ast::Function { name: id.name }, - args, - }) - } - _ => Ok(ast::Expression::MessageReference { id }), - }, - _ => Ok(literal), - } -} - -fn get_call_arg(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let exp = get_selector_expression(ps)?; - - if !ps.current_is(':') { - return Ok(ast::Argument::Expression(exp)); - } - - match exp { - ast::Expression::MessageReference { id } => { - ps.next(); - ps.skip_inline_ws(); - - let val = get_arg_val(ps)?; - Ok(ast::Argument::NamedArgument { name: id, val }) - } - _ => error!(ErrorKind::ForbiddenKey), - } -} - -fn get_call_args(ps: &mut ParserStream) -> Result> -where - I: Iterator, -{ - let mut args = vec![]; - - ps.skip_inline_ws(); - - loop { - if ps.current_is(')') { - break; - } - - let arg = get_call_arg(ps)?; - args.push(arg); - - ps.skip_inline_ws(); - - if ps.current_is(',') { - ps.next(); - ps.skip_inline_ws(); - continue; - } else { - break; - } - } - - Ok(args) -} - -fn get_arg_val(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if ps.is_number_start() { - return Ok(ast::ArgValue::Number(get_number(ps)?)); - } else if ps.current_is('"') { - return Ok(ast::ArgValue::String(get_string(ps)?)); - } - error!(ErrorKind::MissingValue) -} - -fn get_string(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - let mut val = String::new(); - - ps.expect_char('"')?; - - while let Some(ch) = ps.take_char(|x| x != '"' && x != '\n') { - val.push(ch); - } - - if ps.current_is('\n') { - return error!(ErrorKind::UnterminatedStringExpression); - } - - ps.next(); - - Ok(val) -} - -fn get_literal(ps: &mut ParserStream) -> Result -where - I: Iterator, -{ - if let Some(ch) = ps.current() { - let exp = match ch { - '0'...'9' => ast::Expression::NumberExpression { - value: get_number(ps)?, - }, - '-' => { - if let Some('0'...'9') = ps.peek() { - ps.reset_peek(None); - ast::Expression::NumberExpression { - value: get_number(ps)?, - } - } else { - ps.reset_peek(None); - ast::Expression::MessageReference { - id: get_entry_identifier(ps)?, - } - } - } - '"' => ast::Expression::StringExpression { - value: get_string(ps)?, - }, - '$' => { - ps.next(); - ast::Expression::ExternalArgument { - id: get_identifier(ps, false)?, - } - } - _ => ast::Expression::MessageReference { - id: get_entry_identifier(ps)?, - }, - }; - Ok(exp) - } else { - return error!(ErrorKind::MissingLiteral); - } -} - -fn get_junk_entry(ps: &mut ParserStream, source: &str, entry_start: usize) -> ast::Entry -where - I: Iterator, -{ - ps.skip_to_next_entry_start(); - - let slice = get_error_slice(source, entry_start, ps.get_index()); - - ast::Entry::Junk { - content: String::from(slice), - } -} diff --git a/fluent-syntax/src/parser/stream.rs b/fluent-syntax/src/parser/stream.rs deleted file mode 100644 index 816413c8..00000000 --- a/fluent-syntax/src/parser/stream.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::iter::Fuse; - -#[derive(Clone, Debug)] -pub struct ParserStream -where - I: Iterator, -{ - iter: Fuse, - pub buf: Vec, - peek_index: usize, - index: usize, - - pub ch: Option, - - iter_end: bool, - peek_end: bool, -} - -impl> ParserStream { - pub fn new(iterable: I) -> ParserStream { - let mut iter = iterable.into_iter().fuse(); - let ch = iter.next(); - - ParserStream { - iter, - buf: vec![], - peek_index: 0, - index: 0, - ch, - - iter_end: false, - peek_end: false, - } - } - - pub fn current(&mut self) -> Option { - self.ch - } - - pub fn current_is(&mut self, ch: char) -> bool { - self.ch == Some(ch) - } - - pub fn current_peek(&self) -> Option { - if self.peek_end { - return None; - } - - let diff = self.peek_index - self.index; - - if diff == 0 { - self.ch - } else { - Some(self.buf[diff - 1]) - } - } - - pub fn current_peek_is(&mut self, ch: char) -> bool { - self.current_peek() == Some(ch) - } - - pub fn peek(&mut self) -> Option { - if self.peek_end { - return None; - } - - self.peek_index += 1; - - let diff = self.peek_index - self.index; - - if diff > self.buf.len() { - match self.iter.next() { - Some(c) => { - self.buf.push(c); - } - None => { - self.peek_end = true; - return None; - } - } - } - - Some(self.buf[diff - 1]) - } - - pub fn get_index(&self) -> usize { - self.index - } - - pub fn get_peek_index(&self) -> usize { - self.peek_index - } - - pub fn peek_char_is(&mut self, ch: char) -> bool { - if self.peek_end { - return false; - } - - let ret = self.peek() == Some(ch); - - self.peek_index -= 1; - ret - } - - pub fn reset_peek(&mut self, pos: Option) { - match pos { - Some(pos) => { - if pos < self.peek_index { - self.peek_end = false - } - self.peek_index = pos - } - None => { - self.peek_index = self.index; - self.peek_end = self.iter_end; - } - } - } - - pub fn skip_to_peek(&mut self) { - let diff = self.peek_index - self.index; - - for _ in 0..diff { - self.ch = Some(self.buf.remove(0)); - } - - self.index = self.peek_index; - } -} - -impl Iterator for ParserStream -where - I: Iterator, -{ - type Item = char; - - fn next(&mut self) -> Option { - if self.iter_end { - return None; - } - - self.ch = if self.buf.is_empty() { - self.iter.next() - } else { - Some(self.buf.remove(0)) - }; - - self.index += 1; - - if self.ch.is_none() { - self.iter_end = true; - self.peek_end = true; - } - - self.peek_index = self.index; - - self.ch - } -} diff --git a/fluent-syntax/tests/ast/mod.rs b/fluent-syntax/tests/ast/mod.rs new file mode 100644 index 00000000..edf04fab --- /dev/null +++ b/fluent-syntax/tests/ast/mod.rs @@ -0,0 +1,400 @@ +use fluent_syntax::ast; +use serde::ser::SerializeMap; +use serde::ser::SerializeSeq; +use serde::{Serialize, Serializer}; +use serde_derive::Serialize; +use std::error::Error; + +pub fn serialize<'s>(res: &'s ast::Resource) -> Result> { + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); + + let buf = Vec::new(); + let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); + let mut ser = serde_json::Serializer::with_formatter(buf, formatter); + Helper(res).serialize(&mut ser).unwrap(); + Ok(String::from_utf8(ser.into_inner()).unwrap()) +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::Resource")] +struct ResourceDef<'ast> { + #[serde(serialize_with = "serialize_resource_entry_vec")] + body: Vec>, +} + +fn serialize_resource_entry_vec<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + #[serde(tag = "type")] + enum EntryHelper<'ast> { + Junk { + annotations: Vec<&'ast str>, + content: &'ast str, + }, + #[serde(with = "MessageDef")] + Message(&'ast ast::Message<'ast>), + #[serde(with = "TermDef")] + Term(&'ast ast::Term<'ast>), + Comment { + content: String, + }, + GroupComment { + content: String, + }, + ResourceComment { + content: String, + }, + } + + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + let entry = match *e { + ast::ResourceEntry::Entry(ref entry) => match entry { + ast::Entry::Message(ref msg) => EntryHelper::Message(msg), + ast::Entry::Term(ref term) => EntryHelper::Term(term), + ast::Entry::Comment(ast::Comment::Comment { ref content }) => { + EntryHelper::Comment { + content: content.join("\n"), + } + } + ast::Entry::Comment(ast::Comment::GroupComment { ref content }) => { + EntryHelper::GroupComment { + content: content.join("\n"), + } + } + ast::Entry::Comment(ast::Comment::ResourceComment { ref content }) => { + EntryHelper::ResourceComment { + content: content.join("\n"), + } + } + }, + ast::ResourceEntry::Junk(ref junk) => EntryHelper::Junk { + content: junk, + annotations: vec![], + }, + }; + seq.serialize_element(&entry)?; + } + seq.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::Message")] +pub struct MessageDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(serialize_with = "serialize_pattern_option")] + pub value: Option>, + #[serde(serialize_with = "serialize_attribute_vec")] + pub attributes: Vec>, + #[serde(serialize_with = "serialize_comment_option")] + pub comment: Option>, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Term")] +pub struct TermDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(with = "ValueDef")] + pub value: ast::Value<'ast>, + #[serde(serialize_with = "serialize_attribute_vec")] + pub attributes: Vec>, + #[serde(serialize_with = "serialize_comment_option")] + pub comment: Option>, +} + +fn serialize_pattern_option<'se, S>( + v: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "PatternDef")] &'ast ast::Pattern<'ast>); + v.as_ref().map(Helper).serialize(serializer) +} + +fn serialize_attribute_vec<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "AttributeDef")] &'ast ast::Attribute<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_comment_option<'se, S>( + v: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "CommentDef")] &'ast ast::Comment<'ast>); + v.as_ref().map(Helper).serialize(serializer) +} + +#[derive(Serialize)] +#[serde(remote = "ast::Value")] +#[serde(untagged)] +pub enum ValueDef<'ast> { + #[serde(with = "PatternDef")] + Pattern(ast::Pattern<'ast>), + VariantList { + #[serde(serialize_with = "serialize_variants")] + variants: Vec>, + }, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Pattern")] +pub struct PatternDef<'ast> { + #[serde(serialize_with = "serialize_pattern_elements")] + pub elements: Vec>, +} + +fn serialize_pattern_elements<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "PatternElementDef")] &'ast ast::PatternElement<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::PatternElement")] +#[serde(untagged)] +pub enum PatternElementDef<'ast> { + #[serde(serialize_with = "serialize_text_element")] + TextElement(&'ast str), + #[serde(serialize_with = "serialize_placeable")] + Placeable(ast::Expression<'ast>), +} + +fn serialize_text_element<'se, S>(s: &'se str, serializer: S) -> Result +where + S: Serializer, +{ + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "TextElement")?; + map.serialize_entry("value", s)?; + map.end() +} + +fn serialize_placeable<'se, S>(exp: &ast::Expression<'se>, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "ExpressionDef")] &'ast ast::Expression<'ast>); + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "Placeable")?; + map.serialize_entry("expression", &Helper(exp))?; + map.end() +} + +#[derive(Serialize)] +#[serde(remote = "ast::Attribute")] +pub struct AttributeDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(with = "PatternDef")] + pub value: ast::Pattern<'ast>, +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::Identifier")] +struct IdentifierDef<'ast> { + name: &'ast str, +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::Function")] +struct FunctionDef<'ast> { + name: &'ast str, +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::Variant")] +struct VariantDef<'ast> { + #[serde(with = "VariantKeyDef")] + pub key: ast::VariantKey<'ast>, + #[serde(with = "ValueDef")] + pub value: ast::Value<'ast>, + pub default: bool, +} + +#[derive(Serialize, Debug)] +#[serde(remote = "ast::VariantKey")] +#[serde(untagged)] +pub enum VariantKeyDef<'ast> { + Identifier { name: &'ast str }, + NumberLiteral { value: &'ast str }, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Comment")] +#[serde(tag = "type")] +pub enum CommentDef<'ast> { + Comment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, + GroupComment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, + ResourceComment { + #[serde(serialize_with = "serialize_comment_content")] + content: Vec<&'ast str>, + }, +} + +fn serialize_comment_content<'se, S>(v: &Vec<&'se str>, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&v.join("\n")) +} + +#[derive(Serialize)] +#[serde(remote = "ast::InlineExpression")] +#[serde(tag = "type")] +pub enum InlineExpressionDef<'ast> { + StringLiteral { + value: &'ast str, + }, + NumberLiteral { + value: &'ast str, + }, + VariableReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + CallExpression { + #[serde(with = "FunctionDef")] + callee: ast::Function<'ast>, + #[serde(serialize_with = "serialize_inline_expressions")] + positional: Vec>, + #[serde(serialize_with = "serialize_named_arguments")] + named: Vec>, + }, + AttributeExpression { + #[serde(with = "InlineExpressionDef")] + #[serde(rename = "ref")] + reference: ast::InlineExpression<'ast>, + #[serde(with = "IdentifierDef")] + name: ast::Identifier<'ast>, + }, + VariantExpression { + #[serde(with = "InlineExpressionDef")] + #[serde(rename = "ref")] + reference: ast::InlineExpression<'ast>, + #[serde(with = "VariantKeyDef")] + key: ast::VariantKey<'ast>, + }, + MessageReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + TermReference { + #[serde(with = "IdentifierDef")] + id: ast::Identifier<'ast>, + }, + Placeable { + #[serde(with = "ExpressionDef")] + expression: ast::Expression<'ast>, + }, +} + +#[derive(Serialize)] +#[serde(remote = "ast::NamedArgument")] +pub struct NamedArgumentDef<'ast> { + #[serde(with = "IdentifierDef")] + pub name: ast::Identifier<'ast>, + #[serde(with = "InlineExpressionDef")] + pub value: ast::InlineExpression<'ast>, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Expression")] +#[serde(untagged)] +pub enum ExpressionDef<'ast> { + #[serde(with = "InlineExpressionDef")] + InlineExpression(ast::InlineExpression<'ast>), + SelectExpression { + #[serde(with = "InlineExpressionDef")] + selector: ast::InlineExpression<'ast>, + #[serde(serialize_with = "serialize_variants")] + variants: Vec>, + }, +} + +fn serialize_variants<'se, S>(v: &Vec>, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "VariantDef")] &'ast ast::Variant<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_inline_expressions<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "InlineExpressionDef")] &'ast ast::InlineExpression<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} + +fn serialize_named_arguments<'se, S>( + v: &Vec>, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>(#[serde(with = "NamedArgumentDef")] &'ast ast::NamedArgument<'ast>); + let mut seq = serializer.serialize_seq(Some(v.len()))?; + for e in v { + seq.serialize_element(&Helper(e))?; + } + seq.end() +} diff --git a/fluent-syntax/tests/errors.rs b/fluent-syntax/tests/errors.rs deleted file mode 100644 index 76cf5f64..00000000 --- a/fluent-syntax/tests/errors.rs +++ /dev/null @@ -1,291 +0,0 @@ -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::errors::display::annotate_error; -use self::fluent_syntax::parser::errors::ErrorInfo; -use self::fluent_syntax::parser::errors::ErrorKind; -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -#[test] -fn empty_errors() { - let path = "./tests/fixtures/parser/ftl/errors/01-empty.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedEntry, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: " key = value".to_owned(), - line: 0, - pos: 0, - },), - error1.info - ); - } - } -} - -#[test] -fn bad_id_start_errors() { - let path = "./tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedEntry, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "2".to_owned(), - line: 0, - pos: 0, - },), - error1.info - ); - } - } -} - -#[test] -fn just_id_errors() { - let path = "./tests/fixtures/parser/ftl/errors/03-just-id.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key".to_owned(), - line: 0, - pos: 3, - },), - error1.info - ); - } - } -} - -#[test] -fn no_equal_sign_errors() { - let path = "./tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '=' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key Value".to_owned(), - line: 0, - pos: 4, - },), - error1.info - ); - } - } -} - -#[test] -fn wrong_char_in_id_errors() { - let path = "./tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "'a'...'z' | 'A'...'Z'".to_owned(), - }, - error1.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key = Value\n .# = Foo".to_owned(), - line: 0, - pos: 14, - },), - error1.info - ); - } - } -} - -#[test] -fn missing_trait_value_errors() { - let path = "./tests/fixtures/parser/ftl/errors/06-trait-value.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key = Value\n .foo".to_owned(), - line: 0, - pos: 17, - },), - error1.info - ); - } - } -} - -#[test] -fn message_missing_fields_errors() { - let path = "./tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(1, errors.len()); - - let error1 = &errors[0]; - - assert_eq!(ErrorKind::ExpectedToken { token: '\u{2424}' }, error1.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key".to_owned(), - line: 0, - pos: 3, - },), - error1.info - ); - } - } -} - -#[test] -fn private_errors() { - let path = "./tests/fixtures/parser/ftl/errors/08-private.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected errors in the file"), - Err((_, ref errors)) => { - assert_eq!(4, errors.len()); - - let error1 = &errors[0]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "0...9".to_owned(), - }, - error1.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key =\n { $foo ->\n [one] Foo\n *[-other] Foo 2\n }" - .to_owned(), - line: 1, - pos: 48, - },), - error1.info - ); - - let error2 = &errors[1]; - - assert_eq!( - ErrorKind::ExpectedCharRange { - range: "'a'...'z' | 'A'...'Z'".to_owned(), - }, - error2.kind - ); - - assert_eq!( - Some(ErrorInfo { - slice: "key2 = { $-foo }".to_owned(), - line: 7, - pos: 10, - },), - error2.info - ); - - let error3 = &errors[2]; - - assert_eq!(ErrorKind::TermAttributeAsSelector, error3.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key3 = { -brand.gender }".to_owned(), - line: 9, - pos: 23, - },), - error3.info - ); - - let error4 = &errors[3]; - - assert_eq!(ErrorKind::ForbiddenCallee, error4.kind); - - assert_eq!( - Some(ErrorInfo { - slice: "key4 = { -brand() }".to_owned(), - line: 11, - pos: 15, - },), - error4.info - ); - } - } -} - -#[test] -fn test_annotate_errors() { - let input = "key Value"; - - let res = parse(input); - - match res { - Ok(_) => panic!("Should have return an error!"), - Err((_, errors)) => { - assert_eq!(errors.len(), 1); - let err = annotate_error(&errors[0], &None, false); - assert_eq!( - err, - "error[E0003]: expected token `=`\n |\n0 | key Value\n | ^\n |" - ); - } - } -} diff --git a/fluent-syntax/tests/fixtures.rs b/fluent-syntax/tests/fixtures.rs deleted file mode 100644 index b6ceab51..00000000 --- a/fluent-syntax/tests/fixtures.rs +++ /dev/null @@ -1,67 +0,0 @@ -extern crate fluent_syntax; -extern crate glob; - -use self::glob::glob; -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -fn attempt_parse(source: &str) -> Result<(), ()> { - match parse(source) { - Ok(_) => Ok(()), - Err(_) => Err(()), - } -} - -#[test] -fn parse_ftl() { - for entry in glob("./tests/fixtures/parser/ftl/*.ftl").expect("Failed to read glob pattern") { - let p = entry.expect("Error while getting an entry"); - let path = p.to_str().expect("Can't print path"); - - if path.contains("errors") { - continue; - } - - println!("Attempting to parse file: {}", path); - - let string = read_file(path).expect("Failed to read"); - - attempt_parse(&string).expect("Failed to parse"); - } -} - -#[test] -fn error_ftl() { - for entry in glob("./tests/fixtures/parser/ftl/*.ftl").expect("Failed to read glob pattern") { - let p = entry.expect("Error while getting an entry"); - let path = p.to_str().expect("Can't print path"); - - if !path.contains("errors") { - continue; - } - - println!("Attempting to parse error file: {}", path); - - let string = read_file(path).expect("Failed to read"); - - let chunks = string.split("\n\n"); - - for chunk in chunks { - println!("Testing chunk: {:?}", chunk); - match attempt_parse(chunk) { - Ok(_) => panic!("Test didn't fail"), - Err(_) => continue, - } - } - } -} diff --git a/fluent-syntax/tests/fixtures/Makefile b/fluent-syntax/tests/fixtures/Makefile new file mode 100644 index 00000000..49c98e72 --- /dev/null +++ b/fluent-syntax/tests/fixtures/Makefile @@ -0,0 +1,11 @@ +FTL_FIXTURES := $(wildcard *.ftl) +AST_FIXTURES := $(FTL_FIXTURES:%.ftl=%.json) + +all: $(AST_FIXTURES) + +.PHONY: $(AST_FIXTURES) +$(AST_FIXTURES): %.json: %.ftl + @node --experimental-modules ../../bin/parse.mjs $< \ + 2> /dev/null \ + 1> $@; + @echo "$< → $@" diff --git a/fluent-syntax/tests/fixtures/astral.ftl b/fluent-syntax/tests/fixtures/astral.ftl new file mode 100644 index 00000000..b77e32e3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/astral.ftl @@ -0,0 +1,20 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 + +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = {"\uD83D\uDE02"} +surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} + +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = {"A face 😂 with tears of joy."} + +# ERROR Invalid identifier +err-😂 = Value + +# ERROR Invalid expression +err-invalid-expression = { 😂 } + +# ERROR Invalid variant key +err-invalid-variant-key = { $sel -> + *[😂] Value +} diff --git a/fluent-syntax/tests/fixtures/astral.json b/fluent-syntax/tests/fixtures/astral.json new file mode 100644 index 00000000..e3627269 --- /dev/null +++ b/fluent-syntax/tests/fixtures/astral.json @@ -0,0 +1,159 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "face-with-tears-of-joy" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "😂" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "tetragram-for-centre" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "𝌆" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "surrogates-in-text" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "\\uD83D\\uDE02" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "surrogates-in-string" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "surrogates-in-adjacent-strings" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "emoji-in-text" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A face 😂 with tears of joy." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "emoji-in-string" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "A face 😂 with tears of joy." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Invalid identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-😂 = Value\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid expression" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-expression = { 😂 }\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid variant key" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/call_expressions.ftl b/fluent-syntax/tests/fixtures/call_expressions.ftl new file mode 100644 index 00000000..06562642 --- /dev/null +++ b/fluent-syntax/tests/fixtures/call_expressions.ftl @@ -0,0 +1,102 @@ +positional-args = {FUN(1, "a", msg)} +named-args = {FUN(x: 1, y: "Y")} +dense-named-args = {FUN(x:1, y:"Y")} +mixed-args = {FUN(1, "a", msg, x: 1, y: "Y")} + +# ERROR Positional arg must not follow keyword args +shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} + +# ERROR Named arguments must be unique +duplicate-named-args = {FUN(x: 1, x: "X")} + + +## Whitespace around arguments + +sparse-inline-call = {FUN( "a" , msg, x: 1 )} +empty-inline-call = {FUN( )} +multiline-call = {FUN( + "a", + msg, + x: 1 + )} +sparse-multiline-call = {FUN( + + "a" , + msg + , x: 1 + )} +empty-multiline-call = {FUN( + + )} + + +unindented-arg-number = {FUN( +1)} + +unindented-arg-string = {FUN( +"a")} + +unindented-arg-msg-ref = {FUN( +msg)} + +unindented-arg-term-ref = {FUN( +-msg)} + +unindented-arg-var-ref = {FUN( +$var)} + +unindented-arg-call = {FUN( +OTHER())} + +unindented-named-arg = {FUN( +x:1)} + +unindented-closing-paren = {FUN( + x +)} + + + +## Optional trailing comma + +one-argument = {FUN(1,)} +many-arguments = {FUN(1, 2, 3,)} +inline-sparse-args = {FUN( 1, 2, 3, )} +mulitline-args = {FUN( + 1, + 2, + )} +mulitline-sparse-args = {FUN( + + 1 + , + 2 + , + )} + + +## Syntax errors for trailing comma + +one-argument = {FUN(1,,)} +missing-arg = {FUN(,)} +missing-sparse-arg = {FUN( , )} + + +## Whitespace in named arguments + +sparse-named-arg = {FUN( + x : 1, + y : 2, + z + : + 3 + )} + + +unindented-colon = {FUN( + x +:1)} + +unindented-value = {FUN( + x: +1)} diff --git a/fluent-syntax/tests/fixtures/call_expressions.json b/fluent-syntax/tests/fixtures/call_expressions.json new file mode 100644 index 00000000..3a49c4f4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/call_expressions.json @@ -0,0 +1,922 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "positional-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "named-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "name": { + "name": "y" + }, + "value": { + "type": "StringLiteral", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "dense-named-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "name": { + "name": "y" + }, + "value": { + "type": "StringLiteral", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "mixed-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "name": { + "name": "y" + }, + "value": { + "type": "StringLiteral", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Positional arg must not follow keyword args" + }, + { + "type": "Junk", + "annotations": [], + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n" + }, + { + "type": "Comment", + "content": "ERROR Named arguments must be unique" + }, + { + "type": "Junk", + "annotations": [], + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n" + }, + { + "type": "GroupComment", + "content": "Whitespace around arguments" + }, + { + "type": "Message", + "id": { + "name": "sparse-inline-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "StringLiteral", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "empty-inline-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "multiline-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "StringLiteral", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "sparse-multiline-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "StringLiteral", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "empty-multiline-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-number" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-string" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "StringLiteral", + "value": "a" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-msg-ref" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-term-ref" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "TermReference", + "id": { + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-var-ref" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "VariableReference", + "id": { + "name": "var" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-arg-call" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "CallExpression", + "callee": { + "name": "OTHER" + }, + "positional": [], + "named": [] + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-named-arg" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-closing-paren" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "name": "x" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Optional trailing comma" + }, + { + "type": "Message", + "id": { + "name": "one-argument" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "many-arguments" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "inline-sparse-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "mulitline-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "mulitline-sparse-args" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Syntax errors for trailing comma" + }, + { + "type": "Junk", + "annotations": [], + "content": "one-argument = {FUN(1,,)}\nmissing-arg = {FUN(,)}\nmissing-sparse-arg = {FUN( , )}\n" + }, + { + "type": "GroupComment", + "content": "Whitespace in named arguments" + }, + { + "type": "Message", + "id": { + "name": "sparse-named-arg" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "name": { + "name": "y" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + }, + { + "name": { + "name": "z" + }, + "value": { + "type": "NumberLiteral", + "value": "3" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-colon" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unindented-value" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "name": "FUN" + }, + "positional": [], + "named": [ + { + "name": { + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/comments.ftl b/fluent-syntax/tests/fixtures/comments.ftl new file mode 100644 index 00000000..cc3246ea --- /dev/null +++ b/fluent-syntax/tests/fixtures/comments.ftl @@ -0,0 +1,15 @@ +# Standalone Comment + +# Message Comment +foo = Foo + +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent +## Group Comment +### Resource Comment diff --git a/fluent-syntax/tests/fixtures/comments.json b/fluent-syntax/tests/fixtures/comments.json new file mode 100644 index 00000000..edce1b17 --- /dev/null +++ b/fluent-syntax/tests/fixtures/comments.json @@ -0,0 +1,58 @@ +{ + "body": [ + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Message", + "id": { + "name": "foo" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Foo" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Term", + "id": { + "name": "term" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Term" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Term Comment\nwith a blank last line.\n" + } + }, + { + "type": "Comment", + "content": "Another standalone\n\n with indent" + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/convert.js b/fluent-syntax/tests/fixtures/convert.js new file mode 100755 index 00000000..24c8549c --- /dev/null +++ b/fluent-syntax/tests/fixtures/convert.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node + +'use strict'; +const fs = require('fs'); + +fs.readdirSync("./").forEach(file => { + if (file.endsWith(".json")) { + fs.readFile(file, convert.bind(this, file)); + } +}) + +function write(s, fileName) { + const data = new Uint8Array(Buffer.from(s)); + fs.writeFile(fileName, data, (err) => { + if (err) throw err; + console.log(`The file "${fileName}" has been saved!`); + }); +} + + +function convert(fileName, err, data) { + let ast = JSON.parse(data.toString()); + + remove_leading_dash_from_term(ast); + remove_unambigous_types(ast); + split_multiline_text_elements(ast); + + let s = JSON.stringify(ast, null, 4); + write(s, fileName); +} + +function remove_leading_dash_from_term(ast) { + for (let i in ast) { + let node = ast[i]; + if (Array.isArray(node)) { + remove_leading_dash_from_term(node); + } else if (typeof node === "object") { + remove_leading_dash_from_term(node); + } else if (i === "name" && + typeof node == "string" && + node.startsWith("-")) { + ast[i] = node.substr(1); + } + } +} + +function remove_unambigous_types(ast, parent_node = null, parent_key = null) { + for (let i in ast) { + let node = ast[i]; + if (Array.isArray(node)) { + remove_unambigous_types(node, ast); + } else if (typeof node === "object") { + remove_unambigous_types(node, ast, i); + } else if (i === "type" && + parent_key !== "selector" && + ["Resource", + "Pattern", + "Function", + "Variant", + "SelectExpression", + "Attribute", + "NamedArgument", + "VariantList", + "Identifier"].includes(node)) { + ast[i] = undefined; + } else if (parent_key == "key" && + ["NumberLiteral"].includes(node)) { + ast[i] = undefined; + } + } +} + +function split_multiline_text_elements(ast) { + for (let i in ast) { + let node = ast[i]; + if (Array.isArray(node)) { + split_multiline_text_elements(node); + } else if (typeof node === "object" && + node !== null && + node["type"] === "TextElement") { + let parts = node["value"].split("\n"); + let elements = parts.filter(v => v != "").map((v, i) => { + let last = parts.length - 1 === i; + return { + "type": "TextElement", + "value": last ? v : `${v}\n` + }; + }); + ast = ast.splice(i, 1, ...elements); + } else if (typeof node === "object") { + split_multiline_text_elements(node); + } + } +} diff --git a/fluent-syntax/tests/fixtures/crlf.ftl b/fluent-syntax/tests/fixtures/crlf.ftl new file mode 100644 index 00000000..538d7adc --- /dev/null +++ b/fluent-syntax/tests/fixtures/crlf.ftl @@ -0,0 +1,7 @@ +key01 = Value 01 +key02 = + Value 02 + Continued + +# ERROR (Missing value or attributes) +key03 diff --git a/fluent-syntax/tests/fixtures/crlf.json b/fluent-syntax/tests/fixtures/crlf.json new file mode 100644 index 00000000..be7910d3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/crlf.json @@ -0,0 +1,49 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value 01" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value 02\r\n" + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR (Missing value or attributes)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key03\r\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_comment.ftl b/fluent-syntax/tests/fixtures/eof_comment.ftl new file mode 100644 index 00000000..cdeafd90 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_comment.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +# No EOL \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_comment.json b/fluent-syntax/tests/fixtures/eof_comment.json new file mode 100644 index 00000000..ac094a54 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_comment.json @@ -0,0 +1,12 @@ +{ + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Comment", + "content": "No EOL" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_empty.ftl b/fluent-syntax/tests/fixtures/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent-syntax/tests/fixtures/eof_empty.json b/fluent-syntax/tests/fixtures/eof_empty.json new file mode 100644 index 00000000..33f00124 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_empty.json @@ -0,0 +1,3 @@ +{ + "body": [] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id.ftl b/fluent-syntax/tests/fixtures/eof_id.ftl new file mode 100644 index 00000000..63fa86d6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id.json b/fluent-syntax/tests/fixtures/eof_id.json new file mode 100644 index 00000000..33b2ad6d --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id.json @@ -0,0 +1,13 @@ +{ + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id_equals.ftl b/fluent-syntax/tests/fixtures/eof_id_equals.ftl new file mode 100644 index 00000000..7d0d953a --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id_equals.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id = \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_id_equals.json b/fluent-syntax/tests/fixtures/eof_id_equals.json new file mode 100644 index 00000000..d515069b --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_id_equals.json @@ -0,0 +1,13 @@ +{ + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id =" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_junk.ftl b/fluent-syntax/tests/fixtures/eof_junk.ftl new file mode 100644 index 00000000..dbafd3a3 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_junk.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +000 \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_junk.json b/fluent-syntax/tests/fixtures/eof_junk.json new file mode 100644 index 00000000..94f1e77d --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_junk.json @@ -0,0 +1,13 @@ +{ + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "000" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_value.ftl b/fluent-syntax/tests/fixtures/eof_value.ftl new file mode 100644 index 00000000..0d255c57 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/eof_value.json b/fluent-syntax/tests/fixtures/eof_value.json new file mode 100644 index 00000000..bbab1e43 --- /dev/null +++ b/fluent-syntax/tests/fixtures/eof_value.json @@ -0,0 +1,24 @@ +{ + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Message", + "id": { + "name": "no-eol" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "No EOL" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/escaped_characters.ftl b/fluent-syntax/tests/fixtures/escaped_characters.ftl new file mode 100644 index 00000000..d3a5a078 --- /dev/null +++ b/fluent-syntax/tests/fixtures/escaped_characters.ftl @@ -0,0 +1,9 @@ +backslash = Value with \\ (an escaped backslash) +closing-brace = Value with \{ (a closing brace) +unicode-escape = \u0041 +escaped-unicode = \\u0041 + +## String Expressions +quote-in-string = {"\""} +backslash-in-string = {"\\"} +mismatched-quote = {"\\""} diff --git a/fluent-syntax/tests/fixtures/escaped_characters.json b/fluent-syntax/tests/fixtures/escaped_characters.json new file mode 100644 index 00000000..c45748c0 --- /dev/null +++ b/fluent-syntax/tests/fixtures/escaped_characters.json @@ -0,0 +1,115 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "backslash" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value with \\\\ (an escaped backslash)" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "closing-brace" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value with \\{ (a closing brace)" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "unicode-escape" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "escaped-unicode" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "\\\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "String Expressions" + }, + { + "type": "Message", + "id": { + "name": "quote-in-string" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\"" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "backslash-in-string" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\\" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Junk", + "annotations": [], + "content": "mismatched-quote = {\"\\\\\"\"}\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/junk.ftl b/fluent-syntax/tests/fixtures/junk.ftl new file mode 100644 index 00000000..62fc2ea1 --- /dev/null +++ b/fluent-syntax/tests/fixtures/junk.ftl @@ -0,0 +1,4 @@ +ą=Invalid identifier +ć=Another one + +key01 = { diff --git a/fluent-syntax/tests/fixtures/junk.json b/fluent-syntax/tests/fixtures/junk.json new file mode 100644 index 00000000..5cfbfebe --- /dev/null +++ b/fluent-syntax/tests/fixtures/junk.json @@ -0,0 +1,14 @@ +{ + "body": [ + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "key01 = {\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/leading_dots.ftl b/fluent-syntax/tests/fixtures/leading_dots.ftl new file mode 100644 index 00000000..0b9d6693 --- /dev/null +++ b/fluent-syntax/tests/fixtures/leading_dots.ftl @@ -0,0 +1,76 @@ +key01 = .Value +key02 = …Value +key03 = {"."}Value +key04 = + {"."}Value + +key05 = Value + {"."}Continued + +key06 = .Value + {"."}Continued + +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value + .Continued + +# JUNK (attr .Value must have a value) +key08 = + .Value + +# JUNK (attr .Value must have a value) +key09 = + .Value + Continued + +key10 = + .Value = which is an attribute + Continued + +key11 = + {"."}Value = which looks like an attribute + Continued + +key12 = + .accesskey = + A + +key13 = + .attribute = .Value + +key14 = + .attribute = + {"."}Value + +key15 = + { 1 -> + [one] .Value + *[other] + {"."}Value + } + +# JUNK (variant must have a value) +key16 = + { 1 -> + *[one] + .Value + } + +# JUNK (unclosed placeable) +key17 = + { 1 -> + *[one] Value + .Continued + } + +# JUNK (attr .Value must have a value) +key18 = +.Value + +key19 = +.attribute = Value + Continued + +key20 = +{"."}Value diff --git a/fluent-syntax/tests/fixtures/leading_dots.json b/fluent-syntax/tests/fixtures/leading_dots.json new file mode 100644 index 00000000..d4f0702d --- /dev/null +++ b/fluent-syntax/tests/fixtures/leading_dots.json @@ -0,0 +1,443 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "…Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key05" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key06" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": ".Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key07" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "MESSAGE (value = \"Value\", attributes = [])\nJUNK (attr .Continued\" must have a value)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .Continued\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08 =\n .Value\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key09 =\n .Value\n Continued\n" + }, + { + "type": "Message", + "id": { + "name": "key10" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "Value" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "which is an attribute\n" + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key11" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value = which looks like an attribute\n" + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key12" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "accesskey" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key13" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attribute" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key14" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attribute" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key15" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "default": false + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK (variant must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n" + }, + { + "type": "Comment", + "content": "JUNK (unclosed placeable)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key18 =\n.Value\n" + }, + { + "type": "Message", + "id": { + "name": "key19" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attribute" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key20" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/literal_expressions.ftl b/fluent-syntax/tests/fixtures/literal_expressions.ftl new file mode 100644 index 00000000..937b8a17 --- /dev/null +++ b/fluent-syntax/tests/fixtures/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = {"abc"} +number-expression = {123} +number-expression = {-3.14} diff --git a/fluent-syntax/tests/fixtures/literal_expressions.json b/fluent-syntax/tests/fixtures/literal_expressions.json new file mode 100644 index 00000000..a97c61c6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/literal_expressions.json @@ -0,0 +1,61 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "string-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "abc" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "number-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "123" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "number-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "-3.14" + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/member_expressions.ftl b/fluent-syntax/tests/fixtures/member_expressions.ftl new file mode 100644 index 00000000..09e09f3f --- /dev/null +++ b/fluent-syntax/tests/fixtures/member_expressions.ftl @@ -0,0 +1,6 @@ +variant-expression = {-term[case]} +attribute-expression = {msg.attr} + +## Invalid syntax +variant-expression = {msg[case]} +attribute-expression = {-term.attr} diff --git a/fluent-syntax/tests/fixtures/member_expressions.json b/fluent-syntax/tests/fixtures/member_expressions.json new file mode 100644 index 00000000..ba34f217 --- /dev/null +++ b/fluent-syntax/tests/fixtures/member_expressions.json @@ -0,0 +1,67 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "variant-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariantExpression", + "ref": { + "type": "TermReference", + "id": { + "name": "term" + } + }, + "key": { + "name": "case" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "attribute-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "AttributeExpression", + "ref": { + "type": "MessageReference", + "id": { + "name": "msg" + } + }, + "name": { + "name": "attr" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Invalid syntax" + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-expression = {msg[case]}\nattribute-expression = {-term.attr}\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/messages.ftl b/fluent-syntax/tests/fixtures/messages.ftl new file mode 100644 index 00000000..0ade3cac --- /dev/null +++ b/fluent-syntax/tests/fixtures/messages.ftl @@ -0,0 +1,27 @@ +key01 = Value + +key02 = Value + .attr = Attribute + +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +key03 = + .attr = Attribute + +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +# < whitespace > +key05 = + .attr1 = Attribute 1 + +key06 = {""} + +# JUNK Missing value +key07 = + +# JUNK Missing = +key08 diff --git a/fluent-syntax/tests/fixtures/messages.json b/fluent-syntax/tests/fixtures/messages.json new file mode 100644 index 00000000..0ee318b9 --- /dev/null +++ b/fluent-syntax/tests/fixtures/messages.json @@ -0,0 +1,215 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "id": { + "name": "attr1" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "id": { + "name": "attr2" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr1" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "id": { + "name": "attr2" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key05" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr1" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + } + ], + "comment": { + "type": "Comment", + "content": " < whitespace >" + } + }, + { + "type": "Message", + "id": { + "name": "key06" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "key07 =\n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/mixed_entries.ftl b/fluent-syntax/tests/fixtures/mixed_entries.ftl new file mode 100644 index 00000000..99cc023d --- /dev/null +++ b/fluent-syntax/tests/fixtures/mixed_entries.ftl @@ -0,0 +1,24 @@ +# License Comment + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute + +ą=Invalid identifier +ć=Another one + +# Message Comment +key02 = Value + +# Standalone Comment + .attr = Dangling attribute + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 + +key04 = Value 04 diff --git a/fluent-syntax/tests/fixtures/mixed_entries.json b/fluent-syntax/tests/fixtures/mixed_entries.json new file mode 100644 index 00000000..9d9b3cfe --- /dev/null +++ b/fluent-syntax/tests/fixtures/mixed_entries.json @@ -0,0 +1,123 @@ +{ + "body": [ + { + "type": "Comment", + "content": "License Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + }, + { + "type": "Term", + "id": { + "name": "brand-name" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Aurora" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n" + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr = Dangling attribute\n" + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value 03" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are 5 spaces on the line between key03 and key04." + } + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value 04" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/multiline_values.ftl b/fluent-syntax/tests/fixtures/multiline_values.ftl new file mode 100644 index 00000000..4dd81154 --- /dev/null +++ b/fluent-syntax/tests/fixtures/multiline_values.ftl @@ -0,0 +1,35 @@ +key01 = A multiline value + continued on the next line + + and also down here. + +key02 = + A multiline value starting + on a new line. + +key03 = + .attr = A multiline attribute value + continued on the next line + + and also down here. + +key04 = + .attr = + A multiline attribute value + staring on a new line + +key05 = + + A multiline value with non-standard + + indentation. + +key06 = + A multiline value with {"placeables"} + {"at"} the beginning and the end + {"of lines"}{"."} + +key07 = + {"A multiline value"} starting and ending {"with a placeable"} + +key08 = Leading and trailing whitespace. diff --git a/fluent-syntax/tests/fixtures/multiline_values.json b/fluent-syntax/tests/fixtures/multiline_values.json new file mode 100644 index 00000000..1d89c8b0 --- /dev/null +++ b/fluent-syntax/tests/fixtures/multiline_values.json @@ -0,0 +1,236 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline value\n" + }, + { + "type": "TextElement", + "value": "continued on the next line\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "and also down here." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline value starting\n" + }, + { + "type": "TextElement", + "value": "on a new line." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\n" + }, + { + "type": "TextElement", + "value": "continued on the next line\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "and also down here." + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\n" + }, + { + "type": "TextElement", + "value": "staring on a new line" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key05" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with non-standard\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "indentation." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key06" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "placeables" + } + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "at" + } + }, + { + "type": "TextElement", + "value": " the beginning and the end\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "of lines" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key07" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "A multiline value" + } + }, + { + "type": "TextElement", + "value": " starting and ending " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "with a placeable" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key08" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Leading and trailing whitespace." + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl deleted file mode 100644 index 87aebc9b..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic-errors01.ftl +++ /dev/null @@ -1,23 +0,0 @@ - key2 = Value 2 - -= Value - - = Value - -= - - = - - = = - -id - = value - -id -= value - -1 = 1 - --1 = Foo - -19 = Foo2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl deleted file mode 100644 index f7082aec..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic01.ftl +++ /dev/null @@ -1 +0,0 @@ -key1 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl deleted file mode 100644 index c517b5ff..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic02.ftl +++ /dev/null @@ -1,10 +0,0 @@ -key1 = Value 1 -key2 =Value 2 - -key3 = Value 3 - -key4=Value4 -key5= Value 5 -key6= Value 6 -key7 = Value 7 -key8 = Value 8 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl b/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl deleted file mode 100644 index a1e8ae0b..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/01-basic03.ftl +++ /dev/null @@ -1,9 +0,0 @@ -key1 = "Value 1" -key2="Value 2" -key3 = " Value 3 " - -key4 = "" - -key5 = " " - -key6 = " Foo \" Foo2 " diff --git a/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl deleted file mode 100644 index f7f32e63..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/02-multiline01.ftl +++ /dev/null @@ -1,14 +0,0 @@ -key1 = - This is a new line - -key4 = -

- So - Many - Lines -

- -key5 = -

- Foo -

diff --git a/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl b/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl deleted file mode 100644 index de4baa2f..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/03-comments.ftl +++ /dev/null @@ -1,22 +0,0 @@ -### File comment - -# Standalone comment - -# Another standalone comment - -# Comment with a leading space - -## Multi -## Line Section -## -## Comment - -# Comment for entity key1 -key1 = New entity - -# Comment for entity key2 -key2= - | Multi line message - -## Group comment -key3 = Message diff --git a/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl deleted file mode 100644 index 170eeb2a..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/04-sections-errors.ftl +++ /dev/null @@ -1,25 +0,0 @@ -[section]] - -section]] - -[[section - -[section - -[[section] - -[[section]]dd - -[[section -]] - - -[[ ]] - -[[]] - -#doo -[[ - -#foo -[ diff --git a/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl deleted file mode 100644 index caba0259..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/05-variants-errors.ftl +++ /dev/null @@ -1,40 +0,0 @@ -key1 = { - [:key] Value - *[ok] Valid -} - -key3 = { - [] Value - *[ok] Valid -} - -key4 = { - **[f] Foo - *[ok] Valid -} - -key5 = { - *fw] Foo - *[ok] Valid -} - -key6 = { - [ a] A - *[ok] Valid -} - -key7 = { - [ x/a] XA - *[ok] Valid -} - -key8 = { - [x y/a] XYA - *[ok] Valid -} - -key10 = { - [x/a ] XA - [x/a b ] XAB - *[ok] Valid -} diff --git a/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl b/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl deleted file mode 100644 index 3b2ea9c2..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/05-variants.ftl +++ /dev/null @@ -1,37 +0,0 @@ -key1 = Value - .gender = male - -key2 = - .gender = male - -key3 = { - *[masculine] Variant for masculine - [feminine] Variant for feminine - } - -key4 = - .aria-label = Test - -key5 = - .aria-label = Test - .loop = Foo - -key5 = - .m = Foo - -## section - - -key6 = { - *[one] One - [two] Two - [three] Three - } - -key7 = { - *[a b] A - } - -key8 = { - *[a b] A - } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl b/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl deleted file mode 100644 index 984a5bfa..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables-errors.ftl +++ /dev/null @@ -1,87 +0,0 @@ -key1 = { -} - -key2 = { -} - - -key3 = { - $user1 -} - - -key4 = { - - -key6 = {,} - -key7 = {,$user} - -key8 = {$user,,$user2} - -key9 = {$user, ,$user2} - -key10 = {$user, -} - -key11 = { $user - - -key12 = { $user -> - -key13 = { $user -> -} - -key14 = { $user -> $user2 } - -key15 = { $user -> - $user2 -} - -key11 = Foo {} Foo2 - -key12 = Foo { } Foo 2 - -key13 = {$user,} - -key14 = {$user, -} - -key15 = {_,} - -key16 = { foo($user1=) } - -key17 = { foo(1=) } - -key18 = { foo(len()=)} - -key19 = { foo(bar/baz=1)} - -key20 = { len(bar=user) } - -key21 = { len(bar=bar/baz) } - -key22 = { len(bar=foo[key]) } - -key23 = { len(bar=bar/baz[foo:faa]) } - -key24 = { len(bar = baz) } - -key25 = { len( bar = baz ) } - -key27 = { $len -> [foo] Value } - -key28 = { $len -> *[foo] Value } - -key29 = { menu/open } - -key30 = { LEN($u1, $u2, open/brand-name, type:"short") } - -key31 = { menu/brand-name[accusative] } - -key32 = { len(bar/baz) } - -key33 = { len(bar/baz[foo]) } - -key34 = { len(bar/baz[foo/fab]) } - -keyLast = { diff --git a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl b/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl deleted file mode 100644 index 38ec12fa..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/06-placeables01.ftl +++ /dev/null @@ -1,44 +0,0 @@ -key1 = AA { $num } BB - -key2 = { brand-name } - -key4 = { $num -> - *[one] One - [two] Two - } - -key5 = { LEN($num) -> - *[one] One - [two] Two - } - -key6 = { LEN(NEL($num)) -> - *[one] One - [two] Two - } - -key7 = { LIST($user1, $user2) } - -key9 = { LEN(2, 2.5, -3.12, -1.00) } - -key11 = { LEN() } - -key12 = { LEN(1) } - -key13 = { LEN(-1) } - -key14 = { LEN($foo) } - -key15 = { LEN(foo) } - -key19 = { LEN(bar: 1) } - -key20 = { LEN(bar: -1) } - -key21 = { LEN(bar: "user") } - -key22 = { brand-name[masculine] } - -key23 = { NUMBER(style: "percent") } - -key24 = { NUMBER_SPECIAL($num, style: "percent", foo: "bar") } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl b/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl deleted file mode 100644 index 84c89463..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/07-private.ftl +++ /dev/null @@ -1,11 +0,0 @@ --brand-short-name = Firefox - .gender = masculine - -key = Test { -brand-short-name } - -key2 = Test { -brand-short-name.gender -> - [masculine] Foo - *[feminine] Foo 2 - } - -key3 = Test { -brand[one] } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl deleted file mode 100644 index ad8473dd..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/01-empty.ftl +++ /dev/null @@ -1 +0,0 @@ - key = value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl deleted file mode 100644 index 0cfbf088..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/02-bad-id-start.ftl +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl deleted file mode 100644 index 06bfde49..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/03-just-id.ftl +++ /dev/null @@ -1 +0,0 @@ -key diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl deleted file mode 100644 index e4612356..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/04-no-equal-sign.ftl +++ /dev/null @@ -1 +0,0 @@ -key Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl deleted file mode 100644 index 708ef4c5..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/05-bad-char-in-keyword.ftl +++ /dev/null @@ -1,2 +0,0 @@ -key = Value - .# = Foo diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl deleted file mode 100644 index 154f0a4d..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/06-trait-value.ftl +++ /dev/null @@ -1,2 +0,0 @@ -key = Value - .foo diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl deleted file mode 100644 index 1fc4ee77..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/07-message-missing-fields.ftl +++ /dev/null @@ -1,3 +0,0 @@ -key - -key2 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl b/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl deleted file mode 100644 index 2e6e9340..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/errors/08-private.ftl +++ /dev/null @@ -1,12 +0,0 @@ - -key = - { $foo -> - [one] Foo - *[-other] Foo 2 - } - -key2 = { $-foo } - -key3 = { -brand.gender } - -key4 = { -brand() } diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl deleted file mode 100644 index 4c2f1f72..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/01-basic.ftl +++ /dev/null @@ -1,12 +0,0 @@ -key1 = Value -key2 = Value 2 -key3 Value -key4 = Value - -key5 = Value - -key6 Value - -key7 = Value - -key8 = Value diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl deleted file mode 100644 index 54952568..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/02-start.ftl +++ /dev/null @@ -1,2 +0,0 @@ -0 = key -key2 = Value 2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl deleted file mode 100644 index c8015f87..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/03-end.ftl +++ /dev/null @@ -1,5 +0,0 @@ -key1 = Value - -key2 = Value - -key diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl deleted file mode 100644 index ee701588..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/04-multiline.ftl +++ /dev/null @@ -1,11 +0,0 @@ -key1 = Value - -key2 = Value - .one = { OS -> - *[masculine] { LEN($num) -> - *[1] Something - [] Two - } - [feminine] Faa - } -key3 = Value 2 diff --git a/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl b/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl deleted file mode 100644 index 3fe5a742..00000000 --- a/fluent-syntax/tests/fixtures/parser/ftl/junk/05-comment.ftl +++ /dev/null @@ -1,7 +0,0 @@ -key = Value - -key - -# This is a comment - -key2 = Value 2 diff --git a/fluent-syntax/tests/fixtures/placeables.ftl b/fluent-syntax/tests/fixtures/placeables.ftl new file mode 100644 index 00000000..c0e515b6 --- /dev/null +++ b/fluent-syntax/tests/fixtures/placeables.ftl @@ -0,0 +1,3 @@ +nested-placeable = {{{1}}} +padded-placeable = { 1 } +sparse-placeable = { { 1 } } diff --git a/fluent-syntax/tests/fixtures/placeables.json b/fluent-syntax/tests/fixtures/placeables.json new file mode 100644 index 00000000..1a2ef17c --- /dev/null +++ b/fluent-syntax/tests/fixtures/placeables.json @@ -0,0 +1,70 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "nested-placeable" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "padded-placeable" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "sparse-placeable" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/reference_expressions.ftl b/fluent-syntax/tests/fixtures/reference_expressions.ftl new file mode 100644 index 00000000..ab27cbe0 --- /dev/null +++ b/fluent-syntax/tests/fixtures/reference_expressions.ftl @@ -0,0 +1,5 @@ +message-reference = {msg} +term-reference = {-term} +variable-reference = {$var} + +not-call-expression = {FUN} diff --git a/fluent-syntax/tests/fixtures/reference_expressions.json b/fluent-syntax/tests/fixtures/reference_expressions.json new file mode 100644 index 00000000..88dcab73 --- /dev/null +++ b/fluent-syntax/tests/fixtures/reference_expressions.json @@ -0,0 +1,88 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "message-reference" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "name": "msg" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "term-reference" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "variable-reference" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "not-call-expression" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "name": "FUN" + } + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/select_expressions.ftl b/fluent-syntax/tests/fixtures/select_expressions.ftl new file mode 100644 index 00000000..3c54f57c --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_expressions.ftl @@ -0,0 +1,36 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] {""}Other + } + +valid-selector = + { -term.case -> + *[key] value + } + +# ERROR +invalid-selector = + { -term[case] -> + *[key] value + } + +empty-variant = + { 1 -> + *[one] {""} + } + +nested-select = + { 1 -> + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists cannot appear in SelectExpressions +nested-variant-list = + { 1 -> + *[one] { + *[two] Value + } + } diff --git a/fluent-syntax/tests/fixtures/select_expressions.json b/fluent-syntax/tests/fixtures/select_expressions.json new file mode 100644 index 00000000..88325092 --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_expressions.json @@ -0,0 +1,227 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "new-messages" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "CallExpression", + "callee": { + "name": "BUILTIN" + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "key": { + "value": "0" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Zero" + } + ] + }, + "default": false + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "" + } + }, + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "valid-selector" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "name": "term" + } + }, + "name": { + "name": "case" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector =\n { -term[case] ->\n *[key] value\n }\n" + }, + { + "type": "Message", + "id": { + "name": "empty-variant" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "" + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "nested-select" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "key": { + "name": "two" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot appear in SelectExpressions" + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/select_indent.ftl b/fluent-syntax/tests/fixtures/select_indent.ftl new file mode 100644 index 00000000..6c13076b --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_indent.ftl @@ -0,0 +1,95 @@ +select-1tbs-inline = { $selector -> + *[key] Value +} + +select-1tbs-newline = { +$selector -> + *[key] Value +} + +select-1tbs-indent = { + $selector -> + *[key] Value +} + +select-allman-inline = +{ $selector -> + *[key] Value +} + +select-allman-newline = +{ +$selector -> + *[key] Value +} + +select-allman-indent = +{ + $selector -> + *[key] Value +} + +select-gnu-inline = + { $selector -> + *[key] Value + } + +select-gnu-newline = + { +$selector -> + *[key] Value + } + +select-gnu-indent = + { + $selector -> + *[key] Value + } + +select-no-indent = +{ +$selector -> +*[key] Value +[other] Other +} + +select-no-indent-multiline = +{ +$selector -> +*[key] Value + Continued +[other] + Other + Multiline +} + +# ERROR (Multiline text must be indented) +select-no-indent-multiline = { $selector -> + *[key] Value +Continued without indent. +} + +select-flat = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} + +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} diff --git a/fluent-syntax/tests/fixtures/select_indent.json b/fluent-syntax/tests/fixtures/select_indent.json new file mode 100644 index 00000000..8c1dbb63 --- /dev/null +++ b/fluent-syntax/tests/fixtures/select_indent.json @@ -0,0 +1,587 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "select-1tbs-inline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-1tbs-newline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-1tbs-indent" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-allman-inline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-allman-newline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-allman-indent" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-gnu-inline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-gnu-newline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-gnu-indent" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-no-indent" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-no-indent-multiline" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "default": true + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Other\n" + }, + { + "type": "TextElement", + "value": "Multiline" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR (Multiline text must be indented)" + }, + { + "type": "Junk", + "annotations": [], + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\nContinued without indent.\n}\n" + }, + { + "type": "Message", + "id": { + "name": "select-flat" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "select-flat-with-trailing-spaces" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "VariableReference", + "id": { + "name": "selector" + } + }, + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Each line ends with 5 spaces." + } + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/sparse_entries.ftl b/fluent-syntax/tests/fixtures/sparse_entries.ftl new file mode 100644 index 00000000..67920b2c --- /dev/null +++ b/fluent-syntax/tests/fixtures/sparse_entries.ftl @@ -0,0 +1,39 @@ +key01 = + + + Value + +key02 = + + + .attr = Attribute + + +key03 = + Value + Continued + + + Over multiple + Lines + + + + .attr = Attribute + + +key05 = Value + +key06 = { 1 -> + + + [one] One + + + + + *[two] Two + + + + } diff --git a/fluent-syntax/tests/fixtures/sparse_entries.json b/fluent-syntax/tests/fixtures/sparse_entries.json new file mode 100644 index 00000000..73d6707e --- /dev/null +++ b/fluent-syntax/tests/fixtures/sparse_entries.json @@ -0,0 +1,160 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": null, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "TextElement", + "value": "Continued\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "Over multiple\n" + }, + { + "type": "TextElement", + "value": "Lines" + } + ] + }, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key05" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key06" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "One" + } + ] + }, + "default": false + }, + { + "key": { + "name": "two" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Two" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/tab.ftl b/fluent-syntax/tests/fixtures/tab.ftl new file mode 100644 index 00000000..4b23ad87 --- /dev/null +++ b/fluent-syntax/tests/fixtures/tab.ftl @@ -0,0 +1,14 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) +key02 = Value 02 + +# Error (tab is not a valid indent) +key03 = + This line isn't properly indented. + +# Partial Error (tab is not a valid indent) +key04 = + This line is indented by 4 spaces, + whereas this line by 1 tab. diff --git a/fluent-syntax/tests/fixtures/tab.json b/fluent-syntax/tests/fixtures/tab.json new file mode 100644 index 00000000..8d891aa0 --- /dev/null +++ b/fluent-syntax/tests/fixtures/tab.json @@ -0,0 +1,65 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "\tValue 01" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK (tab after = is part of the value)" + } + }, + { + "type": "Comment", + "content": "Error (tab before =)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key02\t= Value 02\n" + }, + { + "type": "Comment", + "content": "Error (tab is not a valid indent)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key03 =\n\tThis line isn't properly indented.\n" + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "This line is indented by 4 spaces," + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Partial Error (tab is not a valid indent)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": "\twhereas this line by 1 tab.\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/terms.ftl b/fluent-syntax/tests/fixtures/terms.ftl new file mode 100644 index 00000000..b8791fcf --- /dev/null +++ b/fluent-syntax/tests/fixtures/terms.ftl @@ -0,0 +1,23 @@ +-term01 = Value + .attr = Attribute + +-term02 = {""} + +# JUNK Missing value +-term03 = + .attr = Attribute + +# JUNK Missing value +# < whitespace > +-term04 = + .attr1 = Attribute 1 + +# JUNK Missing value +-term05 = + +# JUNK Missing value +# < whitespace > +-term06 = + +# JUNK Missing = +-term07 diff --git a/fluent-syntax/tests/fixtures/terms.json b/fluent-syntax/tests/fixtures/terms.json new file mode 100644 index 00000000..2abb8440 --- /dev/null +++ b/fluent-syntax/tests/fixtures/terms.json @@ -0,0 +1,98 @@ +{ + "body": [ + { + "type": "Term", + "id": { + "name": "term01" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "id": { + "name": "attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "term02" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term03 =\n .attr = Attribute\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term04 = \n .attr1 = Attribute 1\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term05 =\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term06 = \n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term07\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/variables.ftl b/fluent-syntax/tests/fixtures/variables.ftl new file mode 100644 index 00000000..6c343692 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variables.ftl @@ -0,0 +1,17 @@ +key01 = {$var} +key02 = { $var } +key03 = { + $var +} +key04 = { +$var} + + +## Errors + +# ERROR Missing variable identifier +err01 = {$} +# ERROR Double $$ +err02 = {$$var} +# ERROR Invalid first char of the identifier +err03 = {$-var} diff --git a/fluent-syntax/tests/fixtures/variables.json b/fluent-syntax/tests/fixtures/variables.json new file mode 100644 index 00000000..d5a79c2d --- /dev/null +++ b/fluent-syntax/tests/fixtures/variables.json @@ -0,0 +1,119 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key01" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key02" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key03" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "name": "key04" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Errors" + }, + { + "type": "Comment", + "content": "ERROR Missing variable identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {$}\n" + }, + { + "type": "Comment", + "content": "ERROR Double $$" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {$$var}\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid first char of the identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {$-var}\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/variant_keys.ftl b/fluent-syntax/tests/fixtures/variant_keys.ftl new file mode 100644 index 00000000..7586d524 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_keys.ftl @@ -0,0 +1,37 @@ +-simple-identifier = + { + *[key] value + } + +-identifier-surrounded-by-whitespace = + { + *[ key ] value + } + +-int-number = + { + *[1] value + } + +-float-number = + { + *[3.14] value + } + +# ERROR +-invalid-identifier = + { + *[two words] value + } + +# ERROR +-invalid-int = + { + *[1 apple] value + } + +# ERROR +-invalid-int = + { + *[3.14 apples] value + } diff --git a/fluent-syntax/tests/fixtures/variant_keys.json b/fluent-syntax/tests/fixtures/variant_keys.json new file mode 100644 index 00000000..68246c92 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_keys.json @@ -0,0 +1,135 @@ +{ + "body": [ + { + "type": "Term", + "id": { + "name": "simple-identifier" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "identifier-surrounded-by-whitespace" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "int-number" + }, + "value": { + "variants": [ + { + "key": { + "value": "1" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "float-number" + }, + "value": { + "variants": [ + { + "key": { + "value": "3.14" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[3.14 apples] value\n }\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/variant_lists.ftl b/fluent-syntax/tests/fixtures/variant_lists.ftl new file mode 100644 index 00000000..d9031d91 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_lists.ftl @@ -0,0 +1,54 @@ +-variant-list-in-term = + { + *[key] Value + } + +# ERROR Attributes of Terms must be Patterns. +-variant-list-in-term-attr = Value + .attr = + { + *[key] Value + } + +# ERROR Message values must be Patterns. +variant-list-in-message = + { + *[key] Value + } + +# ERROR Attributes of Messages must be Patterns. +variant-list-in-message-attr = Value + .attr = + { + *[key] Value + } + +-nested-variant-list-in-term = + { + *[one] { + *[two] Value + } + } + +-nested-select = + { + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists may not appear in SelectExpressions +nested-select-then-variant-list = + { + *[one] { 2 -> + *[two] { + *[three] Value + } + } + } + +# ERROR VariantLists are value types and may not appear in Placeables +variant-list-in-placeable = + A prefix here { + *[key] Value + } and a postfix here make this a Pattern. diff --git a/fluent-syntax/tests/fixtures/variant_lists.json b/fluent-syntax/tests/fixtures/variant_lists.json new file mode 100644 index 00000000..b1855100 --- /dev/null +++ b/fluent-syntax/tests/fixtures/variant_lists.json @@ -0,0 +1,188 @@ +{ + "body": [ + { + "type": "Term", + "id": { + "name": "variant-list-in-term" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "variant-list-in-term-attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Terms must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n" + }, + { + "type": "Comment", + "content": "ERROR Message values must be Patterns." + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n" + }, + { + "type": "Message", + "id": { + "name": "variant-list-in-message-attr" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Messages must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n" + }, + { + "type": "Term", + "id": { + "name": "nested-variant-list-in-term" + }, + "value": { + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "variants": [ + { + "key": { + "name": "two" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "nested-select" + }, + "value": { + "variants": [ + { + "key": { + "name": "one" + }, + "value": { + "elements": [ + { + "type": "Placeable", + "expression": { + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "key": { + "name": "two" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists may not appear in SelectExpressions" + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n" + }, + { + "type": "Comment", + "content": "ERROR VariantLists are value types and may not appear in Placeables" + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-placeable =\n A prefix here {\n *[key] Value\n } and a postfix here make this a Pattern.\n" + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/variants_indent.ftl b/fluent-syntax/tests/fixtures/variants_indent.ftl new file mode 100644 index 00000000..38f5a62e --- /dev/null +++ b/fluent-syntax/tests/fixtures/variants_indent.ftl @@ -0,0 +1,19 @@ +-variants-1tbs = { + *[key] Value +} + +-variants-allman = +{ + *[key] Value +} + +-variants-gnu = + { + *[key] Value + } + +-variants-no-indent = +{ +*[key] Value +[other] Other +} diff --git a/fluent-syntax/tests/fixtures/variants_indent.json b/fluent-syntax/tests/fixtures/variants_indent.json new file mode 100644 index 00000000..a780701d --- /dev/null +++ b/fluent-syntax/tests/fixtures/variants_indent.json @@ -0,0 +1,122 @@ +{ + "body": [ + { + "type": "Term", + "id": { + "name": "variants-1tbs" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "variants-allman" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "variants-gnu" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "name": "variants-no-indent" + }, + "value": { + "variants": [ + { + "key": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "key": { + "name": "other" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + }, + "attributes": [], + "comment": null + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/whitespace_in_value.ftl b/fluent-syntax/tests/fixtures/whitespace_in_value.ftl new file mode 100644 index 00000000..2fba5535 --- /dev/null +++ b/fluent-syntax/tests/fixtures/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/fluent-syntax/tests/fixtures/whitespace_in_value.json b/fluent-syntax/tests/fixtures/whitespace_in_value.json new file mode 100644 index 00000000..0f345b0c --- /dev/null +++ b/fluent-syntax/tests/fixtures/whitespace_in_value.json @@ -0,0 +1,51 @@ +{ + "body": [ + { + "type": "Message", + "id": { + "name": "key" + }, + "value": { + "elements": [ + { + "type": "TextElement", + "value": "first line\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "TextElement", + "value": "last line" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Caution, lines 6 and 7 contain white-space-only lines" + } + } + ] +} \ No newline at end of file diff --git a/fluent-syntax/tests/junk.rs b/fluent-syntax/tests/junk.rs deleted file mode 100644 index c025af6f..00000000 --- a/fluent-syntax/tests/junk.rs +++ /dev/null @@ -1,79 +0,0 @@ -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use self::fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -#[test] -fn basic_junk() { - let path = "./tests/fixtures/parser/ftl/junk/01-basic.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(2, errors.len()); - assert_eq!(8, res.body.len()); - } - } -} - -#[test] -fn start_junk() { - let path = "./tests/fixtures/parser/ftl/junk/02-start.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(2, res.body.len()); - } - } -} - -#[test] -fn end_junk() { - let path = "./tests/fixtures/parser/ftl/junk/03-end.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(3, res.body.len()); - } - } -} - -#[test] -fn multiline_junk() { - let path = "./tests/fixtures/parser/ftl/junk/04-multiline.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(3, res.body.len()); - } - } -} - -#[test] -fn recover_at_comment() { - let path = "./tests/fixtures/parser/ftl/junk/05-comment.ftl"; - let source = read_file(path).expect("Failed to read"); - match parse(&source) { - Ok(_) => panic!("Expected junk in the file"), - Err((res, errors)) => { - assert_eq!(1, errors.len()); - assert_eq!(4, res.body.len()); - } - } -} diff --git a/fluent-syntax/tests/parser_fixtures.rs b/fluent-syntax/tests/parser_fixtures.rs new file mode 100644 index 00000000..f3305110 --- /dev/null +++ b/fluent-syntax/tests/parser_fixtures.rs @@ -0,0 +1,82 @@ +mod ast; + +use glob::glob; +use std::fs; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::process::Command; + +use fluent_syntax::parser::parse; + +fn compare_jsons(value: &str, reference: &str) -> String { + let temp_path = reference.replace(".json", ".candidate.json"); + write_file(&temp_path, value).unwrap(); + + let output = Command::new("sh") + .arg("-c") + .arg(format!("json-diff {} {}", reference, &temp_path)) + .output() + .expect("failed to execute process"); + let s = String::from_utf8_lossy(&output.stdout).to_string(); + fs::remove_file(&temp_path).unwrap(); + s +} + +fn read_file(path: &str, trim: bool) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + if trim { + Ok(s.trim().to_string()) + } else { + Ok(s) + } +} + +fn write_file(path: &str, value: &str) -> std::io::Result<()> { + let mut file = File::create(&path)?; + file.write_all(value.as_bytes())?; + Ok(()) +} + +#[test] +fn parse_fixtures_compare() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let p = entry.expect("Error while getting an entry"); + let path = p.to_str().expect("Can't print path"); + + let reference_path = path.replace(".ftl", ".json"); + let reference_file = read_file(&reference_path, true).unwrap(); + let ftl_file = read_file(&path, false).unwrap(); + + println!("Parsing: {:#?}", path); + let target_ast = match parse(&ftl_file) { + Ok(res) => res, + Err((res, _errors)) => res, + }; + + let target_json = ast::serialize(&target_ast).unwrap(); + + let diff = compare_jsons(&target_json, &reference_path); + assert_eq!( + reference_file, target_json, + "\n=====\nThe diff {} :\n-------\n{}\n-----\n", + path, diff + ); + } +} + +#[test] +fn parse_fixtures() { + for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { + let p = entry.expect("Error while getting an entry"); + let path = p.to_str().expect("Can't print path"); + + println!("Attempting to parse file: {}", path); + + let string = read_file(path, false).expect("Failed to read"); + + let _ = parse(&string); + } +} diff --git a/fluent-syntax/tests/stream.rs b/fluent-syntax/tests/stream.rs deleted file mode 100644 index 9eaca418..00000000 --- a/fluent-syntax/tests/stream.rs +++ /dev/null @@ -1,185 +0,0 @@ -extern crate fluent_syntax; - -use self::fluent_syntax::parser::stream::ParserStream; - -#[test] -fn next() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('a'), ps.current()); - assert_eq!(0, ps.get_index()); - - assert_eq!(Some('b'), ps.next()); - assert_eq!(Some('b'), ps.current()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.next()); - assert_eq!(Some('c'), ps.current()); - assert_eq!(2, ps.get_index()); - - assert_eq!(Some('d'), ps.next()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(3, ps.get_index()); - - assert_eq!(None, ps.next()); - assert_eq!(None, ps.current()); - assert_eq!(4, ps.get_index()); -} - -#[test] -fn peek() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('a'), ps.current_peek()); - assert_eq!(0, ps.get_peek_index()); - - assert_eq!(Some('b'), ps.peek()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - - assert_eq!(None, ps.peek()); - assert_eq!(None, ps.current_peek()); - assert_eq!(4, ps.get_peek_index()); -} - -#[test] -fn peek_and_next() { - let mut ps = ParserStream::new("abcd".chars()); - - assert_eq!(Some('b'), ps.peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(0, ps.get_index()); - - assert_eq!(Some('b'), ps.next()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.next()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - assert_eq!(Some('d'), ps.next()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - - assert_eq!(None, ps.peek()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - assert_eq!(Some('d'), ps.current()); - assert_eq!(None, ps.current_peek()); - - assert_eq!(None, ps.peek()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); - - assert_eq!(None, ps.next()); - assert_eq!(4, ps.get_peek_index()); - assert_eq!(4, ps.get_index()); -} - -#[test] -fn skip_to_peek() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.peek(); - ps.peek(); - - ps.skip_to_peek(); - - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - ps.peek(); - - assert_eq!(Some('c'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(2, ps.get_index()); - - ps.next(); - - assert_eq!(Some('d'), ps.current()); - assert_eq!(Some('d'), ps.current_peek()); - assert_eq!(3, ps.get_peek_index()); - assert_eq!(3, ps.get_index()); -} - -#[test] -fn reset_peek() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.next(); - ps.peek(); - ps.peek(); - ps.reset_peek(None); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - ps.peek(); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - ps.peek(); - ps.peek(); - ps.peek(); - ps.reset_peek(None); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('b'), ps.current_peek()); - assert_eq!(1, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('c'), ps.peek()); - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - assert_eq!(2, ps.get_peek_index()); - assert_eq!(1, ps.get_index()); - - assert_eq!(Some('d'), ps.peek()); - assert_eq!(None, ps.peek()); -} - -#[test] -fn peek_char_is() { - let mut ps = ParserStream::new("abcd".chars()); - - ps.next(); - ps.peek(); - - assert_eq!(ps.peek_char_is('d'), true); - - assert_eq!(Some('b'), ps.current()); - assert_eq!(Some('c'), ps.current_peek()); - - ps.skip_to_peek(); - - assert_eq!(Some('c'), ps.current()); -} From 77c2b3821e41e2c5722925326386c481090a602a Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 29 Oct 2018 14:28:15 -0700 Subject: [PATCH 02/36] Migrate resolver to use copy-free parser This is a massive rewrite of the whole crate. It separates out three crates: fluent-syntax becomes a zero-copy AST+parser(+serialized in the future) compabile with Fluent 0.8 fluent-bundle becomes aligned with fluent-bundle in JS fluent becomes a top-level crate with added simple ResourceManager. All of the code is compatible with current stable Rust 2018. --- .gitignore | 1 + Cargo.toml | 4 +- {fluent => fluent-bundle}/CHANGELOG.md | 0 fluent-bundle/Cargo.toml | 25 + {fluent => fluent-bundle}/README.md | 4 +- {fluent => fluent-bundle}/benches/lib.rs | 20 +- {fluent => fluent-bundle}/benches/menubar.ftl | 0 {fluent => fluent-bundle}/benches/simple.ftl | 0 {fluent => fluent-bundle}/examples/README.md | 0 .../examples/external_arguments.rs | 6 +- .../examples/functions.rs | 6 +- {fluent => fluent-bundle}/examples/hello.rs | 4 +- .../examples/message_reference.rs | 4 +- .../examples/resources/en-US/simple.ftl | 7 + .../examples/resources/pl/simple.ftl | 8 + .../examples/selector.rs | 6 +- fluent-bundle/examples/simple-app.rs | 170 ++++ {fluent => fluent-bundle}/src/bundle.rs | 109 +-- {fluent => fluent-bundle}/src/entry.rs | 8 +- {fluent => fluent-bundle}/src/errors.rs | 2 +- fluent-bundle/src/lib.rs | 48 ++ {fluent => fluent-bundle}/src/resolve.rs | 269 ++++--- fluent-bundle/src/resource.rs | 16 + {fluent => fluent-bundle}/src/types.rs | 0 {fluent => fluent-bundle}/tests/bundle.rs | 4 +- {fluent => fluent-bundle}/tests/format.rs | 6 +- .../tests/format_message.rs | 8 +- .../tests/helpers/mod.rs | 4 +- .../tests/resolve_attribute_expression.rs | 6 +- .../tests/resolve_external_argument.rs | 8 +- .../tests/resolve_message_reference.rs | 6 +- .../tests/resolve_plural_rule.rs | 8 +- .../tests/resolve_select_expression.rs | 41 +- .../tests/resolve_value.rs | 6 +- .../tests/resolve_variant_expression.rs | 10 +- fluent-cli/Cargo.toml | 23 + fluent-cli/src/main.rs | 149 ++++ fluent-syntax/Cargo.toml | 5 +- fluent-syntax/benches/lib.rs | 3 +- fluent-syntax/src/ast.rs | 12 +- fluent-syntax/src/bin/parser.rs | 58 -- fluent-syntax/src/lib.rs | 2 - fluent-syntax/src/parser/errors/mod.rs | 13 +- fluent-syntax/src/parser/ftlstream.rs | 66 +- fluent-syntax/src/parser/mod.rs | 744 ++++++++++-------- fluent-syntax/tests/ast/helper.rs | 67 ++ fluent-syntax/tests/ast/mod.rs | 218 +++-- fluent-syntax/tests/fixtures/any_char.ftl | 8 + fluent-syntax/tests/fixtures/any_char.json | 68 ++ fluent-syntax/tests/fixtures/astral.json | 31 +- .../tests/fixtures/call_expressions.ftl | 2 + .../tests/fixtures/call_expressions.json | 277 ++++++- .../tests/fixtures/callee_expressions.ftl | 46 ++ .../tests/fixtures/callee_expressions.json | 270 +++++++ fluent-syntax/tests/fixtures/comments.json | 7 +- fluent-syntax/tests/fixtures/convert.js | 94 --- fluent-syntax/tests/fixtures/cr.ftl | 1 + fluent-syntax/tests/fixtures/cr.json | 9 + fluent-syntax/tests/fixtures/crlf.ftl | 11 +- fluent-syntax/tests/fixtures/crlf.json | 45 +- fluent-syntax/tests/fixtures/eof_comment.json | 3 +- fluent-syntax/tests/fixtures/eof_empty.json | 3 +- fluent-syntax/tests/fixtures/eof_id.json | 3 +- .../tests/fixtures/eof_id_equals.json | 3 +- fluent-syntax/tests/fixtures/eof_junk.json | 3 +- fluent-syntax/tests/fixtures/eof_value.json | 5 +- .../tests/fixtures/escaped_characters.ftl | 35 +- .../tests/fixtures/escaped_characters.json | 304 ++++++- fluent-syntax/tests/fixtures/junk.ftl | 19 +- fluent-syntax/tests/fixtures/junk.json | 60 +- .../tests/fixtures/leading_dots.json | 88 ++- .../tests/fixtures/literal_expressions.json | 10 +- .../tests/fixtures/member_expressions.ftl | 32 +- .../tests/fixtures/member_expressions.json | 134 +++- fluent-syntax/tests/fixtures/messages.ftl | 2 + fluent-syntax/tests/fixtures/messages.json | 58 +- .../tests/fixtures/mixed_entries.json | 19 +- .../tests/fixtures/multiline_values.ftl | 25 + .../tests/fixtures/multiline_values.json | 185 +++-- fluent-syntax/tests/fixtures/placeables.ftl | 12 + fluent-syntax/tests/fixtures/placeables.json | 45 +- .../tests/fixtures/reference_expressions.ftl | 31 +- .../tests/fixtures/reference_expressions.json | 107 ++- .../tests/fixtures/select_expressions.ftl | 23 +- .../tests/fixtures/select_expressions.json | 79 +- .../tests/fixtures/select_indent.json | 125 ++- .../tests/fixtures/sparse_entries.json | 47 +- fluent-syntax/tests/fixtures/tab.json | 11 +- .../tests/fixtures/term_parameters.ftl | 8 + .../tests/fixtures/term_parameters.json | 203 +++++ fluent-syntax/tests/fixtures/terms.json | 19 +- fluent-syntax/tests/fixtures/variables.json | 15 +- .../tests/fixtures/variant_keys.json | 27 +- .../tests/fixtures/variant_lists.ftl | 3 +- .../tests/fixtures/variant_lists.json | 72 +- .../tests/fixtures/variants_indent.json | 26 +- .../tests/fixtures/whitespace_in_value.json | 35 +- fluent-syntax/tests/parser_fixtures.rs | 33 +- fluent/Cargo.toml | 9 +- fluent/examples/resources/en-US/common.ftl | 1 + fluent/examples/resources/en-US/errors.ftl | 2 + fluent/examples/resources/en-US/simple.ftl | 2 - fluent/examples/resources/pl/common.ftl | 1 + fluent/examples/resources/pl/errors.ftl | 2 + fluent/examples/resources/pl/simple.ftl | 2 - fluent/examples/simple.rs | 50 +- fluent/src/bin/parser.rs | 64 -- fluent/src/lib.rs | 49 +- fluent/src/resource.rs | 16 - fluent/src/resource_manager.rs | 91 +++ 110 files changed, 3867 insertions(+), 1387 deletions(-) rename {fluent => fluent-bundle}/CHANGELOG.md (100%) create mode 100644 fluent-bundle/Cargo.toml rename {fluent => fluent-bundle}/README.md (98%) rename {fluent => fluent-bundle}/benches/lib.rs (77%) rename {fluent => fluent-bundle}/benches/menubar.ftl (100%) rename {fluent => fluent-bundle}/benches/simple.ftl (100%) rename {fluent => fluent-bundle}/examples/README.md (100%) rename {fluent => fluent-bundle}/examples/external_arguments.rs (92%) rename {fluent => fluent-bundle}/examples/functions.rs (95%) rename {fluent => fluent-bundle}/examples/hello.rs (81%) rename {fluent => fluent-bundle}/examples/message_reference.rs (89%) create mode 100644 fluent-bundle/examples/resources/en-US/simple.ftl create mode 100644 fluent-bundle/examples/resources/pl/simple.ftl rename {fluent => fluent-bundle}/examples/selector.rs (88%) create mode 100644 fluent-bundle/examples/simple-app.rs rename {fluent => fluent-bundle}/src/bundle.rs (70%) rename {fluent => fluent-bundle}/src/entry.rs (86%) rename {fluent => fluent-bundle}/src/errors.rs (92%) create mode 100644 fluent-bundle/src/lib.rs rename {fluent => fluent-bundle}/src/resolve.rs (50%) create mode 100644 fluent-bundle/src/resource.rs rename {fluent => fluent-bundle}/src/types.rs (100%) rename {fluent => fluent-bundle}/tests/bundle.rs (95%) rename {fluent => fluent-bundle}/tests/format.rs (82%) rename {fluent => fluent-bundle}/tests/format_message.rs (79%) rename {fluent => fluent-bundle}/tests/helpers/mod.rs (89%) rename {fluent => fluent-bundle}/tests/resolve_attribute_expression.rs (86%) rename {fluent => fluent-bundle}/tests/resolve_external_argument.rs (91%) rename {fluent => fluent-bundle}/tests/resolve_message_reference.rs (93%) rename {fluent => fluent-bundle}/tests/resolve_plural_rule.rs (91%) rename {fluent => fluent-bundle}/tests/resolve_select_expression.rs (85%) rename {fluent => fluent-bundle}/tests/resolve_value.rs (79%) rename {fluent => fluent-bundle}/tests/resolve_variant_expression.rs (82%) create mode 100644 fluent-cli/Cargo.toml create mode 100644 fluent-cli/src/main.rs delete mode 100644 fluent-syntax/src/bin/parser.rs create mode 100644 fluent-syntax/tests/ast/helper.rs create mode 100644 fluent-syntax/tests/fixtures/any_char.ftl create mode 100644 fluent-syntax/tests/fixtures/any_char.json create mode 100644 fluent-syntax/tests/fixtures/callee_expressions.ftl create mode 100644 fluent-syntax/tests/fixtures/callee_expressions.json delete mode 100755 fluent-syntax/tests/fixtures/convert.js create mode 100644 fluent-syntax/tests/fixtures/cr.ftl create mode 100644 fluent-syntax/tests/fixtures/cr.json create mode 100644 fluent-syntax/tests/fixtures/term_parameters.ftl create mode 100644 fluent-syntax/tests/fixtures/term_parameters.json create mode 100644 fluent/examples/resources/en-US/common.ftl create mode 100644 fluent/examples/resources/en-US/errors.ftl create mode 100644 fluent/examples/resources/pl/common.ftl create mode 100644 fluent/examples/resources/pl/errors.ftl delete mode 100644 fluent/src/bin/parser.rs delete mode 100644 fluent/src/resource.rs create mode 100644 fluent/src/resource_manager.rs diff --git a/.gitignore b/.gitignore index 6aa10640..4bf6b093 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target/ +*/target/ **/*.rs.bk Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index de64dbb0..29cf9892 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "fluent-syntax", + "fluent-bundle", + "fluent-cli", "fluent", - "fluent-syntax" ] diff --git a/fluent/CHANGELOG.md b/fluent-bundle/CHANGELOG.md similarity index 100% rename from fluent/CHANGELOG.md rename to fluent-bundle/CHANGELOG.md diff --git a/fluent-bundle/Cargo.toml b/fluent-bundle/Cargo.toml new file mode 100644 index 00000000..59f91b86 --- /dev/null +++ b/fluent-bundle/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "fluent-bundle" +description = """ +A localization system designed to unleash the entire expressive power of +natural language translations. +""" +version = "0.4.3" +edition = "2018" +authors = [ + "Zibi Braniecki ", + "Staś Małolepszy " +] +homepage = "http://www.projectfluent.org" +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] + +[dependencies] +fluent-locale = "^0.4.1" +fluent-syntax = { path = "../fluent-syntax" } +failure = "0.1" +failure_derive = "0.1" +intl_pluralrules = "1.0" diff --git a/fluent/README.md b/fluent-bundle/README.md similarity index 98% rename from fluent/README.md rename to fluent-bundle/README.md index b459ee37..f620b43e 100644 --- a/fluent/README.md +++ b/fluent-bundle/README.md @@ -22,7 +22,9 @@ Usage ----- ```rust -use fluent::bundle::FluentBundle; +extern crate fluent; + +use fluent_bundle::FluentBundle; fn main() { let mut bundle = FluentBundle::new(&["en-US"]); diff --git a/fluent/benches/lib.rs b/fluent-bundle/benches/lib.rs similarity index 77% rename from fluent/benches/lib.rs rename to fluent-bundle/benches/lib.rs index a7abdb30..abcb53ce 100644 --- a/fluent/benches/lib.rs +++ b/fluent-bundle/benches/lib.rs @@ -1,10 +1,8 @@ #![feature(test)] -extern crate fluent; -extern crate fluent_syntax; extern crate test; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; use fluent_syntax::{ast, parser::parse}; use std::fs::File; use std::io; @@ -12,9 +10,9 @@ use std::io::Read; use test::Bencher; fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); + let mut f = File::open(path)?; let mut s = String::new(); - try!(f.read_to_string(&mut s)); + f.read_to_string(&mut s)?; Ok(s) } @@ -27,7 +25,9 @@ fn bench_simple_format(b: &mut Bencher) { for entry in resource.body { match entry { - ast::Entry::Message(ast::Message { id, .. }) => ids.push(id.name), + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { + ids.push(id.name) + } _ => continue, }; } @@ -37,7 +37,7 @@ fn bench_simple_format(b: &mut Bencher) { b.iter(|| { for id in &ids { - bundle.format(id.as_str(), None); + bundle.format(id, None); } }); } @@ -51,7 +51,9 @@ fn bench_menubar_format(b: &mut Bencher) { for entry in resource.body { match entry { - ast::Entry::Message(ast::Message { id, .. }) => ids.push(id.name), + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { + ids.push(id.name) + } _ => continue, }; } @@ -66,7 +68,7 @@ fn bench_menubar_format(b: &mut Bencher) { // widgets may only expect attributes and they shouldn't be forced to display a value. // Here however it doesn't matter because we know for certain that the message for `id` // exists. - bundle.format_message(id.as_str(), None); + bundle.format_message(id, None); } }); } diff --git a/fluent/benches/menubar.ftl b/fluent-bundle/benches/menubar.ftl similarity index 100% rename from fluent/benches/menubar.ftl rename to fluent-bundle/benches/menubar.ftl diff --git a/fluent/benches/simple.ftl b/fluent-bundle/benches/simple.ftl similarity index 100% rename from fluent/benches/simple.ftl rename to fluent-bundle/benches/simple.ftl diff --git a/fluent/examples/README.md b/fluent-bundle/examples/README.md similarity index 100% rename from fluent/examples/README.md rename to fluent-bundle/examples/README.md diff --git a/fluent/examples/external_arguments.rs b/fluent-bundle/examples/external_arguments.rs similarity index 92% rename from fluent/examples/external_arguments.rs rename to fluent-bundle/examples/external_arguments.rs index eda4e350..0b817451 100644 --- a/fluent/examples/external_arguments.rs +++ b/fluent-bundle/examples/external_arguments.rs @@ -1,7 +1,5 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { diff --git a/fluent/examples/functions.rs b/fluent-bundle/examples/functions.rs similarity index 95% rename from fluent/examples/functions.rs rename to fluent-bundle/examples/functions.rs index e0bf1db3..a0955b40 100644 --- a/fluent/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -1,7 +1,5 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; fn main() { let mut bundle = FluentBundle::new(&["en-US"]); diff --git a/fluent/examples/hello.rs b/fluent-bundle/examples/hello.rs similarity index 81% rename from fluent/examples/hello.rs rename to fluent-bundle/examples/hello.rs index b5d2d9b2..2828b333 100644 --- a/fluent/examples/hello.rs +++ b/fluent-bundle/examples/hello.rs @@ -1,6 +1,4 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; fn main() { let mut bundle = FluentBundle::new(&["en-US"]); diff --git a/fluent/examples/message_reference.rs b/fluent-bundle/examples/message_reference.rs similarity index 89% rename from fluent/examples/message_reference.rs rename to fluent-bundle/examples/message_reference.rs index 21e33ac9..bfc512e5 100644 --- a/fluent/examples/message_reference.rs +++ b/fluent-bundle/examples/message_reference.rs @@ -1,6 +1,4 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; fn main() { let mut bundle = FluentBundle::new(&["x-testing"]); diff --git a/fluent-bundle/examples/resources/en-US/simple.ftl b/fluent-bundle/examples/resources/en-US/simple.ftl new file mode 100644 index 00000000..99f0a6bb --- /dev/null +++ b/fluent-bundle/examples/resources/en-US/simple.ftl @@ -0,0 +1,7 @@ +missing-arg-error = Error: Please provide a number as argument. +input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } +response-msg = + { $value -> + [one] "{ $input }" has one Collatz step. + *[other] "{ $input }" has { $value } Collatz steps. + } diff --git a/fluent-bundle/examples/resources/pl/simple.ftl b/fluent-bundle/examples/resources/pl/simple.ftl new file mode 100644 index 00000000..16173dd9 --- /dev/null +++ b/fluent-bundle/examples/resources/pl/simple.ftl @@ -0,0 +1,8 @@ +missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. +input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } +response-msg = + { $value -> + [one] "{ $input }" ma jeden krok Collatza. + [few] "{ $input }" ma { $value } kroki Collatza. + *[many] "{ $input }" ma { $value } kroków Collatza. + } diff --git a/fluent/examples/selector.rs b/fluent-bundle/examples/selector.rs similarity index 88% rename from fluent/examples/selector.rs rename to fluent-bundle/examples/selector.rs index a747f97b..c7e3672b 100644 --- a/fluent/examples/selector.rs +++ b/fluent-bundle/examples/selector.rs @@ -1,7 +1,5 @@ -extern crate fluent; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { diff --git a/fluent-bundle/examples/simple-app.rs b/fluent-bundle/examples/simple-app.rs new file mode 100644 index 00000000..bf7c8854 --- /dev/null +++ b/fluent-bundle/examples/simple-app.rs @@ -0,0 +1,170 @@ +//! This is an example of a simple application +//! which calculates the Collatz conjecture. +//! +//! The function itself is trivial on purpose, +//! so that we can focus on understanding how +//! the application can be made localizable +//! via Fluent. +//! +//! To try the app launch `cargo run --example simple NUM (LOCALES)` +//! +//! NUM is a number to be calculated, and LOCALES is an optional +//! parameter with a comma-separated list of locales requested by the user. +//! +//! Example: +//! +//! caron run --example simple 123 de,pl +//! +//! If the second argument is omitted, `en-US` locale is used as the +//! default one. +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; +use fluent_locale::{negotiate_languages, NegotiationStrategy}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::str::FromStr; + +/// We need a generic file read helper function to +/// read the localization resource file. +/// +/// The resource files are stored in +/// `./examples/resources/{locale}` directory. +fn read_file(path: &str) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +/// This helper function allows us to read the list +/// of available locales by reading the list of +/// directories in `./examples/resources`. +/// +/// It is expected that every directory inside it +/// has a name that is a valid BCP47 language tag. +fn get_available_locales() -> Result, io::Error> { + let mut locales = vec![]; + + let res_dir = fs::read_dir("./examples/resources/")?; + for entry in res_dir { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name() { + if let Some(name) = name.to_str() { + locales.push(String::from(name)); + } + } + } + } + } + return Ok(locales); +} + +/// This function negotiates the locales between available +/// and requested by the user. +/// +/// It uses `fluent-locale` library but one could +/// use any other that will resolve the list of +/// available locales based on the list of +/// requested locales. +fn get_app_locales(requested: &[&str]) -> Result, io::Error> { + let available = get_available_locales()?; + let resolved_locales = negotiate_languages( + requested, + &available, + Some("en-US"), + &NegotiationStrategy::Filtering, + ); + return Ok(resolved_locales + .into_iter() + .map(|s| String::from(s)) + .collect::>()); +} + +static L10N_RESOURCES: &[&str] = &["simple.ftl"]; + +fn main() { + // 1. Get the command line arguments. + let args: Vec = env::args().collect(); + let mut sources: Vec = vec![]; + + // 2. If the argument length is more than 1, + // take the second argument as a comma-separated + // list of requested locales. + // + // Otherwise, take ["en-US"] as the default. + let requested = args + .get(2) + .map_or(vec!["en-US"], |arg| arg.split(",").collect()); + + // 3. Negotiate it against the avialable ones + let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); + + // 4. Create a new Fluent FluentBundle using the + // resolved locales. + let mut bundle = FluentBundle::new(&locales); + + // 5. Load the localization resource + for path in L10N_RESOURCES { + let full_path = format!( + "./examples/resources/{locale}/{path}", + locale = locales[0], + path = path + ); + let source = read_file(&full_path).unwrap(); + sources.push(source); + } + + for source in &sources { + bundle.add_messages(&source).unwrap(); + } + + // 6. Check if the input is provided. + match args.get(1) { + Some(input) => { + // 6.1. Cast it to a number. + match isize::from_str(&input) { + Ok(i) => { + // 6.2. Construct a map of arguments + // to format the message. + let mut args = HashMap::new(); + args.insert("input", FluentValue::from(i)); + args.insert("value", FluentValue::from(collatz(i))); + // 6.3. Format the message. + println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); + } + Err(err) => { + let mut args = HashMap::new(); + args.insert("input", FluentValue::from(input.to_string())); + args.insert("reason", FluentValue::from(err.to_string())); + println!( + "{}", + bundle + .format("input-parse-error-msg", Some(&args)) + .unwrap() + .0 + ); + } + } + } + None => { + println!("{}", bundle.format("missing-arg-error", None).unwrap().0); + } + } +} + +/// Collatz conjecture calculating function. +fn collatz(n: isize) -> isize { + match n { + 1 => 0, + _ => match n % 2 { + 0 => 1 + collatz(n / 2), + _ => 1 + collatz(n * 3 + 1), + }, + } +} diff --git a/fluent/src/bundle.rs b/fluent-bundle/src/bundle.rs similarity index 70% rename from fluent/src/bundle.rs rename to fluent-bundle/src/bundle.rs index 1eff32c7..6c096aad 100644 --- a/fluent/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -33,7 +33,7 @@ pub struct Message { /// Next, call `add_messages` one or more times, supplying translations in the FTL syntax. The /// `FluentBundle` instance is now ready to be used for localization. /// -/// To format a translation, call `get_message` to retrieve a `fluent::bundle::Message` structure +/// To format a translation, call `get_message` to retrieve a `fluent_bundle::bundle::Message` structure /// and then `format` it within the bundle. /// /// The result is an Option wrapping a single string that should be displayed in the UI. It is @@ -57,10 +57,7 @@ pub struct FluentBundle<'bundle> { impl<'bundle> FluentBundle<'bundle> { pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { - let locales = locales - .into_iter() - .map(|s| s.to_string()) - .collect::>(); + let locales = locales.iter().map(|s| s.to_string()).collect::>(); let pr_locale = negotiate_languages( &locales, IntlPluralRules::get_locales(PluralRuleType::CARDINAL), @@ -100,45 +97,53 @@ impl<'bundle> FluentBundle<'bundle> { } } - pub fn add_messages(&mut self, source: &str) -> Result<(), Vec> { - match FluentResource::from_string(source) { - Ok(res) => self.add_resource(res), - Err((res, err)) => { - let mut errors: Vec = - err.into_iter().map(FluentError::ParserError).collect(); - - self.add_resource(res).map_err(|err| { - for e in err { - errors.push(e); - } - errors - }) - } - } - } - - pub fn add_resource(&mut self, res: FluentResource) -> Result<(), Vec> { + //pub fn add_messages(&mut self, source: &'bundle str) -> Result<(), Vec> { + //match FluentResource::from_string(source) { + //Ok(res) => self.add_resource(&res), + //Err((res, err)) => { + //let mut errors: Vec = + //err.into_iter().map(FluentError::ParserError).collect(); + + //self.add_resource(&res).map_err(|err| { + //for e in err { + //errors.push(e); + //} + //errors + //}) + //} + //} + //} + + pub fn add_resource( + &mut self, + res: &'bundle FluentResource<'bundle>, + ) -> Result<(), Vec> { let mut errors = vec![]; - for entry in res.ast.body { + for entry in &res.ast.body { let id = match entry { - ast::Entry::Message(ast::Message { ref id, .. }) => id.name.clone(), - ast::Entry::Term(ast::Term { ref id, .. }) => id.name.clone(), + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { ref id, .. })) + | ast::ResourceEntry::Entry(ast::Entry::Term(ast::Term { ref id, .. })) => id.name, _ => continue, }; let (entry, kind) = match entry { - ast::Entry::Message(message) => (Entry::Message(message), "message"), - ast::Entry::Term(term) => (Entry::Term(term), "term"), + ast::ResourceEntry::Entry(ast::Entry::Message(message)) => { + (Entry::Message(message), "message") + } + ast::ResourceEntry::Entry(ast::Entry::Term(term)) => (Entry::Term(term), "term"), _ => continue, }; - match self.entries.entry(id.clone()) { + match self.entries.entry(id.to_string()) { HashEntry::Vacant(empty) => { empty.insert(entry); } HashEntry::Occupied(_) => { - errors.push(FluentError::Overriding { kind, id }); + errors.push(FluentError::Overriding { + kind, + id: id.to_string(), + }); } } } @@ -167,20 +172,18 @@ impl<'bundle> FluentBundle<'bundle> { let message_id = &path[..ptr_pos]; let message = self.entries.get_message(message_id)?; let attr_name = &path[(ptr_pos + 1)..]; - if let Some(ref attributes) = message.attributes { - for attribute in attributes { - if attribute.id.name == attr_name { - match attribute.to_value(&env) { - Ok(val) => { - return Some((val.format(self), errors)); - } - Err(err) => { - errors.push(FluentError::ResolverError(err)); - // XXX: In the future we'll want to get the partial - // value out of resolver and return it here. - // We also expect to get a Vec or errors out of resolver. - return Some((path.to_string(), errors)); - } + for attribute in message.attributes.iter() { + if attribute.id.name == attr_name { + match attribute.to_value(&env) { + Ok(val) => { + return Some((val.format(self), errors)); + } + Err(err) => { + errors.push(FluentError::ResolverError(err)); + // XXX: In the future we'll want to get the partial + // value out of resolver and return it here. + // We also expect to get a Vec or errors out of resolver. + return Some((path.to_string(), errors)); } } } @@ -227,16 +230,14 @@ impl<'bundle> FluentBundle<'bundle> { let mut attributes = HashMap::new(); - if let Some(ref attrs) = message.attributes { - for attr in attrs { - match attr.to_value(&env) { - Ok(value) => { - let val = value.format(self); - attributes.insert(attr.id.name.to_owned(), val); - } - Err(err) => { - errors.push(FluentError::ResolverError(err)); - } + for attr in message.attributes.iter() { + match attr.to_value(&env) { + Ok(value) => { + let val = value.format(self); + attributes.insert(attr.id.name.to_owned(), val); + } + Err(err) => { + errors.push(FluentError::ResolverError(err)); } } } diff --git a/fluent/src/entry.rs b/fluent-bundle/src/entry.rs similarity index 86% rename from fluent/src/entry.rs rename to fluent-bundle/src/entry.rs index 467599c9..b378dd1d 100644 --- a/fluent/src/entry.rs +++ b/fluent-bundle/src/entry.rs @@ -12,8 +12,8 @@ type FluentFunction<'bundle> = Box< >; pub enum Entry<'bundle> { - Message(ast::Message), - Term(ast::Term), + Message(&'bundle ast::Message<'bundle>), + Term(&'bundle ast::Term<'bundle>), Function(FluentFunction<'bundle>), } @@ -25,14 +25,14 @@ pub trait GetEntry<'bundle> { impl<'bundle> GetEntry<'bundle> for HashMap> { fn get_term(&self, id: &str) -> Option<&ast::Term> { - self.get(id).and_then(|entry| match entry { + self.get(id).and_then(|entry| match *entry { Entry::Term(term) => Some(term), _ => None, }) } fn get_message(&self, id: &str) -> Option<&ast::Message> { - self.get(id).and_then(|entry| match entry { + self.get(id).and_then(|entry| match *entry { Entry::Message(message) => Some(message), _ => None, }) diff --git a/fluent/src/errors.rs b/fluent-bundle/src/errors.rs similarity index 92% rename from fluent/src/errors.rs rename to fluent-bundle/src/errors.rs index 66fafa8d..9b7cf418 100644 --- a/fluent/src/errors.rs +++ b/fluent-bundle/src/errors.rs @@ -1,5 +1,5 @@ use super::resolve::ResolverError; -use fluent_syntax::parser::errors::ParserError; +use fluent_syntax::parser::ParserError; #[derive(Debug, Fail, PartialEq)] pub enum FluentError { diff --git a/fluent-bundle/src/lib.rs b/fluent-bundle/src/lib.rs new file mode 100644 index 00000000..b516ac9c --- /dev/null +++ b/fluent-bundle/src/lib.rs @@ -0,0 +1,48 @@ +//! Fluent is a localization system designed to improve how software is translated. +//! +//! The Rust implementation provides the low level components for syntax operations, like parser +//! and AST, and the core localization struct - `FluentBundle`. +//! +//! `FluentBundle` is the low level container for storing and formatting localization messages. It +//! is expected that implementations will build on top of it by providing language negotiation +//! between user requested languages and available resources and I/O for loading selected +//! resources. +//! +//! # Example +//! +//! ``` +//! use fluent_bundle::bundle::FluentBundle; +//! use fluent_bundle::types::FluentValue; +//! use std::collections::HashMap; +//! +//! let mut bundle = FluentBundle::new(&["en-US"]); +//! bundle.add_messages( +//! " +//! hello-world = Hello, world! +//! intro = Welcome, { $name }. +//! " +//! ); +//! +//! let value = bundle.format("hello-world", None); +//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); +//! +//! let mut args = HashMap::new(); +//! args.insert("name", FluentValue::from("John")); +//! +//! let value = bundle.format("intro", Some(&args)); +//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); +//! ``` + +extern crate failure; +#[macro_use] +extern crate failure_derive; +extern crate fluent_locale; +extern crate fluent_syntax; +extern crate intl_pluralrules; + +pub mod bundle; +pub mod entry; +pub mod errors; +pub mod resolve; +pub mod resource; +pub mod types; diff --git a/fluent/src/resolve.rs b/fluent-bundle/src/resolve.rs similarity index 50% rename from fluent/src/resolve.rs rename to fluent-bundle/src/resolve.rs index e093cfc8..89375951 100644 --- a/fluent/src/resolve.rs +++ b/fluent-bundle/src/resolve.rs @@ -62,7 +62,7 @@ pub trait ResolveValue { fn to_value(&self, env: &Env) -> Result; } -impl ResolveValue for ast::Message { +impl<'resolver> ResolveValue for ast::Message<'resolver> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || { self.value @@ -73,19 +73,31 @@ impl ResolveValue for ast::Message { } } -impl ResolveValue for ast::Term { +impl<'resolver> ResolveValue for ast::Term<'resolver> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl ResolveValue for ast::Attribute { +impl<'resolver> ResolveValue for ast::Attribute<'resolver> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl ResolveValue for ast::Pattern { +impl<'resolver> ResolveValue for ast::Value<'resolver> { + fn to_value(&self, env: &Env) -> Result { + match self { + ast::Value::Pattern(p) => p.to_value(env), + ast::Value::VariantList { variants } => select_default(variants) + .ok_or(ResolverError::None)? + .value + .to_value(env), + } + } +} + +impl<'resolver> ResolveValue for ast::Pattern<'resolver> { fn to_value(&self, env: &Env) -> Result { let mut string = String::with_capacity(128); for elem in &self.elements { @@ -107,79 +119,44 @@ impl ResolveValue for ast::Pattern { } } -impl ResolveValue for ast::PatternElement { +impl<'resolver> ResolveValue for ast::PatternElement<'resolver> { fn to_value(&self, env: &Env) -> Result { match self { - ast::PatternElement::TextElement(s) => Ok(FluentValue::from(s.clone())), + ast::PatternElement::TextElement(s) => Ok(FluentValue::from(*s)), ast::PatternElement::Placeable(p) => p.to_value(env), } } } -impl ResolveValue for ast::Number { - fn to_value(&self, _env: &Env) -> Result { - FluentValue::as_number(&self.value).map_err(|_| ResolverError::None) - } -} - -impl ResolveValue for ast::VariantName { +impl<'resolver> ResolveValue for ast::VariantKey<'resolver> { fn to_value(&self, _env: &Env) -> Result { - Ok(FluentValue::from(self.name.clone())) + match self { + ast::VariantKey::Identifier { name } => Ok(FluentValue::from(*name)), + ast::VariantKey::NumberLiteral { value } => { + FluentValue::as_number(value).map_err(|_| ResolverError::None) + } + } } } -impl ResolveValue for ast::Expression { +impl<'resolver> ResolveValue for ast::Expression<'resolver> { fn to_value(&self, env: &Env) -> Result { match self { - ast::Expression::StringExpression { value } => Ok(FluentValue::from(value.clone())), - ast::Expression::NumberExpression { value } => value.to_value(env), - ast::Expression::MessageReference { ref id } if id.name.starts_with('-') => env - .bundle - .entries - .get_term(&id.name) - .ok_or(ResolverError::None)? - .to_value(env), - ast::Expression::MessageReference { ref id } => env - .bundle - .entries - .get_message(&id.name) - .ok_or(ResolverError::None)? - .to_value(env), - ast::Expression::ExternalArgument { ref id } => env - .args - .and_then(|args| args.get(&id.name.as_ref())) - .cloned() - .ok_or(ResolverError::None), - ast::Expression::SelectExpression { - expression: None, - variants, - } => select_default(variants) - .ok_or(ResolverError::None)? - .value - .to_value(env), - ast::Expression::SelectExpression { - expression, - variants, - } => { - let selector = expression - .as_ref() - .ok_or(ResolverError::None)? - .to_value(env); - - if let Ok(ref selector) = selector { + ast::Expression::InlineExpression(exp) => exp.to_value(env), + ast::Expression::SelectExpression { selector, variants } => { + if let Ok(ref selector) = selector.to_value(env) { for variant in variants { match variant.key { - ast::VarKey::VariantName(ref symbol) => { - let key = FluentValue::from(symbol.name.clone()); + ast::VariantKey::Identifier { name } => { + let key = FluentValue::from(name); if key.matches(env.bundle, selector) { return variant.value.to_value(env); } } - ast::VarKey::Number(ref number) => { - if let Ok(key) = number.to_value(env) { - if key.matches(env.bundle, selector) { - return variant.value.to_value(env); - } + ast::VariantKey::NumberLiteral { value } => { + let key = FluentValue::as_number(value).unwrap(); + if key.matches(env.bundle, selector) { + return variant.value.to_value(env); } } } @@ -191,100 +168,130 @@ impl ResolveValue for ast::Expression { .value .to_value(env) } - ast::Expression::AttributeExpression { id, name } => { - let attributes = if id.name.starts_with('-') { - env.bundle + } + } +} + +impl<'resolver> ResolveValue for ast::InlineExpression<'resolver> { + fn to_value(&self, env: &Env) -> Result { + match self { + ast::InlineExpression::StringLiteral { raw } => { + // XXX: We need to decode the raw into unicode here. + Ok(FluentValue::from(*raw)) + } + ast::InlineExpression::NumberLiteral { value } => { + Ok(FluentValue::as_number(*value).unwrap()) + } + ast::InlineExpression::VariableReference { id } => env + .args + .and_then(|args| args.get(&id.name)) + .cloned() + .ok_or(ResolverError::None), + ast::InlineExpression::CallExpression { + ref callee, + ref positional, + ref named, + } => { + let mut resolved_unnamed_args = Vec::new(); + let mut resolved_named_args = HashMap::new(); + + for expression in positional { + resolved_unnamed_args.push(expression.to_value(env).ok()); + } + + for arg in named { + resolved_named_args + .insert(arg.name.name.to_string(), arg.value.to_value(env).unwrap()); + } + + let func = match **callee { + ast::InlineExpression::FunctionReference { ref id } => { + env.bundle.entries.get_function(id.name) + } + _ => panic!(), + }; + + func.ok_or(ResolverError::None).and_then(|func| { + func(resolved_unnamed_args.as_slice(), &resolved_named_args) + .ok_or(ResolverError::None) + }) + } + ast::InlineExpression::AttributeExpression { reference, name } => { + let attributes: &Vec = match reference.as_ref() { + ast::InlineExpression::MessageReference { ref id } => env + .bundle .entries - .get_term(&id.name) + .get_message(&id.name) .ok_or(ResolverError::None)? .attributes - .as_ref() - } else { - env.bundle + .as_ref(), + ast::InlineExpression::TermReference { ref id } => env + .bundle .entries - .get_message(&id.name) + .get_term(&id.name) .ok_or(ResolverError::None)? .attributes - .as_ref() + .as_ref(), + _ => unimplemented!(), }; - if let Some(attributes) = attributes { - for attribute in attributes { - if attribute.id.name == name.name { - return attribute.to_value(env); - } + for attribute in attributes { + if attribute.id.name == name.name { + return attribute.to_value(env); } } Err(ResolverError::None) } - ast::Expression::VariantExpression { id, key } if id.name.starts_with('-') => { - let term = env - .bundle - .entries - .get_term(&id.name) - .ok_or(ResolverError::None)?; - let variants = match term.value.elements.as_slice() { - [ast::PatternElement::Placeable(ast::Expression::SelectExpression { - expression: None, - ref variants, - })] => variants, - _ => return term.value.to_value(env), - }; - - for variant in variants { - if variant.key == *key { - return variant.value.to_value(env); - } - } - - select_default(variants) - .ok_or(ResolverError::None)? - .value - .to_value(env) - } - ast::Expression::CallExpression { - ref callee, - ref args, - } => { - let resolved_unnamed_args = &mut Vec::new(); - let resolved_named_args = &mut HashMap::new(); - - for arg in args { - env.scope(|| match arg { - ast::Argument::Expression(ref expression) => { - resolved_unnamed_args.push(expression.to_value(env).ok()); - } - ast::Argument::NamedArgument { ref name, ref val } => { - let mut fluent_val: FluentValue; + ast::InlineExpression::VariantExpression { reference, key } => { + if let ast::InlineExpression::TermReference { ref id } = reference.as_ref() { + let term = env + .bundle + .entries + .get_term(&id.name) + .ok_or(ResolverError::None)?; - match val { - ast::ArgValue::Number(ref num) => { - fluent_val = num.to_value(env).unwrap(); - } - ast::ArgValue::String(ref string) => { - fluent_val = FluentValue::from(string.as_str()); + match term.value { + ast::Value::VariantList { ref variants } => { + for variant in variants { + if variant.key == *key { + return variant.value.to_value(env); } - }; + } - resolved_named_args.insert(name.name.clone(), fluent_val); + select_default(variants) + .ok_or(ResolverError::None)? + .value + .to_value(env) } - }); + ast::Value::Pattern(ref p) => p.to_value(env), + } + } else { + unimplemented!() } - - env.bundle - .entries - .get_function(&callee.name) - .ok_or(ResolverError::None) - .and_then(|func| { - func(resolved_unnamed_args.as_slice(), &resolved_named_args) - .ok_or(ResolverError::None) - }) } - _ => unimplemented!(), + ast::InlineExpression::FunctionReference { .. } => panic!(), + ast::InlineExpression::MessageReference { ref id } => env + .bundle + .entries + .get_message(&id.name) + .ok_or(ResolverError::None)? + .to_value(env), + ast::InlineExpression::TermReference { ref id } => env + .bundle + .entries + .get_term(&id.name) + .ok_or(ResolverError::None)? + .to_value(env), + ast::InlineExpression::Placeable { ref expression } => { + let exp = expression.as_ref(); + exp.to_value(env) + } } } } -fn select_default(variants: &[ast::Variant]) -> Option<&ast::Variant> { +fn select_default<'resolver>( + variants: &'resolver [ast::Variant<'resolver>], +) -> Option<&ast::Variant<'resolver>> { for variant in variants { if variant.default { return Some(variant); diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs new file mode 100644 index 00000000..d68dfbc9 --- /dev/null +++ b/fluent-bundle/src/resource.rs @@ -0,0 +1,16 @@ +use fluent_syntax::ast; +use fluent_syntax::parser::parse; +use fluent_syntax::parser::ParserError; + +pub struct FluentResource<'resource> { + pub ast: ast::Resource<'resource>, +} + +impl<'resource> FluentResource<'resource> { + pub fn from_string(source: &'resource str) -> Result)> { + match parse(&source) { + Ok(ast) => Ok(FluentResource { ast }), + Err((ast, errors)) => Err((FluentResource { ast }, errors)), + } + } +} diff --git a/fluent/src/types.rs b/fluent-bundle/src/types.rs similarity index 100% rename from fluent/src/types.rs rename to fluent-bundle/src/types.rs diff --git a/fluent/tests/bundle.rs b/fluent-bundle/tests/bundle.rs similarity index 95% rename from fluent/tests/bundle.rs rename to fluent-bundle/tests/bundle.rs index fc9a9bd3..2dd4ed3a 100644 --- a/fluent/tests/bundle.rs +++ b/fluent-bundle/tests/bundle.rs @@ -1,6 +1,4 @@ -extern crate fluent; - -use self::fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn bundle_new_from_str() { diff --git a/fluent/tests/format.rs b/fluent-bundle/tests/format.rs similarity index 82% rename from fluent/tests/format.rs rename to fluent-bundle/tests/format.rs index 7b5e2f50..f6ee28dc 100644 --- a/fluent/tests/format.rs +++ b/fluent-bundle/tests/format.rs @@ -1,9 +1,7 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; +use fluent_bundle::bundle::FluentBundle; #[test] fn format() { diff --git a/fluent/tests/format_message.rs b/fluent-bundle/tests/format_message.rs similarity index 79% rename from fluent/tests/format_message.rs rename to fluent-bundle/tests/format_message.rs index 472557e1..28394855 100644 --- a/fluent/tests/format_message.rs +++ b/fluent-bundle/tests/format_message.rs @@ -1,10 +1,8 @@ -extern crate fluent; - mod helpers; -use fluent::bundle::FluentBundle; -use fluent::bundle::Message; -use helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::bundle::Message; use std::collections::HashMap; #[test] diff --git a/fluent/tests/helpers/mod.rs b/fluent-bundle/tests/helpers/mod.rs similarity index 89% rename from fluent/tests/helpers/mod.rs rename to fluent-bundle/tests/helpers/mod.rs index 6733bb9e..7b39cf4d 100644 --- a/fluent/tests/helpers/mod.rs +++ b/fluent-bundle/tests/helpers/mod.rs @@ -1,5 +1,5 @@ -use fluent::bundle::FluentError; -use fluent::bundle::Message; +use fluent_bundle::bundle::FluentError; +use fluent_bundle::bundle::Message; #[allow(dead_code)] pub fn assert_format_none(result: Option<(String, Vec)>) { diff --git a/fluent/tests/resolve_attribute_expression.rs b/fluent-bundle/tests/resolve_attribute_expression.rs similarity index 86% rename from fluent/tests/resolve_attribute_expression.rs rename to fluent-bundle/tests/resolve_attribute_expression.rs index cbd5edc8..80c97385 100644 --- a/fluent/tests/resolve_attribute_expression.rs +++ b/fluent-bundle/tests/resolve_attribute_expression.rs @@ -1,9 +1,7 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; #[test] fn attribute_expression() { diff --git a/fluent/tests/resolve_external_argument.rs b/fluent-bundle/tests/resolve_external_argument.rs similarity index 91% rename from fluent/tests/resolve_external_argument.rs rename to fluent-bundle/tests/resolve_external_argument.rs index cb22fa94..ade852d5 100644 --- a/fluent/tests/resolve_external_argument.rs +++ b/fluent-bundle/tests/resolve_external_argument.rs @@ -1,12 +1,10 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_string() { diff --git a/fluent/tests/resolve_message_reference.rs b/fluent-bundle/tests/resolve_message_reference.rs similarity index 93% rename from fluent/tests/resolve_message_reference.rs rename to fluent-bundle/tests/resolve_message_reference.rs index 7a7af037..3ad695ce 100644 --- a/fluent/tests/resolve_message_reference.rs +++ b/fluent-bundle/tests/resolve_message_reference.rs @@ -1,9 +1,7 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; #[test] fn message_reference() { diff --git a/fluent/tests/resolve_plural_rule.rs b/fluent-bundle/tests/resolve_plural_rule.rs similarity index 91% rename from fluent/tests/resolve_plural_rule.rs rename to fluent-bundle/tests/resolve_plural_rule.rs index c22b3483..7c655b1e 100644 --- a/fluent/tests/resolve_plural_rule.rs +++ b/fluent-bundle/tests/resolve_plural_rule.rs @@ -1,12 +1,10 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_number() { diff --git a/fluent/tests/resolve_select_expression.rs b/fluent-bundle/tests/resolve_select_expression.rs similarity index 85% rename from fluent/tests/resolve_select_expression.rs rename to fluent-bundle/tests/resolve_select_expression.rs index 48309369..57a36b3d 100644 --- a/fluent/tests/resolve_select_expression.rs +++ b/fluent-bundle/tests/resolve_select_expression.rs @@ -1,37 +1,10 @@ -extern crate fluent; - mod helpers; use std::collections::HashMap; -use self::fluent::bundle::FluentBundle; -use self::fluent::types::FluentValue; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; - -#[test] -fn select_expression_without_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages( - " -foo = - { - *[nominative] Foo - [genitive] Foo's - } - -bar = - { - [genitive] Bar's - *[nominative] Bar - } -", - )); - - assert_format_no_errors(bundle.format("foo", None), "Foo"); - - assert_format_no_errors(bundle.format("bar", None), "Bar"); -} +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn select_expression_string_selector() { @@ -216,11 +189,11 @@ fn select_expression_message_selector() { assert_add_messages_no_errors(bundle.add_messages( " -bar = Bar - .attr = attr val + .attr = attr_val use-bar = { -bar.attr -> - [attr val] Bar + [attr_val] Bar *[other] Other } ", @@ -235,11 +208,11 @@ fn select_expression_attribute_selector() { assert_add_messages_no_errors(bundle.add_messages( " -foo = Foo - .attr = Foo Attr + .attr = FooAttr use-foo = { -foo.attr -> - [Foo Attr] Foo + [FooAttr] Foo *[other] Other } ", diff --git a/fluent/tests/resolve_value.rs b/fluent-bundle/tests/resolve_value.rs similarity index 79% rename from fluent/tests/resolve_value.rs rename to fluent-bundle/tests/resolve_value.rs index 19cd074c..76f5601c 100644 --- a/fluent/tests/resolve_value.rs +++ b/fluent-bundle/tests/resolve_value.rs @@ -1,9 +1,7 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; #[test] fn format_message() { diff --git a/fluent/tests/resolve_variant_expression.rs b/fluent-bundle/tests/resolve_variant_expression.rs similarity index 82% rename from fluent/tests/resolve_variant_expression.rs rename to fluent-bundle/tests/resolve_variant_expression.rs index 3828b20a..922e1831 100644 --- a/fluent/tests/resolve_variant_expression.rs +++ b/fluent-bundle/tests/resolve_variant_expression.rs @@ -1,9 +1,7 @@ -extern crate fluent; - mod helpers; -use self::fluent::bundle::FluentBundle; -use helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; +use fluent_bundle::bundle::FluentBundle; #[test] fn variant_expression() { @@ -16,7 +14,7 @@ fn variant_expression() { *[nominative] Bar [genitive] Bar's } -bar = { -bar } +baz = { -bar } use-foo = { -foo } use-foo-missing = { -foo[missing] } @@ -30,7 +28,7 @@ missing-missing = { -missing[missing] } ", )); - assert_format_no_errors(bundle.format("bar", None), "Bar"); + assert_format_no_errors(bundle.format("baz", None), "Bar"); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent-cli/Cargo.toml b/fluent-cli/Cargo.toml new file mode 100644 index 00000000..0d19b704 --- /dev/null +++ b/fluent-cli/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "fluent-cli" +description = """ +A collection of command line interface programs +used for Fluent Localization System. +""" +version = "0.0.1" +edition = "2018" +authors = [ + "Zibi Braniecki ", + "Staś Małolepszy " +] +homepage = "http://www.projectfluent.org" +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] + +[dependencies] +annotate-snippets = {version = "0.1", features = ["color"]} +clap = "2.32" +fluent-syntax = { path = "../fluent-syntax" } diff --git a/fluent-cli/src/main.rs b/fluent-cli/src/main.rs new file mode 100644 index 00000000..34013b48 --- /dev/null +++ b/fluent-cli/src/main.rs @@ -0,0 +1,149 @@ +use std::cmp; +use std::fs::File; +use std::io; +use std::io::Read; + +use clap::App; + +use annotate_snippets::display_list::DisplayList; +use annotate_snippets::formatter::DisplayListFormatter; +use annotate_snippets::snippet::{Annotation, AnnotationType, Slice, Snippet, SourceAnnotation}; +use fluent_syntax::ast::Resource; +use fluent_syntax::parser::errors::ErrorKind; +use fluent_syntax::parser::parse; + +fn read_file(path: &str) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn print_entries_resource(res: &Resource) { + println!("{:#?}", res); +} + +fn main() { + let matches = App::new("Fluent Parser") + .version("1.0") + .about("Parses FTL file into an AST") + .args_from_usage( + "-s, --silence 'disable output' + 'Sets the input file to use'", + ) + .get_matches(); + + let input = matches.value_of("INPUT").unwrap(); + + let source = read_file(&input).expect("Read file failed"); + + let res = parse(&source); + + if matches.is_present("silence") { + return; + }; + + match res { + Ok(res) => print_entries_resource(&res), + Err((res, errors)) => { + print_entries_resource(&res); + println!("==============================\n"); + if errors.len() == 1 { + println!("Parser encountered one error:"); + } else { + println!("Parser encountered {} errors:", errors.len()); + } + println!("-----------------------------"); + for err in errors { + println!("{:#?}", err); + if let Some(slice) = err.slice { + let (id, desc) = get_error_info(err.kind); + let end_pos = cmp::min(err.pos.1, slice.1); + let snippet = Snippet { + slices: vec![Slice { + source: source[slice.0..slice.1].to_string(), + line_start: get_line_num(&source, err.pos.0) + 1, + origin: Some(input.to_string()), + fold: false, + annotations: vec![SourceAnnotation { + label: desc.to_string(), + annotation_type: AnnotationType::Error, + range: (err.pos.0 - slice.0, end_pos - slice.0 + 1), + }], + }], + title: Some(Annotation { + label: Some(desc.to_string()), + id: Some(id.to_string()), + annotation_type: AnnotationType::Error, + }), + footer: vec![], + }; + let dl = DisplayList::from(snippet); + let dlf = DisplayListFormatter::new(true); + println!("{}", dlf.format(&dl)); + println!("-----------------------------"); + } + } + } + }; +} + +fn get_line_num(source: &str, pos: usize) -> usize { + let mut ptr = 0; + let mut i = 0; + + let lines = source.lines(); + + for line in lines { + let lnlen = line.chars().count(); + ptr += lnlen + 1; + + if ptr > pos { + break; + } + i += 1; + } + + i +} + +fn get_error_info(kind: ErrorKind) -> (&'static str, String) { + match kind { + ErrorKind::Generic => ("E0001", "Generic error".to_string()), + ErrorKind::ExpectedEntry => ("E0002", "Expected an entry start".to_string()), + ErrorKind::ExpectedToken(ch) => ("E0003", format!("Expected token: \"{}\"", ch)), + ErrorKind::ExpectedCharRange { range } => ( + "E0004", + format!("Expected a character from range: \"{}\"", range), + ), + ErrorKind::ExpectedMessageField { entry_id } => ( + "E0005", + format!( + "Expected message \"{}\" to have a value or attributes", + entry_id + ), + ), + ErrorKind::ExpectedTermField { entry_id } => ( + "E0006", + format!("Expected term \"{}\" to have a value", entry_id), + ), + ErrorKind::ForbiddenWhitespace => { + ("E0007", "Keyword cannot end with a whitespace".to_string()) + } + ErrorKind::ForbiddenCallee => ( + "E0008", + "The callee has to be a simple, upper-case identifier".to_string(), + ), + ErrorKind::ForbiddenKey => ("E0009", "The key has to be a simple identifier".to_string()), + ErrorKind::MissingDefaultVariant => ( + "E0010", + "Expected one of the variants to be marked as default **)".to_string(), + ), + ErrorKind::MissingValue => ("E0012", "Expected value.".to_string()), + ErrorKind::TermAttributeAsPlaceable => ( + "E0019", + "Attributes of terms cannot be used as placeables".to_string(), + ), + _ => ("E0000", "Unknown Error.".to_string()), + } +} diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index ea86cda4..5850be5c 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -16,12 +16,9 @@ readme = "README.md" keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] -[dependencies] -clap = "2.32" -annotate-snippets = {version = "0.1", features = ["color"]} - [dev-dependencies] serde = "1.0" serde_derive = "1.0" serde_json = "1.0" glob = "0.2" +assert-json-diff = "0.2.0" diff --git a/fluent-syntax/benches/lib.rs b/fluent-syntax/benches/lib.rs index f1c2f656..d5736c5e 100644 --- a/fluent-syntax/benches/lib.rs +++ b/fluent-syntax/benches/lib.rs @@ -1,13 +1,12 @@ #![feature(test)] -extern crate fluent_syntax; extern crate test; -use self::test::Bencher; use fluent_syntax::parser::parse; use std::fs::File; use std::io; use std::io::Read; +use test::Bencher; fn read_file(path: &str) -> Result { let mut f = File::open(path)?; diff --git a/fluent-syntax/src/ast.rs b/fluent-syntax/src/ast.rs index dedc0bbe..2959e1d1 100644 --- a/fluent-syntax/src/ast.rs +++ b/fluent-syntax/src/ast.rs @@ -60,11 +60,6 @@ pub struct Identifier<'ast> { pub name: &'ast str, } -#[derive(Debug, PartialEq)] -pub struct Function<'ast> { - pub name: &'ast str, -} - #[derive(Debug, PartialEq)] pub struct Variant<'ast> { pub key: VariantKey<'ast>, @@ -88,7 +83,7 @@ pub enum Comment<'ast> { #[derive(Debug, PartialEq)] pub enum InlineExpression<'ast> { StringLiteral { - value: &'ast str, + raw: &'ast str, }, NumberLiteral { value: &'ast str, @@ -97,7 +92,7 @@ pub enum InlineExpression<'ast> { id: Identifier<'ast>, }, CallExpression { - callee: Function<'ast>, + callee: Box>, positional: Vec>, named: Vec>, }, @@ -115,6 +110,9 @@ pub enum InlineExpression<'ast> { TermReference { id: Identifier<'ast>, }, + FunctionReference { + id: Identifier<'ast>, + }, Placeable { expression: Box>, }, diff --git a/fluent-syntax/src/bin/parser.rs b/fluent-syntax/src/bin/parser.rs deleted file mode 100644 index 0cb35bed..00000000 --- a/fluent-syntax/src/bin/parser.rs +++ /dev/null @@ -1,58 +0,0 @@ -extern crate clap; -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::Read; - -use clap::App; - -use fluent_syntax::ast::Resource; -use fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = File::open(path)?; - let mut s = String::new(); - f.read_to_string(&mut s)?; - Ok(s) -} - -fn print_entries_resource(res: &Resource) { - println!("{:#?}", res); -} - -fn main() { - let matches = App::new("Fluent Parser") - .version("1.0") - .about("Parses FTL file into an AST") - .args_from_usage( - "-s, --silence 'disable output' - 'Sets the input file to use'", - ) - .get_matches(); - - let input = matches.value_of("INPUT").unwrap(); - - let source = read_file(&input).expect("Read file failed"); - - let res = parse(&source); - - if matches.is_present("silence") { - return; - }; - - match res { - Ok(res) => print_entries_resource(&res), - Err((res, errors)) => { - print_entries_resource(&res); - println!("==============================\n"); - if errors.len() == 1 { - println!("Parser encountered one error:"); - } else { - println!("Parser encountered {} errors:", errors.len()); - } - println!("-----------------------------"); - println!("{:#?}", errors); - } - }; -} diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs index 724c7446..a310c76d 100644 --- a/fluent-syntax/src/lib.rs +++ b/fluent-syntax/src/lib.rs @@ -1,4 +1,2 @@ -#![feature(box_syntax, box_patterns)] - pub mod ast; pub mod parser; diff --git a/fluent-syntax/src/parser/errors/mod.rs b/fluent-syntax/src/parser/errors/mod.rs index 0607d912..7b8ab028 100644 --- a/fluent-syntax/src/parser/errors/mod.rs +++ b/fluent-syntax/src/parser/errors/mod.rs @@ -1,14 +1,21 @@ #[derive(Debug, PartialEq)] pub struct ParserError { - pub pos: usize, + pub pos: (usize, usize), pub slice: Option<(usize, usize)>, pub kind: ErrorKind, } macro_rules! error { - ($ps:ident, $kind:expr) => {{ + ($kind:expr, $start:expr) => {{ Err(ParserError { - pos: $ps.ptr, + pos: ($start, $start + 1), + slice: None, + kind: $kind, + }) + }}; + ($kind:expr, $start:expr, $end:expr) => {{ + Err(ParserError { + pos: ($start, $end), slice: None, kind: $kind, }) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index 0e3d8fd8..4205a3eb 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -37,7 +37,7 @@ impl<'p> ParserStream<'p> { pub fn expect_byte(&mut self, b: u8) -> Result<()> { if !self.is_current_byte(b) { - return error!(self, ErrorKind::ExpectedToken(b as char)); + return error!(ErrorKind::ExpectedToken(b as char), self.ptr); } self.ptr += 1; Ok(()) @@ -77,7 +77,7 @@ impl<'p> ParserStream<'p> { } } - pub fn skip_blank_inline(&mut self) -> bool { + pub fn skip_blank_inline(&mut self) -> usize { let start = self.ptr; while self.ptr < self.length { let b = self.source[self.ptr]; @@ -87,7 +87,7 @@ impl<'p> ParserStream<'p> { break; } } - start != self.ptr + self.ptr - start } pub fn skip_to_next_entry_start(&mut self) { @@ -133,53 +133,61 @@ impl<'p> ParserStream<'p> { (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') || b == b'-' } - pub fn skip_to_value_start(&mut self) -> bool { - let start = self.ptr; + pub fn skip_to_value_start(&mut self) -> Option { self.skip_blank_inline(); - if self.ptr >= self.length { - return false; - } if !self.is_eol() { - return true; + return Some(true); } - self.skip_to_next_line_value(start) - } - pub fn skip_to_next_line_value(&mut self, start: usize) -> bool { self.skip_blank_block(); + let inline = self.skip_blank_inline(); if self.is_current_byte(b'{') { - return true; + //self.ptr -= inline; + return Some(true); } - if !inline { - self.ptr = start; - return false; + + if inline == 0 { + return None; } if !self.is_char_pattern_continuation() { - self.ptr = start; - return false; + return None; } - true + self.ptr -= inline; + Some(false) } - pub fn is_char_pattern_continuation(&self) -> bool { - if self.ptr >= self.length { - return false; + pub fn skip_unicode_escape_sequence(&mut self, length: usize) -> Result<()> { + let start = self.ptr; + for _ in 0..length { + match self.source[self.ptr] { + b'0'...b'9' => self.ptr += 1, + b'a'...b'f' => self.ptr += 1, + b'A'...b'F' => self.ptr += 1, + _ => break, + } } - - let b = self.source[self.ptr]; - b != b'}' && b != b'.' && b != b'[' && b != b'*' + if self.ptr - start != length { + return error!( + ErrorKind::InvalidUnicodeEscapeSequence( + self.get_slice(start, self.ptr + 1).to_owned() + ), + self.ptr + ); + } + Ok(()) } - pub fn is_pattern_start(&self) -> bool { + pub fn is_char_pattern_continuation(&self) -> bool { if self.ptr >= self.length { return false; } + let b = self.source[self.ptr]; - b != b'.' && b != b'[' && b != b'*' && b != b'}' + b != b'}' && b != b'.' && b != b'[' && b != b'*' } pub fn is_identifier_start(&self) -> bool { @@ -199,10 +207,6 @@ impl<'p> ParserStream<'p> { } pub fn is_eol(&self) -> bool { - if self.ptr >= self.length { - return false; - } - if self.is_current_byte(b'\n') { return true; } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 79db6404..56635aa7 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -2,6 +2,7 @@ pub mod errors; mod ftlstream; +use std::cmp; use std::result; use std::str; @@ -20,107 +21,48 @@ pub fn parse(source: &str) -> result::Result { - if last_entry_end != entry_start { - let mut te = 0; - while ps.is_byte_at(b'\n', entry_start - te - 1) { - te += 1; - } - let te = if te > 1 { te - 1 } else { 0 }; - let slice = ps.get_slice(last_entry_end, entry_start - te); - body.push(ast::ResourceEntry::Junk(slice)); + let mut entry = get_entry(&mut ps, entry_start); + + if let Some(comment) = last_comment.take() { + match entry { + Ok(ast::Entry::Message(ref mut msg)) if last_blank_count < 2 => { + msg.comment = Some(comment); } - if let Some(content) = last_comment { - match entry { - ast::Entry::Message(mut msg) => { - msg.comment = Some(ast::Comment::Comment { content }); - body.push(ast::ResourceEntry::Entry(ast::Entry::Message(msg))); - last_comment = None; - } - ast::Entry::Term(mut term) => { - term.comment = Some(ast::Comment::Comment { content }); - body.push(ast::ResourceEntry::Entry(ast::Entry::Term(term))); - last_comment = None; - } - ast::Entry::Comment(new_comment) => { - body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( - ast::Comment::Comment { content }, - ))); - if let ast::Comment::Comment { content } = new_comment { - last_comment = Some(content); - } else { - body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( - new_comment, - ))); - last_comment = None; - } - } - } - } else { - match entry { - ast::Entry::Comment(ast::Comment::Comment { content }) => { - last_comment = Some(content); - } - _ => { - body.push(ast::ResourceEntry::Entry(entry)); - } - } + Ok(ast::Entry::Term(ref mut term)) if last_blank_count < 2 => { + term.comment = Some(comment); } - ps.skip_eol(); - if ps.skip_blank_block() > 0 { - if let Some(content) = last_comment { - body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( - ast::Comment::Comment { content }, - ))); - last_comment = None; - } + _ => { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment(comment))); } - last_entry_end = ps.ptr; + } + } + + match entry { + Ok(ast::Entry::Comment(comment @ ast::Comment::Comment { .. })) => { + last_comment = Some(comment); + } + Ok(entry) => { + body.push(ast::ResourceEntry::Entry(entry)); } Err(mut err) => { - if let Some(content) = last_comment { - body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( - ast::Comment::Comment { content }, - ))); - last_comment = None; - } ps.skip_to_next_entry_start(); - let mut te = 0; - while ps.is_byte_at(b'\n', ps.ptr - te - 1) { - te += 1; - } - err.slice = Some((last_entry_end, ps.ptr - te)); + err.slice = Some((entry_start, ps.ptr)); errors.push(err); - if te > 1 { - let slice = ps.get_slice(last_entry_end, ps.ptr - te + 1); - body.push(ast::ResourceEntry::Junk(slice)); - last_entry_end = ps.ptr; - } + let slice = ps.get_slice(entry_start, ps.ptr); + body.push(ast::ResourceEntry::Junk(slice)); } } - } - if let Some(content) = last_comment { - body.push(ast::ResourceEntry::Entry(ast::Entry::Comment( - ast::Comment::Comment { content }, - ))); - } - if last_entry_end != ps.ptr { - let mut te = 0; - while ps.is_byte_at(b'\n', ps.ptr - te - 1) { - te += 1; - } - let te = if te > 1 { te - 1 } else { 0 }; - let slice = ps.get_slice(last_entry_end, ps.ptr - te); - body.push(ast::ResourceEntry::Junk(slice)); + last_blank_count = ps.skip_blank_block(); } + if let Some(last_comment) = last_comment.take() { + body.push(ast::ResourceEntry::Entry(ast::Entry::Comment(last_comment))); + } if errors.is_empty() { Ok(ast::Resource { body }) } else { @@ -128,26 +70,21 @@ pub fn parse(source: &str) -> result::Result(ps: &mut ParserStream<'p>) -> Result> { +fn get_entry<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { let entry = match ps.source[ps.ptr] { b'#' => ast::Entry::Comment(get_comment(ps)?), - b'-' => ast::Entry::Term(get_term(ps)?), - _ => ast::Entry::Message(get_message(ps)?), + b'-' => ast::Entry::Term(get_term(ps, entry_start)?), + _ => ast::Entry::Message(get_message(ps, entry_start)?), }; Ok(entry) } -fn get_message<'p>(ps: &mut ParserStream<'p>) -> Result> { +fn get_message<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { let id = get_identifier(ps)?; ps.skip_blank_inline(); ps.expect_byte(b'=')?; - ps.skip_blank_inline(); - let pattern = if ps.skip_to_value_start() { - get_pattern(ps)? - } else { - None - }; + let pattern = get_pattern(ps)?; ps.skip_blank_block(); @@ -162,10 +99,10 @@ fn get_message<'p>(ps: &mut ParserStream<'p>) -> Result> { if pattern.is_none() && attributes.is_empty() { return error!( - ps, ErrorKind::ExpectedMessageField { entry_id: id.name.to_string() - } + }, + entry_start, ps.ptr ); } @@ -177,7 +114,7 @@ fn get_message<'p>(ps: &mut ParserStream<'p>) -> Result> { }) } -fn get_term<'p>(ps: &mut ParserStream<'p>) -> Result> { +fn get_term<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result> { ps.expect_byte(b'-')?; let id = get_identifier(ps)?; ps.skip_blank_inline(); @@ -206,16 +143,16 @@ fn get_term<'p>(ps: &mut ParserStream<'p>) -> Result> { }) } else { error!( - ps, ErrorKind::ExpectedTermField { entry_id: id.name.to_string() - } + }, + entry_start, ps.ptr ) } } fn get_value<'p>(ps: &mut ParserStream<'p>) -> Result>> { - if !ps.skip_to_value_start() { + if ps.skip_to_value_start().is_none() { return Ok(None); } @@ -224,7 +161,7 @@ fn get_value<'p>(ps: &mut ParserStream<'p>) -> Result>> { ps.ptr += 1; ps.skip_blank(); if ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { - let variants = get_variants(ps, true)?; + let variants = get_variants(ps)?; ps.expect_byte(b'}')?; return Ok(Some(ast::Value::VariantList { variants })); } @@ -241,29 +178,37 @@ fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Result(ps: &mut ParserStream<'p>) -> Result> { + ps.expect_byte(b'.')?; + let id = get_identifier(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'=')?; + let pattern = get_pattern(ps)?; + + match pattern { + Some(pattern) => Ok(ast::Attribute { id, value: pattern }), + None => error!(ErrorKind::MissingValue, ps.ptr), + } +} + fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> { let start_pos = ps.ptr; @@ -274,10 +219,10 @@ fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> ps.ptr += 1; } else { return error!( - ps, ErrorKind::ExpectedCharRange { range: "a-zA-Z".to_string() - } + }, + ps.ptr ); } } else if (b >= b'a' && b <= b'z') @@ -298,7 +243,7 @@ fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> fn get_variant_key<'p>(ps: &mut ParserStream<'p>) -> Result> { if !ps.take_if(b'[') { - return error!(ps, ErrorKind::ExpectedToken('[')); + return error!(ErrorKind::ExpectedToken('['), ps.ptr); } ps.skip_blank(); @@ -319,10 +264,7 @@ fn get_variant_key<'p>(ps: &mut ParserStream<'p>) -> Result> Ok(key) } -fn get_variants<'p>( - ps: &mut ParserStream<'p>, - variant_lists: bool, -) -> Result>> { +fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> { let mut variants = vec![]; let mut has_default = false; @@ -331,7 +273,7 @@ fn get_variants<'p>( if default { if has_default { - return error!(ps, ErrorKind::MultipleDefaultVariants); + return error!(ErrorKind::MultipleDefaultVariants, ps.ptr); } else { has_default = true; } @@ -339,13 +281,7 @@ fn get_variants<'p>( let key = get_variant_key(ps)?; - ps.skip_blank_inline(); - - let value = if variant_lists { - get_value(ps)? - } else { - get_pattern(ps)?.map(ast::Value::Pattern) - }; + let value = get_pattern(ps)?.map(ast::Value::Pattern); if let Some(value) = value { variants.push(ast::Variant { @@ -355,126 +291,217 @@ fn get_variants<'p>( }); ps.skip_blank(); } else { - return error!(ps, ErrorKind::MissingValue); + return error!(ErrorKind::MissingValue, ps.ptr); } } if !has_default { - error!(ps, ErrorKind::MissingDefaultVariant) + error!(ErrorKind::MissingDefaultVariant, ps.ptr) } else { Ok(variants) } } -fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result>> { - let start = ps.ptr; - if ps.skip_eol() { - ps.skip_blank_block(); - if !ps.skip_blank_inline() || !ps.is_pattern_start() { - ps.ptr = start; - return Ok(None); - } - } +#[derive(Debug, PartialEq)] +enum TextElementTermination { + LineFeed, + CarriageReturn, + PlaceableStart, + EOF, +} +#[derive(Debug)] +enum PatternElementPointers<'a> { + Placeable(ast::Expression<'a>), + TextElement(usize, usize, usize, TextElementPosition), +} + +#[derive(Debug, PartialEq)] +enum TextElementPosition { + InitialLineStart, + LineStart, + Continuation, +} + +#[derive(Debug, PartialEq)] +enum TextElementType { + Blank, + NonBlank, +} + +fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result>> { let mut elements = vec![]; + let mut last_non_blank = None; + let mut common_indent = 10; - loop { - let mut start_pos = ps.ptr; + ps.skip_blank_inline(); - while ps.ptr < ps.length { - if ps.skip_eol() { - break; + let mut text_element_role = if ps.skip_eol() { + ps.skip_blank_block(); + TextElementPosition::LineStart + } else { + TextElementPosition::InitialLineStart + }; + + while ps.ptr < ps.length { + if ps.source[ps.ptr] == b'{' { + if text_element_role == TextElementPosition::LineStart { + common_indent = 0; } - let b = ps.source[ps.ptr]; - match b { - b'\\' => { - ps.ptr += 1; - let b = ps.source[ps.ptr]; - match b { - b'{' => ps.ptr += 1, - b'\\' => ps.ptr += 1, - b'u' => { - ps.ptr += 2; - let start = ps.ptr; - for _ in 0..4 { - match ps.source[ps.ptr] { - b'0'...b'9' => ps.ptr += 1, - b'a'...b'f' => ps.ptr += 1, - b'A'...b'F' => ps.ptr += 1, - _ => break, - } - } - if start == ps.ptr { - return error!( - ps, - ErrorKind::InvalidUnicodeEscapeSequence( - ps.get_slice(start, ps.ptr + 1).to_owned() - ) - ); - } - } - _ => panic!(), + let exp = get_placeable(ps)?; + last_non_blank = Some(elements.len()); + elements.push(PatternElementPointers::Placeable(exp)); + text_element_role = TextElementPosition::Continuation; + } else { + let slice_start = ps.ptr; + let mut indent = 0; + if text_element_role == TextElementPosition::LineStart { + indent = ps.skip_blank_inline(); + if indent == 0 { + if ps.source[ps.ptr] != b'\n' && ps.source[ps.ptr] != b'\r' { + break; } + } else if ps.source[ps.ptr] == b'.' + || ps.source[ps.ptr] == b'}' + || ps.source[ps.ptr] == b'*' + || ps.source[ps.ptr] == b'[' + { + ps.ptr = slice_start; + break; } - b'{' => { - if start_pos != ps.ptr { - let value = ps.get_slice(start_pos, ps.ptr); - elements.push(ast::PatternElement::TextElement(value)); + } + let (start, end, text_element_type, termination_reason) = get_text_slice(ps)?; + if start != end { + if text_element_role == TextElementPosition::LineStart + && text_element_type == TextElementType::NonBlank + && indent < common_indent + { + common_indent = indent; + } + if text_element_role != TextElementPosition::LineStart + || text_element_type == TextElementType::NonBlank + || termination_reason == TextElementTermination::LineFeed + { + if text_element_type == TextElementType::NonBlank { + last_non_blank = Some(elements.len()); } - ps.ptr += 1; // { - ps.skip_blank(); - let exp = get_expression(ps)?; - elements.push(ast::PatternElement::Placeable(exp)); - ps.skip_blank_inline(); - ps.expect_byte(b'}')?; - start_pos = ps.ptr; + elements.push(PatternElementPointers::TextElement( + slice_start, + end, + indent, + text_element_role, + )); } - _ => ps.ptr += 1, } - } - if start_pos != ps.ptr { - let value = ps.get_slice(start_pos, ps.ptr); - elements.push(ast::PatternElement::TextElement(value)); + text_element_role = match termination_reason { + TextElementTermination::LineFeed => TextElementPosition::LineStart, + TextElementTermination::CarriageReturn => TextElementPosition::Continuation, + TextElementTermination::PlaceableStart => TextElementPosition::Continuation, + TextElementTermination::EOF => TextElementPosition::Continuation, + }; } + } - let end_of_line = ps.ptr; - - let bl = ps.skip_blank_block(); - - if !ps.skip_blank_inline() || !ps.is_pattern_start() { - ps.ptr = end_of_line; - break; - } else { - for _ in 0..bl { - elements.push(ast::PatternElement::TextElement("\n")); + if let Some(last_non_blank) = last_non_blank { + let mut collected_elems = 0; + let elements = elements.into_iter().filter(|_| { + if collected_elems > last_non_blank { + return false; } - } + collected_elems += 1; + true + }); + + let elements = elements + .enumerate() + .filter_map(|(i, elem)| match elem { + PatternElementPointers::Placeable(exp) => Some(ast::PatternElement::Placeable(exp)), + PatternElementPointers::TextElement(start, end, indent, role) => { + let start = if role == TextElementPosition::LineStart { + start + cmp::min(indent, common_indent) + } else { + start + }; + let slice = ps.get_slice(start, end); + if last_non_blank == i { + if slice == "\n" { + return None; + } + Some(ast::PatternElement::TextElement(slice.trim_end())) + } else { + Some(ast::PatternElement::TextElement(slice)) + } + } + }) + .collect(); + return Ok(Some(ast::Pattern { elements })); } - if !elements.is_empty() { - let last_pos = elements.len() - 1; - let val = &elements[last_pos]; - let mut new_val = ""; - let mut modified = false; - if let ast::PatternElement::TextElement(te) = val { - new_val = te.trim_right(); - modified = &new_val != te; - } - if modified { - elements.pop(); - if !new_val.is_empty() { - elements.insert(last_pos, ast::PatternElement::TextElement(new_val)); - ps.ptr -= 1; // move before last \n + Ok(None) +} + +fn get_text_slice<'p>( + ps: &mut ParserStream<'p>, +) -> Result<(usize, usize, TextElementType, TextElementTermination)> { + let start_pos = ps.ptr; + let mut text_element_type = TextElementType::Blank; + + while ps.ptr < ps.length { + if ps.source[ps.ptr] == b'\n' { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::LineFeed, + )); + } else if ps.source[ps.ptr] == b'\r' && ps.is_byte_at(b'\n', ps.ptr + 1) { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr - 1, + text_element_type, + TextElementTermination::CarriageReturn, + )); + } + match ps.source[ps.ptr] { + b'{' => { + return Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::PlaceableStart, + )); + } + b'}' => { + return error!(ErrorKind::Generic, ps.ptr); + } + b'\\' => { + text_element_type = TextElementType::NonBlank; + ps.ptr += 1; + match ps.source[ps.ptr] { + b'\\' => ps.ptr += 1, + b'u' => { + ps.ptr += 1; + ps.skip_unicode_escape_sequence(4)?; + } + _ => {} + } + } + b' ' => ps.ptr += 1, + _ => { + text_element_type = TextElementType::NonBlank; + ps.ptr += 1 } } } - - if elements.is_empty() { - Ok(None) - } else { - Ok(Some(ast::Pattern { elements })) - } + Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::EOF, + )) } fn get_comment<'p>(ps: &mut ParserStream<'p>) -> Result> { @@ -533,33 +560,77 @@ fn get_comment_line<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { Ok(str::from_utf8(&ps.source[start_pos..ps.ptr]).unwrap()) } +fn get_placeable<'p>(ps: &mut ParserStream<'p>) -> Result> { + ps.expect_byte(b'{')?; + ps.skip_blank(); + let exp = get_expression(ps)?; + ps.skip_blank_inline(); + ps.expect_byte(b'}')?; + + let invalid_expression_found = match &exp { + ast::Expression::InlineExpression(ast::InlineExpression::AttributeExpression { + ref reference, + .. + }) => { + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false + } + } + ast::Expression::InlineExpression(ast::InlineExpression::CallExpression { + callee, .. + }) => { + if let ast::InlineExpression::AttributeExpression { .. } = **callee { + true + } else { + false + } + } + _ => false, + }; + if invalid_expression_found { + return error!(ErrorKind::TermAttributeAsPlaceable, ps.ptr); + } + + Ok(exp) +} + fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { - let exp = get_inline_expression(ps)?; + let exp = get_call_expression(ps)?; ps.skip_blank(); if !ps.is_current_byte(b'-') || !ps.is_byte_at(b'>', ps.ptr + 1) { - if let ast::InlineExpression::AttributeExpression { ref reference, .. } = exp { - if let box ast::InlineExpression::TermReference { .. } = reference { - return error!(ps, ErrorKind::TermAttributeAsPlaceable); - } - } return Ok(ast::Expression::InlineExpression(exp)); } - match exp { - ast::InlineExpression::MessageReference { .. } => { - return error!(ps, ErrorKind::MessageReferenceAsSelector); - } + let is_valid = match exp { + ast::InlineExpression::StringLiteral { .. } => true, + ast::InlineExpression::NumberLiteral { .. } => true, + ast::InlineExpression::VariableReference { .. } => true, ast::InlineExpression::AttributeExpression { ref reference, .. } => { - if let box ast::InlineExpression::MessageReference { .. } = reference { - return error!(ps, ErrorKind::MessageAttributeAsSelector); + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false } } - ast::InlineExpression::VariantExpression { .. } => { - return error!(ps, ErrorKind::VariantAsSelector); + ast::InlineExpression::CallExpression { ref callee, .. } => { + if let ast::InlineExpression::FunctionReference { .. } = **callee { + true + } else if let ast::InlineExpression::AttributeExpression { .. } = **callee { + true + } else { + false + } } - _ => {} + _ => false, + }; + + if !is_valid { + //XXX: Generalize error type + return error!(ErrorKind::MessageReferenceAsSelector, ps.ptr); } ps.ptr += 2; // -> @@ -568,7 +639,7 @@ fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> ps.expect_byte(b'\n')?; ps.skip_blank(); - let variants = get_variants(ps, false)?; + let variants = get_variants(ps)?; Ok(ast::Expression::SelectExpression { selector: exp, @@ -576,7 +647,67 @@ fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> }) } -fn get_inline_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { +fn get_call_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let mut callee = get_attribute_expression(ps)?; + + let expr = if ps.is_current_byte(b'(') { + let is_valid = match callee { + ast::InlineExpression::AttributeExpression { ref reference, .. } => { + if let ast::InlineExpression::TermReference { .. } = **reference { + true + } else { + false + } + } + ast::InlineExpression::TermReference { .. } => true, + ast::InlineExpression::MessageReference { ref id, .. } => { + id.name.find(|c: char| c.is_ascii_lowercase()).is_none() + } + _ => false, + }; + + if is_valid { + if let ast::InlineExpression::MessageReference { id } = callee { + callee = ast::InlineExpression::FunctionReference { id }; + } + let (positional, named) = get_call_args(ps)?; + ast::InlineExpression::CallExpression { + callee: Box::new(callee), + positional, + named, + } + } else { + return error!(ErrorKind::ForbiddenCallee, ps.ptr); + } + } else { + callee + }; + + Ok(expr) +} + +fn get_attribute_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { + let reference = get_literal(ps)?; + + match reference { + ast::InlineExpression::MessageReference { .. } + | ast::InlineExpression::TermReference { .. } => { + if ps.is_current_byte(b'.') { + ps.ptr += 1; // . + let attr = get_identifier(ps)?; + Ok(ast::InlineExpression::AttributeExpression { + reference: Box::new(reference), + name: attr, + }) + } else { + Ok(reference) + } + } + _ => Ok(reference), + } +} + +fn get_literal<'p>(ps: &mut ParserStream<'p>) -> Result> { match ps.source.get(ps.ptr) { Some(b'"') => { ps.ptr += 1; // " @@ -589,37 +720,27 @@ fn get_inline_expression<'p>(ps: &mut ParserStream<'p>) -> Result ps.ptr += 2, b'u' => { ps.ptr += 2; - let start = ps.ptr; - for _ in 0..4 { - match ps.source[ps.ptr] { - b'0'...b'9' => ps.ptr += 1, - b'a'...b'f' => ps.ptr += 1, - b'A'...b'F' => ps.ptr += 1, - _ => break, - } - } - if start == ps.ptr { - return error!( - ps, - ErrorKind::InvalidUnicodeEscapeSequence( - ps.get_slice(start, ps.ptr + 1).to_owned() - ) - ); - } + ps.skip_unicode_escape_sequence(4)?; + } + b'U' => { + ps.ptr += 2; + ps.skip_unicode_escape_sequence(6)?; } - _ => panic!(), + _ => return error!(ErrorKind::Generic, ps.ptr), }, b'"' => { break; } + b'\n' => { + return error!(ErrorKind::Generic, ps.ptr); + } _ => ps.ptr += 1, } } ps.expect_byte(b'"')?; - Ok(ast::InlineExpression::StringLiteral { - value: ps.get_slice(start, ps.ptr - 1), - }) + let slice = ps.get_slice(start, ps.ptr - 1); + Ok(ast::InlineExpression::StringLiteral { raw: slice }) } Some(b'0'...b'9') => { let num = get_number_literal(ps)?; @@ -630,14 +751,6 @@ fn get_inline_expression<'p>(ps: &mut ParserStream<'p>) -> Result { - ps.ptr += 1; // . - let attr = get_identifier(ps)?; - Ok(ast::InlineExpression::AttributeExpression { - reference: Box::new(ast::InlineExpression::TermReference { id }), - name: attr, - }) - } b'[' => { let key = get_variant_key(ps)?; Ok(ast::InlineExpression::VariantExpression { @@ -660,50 +773,18 @@ fn get_inline_expression<'p>(ps: &mut ParserStream<'p>) -> Result { let id = get_identifier(ps)?; - - match ps.source[ps.ptr] { - b'(' => get_call_expression(ps, Some(id)), - b'.' => { - ps.ptr += 1; // . - let attr = get_identifier(ps)?; - Ok(ast::InlineExpression::AttributeExpression { - reference: Box::new(ast::InlineExpression::MessageReference { id }), - name: attr, - }) - } - _ => Ok(ast::InlineExpression::MessageReference { id }), - } + Ok(ast::InlineExpression::MessageReference { id }) } Some(b'{') => { - ps.ptr += 1; // { - ps.skip_blank(); - let exp = get_expression(ps)?; - ps.skip_blank_inline(); - ps.expect_byte(b'}')?; + let exp = get_placeable(ps)?; Ok(ast::InlineExpression::Placeable { expression: Box::new(exp), }) } - _ => error!(ps, ErrorKind::MissingLiteral), + _ => error!(ErrorKind::MissingLiteral, ps.ptr), } } -fn get_call_expression<'p>( - ps: &mut ParserStream<'p>, - id: Option>, -) -> Result> { - let id = match id { - Some(id) => id, - None => get_identifier(ps)?, - }; - let (positional, named) = get_call_args(ps)?; - Ok(ast::InlineExpression::CallExpression { - callee: ast::Function { name: id.name }, - positional, - named, - }) -} - fn get_call_args<'p>( ps: &mut ParserStream<'p>, ) -> Result<(Vec>, Vec>)> { @@ -719,39 +800,40 @@ fn get_call_args<'p>( if b == b')' { break; } - let id = if ps.is_identifier_start() { - Some(get_identifier(ps)?) - } else { - None - }; - if let Some(id) = id { - ps.skip_blank(); - if ps.is_current_byte(b':') { - if argument_names.contains(&id.name.to_owned()) { - return error!(ps, ErrorKind::DuplicatedNamedArgument(id.name.to_owned())); - } - ps.ptr += 1; + let expr = get_call_expression(ps)?; + + match expr { + ast::InlineExpression::MessageReference { ref id } => { ps.skip_blank(); - let val = get_inline_expression(ps)?; - argument_names.push(id.name.to_owned()); - named.push(ast::NamedArgument { - name: id, - value: val, - }); - } else if ps.is_current_byte(b'(') { - positional.push(get_call_expression(ps, Some(id))?); - } else { - if !argument_names.is_empty() { - return error!(ps, ErrorKind::PositionalArgumentFollowsNamed); + if ps.is_current_byte(b':') { + if argument_names.contains(&id.name.to_owned()) { + return error!( + ErrorKind::DuplicatedNamedArgument(id.name.to_owned()), + ps.ptr + ); + } + ps.ptr += 1; + ps.skip_blank(); + let val = get_call_expression(ps)?; + argument_names.push(id.name.to_owned()); + named.push(ast::NamedArgument { + name: ast::Identifier { name: id.name }, + value: val, + }); + } else { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, ps.ptr); + } + positional.push(expr); } - positional.push(ast::InlineExpression::MessageReference { id }); } - } else { - if !argument_names.is_empty() { - return error!(ps, ErrorKind::PositionalArgumentFollowsNamed); + _ => { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, ps.ptr); + } + positional.push(expr); } - positional.push(get_inline_expression(ps)?); } ps.skip_blank(); diff --git a/fluent-syntax/tests/ast/helper.rs b/fluent-syntax/tests/ast/helper.rs new file mode 100644 index 00000000..243a33d8 --- /dev/null +++ b/fluent-syntax/tests/ast/helper.rs @@ -0,0 +1,67 @@ +use std::char; +use std::collections::VecDeque; + +fn encode_unicode(s: &str, l: usize) -> char { + if s.len() != l { + return '�'; + } + let u = match u32::from_str_radix(s, 16) { + Ok(u) => u, + Err(_) => return '�', + }; + char::from_u32(u).unwrap_or('�') +} + +pub fn unescape_unicode(s: &str) -> String { + let mut queue: VecDeque<_> = String::from(s).chars().collect(); + let mut result = String::new(); + + while let Some(c) = queue.pop_front() { + if c != '\\' { + result.push(c); + continue; + } + + match queue.pop_front() { + Some('u') => { + let mut buffer = String::new(); + for _ in 0..4 { + if let Some(c) = queue.pop_front() { + match c { + '0'...'9' | 'a'...'f' | 'A'...'F' => { + buffer.push(c); + } + _ => break, + } + } else { + break; + } + } + let new_char = encode_unicode(&buffer, 4); + result.push(new_char); + } + Some('U') => { + let mut buffer = String::new(); + for _ in 0..6 { + if let Some(c) = queue.pop_front() { + match c { + '0'...'9' | 'a'...'f' | 'A'...'F' => { + buffer.push(c); + } + _ => break, + } + } else { + break; + } + } + let new_char = encode_unicode(&buffer, 6); + result.push(new_char); + } + Some(c) => { + result.push(c); + } + None => break, + } + } + result +} diff --git a/fluent-syntax/tests/ast/mod.rs b/fluent-syntax/tests/ast/mod.rs index edf04fab..65a67f45 100644 --- a/fluent-syntax/tests/ast/mod.rs +++ b/fluent-syntax/tests/ast/mod.rs @@ -1,3 +1,5 @@ +mod helper; + use fluent_syntax::ast; use serde::ser::SerializeMap; use serde::ser::SerializeSeq; @@ -7,7 +9,13 @@ use std::error::Error; pub fn serialize<'s>(res: &'s ast::Resource) -> Result> { #[derive(Serialize)] - struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); + struct Helper<'ast>(#[serde(serialize_with = "serialize_resource")] &'ast ast::Resource<'ast>); + Ok(serde_json::to_string(&Helper(res)).unwrap()) +} + +pub fn _serialize_to_pretty_json<'s>(res: &'s ast::Resource) -> Result> { + #[derive(Serialize)] + struct Helper<'ast>(#[serde(serialize_with = "serialize_resource")] &'ast ast::Resource<'ast>); let buf = Vec::new(); let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); @@ -16,11 +24,20 @@ pub fn serialize<'s>(res: &'s ast::Resource) -> Result> { Ok(String::from_utf8(ser.into_inner()).unwrap()) } -#[derive(Serialize, Debug)] -#[serde(remote = "ast::Resource")] -struct ResourceDef<'ast> { - #[serde(serialize_with = "serialize_resource_entry_vec")] - body: Vec>, +fn serialize_resource<'se, S>(res: &'se ast::Resource, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>( + #[serde(serialize_with = "serialize_resource_entry_vec")] + &'ast Vec>, + ); + + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "Resource")?; + map.serialize_entry("body", &Helper(&res.body))?; + map.end() } fn serialize_resource_entry_vec<'se, S>( @@ -87,7 +104,7 @@ where #[derive(Serialize)] #[serde(remote = "ast::Message")] pub struct MessageDef<'ast> { - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] pub id: ast::Identifier<'ast>, #[serde(serialize_with = "serialize_pattern_option")] pub value: Option>, @@ -100,7 +117,7 @@ pub struct MessageDef<'ast> { #[derive(Serialize)] #[serde(remote = "ast::Term")] pub struct TermDef<'ast> { - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] pub id: ast::Identifier<'ast>, #[serde(with = "ValueDef")] pub value: ast::Value<'ast>, @@ -118,7 +135,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(with = "PatternDef")] &'ast ast::Pattern<'ast>); + struct Helper<'ast>(#[serde(serialize_with = "serialize_pattern")] &'ast ast::Pattern<'ast>); v.as_ref().map(Helper).serialize(serializer) } @@ -130,7 +147,9 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(with = "AttributeDef")] &'ast ast::Attribute<'ast>); + struct Helper<'ast>( + #[serde(serialize_with = "serialize_attribute")] &'ast ast::Attribute<'ast>, + ); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; @@ -152,9 +171,9 @@ where #[derive(Serialize)] #[serde(remote = "ast::Value")] -#[serde(untagged)] +#[serde(tag = "type")] pub enum ValueDef<'ast> { - #[serde(with = "PatternDef")] + #[serde(serialize_with = "serialize_pattern")] Pattern(ast::Pattern<'ast>), VariantList { #[serde(serialize_with = "serialize_variants")] @@ -162,11 +181,20 @@ pub enum ValueDef<'ast> { }, } -#[derive(Serialize)] -#[serde(remote = "ast::Pattern")] -pub struct PatternDef<'ast> { - #[serde(serialize_with = "serialize_pattern_elements")] - pub elements: Vec>, +fn serialize_pattern<'se, S>(pattern: &'se ast::Pattern, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct Helper<'ast>( + #[serde(serialize_with = "serialize_pattern_elements")] + &'ast Vec>, + ); + + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "Pattern")?; + map.serialize_entry("elements", &Helper(&pattern.elements))?; + map.end() } fn serialize_pattern_elements<'se, S>( @@ -179,8 +207,24 @@ where #[derive(Serialize)] struct Helper<'ast>(#[serde(with = "PatternElementDef")] &'ast ast::PatternElement<'ast>); let mut seq = serializer.serialize_seq(Some(v.len()))?; + let mut buffer = String::new(); for e in v { - seq.serialize_element(&Helper(e))?; + match e { + ast::PatternElement::TextElement(e) => { + buffer.push_str(e); + } + _ => { + if !buffer.is_empty() { + seq.serialize_element(&Helper(&ast::PatternElement::TextElement(&buffer)))?; + buffer = String::new(); + } + + seq.serialize_element(&Helper(e))?; + } + } + } + if !buffer.is_empty() { + seq.serialize_element(&Helper(&ast::PatternElement::TextElement(&buffer)))?; } seq.end() } @@ -217,40 +261,61 @@ where map.end() } -#[derive(Serialize)] -#[serde(remote = "ast::Attribute")] -pub struct AttributeDef<'ast> { - #[serde(with = "IdentifierDef")] - pub id: ast::Identifier<'ast>, - #[serde(with = "PatternDef")] - pub value: ast::Pattern<'ast>, -} +fn serialize_attribute<'se, S>( + attribute: &'se ast::Attribute, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct IdHelper<'ast>( + #[serde(serialize_with = "serialize_identifier")] &'ast ast::Identifier<'ast>, + ); -#[derive(Serialize, Debug)] -#[serde(remote = "ast::Identifier")] -struct IdentifierDef<'ast> { - name: &'ast str, + #[derive(Serialize)] + struct ValueHelper<'ast>( + #[serde(serialize_with = "serialize_pattern")] &'ast ast::Pattern<'ast>, + ); + + let mut map = serializer.serialize_map(Some(3))?; + map.serialize_entry("type", "Attribute")?; + map.serialize_entry("id", &IdHelper(&attribute.id))?; + map.serialize_entry("value", &ValueHelper(&attribute.value))?; + map.end() } -#[derive(Serialize, Debug)] -#[serde(remote = "ast::Function")] -struct FunctionDef<'ast> { - name: &'ast str, +fn serialize_identifier<'se, S>(id: &'se ast::Identifier, serializer: S) -> Result +where + S: Serializer, +{ + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("type", "Identifier")?; + map.serialize_entry("name", id.name)?; + map.end() } -#[derive(Serialize, Debug)] -#[serde(remote = "ast::Variant")] -struct VariantDef<'ast> { - #[serde(with = "VariantKeyDef")] - pub key: ast::VariantKey<'ast>, - #[serde(with = "ValueDef")] - pub value: ast::Value<'ast>, - pub default: bool, +fn serialize_variant<'se, S>(variant: &'se ast::Variant, serializer: S) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct KeyHelper<'ast>(#[serde(with = "VariantKeyDef")] &'ast ast::VariantKey<'ast>); + + #[derive(Serialize)] + struct ValueHelper<'ast>(#[serde(with = "ValueDef")] &'ast ast::Value<'ast>); + + let mut map = serializer.serialize_map(Some(4))?; + map.serialize_entry("type", "Variant")?; + map.serialize_entry("key", &KeyHelper(&variant.key))?; + map.serialize_entry("value", &ValueHelper(&variant.value))?; + map.serialize_entry("default", &variant.default)?; + map.end() } #[derive(Serialize, Debug)] #[serde(remote = "ast::VariantKey")] -#[serde(untagged)] +#[serde(tag = "type")] pub enum VariantKeyDef<'ast> { Identifier { name: &'ast str }, NumberLiteral { value: &'ast str }, @@ -285,19 +350,20 @@ where #[serde(remote = "ast::InlineExpression")] #[serde(tag = "type")] pub enum InlineExpressionDef<'ast> { + #[serde(serialize_with = "serialize_string_literal")] StringLiteral { - value: &'ast str, + raw: &'ast str, }, NumberLiteral { value: &'ast str, }, VariableReference { - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] id: ast::Identifier<'ast>, }, CallExpression { - #[serde(with = "FunctionDef")] - callee: ast::Function<'ast>, + #[serde(with = "InlineExpressionDef")] + callee: ast::InlineExpression<'ast>, #[serde(serialize_with = "serialize_inline_expressions")] positional: Vec>, #[serde(serialize_with = "serialize_named_arguments")] @@ -307,7 +373,7 @@ pub enum InlineExpressionDef<'ast> { #[serde(with = "InlineExpressionDef")] #[serde(rename = "ref")] reference: ast::InlineExpression<'ast>, - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] name: ast::Identifier<'ast>, }, VariantExpression { @@ -318,11 +384,15 @@ pub enum InlineExpressionDef<'ast> { key: ast::VariantKey<'ast>, }, MessageReference { - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] id: ast::Identifier<'ast>, }, TermReference { - #[serde(with = "IdentifierDef")] + #[serde(serialize_with = "serialize_identifier")] + id: ast::Identifier<'ast>, + }, + FunctionReference { + #[serde(serialize_with = "serialize_identifier")] id: ast::Identifier<'ast>, }, Placeable { @@ -331,18 +401,44 @@ pub enum InlineExpressionDef<'ast> { }, } -#[derive(Serialize)] -#[serde(remote = "ast::NamedArgument")] -pub struct NamedArgumentDef<'ast> { - #[serde(with = "IdentifierDef")] - pub name: ast::Identifier<'ast>, - #[serde(with = "InlineExpressionDef")] - pub value: ast::InlineExpression<'ast>, +fn serialize_string_literal<'se, S>(raw: &'se str, serializer: S) -> Result +where + S: Serializer, +{ + let mut map = serializer.serialize_map(Some(3))?; + map.serialize_entry("type", "StringLiteral")?; + map.serialize_entry("raw", raw)?; + map.serialize_entry("value", &helper::unescape_unicode(&raw))?; + map.end() +} + +fn serialize_named_argument<'se, S>( + arg: &'se ast::NamedArgument, + serializer: S, +) -> Result +where + S: Serializer, +{ + #[derive(Serialize)] + struct IdentifierHelper<'ast>( + #[serde(serialize_with = "serialize_identifier")] &'ast ast::Identifier<'ast>, + ); + + #[derive(Serialize)] + struct InlineExpressionHelper<'ast>( + #[serde(with = "InlineExpressionDef")] &'ast ast::InlineExpression<'ast>, + ); + + let mut map = serializer.serialize_map(Some(3))?; + map.serialize_entry("type", "NamedArgument")?; + map.serialize_entry("name", &IdentifierHelper(&arg.name))?; + map.serialize_entry("value", &InlineExpressionHelper(&arg.value))?; + map.end() } #[derive(Serialize)] #[serde(remote = "ast::Expression")] -#[serde(untagged)] +#[serde(tag = "type")] pub enum ExpressionDef<'ast> { #[serde(with = "InlineExpressionDef")] InlineExpression(ast::InlineExpression<'ast>), @@ -359,7 +455,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(with = "VariantDef")] &'ast ast::Variant<'ast>); + struct Helper<'ast>(#[serde(serialize_with = "serialize_variant")] &'ast ast::Variant<'ast>); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; @@ -391,7 +487,9 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(with = "NamedArgumentDef")] &'ast ast::NamedArgument<'ast>); + struct Helper<'ast>( + #[serde(serialize_with = "serialize_named_argument")] &'ast ast::NamedArgument<'ast>, + ); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; diff --git a/fluent-syntax/tests/fixtures/any_char.ftl b/fluent-syntax/tests/fixtures/any_char.ftl new file mode 100644 index 00000000..6966a0da --- /dev/null +++ b/fluent-syntax/tests/fixtures/any_char.ftl @@ -0,0 +1,8 @@ +# ↓ BEL, U+0007 +control0 = abcdef + +# ↓ DEL, U+007F +delete = abcdef + +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/fluent-syntax/tests/fixtures/any_char.json b/fluent-syntax/tests/fixtures/any_char.json new file mode 100644 index 00000000..07e7dc4b --- /dev/null +++ b/fluent-syntax/tests/fixtures/any_char.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc\u0007def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BEL, U+0007" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "delete" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcdef" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ DEL, U+007F" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc‚def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BPM, U+0082" + } + } + ] +} diff --git a/fluent-syntax/tests/fixtures/astral.json b/fluent-syntax/tests/fixtures/astral.json index e3627269..b69743ca 100644 --- a/fluent-syntax/tests/fixtures/astral.json +++ b/fluent-syntax/tests/fixtures/astral.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "face-with-tears-of-joy" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -19,9 +22,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "tetragram-for-centre" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -35,9 +40,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "surrogates-in-text" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -51,15 +58,18 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "surrogates-in-string" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D\\uDE02" + "raw": "\\uD83D\\uDE02", + "value": "��" } } ] @@ -70,22 +80,26 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "surrogates-in-adjacent-strings" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D" + "raw": "\\uD83D", + "value": "�" } }, { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uDE02" + "raw": "\\uDE02", + "value": "�" } } ] @@ -96,9 +110,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "emoji-in-text" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -112,14 +128,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "emoji-in-string" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A face 😂 with tears of joy.", "value": "A face 😂 with tears of joy." } } @@ -135,7 +154,7 @@ { "type": "Junk", "annotations": [], - "content": "err-😂 = Value\n" + "content": "err-😂 = Value\n\n" }, { "type": "Comment", @@ -144,7 +163,7 @@ { "type": "Junk", "annotations": [], - "content": "err-invalid-expression = { 😂 }\n" + "content": "err-invalid-expression = { 😂 }\n\n" }, { "type": "Comment", @@ -156,4 +175,4 @@ "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/call_expressions.ftl b/fluent-syntax/tests/fixtures/call_expressions.ftl index 06562642..19e76b7e 100644 --- a/fluent-syntax/tests/fixtures/call_expressions.ftl +++ b/fluent-syntax/tests/fixtures/call_expressions.ftl @@ -1,3 +1,5 @@ +## Arguments + positional-args = {FUN(1, "a", msg)} named-args = {FUN(x: 1, y: "Y")} dense-named-args = {FUN(x:1, y:"Y")} diff --git a/fluent-syntax/tests/fixtures/call_expressions.json b/fluent-syntax/tests/fixtures/call_expressions.json index 3a49c4f4..4dd73483 100644 --- a/fluent-syntax/tests/fixtures/call_expressions.json +++ b/fluent-syntax/tests/fixtures/call_expressions.json @@ -1,18 +1,29 @@ { + "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Arguments" + }, { "type": "Message", "id": { + "type": "Identifier", "name": "positional-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -21,11 +32,13 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } @@ -41,21 +54,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "named-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -64,11 +85,14 @@ } }, { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "y" }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -83,21 +107,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "dense-named-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -106,11 +138,14 @@ } }, { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "y" }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -125,16 +160,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "mixed-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -143,18 +184,22 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } ], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -163,11 +208,14 @@ } }, { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "y" }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -186,7 +234,7 @@ { "type": "Junk", "annotations": [], - "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n" + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n\n" }, { "type": "Comment", @@ -195,7 +243,7 @@ { "type": "Junk", "annotations": [], - "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n" + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n\n\n" }, { "type": "GroupComment", @@ -204,32 +252,42 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "sparse-inline-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } ], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -248,16 +306,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "empty-inline-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -271,32 +335,42 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "multiline-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } ], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -315,32 +389,42 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "sparse-multiline-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } ], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -359,16 +443,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "empty-multiline-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -382,16 +472,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-number" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -410,20 +506,27 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-string" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" } ], @@ -438,21 +541,28 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-msg-ref" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } @@ -468,21 +578,28 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-term-ref" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "TermReference", "id": { + "type": "Identifier", "name": "msg" } } @@ -498,21 +615,28 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-var-ref" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -528,22 +652,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-arg-call" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "CallExpression", "callee": { - "name": "OTHER" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "OTHER" + } }, "positional": [], "named": [] @@ -560,21 +694,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-named-arg" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -593,21 +735,28 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-closing-paren" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "MessageReference", "id": { + "type": "Identifier", "name": "x" } } @@ -627,16 +776,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "one-argument" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -655,16 +810,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "many-arguments" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -691,16 +852,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "inline-sparse-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -727,16 +894,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "mulitline-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -759,16 +932,22 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "mulitline-sparse-args" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -795,7 +974,17 @@ { "type": "Junk", "annotations": [], - "content": "one-argument = {FUN(1,,)}\nmissing-arg = {FUN(,)}\nmissing-sparse-arg = {FUN( , )}\n" + "content": "one-argument = {FUN(1,,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-arg = {FUN(,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-sparse-arg = {FUN( , )}\n\n\n" }, { "type": "GroupComment", @@ -804,21 +993,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "sparse-named-arg" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -827,7 +1024,9 @@ } }, { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "y" }, "value": { @@ -836,7 +1035,9 @@ } }, { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "z" }, "value": { @@ -855,21 +1056,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-colon" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -888,21 +1097,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "unindented-value" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "CallExpression", "callee": { - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ { + "type": "NamedArgument", "name": { + "type": "Identifier", "name": "x" }, "value": { @@ -919,4 +1136,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/callee_expressions.ftl b/fluent-syntax/tests/fixtures/callee_expressions.ftl new file mode 100644 index 00000000..637a2e4d --- /dev/null +++ b/fluent-syntax/tests/fixtures/callee_expressions.ftl @@ -0,0 +1,46 @@ +## Callees in placeables. + +function-callee-placeable = {FUNCTION()} +term-callee-placeable = {-term()} + +# ERROR Messages cannot be parameterized. +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = {FUNCTION() -> + *[key] Value +} +term-attr-callee-selector = {-term.attr() -> + *[key] Value +} + +# ERROR Messages cannot be parameterized. +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/callee_expressions.json b/fluent-syntax/tests/fixtures/callee_expressions.json new file mode 100644 index 00000000..50cdaeb4 --- /dev/null +++ b/fluent-syntax/tests/fixtures/callee_expressions.json @@ -0,0 +1,270 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Callees in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-placeable = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-placeable = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-placeable = {message.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used in Placeables." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attr-callee-placeable = {-term.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-placeable = {$variable()}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Callees in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attr-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-selector = {message() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-selector = {Function() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-selector = {message.attr() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee-selector = {-term() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-selector = {$variable() ->\n *[key] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/comments.json b/fluent-syntax/tests/fixtures/comments.json index edce1b17..c28115ac 100644 --- a/fluent-syntax/tests/fixtures/comments.json +++ b/fluent-syntax/tests/fixtures/comments.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "Comment", @@ -7,9 +8,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "foo" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -26,9 +29,11 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "term" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -55,4 +60,4 @@ "content": "Resource Comment" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/convert.js b/fluent-syntax/tests/fixtures/convert.js deleted file mode 100755 index 24c8549c..00000000 --- a/fluent-syntax/tests/fixtures/convert.js +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env node - -'use strict'; -const fs = require('fs'); - -fs.readdirSync("./").forEach(file => { - if (file.endsWith(".json")) { - fs.readFile(file, convert.bind(this, file)); - } -}) - -function write(s, fileName) { - const data = new Uint8Array(Buffer.from(s)); - fs.writeFile(fileName, data, (err) => { - if (err) throw err; - console.log(`The file "${fileName}" has been saved!`); - }); -} - - -function convert(fileName, err, data) { - let ast = JSON.parse(data.toString()); - - remove_leading_dash_from_term(ast); - remove_unambigous_types(ast); - split_multiline_text_elements(ast); - - let s = JSON.stringify(ast, null, 4); - write(s, fileName); -} - -function remove_leading_dash_from_term(ast) { - for (let i in ast) { - let node = ast[i]; - if (Array.isArray(node)) { - remove_leading_dash_from_term(node); - } else if (typeof node === "object") { - remove_leading_dash_from_term(node); - } else if (i === "name" && - typeof node == "string" && - node.startsWith("-")) { - ast[i] = node.substr(1); - } - } -} - -function remove_unambigous_types(ast, parent_node = null, parent_key = null) { - for (let i in ast) { - let node = ast[i]; - if (Array.isArray(node)) { - remove_unambigous_types(node, ast); - } else if (typeof node === "object") { - remove_unambigous_types(node, ast, i); - } else if (i === "type" && - parent_key !== "selector" && - ["Resource", - "Pattern", - "Function", - "Variant", - "SelectExpression", - "Attribute", - "NamedArgument", - "VariantList", - "Identifier"].includes(node)) { - ast[i] = undefined; - } else if (parent_key == "key" && - ["NumberLiteral"].includes(node)) { - ast[i] = undefined; - } - } -} - -function split_multiline_text_elements(ast) { - for (let i in ast) { - let node = ast[i]; - if (Array.isArray(node)) { - split_multiline_text_elements(node); - } else if (typeof node === "object" && - node !== null && - node["type"] === "TextElement") { - let parts = node["value"].split("\n"); - let elements = parts.filter(v => v != "").map((v, i) => { - let last = parts.length - 1 === i; - return { - "type": "TextElement", - "value": last ? v : `${v}\n` - }; - }); - ast = ast.splice(i, 1, ...elements); - } else if (typeof node === "object") { - split_multiline_text_elements(node); - } - } -} diff --git a/fluent-syntax/tests/fixtures/cr.ftl b/fluent-syntax/tests/fixtures/cr.ftl new file mode 100644 index 00000000..549c662a --- /dev/null +++ b/fluent-syntax/tests/fixtures/cr.ftl @@ -0,0 +1 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } \ No newline at end of file diff --git a/fluent-syntax/tests/fixtures/cr.json b/fluent-syntax/tests/fixtures/cr.json new file mode 100644 index 00000000..44eab75f --- /dev/null +++ b/fluent-syntax/tests/fixtures/cr.json @@ -0,0 +1,9 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + } + ] +} diff --git a/fluent-syntax/tests/fixtures/crlf.ftl b/fluent-syntax/tests/fixtures/crlf.ftl index 538d7adc..133b6c2c 100644 --- a/fluent-syntax/tests/fixtures/crlf.ftl +++ b/fluent-syntax/tests/fixtures/crlf.ftl @@ -1,7 +1,14 @@ + key01 = Value 01 key02 = + Value 02 Continued -# ERROR (Missing value or attributes) -key03 + .title = Title + +# ERROR Unclosed StringLiteral +err03 = { "str + +# ERROR Missing newline after ->. +err04 = { $sel -> } diff --git a/fluent-syntax/tests/fixtures/crlf.json b/fluent-syntax/tests/fixtures/crlf.json index be7910d3..58d26a77 100644 --- a/fluent-syntax/tests/fixtures/crlf.json +++ b/fluent-syntax/tests/fixtures/crlf.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -19,31 +22,55 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value 02\r\n" - }, - { - "type": "TextElement", - "value": "Continued" + "value": "Value 02\nContinued" } ] }, - "attributes": [], + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "title" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Title" + } + ] + } + } + ], "comment": null }, { "type": "Comment", - "content": "ERROR (Missing value or attributes)" + "content": "ERROR Unclosed StringLiteral" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = { \"str\r\n\r\n" + }, + { + "type": "Comment", + "content": "ERROR Missing newline after ->." }, { "type": "Junk", "annotations": [], - "content": "key03\r\n" + "content": "err04 = { $sel -> }\r\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_comment.json b/fluent-syntax/tests/fixtures/eof_comment.json index ac094a54..9483a1eb 100644 --- a/fluent-syntax/tests/fixtures/eof_comment.json +++ b/fluent-syntax/tests/fixtures/eof_comment.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "ResourceComment", @@ -9,4 +10,4 @@ "content": "No EOL" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_empty.json b/fluent-syntax/tests/fixtures/eof_empty.json index 33f00124..b1992785 100644 --- a/fluent-syntax/tests/fixtures/eof_empty.json +++ b/fluent-syntax/tests/fixtures/eof_empty.json @@ -1,3 +1,4 @@ { + "type": "Resource", "body": [] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_id.json b/fluent-syntax/tests/fixtures/eof_id.json index 33b2ad6d..93693f94 100644 --- a/fluent-syntax/tests/fixtures/eof_id.json +++ b/fluent-syntax/tests/fixtures/eof_id.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "ResourceComment", @@ -10,4 +11,4 @@ "content": "message-id" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_id_equals.json b/fluent-syntax/tests/fixtures/eof_id_equals.json index d515069b..eefd3285 100644 --- a/fluent-syntax/tests/fixtures/eof_id_equals.json +++ b/fluent-syntax/tests/fixtures/eof_id_equals.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "ResourceComment", @@ -10,4 +11,4 @@ "content": "message-id =" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_junk.json b/fluent-syntax/tests/fixtures/eof_junk.json index 94f1e77d..d7d12824 100644 --- a/fluent-syntax/tests/fixtures/eof_junk.json +++ b/fluent-syntax/tests/fixtures/eof_junk.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "ResourceComment", @@ -10,4 +11,4 @@ "content": "000" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/eof_value.json b/fluent-syntax/tests/fixtures/eof_value.json index bbab1e43..502435fc 100644 --- a/fluent-syntax/tests/fixtures/eof_value.json +++ b/fluent-syntax/tests/fixtures/eof_value.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "ResourceComment", @@ -7,9 +8,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "no-eol" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -21,4 +24,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/escaped_characters.ftl b/fluent-syntax/tests/fixtures/escaped_characters.ftl index d3a5a078..ec862320 100644 --- a/fluent-syntax/tests/fixtures/escaped_characters.ftl +++ b/fluent-syntax/tests/fixtures/escaped_characters.ftl @@ -1,9 +1,34 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (a closing brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-4digits = {"\u0041"} +escape-unicode-4digits = {"\\u0041"} +string-unicode-6digits = {"\U01F602"} +escape-unicode-6digits = {"\\U01F602"} + +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = {"\u004100"} +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = {"\U01F60200"} + +# ERROR Too few hex digits after \u. +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. +string-too-few-6digits = {"\U1F602"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/tests/fixtures/escaped_characters.json b/fluent-syntax/tests/fixtures/escaped_characters.json index c45748c0..a3220996 100644 --- a/fluent-syntax/tests/fixtures/escaped_characters.json +++ b/fluent-syntax/tests/fixtures/escaped_characters.json @@ -1,15 +1,40 @@ { + "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\ a backslash" + } + ] + }, + "attributes": [], + "comment": null + }, { "type": "Message", "id": { - "name": "backslash" + "type": "Identifier", + "name": "text-backslash-two" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)" + "value": "Value with \\\\ two backslashes" } ] }, @@ -19,13 +44,25 @@ { "type": "Message", "id": { - "name": "closing-brace" + "type": "Identifier", + "name": "text-backslash-brace" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\{ (a closing brace)" + "value": "Value with \\" + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + } } ] }, @@ -35,9 +72,11 @@ { "type": "Message", "id": { - "name": "unicode-escape" + "type": "Identifier", + "name": "text-backslash-u" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -51,9 +90,11 @@ { "type": "Message", "id": { - "name": "escaped-unicode" + "type": "Identifier", + "name": "text-backslash-backslash-u" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -66,20 +107,23 @@ }, { "type": "GroupComment", - "content": "String Expressions" + "content": "String literals" }, { "type": "Message", "id": { + "type": "Identifier", "name": "quote-in-string" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"" + "raw": "\\\"", + "value": "\"" } } ] @@ -90,15 +134,18 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "backslash-in-string" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\" + "raw": "\\\\", + "value": "\\" } } ] @@ -106,10 +153,247 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Mismatched quote" + }, { "type": "Junk", "annotations": [], "content": "mismatched-quote = {\"\\\\\"\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Unknown escape" + }, + { + "type": "Junk", + "annotations": [], + "content": "unknown-escape = {\"\\x\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Unicode escapes" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u0041", + "value": "A" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\u0041", + "value": "\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F602", + "value": "😂" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\U01F602", + "value": "\\U01F602" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u004100", + "value": "A00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F60200", + "value": "😂00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\u." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-4digits = {\"\\u41\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\U." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-6digits = {\"\\U1F602\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Literal braces" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "{", + "value": "{" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "}", + "value": "}" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/junk.ftl b/fluent-syntax/tests/fixtures/junk.ftl index 62fc2ea1..b0b0c5f3 100644 --- a/fluent-syntax/tests/fixtures/junk.ftl +++ b/fluent-syntax/tests/fixtures/junk.ftl @@ -1,4 +1,21 @@ +## Two adjacent Junks. +err01 = {1x} +err02 = {2x} + +# A single Junk. +err03 = {1x +2 + +# A single Junk. ą=Invalid identifier ć=Another one -key01 = { +# The COMMENT ends this junk. +err04 = { +# COMMENT + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. +err04 = { +# COMMENT +} diff --git a/fluent-syntax/tests/fixtures/junk.json b/fluent-syntax/tests/fixtures/junk.json index 5cfbfebe..15e62a4d 100644 --- a/fluent-syntax/tests/fixtures/junk.json +++ b/fluent-syntax/tests/fixtures/junk.json @@ -1,14 +1,68 @@ { + "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Two adjacent Junks." + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {1x}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {2x}\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {1x\n2\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, { "type": "Junk", "annotations": [], - "content": "ą=Invalid identifier\nć=Another one\n" + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" }, { "type": "Junk", "annotations": [], - "content": "key01 = {\n" + "content": "}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/leading_dots.json b/fluent-syntax/tests/fixtures/leading_dots.json index d4f0702d..a185d489 100644 --- a/fluent-syntax/tests/fixtures/leading_dots.json +++ b/fluent-syntax/tests/fixtures/leading_dots.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -19,9 +22,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -35,14 +40,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -58,14 +66,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -81,9 +92,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key05" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -93,6 +106,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -108,9 +122,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key06" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -120,6 +136,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -135,9 +152,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key07" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -154,7 +173,7 @@ { "type": "Junk", "annotations": [], - "content": " .Continued\n" + "content": " .Continued\n\n" }, { "type": "Comment", @@ -163,7 +182,7 @@ { "type": "Junk", "annotations": [], - "content": "key08 =\n .Value\n" + "content": "key08 =\n .Value\n\n" }, { "type": "Comment", @@ -172,28 +191,28 @@ { "type": "Junk", "annotations": [], - "content": "key09 =\n .Value\n Continued\n" + "content": "key09 =\n .Value\n Continued\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "key10" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "Value" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "which is an attribute\n" - }, - { - "type": "TextElement", - "value": "Continued" + "value": "which is an attribute\nContinued" } ] } @@ -204,24 +223,23 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key11" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, { "type": "TextElement", - "value": "Value = which looks like an attribute\n" - }, - { - "type": "TextElement", - "value": "Continued" + "value": "Value = which looks like an attribute\nContinued" } ] }, @@ -231,15 +249,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key12" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "accesskey" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -254,15 +276,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key13" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attribute" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -277,20 +303,25 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key14" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attribute" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -307,23 +338,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key15" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "1" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "one" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -334,15 +371,19 @@ "default": false }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -369,7 +410,7 @@ { "type": "Junk", "annotations": [], - "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n" + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n\n" }, { "type": "Comment", @@ -378,7 +419,7 @@ { "type": "Junk", "annotations": [], - "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n" + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n\n" }, { "type": "Comment", @@ -387,28 +428,28 @@ { "type": "Junk", "annotations": [], - "content": "key18 =\n.Value\n" + "content": "key18 =\n.Value\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "key19" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attribute" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value\n" - }, - { - "type": "TextElement", - "value": "Continued" + "value": "Value\nContinued" } ] } @@ -419,14 +460,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key20" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -440,4 +484,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/literal_expressions.json b/fluent-syntax/tests/fixtures/literal_expressions.json index a97c61c6..da73e131 100644 --- a/fluent-syntax/tests/fixtures/literal_expressions.json +++ b/fluent-syntax/tests/fixtures/literal_expressions.json @@ -1,16 +1,20 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "string-expression" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "abc", "value": "abc" } } @@ -22,9 +26,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "number-expression" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -41,9 +47,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "number-expression" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -58,4 +66,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/member_expressions.ftl b/fluent-syntax/tests/fixtures/member_expressions.ftl index 09e09f3f..b4a93922 100644 --- a/fluent-syntax/tests/fixtures/member_expressions.ftl +++ b/fluent-syntax/tests/fixtures/member_expressions.ftl @@ -1,6 +1,28 @@ -variant-expression = {-term[case]} -attribute-expression = {msg.attr} +## Member expressions in placeables. -## Invalid syntax -variant-expression = {msg[case]} -attribute-expression = {-term.attr} +message-attribute-expression-placeable = {msg.attr} +term-variant-expression-placeable = {-term[case]} + +# ERROR Message values cannot be VariantLists +message-variant-expression-placeable = {msg[case]} +# ERROR Term attributes may not be used for interpolation. +term-attribute-expression-placeable = {-term.attr} + +## Member expressions in selectors. + +term-attribute-expression-selector = {-term.attr -> + *[key] Value +} + +# ERROR Message attributes may not be used as selector. +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} +# ERROR Term values may not be used as selector. +term-variant-expression-selector = {-term[case] -> + *[key] Value +} +# ERROR Message values cannot be VariantLists +message-variant-expression-selector = {msg[case] -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/member_expressions.json b/fluent-syntax/tests/fixtures/member_expressions.json index ba34f217..f6890e56 100644 --- a/fluent-syntax/tests/fixtures/member_expressions.json +++ b/fluent-syntax/tests/fixtures/member_expressions.json @@ -1,11 +1,49 @@ { + "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Member expressions in placeables." + }, { "type": "Message", "id": { - "name": "variant-expression" + "type": "Identifier", + "name": "message-attribute-expression-placeable" }, "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "AttributeExpression", + "ref": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-variant-expression-placeable" + }, + "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -14,10 +52,12 @@ "ref": { "type": "TermReference", "id": { + "type": "Identifier", "name": "term" } }, "key": { + "type": "Identifier", "name": "case" } } @@ -27,26 +67,74 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-placeable = {msg[case]}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used for interpolation." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attribute-expression-placeable = {-term.attr}\n\n" + }, + { + "type": "GroupComment", + "content": "Member expressions in selectors." + }, { "type": "Message", "id": { - "name": "attribute-expression" + "type": "Identifier", + "name": "term-attribute-expression-selector" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { - "type": "AttributeExpression", - "ref": { - "type": "MessageReference", - "id": { - "name": "msg" + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" } }, - "name": { - "name": "attr" - } + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] } } ] @@ -55,13 +143,31 @@ "comment": null }, { - "type": "GroupComment", - "content": "Invalid syntax" + "type": "Comment", + "content": "ERROR Message attributes may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attribute-expression-selector = {msg.attr ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-variant-expression-selector = {-term[case] ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" }, { "type": "Junk", "annotations": [], - "content": "variant-expression = {msg[case]}\nattribute-expression = {-term.attr}\n" + "content": "message-variant-expression-selector = {msg[case] ->\n *[key] Value\n}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/messages.ftl b/fluent-syntax/tests/fixtures/messages.ftl index 0ade3cac..00b4ab46 100644 --- a/fluent-syntax/tests/fixtures/messages.ftl +++ b/fluent-syntax/tests/fixtures/messages.ftl @@ -25,3 +25,5 @@ key07 = # JUNK Missing = key08 + +KEY09 = Value 09 diff --git a/fluent-syntax/tests/fixtures/messages.json b/fluent-syntax/tests/fixtures/messages.json index 0ee318b9..cdbe5c93 100644 --- a/fluent-syntax/tests/fixtures/messages.json +++ b/fluent-syntax/tests/fixtures/messages.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -19,9 +22,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -31,10 +36,13 @@ }, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -49,9 +57,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -61,10 +71,13 @@ }, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr1" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -74,10 +87,13 @@ } }, { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr2" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -92,15 +108,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -115,15 +135,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr1" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -133,10 +157,13 @@ } }, { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr2" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -151,15 +178,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key05" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr1" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -177,14 +208,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key06" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } @@ -200,7 +234,7 @@ { "type": "Junk", "annotations": [], - "content": "key07 =\n" + "content": "key07 =\n\n" }, { "type": "Comment", @@ -209,7 +243,25 @@ { "type": "Junk", "annotations": [], - "content": "key08\n" + "content": "key08\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "KEY09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 09" + } + ] + }, + "attributes": [], + "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/mixed_entries.json b/fluent-syntax/tests/fixtures/mixed_entries.json index 9d9b3cfe..a9dc501f 100644 --- a/fluent-syntax/tests/fixtures/mixed_entries.json +++ b/fluent-syntax/tests/fixtures/mixed_entries.json @@ -1,4 +1,5 @@ { + "type": "Resource", "body": [ { "type": "Comment", @@ -11,9 +12,11 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "brand-name" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -31,15 +34,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -54,14 +61,16 @@ { "type": "Junk", "annotations": [], - "content": "ą=Invalid identifier\nć=Another one\n" + "content": "ą=Invalid identifier\nć=Another one\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -82,14 +91,16 @@ { "type": "Junk", "annotations": [], - "content": " .attr = Dangling attribute\n" + "content": " .attr = Dangling attribute\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -106,9 +117,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -120,4 +133,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/multiline_values.ftl b/fluent-syntax/tests/fixtures/multiline_values.ftl index 4dd81154..e3739bb5 100644 --- a/fluent-syntax/tests/fixtures/multiline_values.ftl +++ b/fluent-syntax/tests/fixtures/multiline_values.ftl @@ -33,3 +33,28 @@ key07 = {"A multiline value"} starting and ending {"with a placeable"} key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/fluent-syntax/tests/fixtures/multiline_values.json b/fluent-syntax/tests/fixtures/multiline_values.json index 1d89c8b0..4d3dd033 100644 --- a/fluent-syntax/tests/fixtures/multiline_values.json +++ b/fluent-syntax/tests/fixtures/multiline_values.json @@ -1,27 +1,18 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "A multiline value\n" - }, - { - "type": "TextElement", - "value": "continued on the next line\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "and also down here." + "value": "A multiline value\ncontinued on the next line\n\nand also down here." } ] }, @@ -31,17 +22,15 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "A multiline value starting\n" - }, - { - "type": "TextElement", - "value": "on a new line." + "value": "A multiline value starting\non a new line." } ] }, @@ -51,31 +40,23 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "A multiline attribute value\n" - }, - { - "type": "TextElement", - "value": "continued on the next line\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "and also down here." + "value": "A multiline attribute value\ncontinued on the next line\n\nand also down here." } ] } @@ -86,23 +67,23 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "A multiline attribute value\n" - }, - { - "type": "TextElement", - "value": "staring on a new line" + "value": "A multiline attribute value\nstaring on a new line" } ] } @@ -113,21 +94,15 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key05" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "A multiline value with non-standard\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "indentation." + "value": "A multiline value with non-standard\n\n indentation." } ] }, @@ -137,9 +112,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key06" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -149,6 +126,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "placeables", "value": "placeables" } }, @@ -160,6 +138,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "at", "value": "at" } }, @@ -171,6 +150,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "of lines", "value": "of lines" } }, @@ -178,6 +158,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } } @@ -189,14 +170,17 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key07" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A multiline value", "value": "A multiline value" } }, @@ -208,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "with a placeable", "value": "with a placeable" } } @@ -219,9 +204,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key08" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -231,6 +218,112 @@ }, "attributes": [], "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/placeables.ftl b/fluent-syntax/tests/fixtures/placeables.ftl index c0e515b6..7a1b280f 100644 --- a/fluent-syntax/tests/fixtures/placeables.ftl +++ b/fluent-syntax/tests/fixtures/placeables.ftl @@ -1,3 +1,15 @@ nested-placeable = {{{1}}} padded-placeable = { 1 } sparse-placeable = { { 1 } } + +# ERROR Unmatched opening brace +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/tests/fixtures/placeables.json b/fluent-syntax/tests/fixtures/placeables.json index 1a2ef17c..7d67d940 100644 --- a/fluent-syntax/tests/fixtures/placeables.json +++ b/fluent-syntax/tests/fixtures/placeables.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "nested-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -28,9 +31,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "padded-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -47,9 +52,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "sparse-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", @@ -65,6 +72,42 @@ }, "attributes": [], "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open1 = { 1\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open2 = {{ 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close1 = 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close2 = { 1 }}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/reference_expressions.ftl b/fluent-syntax/tests/fixtures/reference_expressions.ftl index ab27cbe0..9c2e9c54 100644 --- a/fluent-syntax/tests/fixtures/reference_expressions.ftl +++ b/fluent-syntax/tests/fixtures/reference_expressions.ftl @@ -1,5 +1,28 @@ -message-reference = {msg} -term-reference = {-term} -variable-reference = {$var} +## Reference expressions in placeables. -not-call-expression = {FUN} +message-reference-placeable = {msg} +term-reference-placeable = {-term} +variable-reference-placeable = {$var} + +# ERROR Function references are invalid outside of call expressions. +function-reference-placeable = {FUN} + + +## Reference expressions in selectors. + +variable-reference-selector = {$var -> + *[key] Value +} + +# ERROR Message values may not be used as selectors. +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions. +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/fluent-syntax/tests/fixtures/reference_expressions.json b/fluent-syntax/tests/fixtures/reference_expressions.json index 88dcab73..65c9d4cc 100644 --- a/fluent-syntax/tests/fixtures/reference_expressions.json +++ b/fluent-syntax/tests/fixtures/reference_expressions.json @@ -1,17 +1,25 @@ { + "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Reference expressions in placeables." + }, { "type": "Message", "id": { - "name": "message-reference" + "type": "Identifier", + "name": "message-reference-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "MessageReference", "id": { + "type": "Identifier", "name": "msg" } } @@ -24,15 +32,18 @@ { "type": "Message", "id": { - "name": "term-reference" + "type": "Identifier", + "name": "term-reference-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "TermReference", "id": { + "type": "Identifier", "name": "term" } } @@ -45,15 +56,18 @@ { "type": "Message", "id": { - "name": "variable-reference" + "type": "Identifier", + "name": "variable-reference-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -66,15 +80,18 @@ { "type": "Message", "id": { - "name": "not-call-expression" + "type": "Identifier", + "name": "function-reference-placeable" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "MessageReference", "id": { + "type": "Identifier", "name": "FUN" } } @@ -82,7 +99,87 @@ ] }, "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + } + }, + { + "type": "GroupComment", + "content": "Reference expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-reference-selector = {msg ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-reference-selector = {-term ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + }, + { + "type": "Junk", + "annotations": [], + "content": "function-expression-selector = {FUN ->\n *[key] Value\n}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/select_expressions.ftl b/fluent-syntax/tests/fixtures/select_expressions.ftl index 3c54f57c..7a1fb820 100644 --- a/fluent-syntax/tests/fixtures/select_expressions.ftl +++ b/fluent-syntax/tests/fixtures/select_expressions.ftl @@ -4,17 +4,29 @@ new-messages = *[other] {""}Other } -valid-selector = +valid-selector-term-attribute = { -term.case -> *[key] value } # ERROR -invalid-selector = +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR +invalid-selector-term-variant = { -term[case] -> *[key] value } +# ERROR +invalid-selector-term-call = + { -term(case: "nominative") -> + *[key] value + } + empty-variant = { 1 -> *[one] {""} @@ -27,10 +39,15 @@ nested-select = } } -# ERROR VariantLists cannot appear in SelectExpressions +# ERROR VariantLists cannot be Variant values. nested-variant-list = { 1 -> *[one] { *[two] Value } } + +# ERROR Missing line end after variant list +missing-line-end = + { 1 -> + *[one] One} diff --git a/fluent-syntax/tests/fixtures/select_expressions.json b/fluent-syntax/tests/fixtures/select_expressions.json index 88325092..a3dc5730 100644 --- a/fluent-syntax/tests/fixtures/select_expressions.json +++ b/fluent-syntax/tests/fixtures/select_expressions.json @@ -1,29 +1,40 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "new-messages" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "CallExpression", "callee": { - "name": "BUILTIN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "BUILTIN" + } }, "positional": [], "named": [] }, "variants": [ { + "type": "Variant", "key": { + "type": "NumberLiteral", "value": "0" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -34,15 +45,19 @@ "default": false }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } }, @@ -65,31 +80,39 @@ { "type": "Message", "id": { - "name": "valid-selector" + "type": "Identifier", + "name": "valid-selector-term-attribute" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "AttributeExpression", "ref": { "type": "TermReference", "id": { + "type": "Identifier", "name": "term" } }, "name": { + "type": "Identifier", "name": "case" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -114,33 +137,58 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector =\n { -term[case] ->\n *[key] value\n }\n" + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "empty-variant" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "1" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "one" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } @@ -159,37 +207,47 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "nested-select" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "1" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "one" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "2" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "two" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -216,12 +274,21 @@ }, { "type": "Comment", - "content": "ERROR VariantLists cannot appear in SelectExpressions" + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Missing line end after variant list" }, { "type": "Junk", "annotations": [], - "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + "content": "missing-line-end =\n { 1 ->\n *[one] One}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/select_indent.json b/fluent-syntax/tests/fixtures/select_indent.json index 8c1dbb63..d8c0fa88 100644 --- a/fluent-syntax/tests/fixtures/select_indent.json +++ b/fluent-syntax/tests/fixtures/select_indent.json @@ -1,27 +1,35 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "select-1tbs-inline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -42,25 +50,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-1tbs-newline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -81,25 +96,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-1tbs-indent" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -120,25 +142,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-allman-inline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -159,25 +188,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-allman-newline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -198,25 +234,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-allman-indent" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -237,25 +280,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-gnu-inline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -276,25 +326,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-gnu-newline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -315,25 +372,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-gnu-indent" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -354,25 +418,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-no-indent" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -383,10 +454,13 @@ "default": true }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -407,51 +481,53 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-no-indent-multiline" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value\n" - }, - { - "type": "TextElement", - "value": "Continued" + "value": "Value\nContinued" } ] }, "default": true }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Other\n" - }, - { - "type": "TextElement", - "value": "Multiline" + "value": "Other\nMultiline" } ] }, @@ -472,30 +548,42 @@ { "type": "Junk", "annotations": [], - "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\nContinued without indent.\n}\n" + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "Continued without indent.\n}\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "select-flat" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -506,10 +594,13 @@ "default": true }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -530,25 +621,32 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "select-flat-with-trailing-spaces" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "selector" } }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -559,10 +657,13 @@ "default": true }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -584,4 +685,4 @@ } } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/sparse_entries.json b/fluent-syntax/tests/fixtures/sparse_entries.json index 73d6707e..0c4a3686 100644 --- a/fluent-syntax/tests/fixtures/sparse_entries.json +++ b/fluent-syntax/tests/fixtures/sparse_entries.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -19,15 +22,19 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": null, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -42,42 +49,27 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value\n" - }, - { - "type": "TextElement", - "value": "Continued\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "Over multiple\n" - }, - { - "type": "TextElement", - "value": "Lines" + "value": "Value\nContinued\n\n\nOver multiple\nLines" } ] }, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -92,9 +84,11 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key05" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -108,23 +102,29 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key06" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "1" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "one" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -135,10 +135,13 @@ "default": false }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "two" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -157,4 +160,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/tab.json b/fluent-syntax/tests/fixtures/tab.json index 8d891aa0..714eb947 100644 --- a/fluent-syntax/tests/fixtures/tab.json +++ b/fluent-syntax/tests/fixtures/tab.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -26,7 +29,7 @@ { "type": "Junk", "annotations": [], - "content": "key02\t= Value 02\n" + "content": "key02\t= Value 02\n\n" }, { "type": "Comment", @@ -35,14 +38,16 @@ { "type": "Junk", "annotations": [], - "content": "key03 =\n\tThis line isn't properly indented.\n" + "content": "key03 =\n\tThis line isn't properly indented.\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -62,4 +67,4 @@ "content": "\twhereas this line by 1 tab.\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/term_parameters.ftl b/fluent-syntax/tests/fixtures/term_parameters.ftl new file mode 100644 index 00000000..61442361 --- /dev/null +++ b/fluent-syntax/tests/fixtures/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = { $arg -> + *[key] Value +} + +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/tests/fixtures/term_parameters.json b/fluent-syntax/tests/fixtures/term_parameters.json new file mode 100644 index 00000000..f9f09613 --- /dev/null +++ b/fluent-syntax/tests/fixtures/term_parameters.json @@ -0,0 +1,203 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "arg" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "arg" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "positional", + "value": "positional" + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg1" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg2" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/tests/fixtures/terms.json b/fluent-syntax/tests/fixtures/terms.json index 2abb8440..2321283f 100644 --- a/fluent-syntax/tests/fixtures/terms.json +++ b/fluent-syntax/tests/fixtures/terms.json @@ -1,11 +1,14 @@ { + "type": "Resource", "body": [ { "type": "Term", "id": { + "type": "Identifier", "name": "term01" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -15,10 +18,13 @@ }, "attributes": [ { + "type": "Attribute", "id": { + "type": "Identifier", "name": "attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -33,14 +39,17 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "term02" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } @@ -56,7 +65,7 @@ { "type": "Junk", "annotations": [], - "content": "-term03 =\n .attr = Attribute\n" + "content": "-term03 =\n .attr = Attribute\n\n" }, { "type": "Comment", @@ -65,7 +74,7 @@ { "type": "Junk", "annotations": [], - "content": "-term04 = \n .attr1 = Attribute 1\n" + "content": "-term04 = \n .attr1 = Attribute 1\n\n" }, { "type": "Comment", @@ -74,7 +83,7 @@ { "type": "Junk", "annotations": [], - "content": "-term05 =\n" + "content": "-term05 =\n\n" }, { "type": "Comment", @@ -83,7 +92,7 @@ { "type": "Junk", "annotations": [], - "content": "-term06 = \n" + "content": "-term06 = \n\n" }, { "type": "Comment", @@ -95,4 +104,4 @@ "content": "-term07\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/variables.json b/fluent-syntax/tests/fixtures/variables.json index d5a79c2d..58682e5b 100644 --- a/fluent-syntax/tests/fixtures/variables.json +++ b/fluent-syntax/tests/fixtures/variables.json @@ -1,17 +1,21 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key01" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -24,15 +28,18 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key02" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -45,15 +52,18 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key03" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -66,15 +76,18 @@ { "type": "Message", "id": { + "type": "Identifier", "name": "key04" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { "type": "VariableReference", "id": { + "type": "Identifier", "name": "var" } } @@ -116,4 +129,4 @@ "content": "err03 = {$-var}\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/variant_keys.json b/fluent-syntax/tests/fixtures/variant_keys.json index 68246c92..cf752e43 100644 --- a/fluent-syntax/tests/fixtures/variant_keys.json +++ b/fluent-syntax/tests/fixtures/variant_keys.json @@ -1,17 +1,23 @@ { + "type": "Resource", "body": [ { "type": "Term", "id": { + "type": "Identifier", "name": "simple-identifier" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -29,15 +35,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "identifier-surrounded-by-whitespace" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -55,15 +66,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "int-number" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "NumberLiteral", "value": "1" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -81,15 +97,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "float-number" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "NumberLiteral", "value": "3.14" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -111,7 +132,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-identifier =\n {\n *[two words] value\n }\n" + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n\n" }, { "type": "Comment", @@ -120,7 +141,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-int =\n {\n *[1 apple] value\n }\n" + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n\n" }, { "type": "Comment", @@ -132,4 +153,4 @@ "content": "-invalid-int =\n {\n *[3.14 apples] value\n }\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/variant_lists.ftl b/fluent-syntax/tests/fixtures/variant_lists.ftl index d9031d91..e5c61dd8 100644 --- a/fluent-syntax/tests/fixtures/variant_lists.ftl +++ b/fluent-syntax/tests/fixtures/variant_lists.ftl @@ -23,6 +23,7 @@ variant-list-in-message-attr = Value *[key] Value } +# ERROR VariantLists cannot be Variant values. -nested-variant-list-in-term = { *[one] { @@ -37,7 +38,7 @@ variant-list-in-message-attr = Value } } -# ERROR VariantLists may not appear in SelectExpressions +# ERROR VariantLists cannot be Variant values. nested-select-then-variant-list = { *[one] { 2 -> diff --git a/fluent-syntax/tests/fixtures/variant_lists.json b/fluent-syntax/tests/fixtures/variant_lists.json index b1855100..83f16662 100644 --- a/fluent-syntax/tests/fixtures/variant_lists.json +++ b/fluent-syntax/tests/fixtures/variant_lists.json @@ -1,17 +1,23 @@ { + "type": "Resource", "body": [ { "type": "Term", "id": { + "type": "Identifier", "name": "variant-list-in-term" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -29,9 +35,11 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "variant-list-in-term-attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -48,7 +56,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { "type": "Comment", @@ -57,14 +65,16 @@ { "type": "Junk", "annotations": [], - "content": "variant-list-in-message =\n {\n *[key] Value\n }\n" + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n\n" }, { "type": "Message", "id": { + "type": "Identifier", "name": "variant-list-in-message-attr" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -81,70 +91,52 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { - "type": "Term", - "id": { - "name": "nested-variant-list-in-term" - }, - "value": { - "variants": [ - { - "key": { - "name": "one" - }, - "value": { - "variants": [ - { - "key": { - "name": "two" - }, - "value": { - "elements": [ - { - "type": "TextElement", - "value": "Value" - } - ] - }, - "default": true - } - ] - }, - "default": true - } - ] - }, - "attributes": [], - "comment": null + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "-nested-variant-list-in-term =\n {\n *[one] {\n *[two] Value\n }\n }\n\n" }, { "type": "Term", "id": { + "type": "Identifier", "name": "nested-select" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "one" }, "value": { + "type": "Pattern", "elements": [ { "type": "Placeable", "expression": { + "type": "SelectExpression", "selector": { "type": "NumberLiteral", "value": "2" }, "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "two" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -168,12 +160,12 @@ }, { "type": "Comment", - "content": "ERROR VariantLists may not appear in SelectExpressions" + "content": "ERROR VariantLists cannot be Variant values." }, { "type": "Junk", "annotations": [], - "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n" + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n\n" }, { "type": "Comment", @@ -185,4 +177,4 @@ "content": "variant-list-in-placeable =\n A prefix here {\n *[key] Value\n } and a postfix here make this a Pattern.\n" } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/variants_indent.json b/fluent-syntax/tests/fixtures/variants_indent.json index a780701d..7b987d48 100644 --- a/fluent-syntax/tests/fixtures/variants_indent.json +++ b/fluent-syntax/tests/fixtures/variants_indent.json @@ -1,17 +1,23 @@ { + "type": "Resource", "body": [ { "type": "Term", "id": { + "type": "Identifier", "name": "variants-1tbs" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -29,15 +35,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "variants-allman" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -55,15 +66,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "variants-gnu" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -81,15 +97,20 @@ { "type": "Term", "id": { + "type": "Identifier", "name": "variants-no-indent" }, "value": { + "type": "VariantList", "variants": [ { + "type": "Variant", "key": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -100,10 +121,13 @@ "default": true }, { + "type": "Variant", "key": { + "type": "Identifier", "name": "other" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", @@ -119,4 +143,4 @@ "comment": null } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/fixtures/whitespace_in_value.json b/fluent-syntax/tests/fixtures/whitespace_in_value.json index 0f345b0c..077e6867 100644 --- a/fluent-syntax/tests/fixtures/whitespace_in_value.json +++ b/fluent-syntax/tests/fixtures/whitespace_in_value.json @@ -1,43 +1,18 @@ { + "type": "Resource", "body": [ { "type": "Message", "id": { + "type": "Identifier", "name": "key" }, "value": { + "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "first line\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "\n" - }, - { - "type": "TextElement", - "value": "last line" + "value": "first line\n\n\n\n\n\n\nlast line" } ] }, @@ -48,4 +23,4 @@ } } ] -} \ No newline at end of file +} diff --git a/fluent-syntax/tests/parser_fixtures.rs b/fluent-syntax/tests/parser_fixtures.rs index f3305110..1eee7949 100644 --- a/fluent-syntax/tests/parser_fixtures.rs +++ b/fluent-syntax/tests/parser_fixtures.rs @@ -1,26 +1,20 @@ mod ast; +use assert_json_diff::assert_json_include; use glob::glob; -use std::fs; +use serde_json::Value; use std::fs::File; use std::io; use std::io::prelude::*; -use std::process::Command; use fluent_syntax::parser::parse; -fn compare_jsons(value: &str, reference: &str) -> String { - let temp_path = reference.replace(".json", ".candidate.json"); - write_file(&temp_path, value).unwrap(); +fn compare_jsons(value: &str, reference: &str) { + let a: Value = serde_json::from_str(value).unwrap(); - let output = Command::new("sh") - .arg("-c") - .arg(format!("json-diff {} {}", reference, &temp_path)) - .output() - .expect("failed to execute process"); - let s = String::from_utf8_lossy(&output.stdout).to_string(); - fs::remove_file(&temp_path).unwrap(); - s + let b: Value = serde_json::from_str(reference).unwrap(); + + assert_json_include!(actual: a, expected: b); } fn read_file(path: &str, trim: bool) -> Result { @@ -34,12 +28,6 @@ fn read_file(path: &str, trim: bool) -> Result { } } -fn write_file(path: &str, value: &str) -> std::io::Result<()> { - let mut file = File::create(&path)?; - file.write_all(value.as_bytes())?; - Ok(()) -} - #[test] fn parse_fixtures_compare() { for entry in glob("./tests/fixtures/*.ftl").expect("Failed to read glob pattern") { @@ -58,12 +46,7 @@ fn parse_fixtures_compare() { let target_json = ast::serialize(&target_ast).unwrap(); - let diff = compare_jsons(&target_json, &reference_path); - assert_eq!( - reference_file, target_json, - "\n=====\nThe diff {} :\n-------\n{}\n-----\n", - path, diff - ); + compare_jsons(&target_json, &reference_file); } } diff --git a/fluent/Cargo.toml b/fluent/Cargo.toml index d9102f09..ade4bb98 100644 --- a/fluent/Cargo.toml +++ b/fluent/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "fluent" description = """ -A localization library designed to unleash the entire expressive power of +A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" @@ -9,6 +9,7 @@ authors = [ "Zibi Braniecki ", "Staś Małolepszy " ] +edition = "2018" homepage = "http://www.projectfluent.org" license = "Apache-2.0/MIT" repository = "https://github.com/projectfluent/fluent-rs" @@ -17,9 +18,5 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] -clap = "2.32" +fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" -fluent-syntax = "0.1.1" -failure = "0.1" -failure_derive = "0.1" -intl_pluralrules = "1.0" diff --git a/fluent/examples/resources/en-US/common.ftl b/fluent/examples/resources/en-US/common.ftl new file mode 100644 index 00000000..09e3bdaa --- /dev/null +++ b/fluent/examples/resources/en-US/common.ftl @@ -0,0 +1 @@ +hello-world = Hello World! diff --git a/fluent/examples/resources/en-US/errors.ftl b/fluent/examples/resources/en-US/errors.ftl new file mode 100644 index 00000000..6a228bbe --- /dev/null +++ b/fluent/examples/resources/en-US/errors.ftl @@ -0,0 +1,2 @@ +missing-arg-error = Error: Please provide a number as argument. +input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } diff --git a/fluent/examples/resources/en-US/simple.ftl b/fluent/examples/resources/en-US/simple.ftl index 99f0a6bb..104d3e30 100644 --- a/fluent/examples/resources/en-US/simple.ftl +++ b/fluent/examples/resources/en-US/simple.ftl @@ -1,5 +1,3 @@ -missing-arg-error = Error: Please provide a number as argument. -input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } response-msg = { $value -> [one] "{ $input }" has one Collatz step. diff --git a/fluent/examples/resources/pl/common.ftl b/fluent/examples/resources/pl/common.ftl new file mode 100644 index 00000000..5f6a3d9f --- /dev/null +++ b/fluent/examples/resources/pl/common.ftl @@ -0,0 +1 @@ +hello-world = Witaj, Świecie! diff --git a/fluent/examples/resources/pl/errors.ftl b/fluent/examples/resources/pl/errors.ftl new file mode 100644 index 00000000..f27124a3 --- /dev/null +++ b/fluent/examples/resources/pl/errors.ftl @@ -0,0 +1,2 @@ +missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. +input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } diff --git a/fluent/examples/resources/pl/simple.ftl b/fluent/examples/resources/pl/simple.ftl index 16173dd9..7a17d125 100644 --- a/fluent/examples/resources/pl/simple.ftl +++ b/fluent/examples/resources/pl/simple.ftl @@ -1,5 +1,3 @@ -missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. -input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } response-msg = { $value -> [one] "{ $input }" ma jeden krok Collatza. diff --git a/fluent/examples/simple.rs b/fluent/examples/simple.rs index ff3b597b..0b0d8806 100644 --- a/fluent/examples/simple.rs +++ b/fluent/examples/simple.rs @@ -17,32 +17,15 @@ //! //! If the second argument is omitted, `en-US` locale is used as the //! default one. -extern crate fluent; -extern crate fluent_locale; - -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent::resource_manager::ResourceManager; +use fluent_bundle::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; use std::fs; -use std::fs::File; use std::io; -use std::io::prelude::*; use std::str::FromStr; -/// We need a generic file read helper function to -/// read the localization resource file. -/// -/// The resource files are stored in -/// `./examples/resources/{locale}` directory. -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - /// This helper function allows us to read the list /// of available locales by reading the list of /// directories in `./examples/resources`. @@ -89,12 +72,14 @@ fn get_app_locales(requested: &[&str]) -> Result, io::Error> { .collect::>()); } -static L10N_RESOURCES: &[&str] = &["simple.ftl"]; +static L10N_RESOURCES: &[&str] = &["simple.ftl", "errors.ftl"]; fn main() { // 1. Get the command line arguments. let args: Vec = env::args().collect(); + let mgr = ResourceManager::new(); + // 2. If the argument length is more than 1, // take the second argument as a comma-separated // list of requested locales. @@ -109,20 +94,19 @@ fn main() { // 4. Create a new Fluent FluentBundle using the // resolved locales. - let mut bundle = FluentBundle::new(&locales); + let paths = L10N_RESOURCES + .iter() + .map(|path| { + format!( + "./examples/resources/{locale}/{path}", + locale = locales[0], + path = path + ) + }) + .collect(); - // 5. Load the localization resource - for path in L10N_RESOURCES { - let full_path = format!( - "./examples/resources/{locale}/{path}", - locale = locales[0], - path = path - ); - let res = read_file(&full_path).unwrap(); - // 5.1 Insert the loaded resource into the - // Fluent FluentBundle. - bundle.add_messages(&res).unwrap(); - } + // 5. Get a bundle for given paths and locales. + let bundle = mgr.get_bundle(&locales, &paths); // 6. Check if the input is provided. match args.get(1) { diff --git a/fluent/src/bin/parser.rs b/fluent/src/bin/parser.rs deleted file mode 100644 index afbab0e8..00000000 --- a/fluent/src/bin/parser.rs +++ /dev/null @@ -1,64 +0,0 @@ -extern crate clap; -extern crate fluent; -extern crate fluent_syntax; - -use std::fs::File; -use std::io; -use std::io::Read; - -use clap::App; - -use fluent_syntax::ast::Resource; -use fluent_syntax::parser::errors::display::annotate_error; -use fluent_syntax::parser::parse; - -fn read_file(path: &str) -> Result { - let mut f = try!(File::open(path)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); - Ok(s) -} - -fn print_entries_resource(res: &Resource) { - println!("{:#?}", res); -} - -fn main() { - let matches = App::new("Fluent Parser") - .version("1.0") - .about("Parses FTL file into an AST") - .args_from_usage( - "-s, --silence 'disable output' - 'Sets the input file to use'", - ) - .get_matches(); - - let input = matches.value_of("INPUT").unwrap(); - - let source = read_file(&input).expect("Read file failed"); - - let res = parse(&source); - - if matches.is_present("silence") { - return; - }; - - match res { - Ok(res) => print_entries_resource(&res), - Err((res, errors)) => { - print_entries_resource(&res); - println!("==============================\n"); - if errors.len() == 1 { - println!("Parser encountered one error:"); - } else { - println!("Parser encountered {} errors:", errors.len()); - } - println!("-----------------------------"); - for err in errors { - let f = annotate_error(&err, &Some(input.to_string()), true); - println!("{}", f); - println!("-----------------------------"); - } - } - }; -} diff --git a/fluent/src/lib.rs b/fluent/src/lib.rs index 019069a3..4f362414 100644 --- a/fluent/src/lib.rs +++ b/fluent/src/lib.rs @@ -1,48 +1 @@ -//! Fluent is a localization system designed to improve how software is translated. -//! -//! The Rust implementation provides the low level components for syntax operations, like parser -//! and AST, and the core localization struct - `FluentBundle`. -//! -//! `FluentBundle` is the low level container for storing and formatting localization messages. It -//! is expected that implementations will build on top of it by providing language negotiation -//! between user requested languages and available resources and I/O for loading selected -//! resources. -//! -//! # Example -//! -//! ``` -//! use fluent::bundle::FluentBundle; -//! use fluent::types::FluentValue; -//! use std::collections::HashMap; -//! -//! let mut bundle = FluentBundle::new(&["en-US"]); -//! bundle.add_messages( -//! " -//! hello-world = Hello, world! -//! intro = Welcome, { $name }. -//! " -//! ); -//! -//! let value = bundle.format("hello-world", None); -//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); -//! -//! let mut args = HashMap::new(); -//! args.insert("name", FluentValue::from("John")); -//! -//! let value = bundle.format("intro", Some(&args)); -//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); -//! ``` - -extern crate failure; -#[macro_use] -extern crate failure_derive; -extern crate fluent_locale; -extern crate fluent_syntax; -extern crate intl_pluralrules; - -pub mod bundle; -pub mod entry; -pub mod errors; -pub mod resolve; -pub mod resource; -pub mod types; +pub mod resource_manager; diff --git a/fluent/src/resource.rs b/fluent/src/resource.rs deleted file mode 100644 index 9521b486..00000000 --- a/fluent/src/resource.rs +++ /dev/null @@ -1,16 +0,0 @@ -use fluent_syntax::ast; -use fluent_syntax::parser::errors::ParserError; -use fluent_syntax::parser::parse; - -pub struct FluentResource { - pub ast: ast::Resource, -} - -impl FluentResource { - pub fn from_string(source: &str) -> Result)> { - match parse(source) { - Ok(ast) => Ok(FluentResource { ast }), - Err((ast, errors)) => Err((FluentResource { ast }, errors)), - } - } -} diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs new file mode 100644 index 00000000..2910db29 --- /dev/null +++ b/fluent/src/resource_manager.rs @@ -0,0 +1,91 @@ +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; +use std::cell::UnsafeCell; +use std::collections::hash_map::HashMap; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::ops::Deref; + +fn read_file(path: &str) -> Result { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +unsafe trait Allocated: Deref {} + +unsafe impl Allocated for String {} +unsafe impl Allocated for Box {} + +struct FrozenMap { + map: UnsafeCell>, +} + +impl FrozenMap { + // under no circumstances implement delete() on this + // under no circumstances return an &T + pub fn new() -> Self { + Self { + map: UnsafeCell::new(Default::default()), + } + } + + pub fn insert(&self, k: String, v: T) -> &T::Target { + unsafe { + let map = self.map.get(); + &*(*map).entry(k).or_insert(v) + } + } + + pub fn get(&self, k: &str) -> Option<&T::Target> { + unsafe { + let map = self.map.get(); + (*map).get(k).map(|x| &**x) + } + } +} + +pub struct ResourceManager<'mgr> { + strings: FrozenMap, + resources: FrozenMap>>, +} + +impl<'mgr> ResourceManager<'mgr> { + pub fn new() -> Self { + ResourceManager { + strings: FrozenMap::new(), + resources: FrozenMap::new(), + } + } + + pub fn get_resource(&'mgr self, path: &str) -> &'mgr FluentResource<'mgr> { + let strings = &self.strings; + + if strings.get(path).is_some() { + return self.resources.get(path).unwrap(); + } else { + let string = read_file(path).unwrap(); + let val = self.strings.insert(path.to_string(), string); + let res = match FluentResource::from_string(val) { + Ok(res) => res, + Err((res, _err)) => res, + }; + self.resources.insert(path.to_string(), Box::new(res)) + } + } + + pub fn get_bundle( + &'mgr self, + locales: &Vec, + paths: &Vec, + ) -> FluentBundle<'mgr> { + let mut bundle = FluentBundle::new(locales); + for path in paths { + let res = self.get_resource(path); + bundle.add_resource(res).unwrap(); + } + return bundle; + } +} From bdbdaad2392d0edbda263cdfe20133d2dfb8906f Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Fri, 28 Dec 2018 14:17:59 -0800 Subject: [PATCH 03/36] Switch to elsa --- fluent/Cargo.toml | 1 + fluent/src/resource_manager.rs | 41 +++------------------------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/fluent/Cargo.toml b/fluent/Cargo.toml index ade4bb98..277631f4 100644 --- a/fluent/Cargo.toml +++ b/fluent/Cargo.toml @@ -20,3 +20,4 @@ categories = ["localization", "internationalization"] [dependencies] fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" +elsa = "^0.1.2" diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs index 2910db29..4798aedd 100644 --- a/fluent/src/resource_manager.rs +++ b/fluent/src/resource_manager.rs @@ -1,11 +1,9 @@ +use elsa::FrozenMap; use fluent_bundle::bundle::FluentBundle; use fluent_bundle::resource::FluentResource; -use std::cell::UnsafeCell; -use std::collections::hash_map::HashMap; use std::fs::File; use std::io; use std::io::prelude::*; -use std::ops::Deref; fn read_file(path: &str) -> Result { let mut f = File::open(path)?; @@ -14,42 +12,9 @@ fn read_file(path: &str) -> Result { Ok(s) } -unsafe trait Allocated: Deref {} - -unsafe impl Allocated for String {} -unsafe impl Allocated for Box {} - -struct FrozenMap { - map: UnsafeCell>, -} - -impl FrozenMap { - // under no circumstances implement delete() on this - // under no circumstances return an &T - pub fn new() -> Self { - Self { - map: UnsafeCell::new(Default::default()), - } - } - - pub fn insert(&self, k: String, v: T) -> &T::Target { - unsafe { - let map = self.map.get(); - &*(*map).entry(k).or_insert(v) - } - } - - pub fn get(&self, k: &str) -> Option<&T::Target> { - unsafe { - let map = self.map.get(); - (*map).get(k).map(|x| &**x) - } - } -} - pub struct ResourceManager<'mgr> { - strings: FrozenMap, - resources: FrozenMap>>, + strings: FrozenMap, + resources: FrozenMap>>, } impl<'mgr> ResourceManager<'mgr> { From 6e022726350cbbd780bbcdb50d980e57bc958ae9 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 31 Dec 2018 16:07:43 -0800 Subject: [PATCH 04/36] Merge fluent-bundle and fluent and attempt to use ResourceManager via Cow --- Cargo.toml | 1 - fluent-bundle/src/lib.rs | 48 ------ {fluent-bundle => fluent-res}/Cargo.toml | 10 +- .../examples/resources/en-US/common.ftl | 1 + .../examples/resources/en-US/errors.ftl | 5 - .../examples/resources/en-US/simple.ftl | 5 + fluent-res/examples/resources/pl/common.ftl | 1 + fluent-res/examples/resources/pl/errors.ftl | 2 + .../examples/resources/pl/simple.ftl | 2 - fluent-res/examples/simple.rs | 154 ++++++++++++++++++ fluent-res/src/lib.rs | 1 + {fluent-bundle => fluent}/CHANGELOG.md | 0 fluent/Cargo.toml | 7 +- {fluent-bundle => fluent}/README.md | 0 {fluent-bundle => fluent}/benches/lib.rs | 0 {fluent-bundle => fluent}/benches/menubar.ftl | 0 {fluent-bundle => fluent}/benches/simple.ftl | 0 {fluent-bundle => fluent}/examples/README.md | 0 .../examples/external_arguments.rs | 6 +- .../examples/functions.rs | 8 +- {fluent-bundle => fluent}/examples/hello.rs | 4 +- .../examples/message_reference.rs | 4 +- .../examples/selector.rs | 6 +- .../examples/simple-app.rs | 6 +- fluent/examples/simple.rs | 2 +- {fluent-bundle => fluent}/src/bundle.rs | 74 ++++++--- {fluent-bundle => fluent}/src/entry.rs | 0 {fluent-bundle => fluent}/src/errors.rs | 3 +- fluent/src/lib.rs | 41 +++++ {fluent-bundle => fluent}/src/resolve.rs | 0 {fluent-bundle => fluent}/src/resource.rs | 0 fluent/src/resource_manager.rs | 27 ++- {fluent-bundle => fluent}/src/types.rs | 0 {fluent-bundle => fluent}/tests/bundle.rs | 28 ++-- {fluent-bundle => fluent}/tests/format.rs | 4 +- .../tests/format_message.rs | 6 +- .../tests/helpers/mod.rs | 4 +- .../tests/resolve_attribute_expression.rs | 4 +- .../tests/resolve_external_argument.rs | 4 +- .../tests/resolve_message_reference.rs | 2 +- .../tests/resolve_plural_rule.rs | 8 +- .../tests/resolve_select_expression.rs | 46 +++--- .../tests/resolve_value.rs | 6 +- .../tests/resolve_variant_expression.rs | 4 +- 44 files changed, 368 insertions(+), 166 deletions(-) delete mode 100644 fluent-bundle/src/lib.rs rename {fluent-bundle => fluent-res}/Cargo.toml (81%) create mode 100644 fluent-res/examples/resources/en-US/common.ftl rename fluent-bundle/examples/resources/en-US/simple.ftl => fluent-res/examples/resources/en-US/errors.ftl (50%) create mode 100644 fluent-res/examples/resources/en-US/simple.ftl create mode 100644 fluent-res/examples/resources/pl/common.ftl create mode 100644 fluent-res/examples/resources/pl/errors.ftl rename {fluent-bundle => fluent-res}/examples/resources/pl/simple.ftl (56%) create mode 100644 fluent-res/examples/simple.rs create mode 100644 fluent-res/src/lib.rs rename {fluent-bundle => fluent}/CHANGELOG.md (100%) rename {fluent-bundle => fluent}/README.md (100%) rename {fluent-bundle => fluent}/benches/lib.rs (100%) rename {fluent-bundle => fluent}/benches/menubar.ftl (100%) rename {fluent-bundle => fluent}/benches/simple.ftl (100%) rename {fluent-bundle => fluent}/examples/README.md (100%) rename {fluent-bundle => fluent}/examples/external_arguments.rs (88%) rename {fluent-bundle => fluent}/examples/functions.rs (92%) rename {fluent-bundle => fluent}/examples/hello.rs (66%) rename {fluent-bundle => fluent}/examples/message_reference.rs (80%) rename {fluent-bundle => fluent}/examples/selector.rs (82%) rename {fluent-bundle => fluent}/examples/simple-app.rs (97%) rename {fluent-bundle => fluent}/src/bundle.rs (79%) rename {fluent-bundle => fluent}/src/entry.rs (100%) rename {fluent-bundle => fluent}/src/errors.rs (91%) rename {fluent-bundle => fluent}/src/resolve.rs (100%) rename {fluent-bundle => fluent}/src/resource.rs (100%) rename {fluent-bundle => fluent}/src/types.rs (100%) rename {fluent-bundle => fluent}/tests/bundle.rs (53%) rename {fluent-bundle => fluent}/tests/format.rs (88%) rename {fluent-bundle => fluent}/tests/format_message.rs (82%) rename {fluent-bundle => fluent}/tests/helpers/mod.rs (89%) rename {fluent-bundle => fluent}/tests/resolve_attribute_expression.rs (90%) rename {fluent-bundle => fluent}/tests/resolve_external_argument.rs (95%) rename {fluent-bundle => fluent}/tests/resolve_message_reference.rs (98%) rename {fluent-bundle => fluent}/tests/resolve_plural_rule.rs (91%) rename {fluent-bundle => fluent}/tests/resolve_select_expression.rs (86%) rename {fluent-bundle => fluent}/tests/resolve_value.rs (75%) rename {fluent-bundle => fluent}/tests/resolve_variant_expression.rs (92%) diff --git a/Cargo.toml b/Cargo.toml index 29cf9892..dd3d5a8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ "fluent-syntax", - "fluent-bundle", "fluent-cli", "fluent", ] diff --git a/fluent-bundle/src/lib.rs b/fluent-bundle/src/lib.rs deleted file mode 100644 index b516ac9c..00000000 --- a/fluent-bundle/src/lib.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Fluent is a localization system designed to improve how software is translated. -//! -//! The Rust implementation provides the low level components for syntax operations, like parser -//! and AST, and the core localization struct - `FluentBundle`. -//! -//! `FluentBundle` is the low level container for storing and formatting localization messages. It -//! is expected that implementations will build on top of it by providing language negotiation -//! between user requested languages and available resources and I/O for loading selected -//! resources. -//! -//! # Example -//! -//! ``` -//! use fluent_bundle::bundle::FluentBundle; -//! use fluent_bundle::types::FluentValue; -//! use std::collections::HashMap; -//! -//! let mut bundle = FluentBundle::new(&["en-US"]); -//! bundle.add_messages( -//! " -//! hello-world = Hello, world! -//! intro = Welcome, { $name }. -//! " -//! ); -//! -//! let value = bundle.format("hello-world", None); -//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); -//! -//! let mut args = HashMap::new(); -//! args.insert("name", FluentValue::from("John")); -//! -//! let value = bundle.format("intro", Some(&args)); -//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); -//! ``` - -extern crate failure; -#[macro_use] -extern crate failure_derive; -extern crate fluent_locale; -extern crate fluent_syntax; -extern crate intl_pluralrules; - -pub mod bundle; -pub mod entry; -pub mod errors; -pub mod resolve; -pub mod resource; -pub mod types; diff --git a/fluent-bundle/Cargo.toml b/fluent-res/Cargo.toml similarity index 81% rename from fluent-bundle/Cargo.toml rename to fluent-res/Cargo.toml index 59f91b86..277631f4 100644 --- a/fluent-bundle/Cargo.toml +++ b/fluent-res/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "fluent-bundle" +name = "fluent" description = """ A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" -edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " ] +edition = "2018" homepage = "http://www.projectfluent.org" license = "Apache-2.0/MIT" repository = "https://github.com/projectfluent/fluent-rs" @@ -18,8 +18,6 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] +fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" -fluent-syntax = { path = "../fluent-syntax" } -failure = "0.1" -failure_derive = "0.1" -intl_pluralrules = "1.0" +elsa = "^0.1.2" diff --git a/fluent-res/examples/resources/en-US/common.ftl b/fluent-res/examples/resources/en-US/common.ftl new file mode 100644 index 00000000..09e3bdaa --- /dev/null +++ b/fluent-res/examples/resources/en-US/common.ftl @@ -0,0 +1 @@ +hello-world = Hello World! diff --git a/fluent-bundle/examples/resources/en-US/simple.ftl b/fluent-res/examples/resources/en-US/errors.ftl similarity index 50% rename from fluent-bundle/examples/resources/en-US/simple.ftl rename to fluent-res/examples/resources/en-US/errors.ftl index 99f0a6bb..6a228bbe 100644 --- a/fluent-bundle/examples/resources/en-US/simple.ftl +++ b/fluent-res/examples/resources/en-US/errors.ftl @@ -1,7 +1,2 @@ missing-arg-error = Error: Please provide a number as argument. input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } -response-msg = - { $value -> - [one] "{ $input }" has one Collatz step. - *[other] "{ $input }" has { $value } Collatz steps. - } diff --git a/fluent-res/examples/resources/en-US/simple.ftl b/fluent-res/examples/resources/en-US/simple.ftl new file mode 100644 index 00000000..104d3e30 --- /dev/null +++ b/fluent-res/examples/resources/en-US/simple.ftl @@ -0,0 +1,5 @@ +response-msg = + { $value -> + [one] "{ $input }" has one Collatz step. + *[other] "{ $input }" has { $value } Collatz steps. + } diff --git a/fluent-res/examples/resources/pl/common.ftl b/fluent-res/examples/resources/pl/common.ftl new file mode 100644 index 00000000..5f6a3d9f --- /dev/null +++ b/fluent-res/examples/resources/pl/common.ftl @@ -0,0 +1 @@ +hello-world = Witaj, Świecie! diff --git a/fluent-res/examples/resources/pl/errors.ftl b/fluent-res/examples/resources/pl/errors.ftl new file mode 100644 index 00000000..f27124a3 --- /dev/null +++ b/fluent-res/examples/resources/pl/errors.ftl @@ -0,0 +1,2 @@ +missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. +input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } diff --git a/fluent-bundle/examples/resources/pl/simple.ftl b/fluent-res/examples/resources/pl/simple.ftl similarity index 56% rename from fluent-bundle/examples/resources/pl/simple.ftl rename to fluent-res/examples/resources/pl/simple.ftl index 16173dd9..7a17d125 100644 --- a/fluent-bundle/examples/resources/pl/simple.ftl +++ b/fluent-res/examples/resources/pl/simple.ftl @@ -1,5 +1,3 @@ -missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. -input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } response-msg = { $value -> [one] "{ $input }" ma jeden krok Collatza. diff --git a/fluent-res/examples/simple.rs b/fluent-res/examples/simple.rs new file mode 100644 index 00000000..0b0d8806 --- /dev/null +++ b/fluent-res/examples/simple.rs @@ -0,0 +1,154 @@ +//! This is an example of a simple application +//! which calculates the Collatz conjecture. +//! +//! The function itself is trivial on purpose, +//! so that we can focus on understanding how +//! the application can be made localizable +//! via Fluent. +//! +//! To try the app launch `cargo run --example simple NUM (LOCALES)` +//! +//! NUM is a number to be calculated, and LOCALES is an optional +//! parameter with a comma-separated list of locales requested by the user. +//! +//! Example: +//! +//! caron run --example simple 123 de,pl +//! +//! If the second argument is omitted, `en-US` locale is used as the +//! default one. +use fluent::resource_manager::ResourceManager; +use fluent_bundle::types::FluentValue; +use fluent_locale::{negotiate_languages, NegotiationStrategy}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::io; +use std::str::FromStr; + +/// This helper function allows us to read the list +/// of available locales by reading the list of +/// directories in `./examples/resources`. +/// +/// It is expected that every directory inside it +/// has a name that is a valid BCP47 language tag. +fn get_available_locales() -> Result, io::Error> { + let mut locales = vec![]; + + let res_dir = fs::read_dir("./examples/resources/")?; + for entry in res_dir { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name() { + if let Some(name) = name.to_str() { + locales.push(String::from(name)); + } + } + } + } + } + return Ok(locales); +} + +/// This function negotiates the locales between available +/// and requested by the user. +/// +/// It uses `fluent-locale` library but one could +/// use any other that will resolve the list of +/// available locales based on the list of +/// requested locales. +fn get_app_locales(requested: &[&str]) -> Result, io::Error> { + let available = get_available_locales()?; + let resolved_locales = negotiate_languages( + requested, + &available, + Some("en-US"), + &NegotiationStrategy::Filtering, + ); + return Ok(resolved_locales + .into_iter() + .map(|s| String::from(s)) + .collect::>()); +} + +static L10N_RESOURCES: &[&str] = &["simple.ftl", "errors.ftl"]; + +fn main() { + // 1. Get the command line arguments. + let args: Vec = env::args().collect(); + + let mgr = ResourceManager::new(); + + // 2. If the argument length is more than 1, + // take the second argument as a comma-separated + // list of requested locales. + // + // Otherwise, take ["en-US"] as the default. + let requested = args + .get(2) + .map_or(vec!["en-US"], |arg| arg.split(",").collect()); + + // 3. Negotiate it against the avialable ones + let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); + + // 4. Create a new Fluent FluentBundle using the + // resolved locales. + let paths = L10N_RESOURCES + .iter() + .map(|path| { + format!( + "./examples/resources/{locale}/{path}", + locale = locales[0], + path = path + ) + }) + .collect(); + + // 5. Get a bundle for given paths and locales. + let bundle = mgr.get_bundle(&locales, &paths); + + // 6. Check if the input is provided. + match args.get(1) { + Some(input) => { + // 6.1. Cast it to a number. + match isize::from_str(&input) { + Ok(i) => { + // 6.2. Construct a map of arguments + // to format the message. + let mut args = HashMap::new(); + args.insert("input", FluentValue::from(i)); + args.insert("value", FluentValue::from(collatz(i))); + // 6.3. Format the message. + println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); + } + Err(err) => { + let mut args = HashMap::new(); + args.insert("input", FluentValue::from(input.to_string())); + args.insert("reason", FluentValue::from(err.to_string())); + println!( + "{}", + bundle + .format("input-parse-error-msg", Some(&args)) + .unwrap() + .0 + ); + } + } + } + None => { + println!("{}", bundle.format("missing-arg-error", None).unwrap().0); + } + } +} + +/// Collatz conjecture calculating function. +fn collatz(n: isize) -> isize { + match n { + 1 => 0, + _ => match n % 2 { + 0 => 1 + collatz(n / 2), + _ => 1 + collatz(n * 3 + 1), + }, + } +} diff --git a/fluent-res/src/lib.rs b/fluent-res/src/lib.rs new file mode 100644 index 00000000..4f362414 --- /dev/null +++ b/fluent-res/src/lib.rs @@ -0,0 +1 @@ +pub mod resource_manager; diff --git a/fluent-bundle/CHANGELOG.md b/fluent/CHANGELOG.md similarity index 100% rename from fluent-bundle/CHANGELOG.md rename to fluent/CHANGELOG.md diff --git a/fluent/Cargo.toml b/fluent/Cargo.toml index 277631f4..b8e1fae3 100644 --- a/fluent/Cargo.toml +++ b/fluent/Cargo.toml @@ -5,11 +5,11 @@ A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" +edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " ] -edition = "2018" homepage = "http://www.projectfluent.org" license = "Apache-2.0/MIT" repository = "https://github.com/projectfluent/fluent-rs" @@ -18,6 +18,9 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] -fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" +fluent-syntax = { path = "../fluent-syntax" } +failure = "0.1" +failure_derive = "0.1" +intl_pluralrules = "^1.0" elsa = "^0.1.2" diff --git a/fluent-bundle/README.md b/fluent/README.md similarity index 100% rename from fluent-bundle/README.md rename to fluent/README.md diff --git a/fluent-bundle/benches/lib.rs b/fluent/benches/lib.rs similarity index 100% rename from fluent-bundle/benches/lib.rs rename to fluent/benches/lib.rs diff --git a/fluent-bundle/benches/menubar.ftl b/fluent/benches/menubar.ftl similarity index 100% rename from fluent-bundle/benches/menubar.ftl rename to fluent/benches/menubar.ftl diff --git a/fluent-bundle/benches/simple.ftl b/fluent/benches/simple.ftl similarity index 100% rename from fluent-bundle/benches/simple.ftl rename to fluent/benches/simple.ftl diff --git a/fluent-bundle/examples/README.md b/fluent/examples/README.md similarity index 100% rename from fluent-bundle/examples/README.md rename to fluent/examples/README.md diff --git a/fluent-bundle/examples/external_arguments.rs b/fluent/examples/external_arguments.rs similarity index 88% rename from fluent-bundle/examples/external_arguments.rs rename to fluent/examples/external_arguments.rs index 0b817451..e8b4b2e8 100644 --- a/fluent-bundle/examples/external_arguments.rs +++ b/fluent/examples/external_arguments.rs @@ -1,9 +1,9 @@ -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["en"]); + let mut bundle = FluentBundle::new(&["en"], None); bundle .add_messages( " diff --git a/fluent-bundle/examples/functions.rs b/fluent/examples/functions.rs similarity index 92% rename from fluent-bundle/examples/functions.rs rename to fluent/examples/functions.rs index a0955b40..f80c7535 100644 --- a/fluent-bundle/examples/functions.rs +++ b/fluent/examples/functions.rs @@ -1,8 +1,8 @@ -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; fn main() { - let mut bundle = FluentBundle::new(&["en-US"]); + let mut bundle = FluentBundle::new(&["en-US"], None); // Test for a simple function that returns a string bundle @@ -45,7 +45,7 @@ fn main() { .add_messages("meaning-of-life = { MEANING_OF_LIFE(42) }") .unwrap(); bundle - .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") + .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: "us") }") .unwrap(); let value = bundle.format("hello-world", None); diff --git a/fluent-bundle/examples/hello.rs b/fluent/examples/hello.rs similarity index 66% rename from fluent-bundle/examples/hello.rs rename to fluent/examples/hello.rs index 2828b333..5bfe083b 100644 --- a/fluent-bundle/examples/hello.rs +++ b/fluent/examples/hello.rs @@ -1,7 +1,7 @@ -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; fn main() { - let mut bundle = FluentBundle::new(&["en-US"]); + let mut bundle = FluentBundle::new(&["en-US"], None); bundle.add_messages("hello-world = Hello, world!").unwrap(); let (value, _) = bundle.format("hello-world", None).unwrap(); assert_eq!(&value, "Hello, world!"); diff --git a/fluent-bundle/examples/message_reference.rs b/fluent/examples/message_reference.rs similarity index 80% rename from fluent-bundle/examples/message_reference.rs rename to fluent/examples/message_reference.rs index bfc512e5..da1cf50b 100644 --- a/fluent-bundle/examples/message_reference.rs +++ b/fluent/examples/message_reference.rs @@ -1,7 +1,7 @@ -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); bundle .add_messages( " diff --git a/fluent-bundle/examples/selector.rs b/fluent/examples/selector.rs similarity index 82% rename from fluent-bundle/examples/selector.rs rename to fluent/examples/selector.rs index c7e3672b..01b8012d 100644 --- a/fluent-bundle/examples/selector.rs +++ b/fluent/examples/selector.rs @@ -1,9 +1,9 @@ -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); bundle .add_messages( " diff --git a/fluent-bundle/examples/simple-app.rs b/fluent/examples/simple-app.rs similarity index 97% rename from fluent-bundle/examples/simple-app.rs rename to fluent/examples/simple-app.rs index bf7c8854..f08d6e7e 100644 --- a/fluent-bundle/examples/simple-app.rs +++ b/fluent/examples/simple-app.rs @@ -17,8 +17,8 @@ //! //! If the second argument is omitted, `en-US` locale is used as the //! default one. -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; @@ -107,7 +107,7 @@ fn main() { // 4. Create a new Fluent FluentBundle using the // resolved locales. - let mut bundle = FluentBundle::new(&locales); + let mut bundle = FluentBundle::new(&locales, None); // 5. Load the localization resource for path in L10N_RESOURCES { diff --git a/fluent/examples/simple.rs b/fluent/examples/simple.rs index 0b0d8806..3def15ab 100644 --- a/fluent/examples/simple.rs +++ b/fluent/examples/simple.rs @@ -18,7 +18,7 @@ //! If the second argument is omitted, `en-US` locale is used as the //! default one. use fluent::resource_manager::ResourceManager; -use fluent_bundle::types::FluentValue; +use fluent::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; diff --git a/fluent-bundle/src/bundle.rs b/fluent/src/bundle.rs similarity index 79% rename from fluent-bundle/src/bundle.rs rename to fluent/src/bundle.rs index 6c096aad..e593fe99 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent/src/bundle.rs @@ -6,6 +6,7 @@ use std::cell::RefCell; use std::collections::hash_map::{Entry as HashEntry, HashMap}; +use std::borrow::Cow; use super::entry::{Entry, GetEntry}; pub use super::errors::FluentError; @@ -14,6 +15,7 @@ use super::resource::FluentResource; use super::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use fluent_syntax::ast; +use crate::resource_manager::ResourceManager; use intl_pluralrules::{IntlPluralRules, PluralRuleType}; #[derive(Debug, PartialEq)] @@ -48,15 +50,15 @@ pub struct Message { /// purpose of language negotiation with i18n formatters. For instance, if date and time formatting /// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain /// to negotiate a sensible fallback for date and time formatting. -#[allow(dead_code)] pub struct FluentBundle<'bundle> { pub locales: Vec, pub entries: HashMap>, pub plural_rules: IntlPluralRules, + pub resource_manager: Cow<'bundle, ResourceManager<'bundle>>, } impl<'bundle> FluentBundle<'bundle> { - pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { + pub fn new<'a, S: ToString>(locales: &'a [S], res_mgr: Option<&'bundle ResourceManager<'bundle>>) -> FluentBundle<'bundle> { let locales = locales.iter().map(|s| s.to_string()).collect::>(); let pr_locale = negotiate_languages( &locales, @@ -67,10 +69,15 @@ impl<'bundle> FluentBundle<'bundle> { .to_owned(); let pr = IntlPluralRules::create(&pr_locale, PluralRuleType::CARDINAL).unwrap(); + let res_mgr = match res_mgr { + Some(res_mgr) => Cow::Borrowed(res_mgr), + None => Cow::Owned(ResourceManager::new()) + }; FluentBundle { locales, entries: HashMap::new(), plural_rules: pr, + resource_manager: res_mgr } } @@ -97,22 +104,45 @@ impl<'bundle> FluentBundle<'bundle> { } } - //pub fn add_messages(&mut self, source: &'bundle str) -> Result<(), Vec> { - //match FluentResource::from_string(source) { - //Ok(res) => self.add_resource(&res), - //Err((res, err)) => { - //let mut errors: Vec = - //err.into_iter().map(FluentError::ParserError).collect(); - - //self.add_resource(&res).map_err(|err| { - //for e in err { - //errors.push(e); - //} - //errors - //}) - //} - //} - //} + pub fn add_messages(&'bundle mut self, source: &str) -> Result<(), Vec> { + let res_mgr = &self.resource_manager; + let res = res_mgr.get_resource_for_string(source); + let mut errors = vec![]; + + for entry in &res.ast.body { + let id = match entry { + ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { ref id, .. })) + | ast::ResourceEntry::Entry(ast::Entry::Term(ast::Term { ref id, .. })) => id.name, + _ => continue, + }; + + let (entry, kind) = match entry { + ast::ResourceEntry::Entry(ast::Entry::Message(message)) => { + (Entry::Message(message), "message") + } + ast::ResourceEntry::Entry(ast::Entry::Term(term)) => (Entry::Term(term), "term"), + _ => continue, + }; + + match self.entries.entry(id.to_string()) { + HashEntry::Vacant(empty) => { + empty.insert(entry); + } + HashEntry::Occupied(_) => { + errors.push(FluentError::Overriding { + kind, + id: id.to_string(), + }); + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } pub fn add_resource( &mut self, @@ -156,9 +186,9 @@ impl<'bundle> FluentBundle<'bundle> { } pub fn format( - &self, + &'bundle self, path: &str, - args: Option<&HashMap<&str, FluentValue>>, + args: Option<&'bundle HashMap<&str, FluentValue>>, ) -> Option<(String, Vec)> { let env = Env { bundle: self, @@ -207,9 +237,9 @@ impl<'bundle> FluentBundle<'bundle> { } pub fn format_message( - &self, + &'bundle self, message_id: &str, - args: Option<&HashMap<&str, FluentValue>>, + args: Option<&'bundle HashMap<&str, FluentValue>>, ) -> Option<(Message, Vec)> { let mut errors = vec![]; diff --git a/fluent-bundle/src/entry.rs b/fluent/src/entry.rs similarity index 100% rename from fluent-bundle/src/entry.rs rename to fluent/src/entry.rs diff --git a/fluent-bundle/src/errors.rs b/fluent/src/errors.rs similarity index 91% rename from fluent-bundle/src/errors.rs rename to fluent/src/errors.rs index 9b7cf418..9c430c3d 100644 --- a/fluent-bundle/src/errors.rs +++ b/fluent/src/errors.rs @@ -1,5 +1,6 @@ -use super::resolve::ResolverError; +use crate::resolve::ResolverError; use fluent_syntax::parser::ParserError; +use failure_derive::Fail; #[derive(Debug, Fail, PartialEq)] pub enum FluentError { diff --git a/fluent/src/lib.rs b/fluent/src/lib.rs index 4f362414..885a4f97 100644 --- a/fluent/src/lib.rs +++ b/fluent/src/lib.rs @@ -1 +1,42 @@ +//! Fluent is a localization system designed to improve how software is translated. +//! +//! The Rust implementation provides the low level components for syntax operations, like parser +//! and AST, and the core localization struct - `FluentBundle`. +//! +//! `FluentBundle` is the low level container for storing and formatting localization messages. It +//! is expected that implementations will build on top of it by providing language negotiation +//! between user requested languages and available resources and I/O for loading selected +//! resources. +//! +//! # Example +//! +//! ``` +//! use fluent_bundle::bundle::FluentBundle; +//! use fluent_bundle::types::FluentValue; +//! use std::collections::HashMap; +//! +//! let mut bundle = FluentBundle::new(&["en-US"]); +//! bundle.add_messages( +//! " +//! hello-world = Hello, world! +//! intro = Welcome, { $name }. +//! " +//! ); +//! +//! let value = bundle.format("hello-world", None); +//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); +//! +//! let mut args = HashMap::new(); +//! args.insert("name", FluentValue::from("John")); +//! +//! let value = bundle.format("intro", Some(&args)); +//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); +//! ``` + +pub mod bundle; +pub mod entry; +pub mod errors; +pub mod resolve; +pub mod resource; +pub mod types; pub mod resource_manager; diff --git a/fluent-bundle/src/resolve.rs b/fluent/src/resolve.rs similarity index 100% rename from fluent-bundle/src/resolve.rs rename to fluent/src/resolve.rs diff --git a/fluent-bundle/src/resource.rs b/fluent/src/resource.rs similarity index 100% rename from fluent-bundle/src/resource.rs rename to fluent/src/resource.rs diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs index 4798aedd..e81dc09a 100644 --- a/fluent/src/resource_manager.rs +++ b/fluent/src/resource_manager.rs @@ -1,6 +1,6 @@ use elsa::FrozenMap; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::resource::FluentResource; +use crate::bundle::FluentBundle; +use crate::resource::FluentResource; use std::fs::File; use std::io; use std::io::prelude::*; @@ -17,6 +17,12 @@ pub struct ResourceManager<'mgr> { resources: FrozenMap>>, } +impl<'mgr> Clone for ResourceManager<'mgr> { + fn clone(&self) -> ResourceManager<'mgr> { + unimplemented!() + } +} + impl<'mgr> ResourceManager<'mgr> { pub fn new() -> Self { ResourceManager { @@ -41,12 +47,27 @@ impl<'mgr> ResourceManager<'mgr> { } } + pub fn get_resource_for_string(&'mgr self, string: &str) -> &'mgr FluentResource<'mgr> { + let strings = &self.strings; + + if strings.get(string).is_some() { + return self.resources.get(string).unwrap(); + } else { + let val = self.strings.insert(string.to_string(), string.to_owned()); + let res = match FluentResource::from_string(val) { + Ok(res) => res, + Err((res, _err)) => res, + }; + self.resources.insert(string.to_string(), Box::new(res)) + } + } + pub fn get_bundle( &'mgr self, locales: &Vec, paths: &Vec, ) -> FluentBundle<'mgr> { - let mut bundle = FluentBundle::new(locales); + let mut bundle = FluentBundle::new(locales, Some(&self)); for path in paths { let res = self.get_resource(path); bundle.add_resource(res).unwrap(); diff --git a/fluent-bundle/src/types.rs b/fluent/src/types.rs similarity index 100% rename from fluent-bundle/src/types.rs rename to fluent/src/types.rs diff --git a/fluent-bundle/tests/bundle.rs b/fluent/tests/bundle.rs similarity index 53% rename from fluent-bundle/tests/bundle.rs rename to fluent/tests/bundle.rs index 2dd4ed3a..71b5e935 100644 --- a/fluent-bundle/tests/bundle.rs +++ b/fluent/tests/bundle.rs @@ -1,19 +1,19 @@ -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn bundle_new_from_str() { let arr_of_str = ["x-testing"]; - let _ = FluentBundle::new(&arr_of_str); - let _ = FluentBundle::new(&arr_of_str[..]); + let _ = FluentBundle::new(&arr_of_str, None); + let _ = FluentBundle::new(&arr_of_str[..], None); let vec_of_str = vec!["x-testing"]; - let _ = FluentBundle::new(&vec_of_str); - let _ = FluentBundle::new(&vec_of_str[..]); + let _ = FluentBundle::new(&vec_of_str, None); + let _ = FluentBundle::new(&vec_of_str[..], None); let iter_of_str = ["x-testing"].iter(); let vec_from_iter = iter_of_str.cloned().collect::>(); - let _ = FluentBundle::new(&vec_from_iter); - let _ = FluentBundle::new(&vec_from_iter[..]); + let _ = FluentBundle::new(&vec_from_iter, None); + let _ = FluentBundle::new(&vec_from_iter[..], None); } #[test] @@ -21,25 +21,25 @@ fn bundle_new_from_strings() { let arr_of_strings = ["x-testing".to_string()]; let arr_of_str = [arr_of_strings[0].as_str()]; - let _ = FluentBundle::new(&arr_of_str); - let _ = FluentBundle::new(&arr_of_str[..]); + let _ = FluentBundle::new(&arr_of_str, None); + let _ = FluentBundle::new(&arr_of_str[..], None); let vec_of_strings = ["x-testing".to_string()]; let vec_of_str = [vec_of_strings[0].as_str()]; - let _ = FluentBundle::new(&vec_of_str); - let _ = FluentBundle::new(&vec_of_str[..]); + let _ = FluentBundle::new(&vec_of_str, None); + let _ = FluentBundle::new(&vec_of_str[..], None); let iter_of_strings = arr_of_strings.iter(); let vec_from_iter = iter_of_strings .map(|elem| elem.as_str()) .collect::>(); - let _ = FluentBundle::new(&vec_from_iter); - let _ = FluentBundle::new(&vec_from_iter[..]); + let _ = FluentBundle::new(&vec_from_iter, None); + let _ = FluentBundle::new(&vec_from_iter[..], None); } fn create_bundle<'a, 'b>(locales: &'b Vec<&'b str>) -> FluentBundle<'a> { - FluentBundle::new(locales) + FluentBundle::new(locales, None) } #[test] diff --git a/fluent-bundle/tests/format.rs b/fluent/tests/format.rs similarity index 88% rename from fluent-bundle/tests/format.rs rename to fluent/tests/format.rs index f6ee28dc..acaa08b2 100644 --- a/fluent-bundle/tests/format.rs +++ b/fluent/tests/format.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent-bundle/tests/format_message.rs b/fluent/tests/format_message.rs similarity index 82% rename from fluent-bundle/tests/format_message.rs rename to fluent/tests/format_message.rs index 28394855..fd7b4919 100644 --- a/fluent-bundle/tests/format_message.rs +++ b/fluent/tests/format_message.rs @@ -1,13 +1,13 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::bundle::Message; +use fluent::bundle::FluentBundle; +use fluent::bundle::Message; use std::collections::HashMap; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent-bundle/tests/helpers/mod.rs b/fluent/tests/helpers/mod.rs similarity index 89% rename from fluent-bundle/tests/helpers/mod.rs rename to fluent/tests/helpers/mod.rs index 7b39cf4d..6733bb9e 100644 --- a/fluent-bundle/tests/helpers/mod.rs +++ b/fluent/tests/helpers/mod.rs @@ -1,5 +1,5 @@ -use fluent_bundle::bundle::FluentError; -use fluent_bundle::bundle::Message; +use fluent::bundle::FluentError; +use fluent::bundle::Message; #[allow(dead_code)] pub fn assert_format_none(result: Option<(String, Vec)>) { diff --git a/fluent-bundle/tests/resolve_attribute_expression.rs b/fluent/tests/resolve_attribute_expression.rs similarity index 90% rename from fluent-bundle/tests/resolve_attribute_expression.rs rename to fluent/tests/resolve_attribute_expression.rs index 80c97385..d1be881b 100644 --- a/fluent-bundle/tests/resolve_attribute_expression.rs +++ b/fluent/tests/resolve_attribute_expression.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn attribute_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " diff --git a/fluent-bundle/tests/resolve_external_argument.rs b/fluent/tests/resolve_external_argument.rs similarity index 95% rename from fluent-bundle/tests/resolve_external_argument.rs rename to fluent/tests/resolve_external_argument.rs index ade852d5..56c38948 100644 --- a/fluent-bundle/tests/resolve_external_argument.rs +++ b/fluent/tests/resolve_external_argument.rs @@ -3,8 +3,8 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; #[test] fn external_argument_string() { diff --git a/fluent-bundle/tests/resolve_message_reference.rs b/fluent/tests/resolve_message_reference.rs similarity index 98% rename from fluent-bundle/tests/resolve_message_reference.rs rename to fluent/tests/resolve_message_reference.rs index 3ad695ce..3166c0e9 100644 --- a/fluent-bundle/tests/resolve_message_reference.rs +++ b/fluent/tests/resolve_message_reference.rs @@ -1,7 +1,7 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn message_reference() { diff --git a/fluent-bundle/tests/resolve_plural_rule.rs b/fluent/tests/resolve_plural_rule.rs similarity index 91% rename from fluent-bundle/tests/resolve_plural_rule.rs rename to fluent/tests/resolve_plural_rule.rs index 7c655b1e..36058826 100644 --- a/fluent-bundle/tests/resolve_plural_rule.rs +++ b/fluent/tests/resolve_plural_rule.rs @@ -3,12 +3,12 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["en"]); + let mut bundle = FluentBundle::new(&["en"], None); assert_add_messages_no_errors(bundle.add_messages( " unread-emails = @@ -43,7 +43,7 @@ unread-emails-dec = #[test] fn exact_match() { - let mut bundle = FluentBundle::new(&["en"]); + let mut bundle = FluentBundle::new(&["en"], None); assert_add_messages_no_errors(bundle.add_messages( " unread-emails = diff --git a/fluent-bundle/tests/resolve_select_expression.rs b/fluent/tests/resolve_select_expression.rs similarity index 86% rename from fluent-bundle/tests/resolve_select_expression.rs rename to fluent/tests/resolve_select_expression.rs index 57a36b3d..9b8bd35d 100644 --- a/fluent-bundle/tests/resolve_select_expression.rs +++ b/fluent/tests/resolve_select_expression.rs @@ -3,26 +3,26 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::types::FluentValue; +use fluent::bundle::FluentBundle; +use fluent::types::FluentValue; #[test] fn select_expression_string_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" foo = - { \"genitive\" -> + { "genitive" -> *[nominative] Foo [genitive] Foo's } bar = - { \"missing\" -> + { "missing" -> *[nominative] Bar [genitive] Bar's } -", +"#, )); assert_format_no_errors(bundle.format("foo", None), "Foo's"); @@ -32,9 +32,9 @@ bar = #[test] fn select_expression_number_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" foo = { 3 -> *[1] Foo 1 @@ -53,7 +53,7 @@ baz = [3] Baz 3 [3.14] Baz Pi } -", +"#, )); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -65,9 +65,9 @@ baz = #[test] fn select_expression_plurals() { - let mut bundle = FluentBundle::new(&["en"]); + let mut bundle = FluentBundle::new(&["en"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" foo = { 3 -> [one] Foo One @@ -83,12 +83,12 @@ bar = } baz = - { \"one\" -> + { "one" -> [1] Bar One [3] Bar 3 *[other] Bar Other } -", +"#, )); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -100,9 +100,9 @@ baz = #[test] fn select_expression_external_argument_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" foo-hit = { $str -> *[foo] Foo @@ -156,7 +156,7 @@ baz-unknown = *[1] Baz 1 [2] Baz 2 } -", +"#, )); let mut args = HashMap::new(); @@ -185,9 +185,9 @@ baz-unknown = #[test] fn select_expression_message_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" -bar = Bar .attr = attr_val @@ -196,7 +196,7 @@ use-bar = [attr_val] Bar *[other] Other } -", +"#, )); assert_format_no_errors(bundle.format("use-bar", None), "Bar"); @@ -204,9 +204,9 @@ use-bar = #[test] fn select_expression_attribute_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( - " + r#" -foo = Foo .attr = FooAttr @@ -215,7 +215,7 @@ use-foo = [FooAttr] Foo *[other] Other } -", +"#, )); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent-bundle/tests/resolve_value.rs b/fluent/tests/resolve_value.rs similarity index 75% rename from fluent-bundle/tests/resolve_value.rs rename to fluent/tests/resolve_value.rs index 76f5601c..4e6259af 100644 --- a/fluent-bundle/tests/resolve_value.rs +++ b/fluent/tests/resolve_value.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn format_message() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo @@ -17,7 +17,7 @@ foo = Foo #[test] fn format_attribute() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent-bundle/tests/resolve_variant_expression.rs b/fluent/tests/resolve_variant_expression.rs similarity index 92% rename from fluent-bundle/tests/resolve_variant_expression.rs rename to fluent/tests/resolve_variant_expression.rs index 922e1831..26b2b7c5 100644 --- a/fluent-bundle/tests/resolve_variant_expression.rs +++ b/fluent/tests/resolve_variant_expression.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use fluent::bundle::FluentBundle; #[test] fn variant_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); + let mut bundle = FluentBundle::new(&["x-testing"], None); assert_add_messages_no_errors(bundle.add_messages( " -foo = Foo From 652b62377ed7593ebde54b41132aa9b39478b461 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 31 Dec 2018 16:26:37 -0800 Subject: [PATCH 05/36] Revert "Merge fluent-bundle and fluent and attempt to use ResourceManager via Cow" This reverts commit 6e022726350cbbd780bbcdb50d980e57bc958ae9. --- Cargo.toml | 1 + {fluent => fluent-bundle}/CHANGELOG.md | 0 {fluent-res => fluent-bundle}/Cargo.toml | 10 +- {fluent => fluent-bundle}/README.md | 0 {fluent => fluent-bundle}/benches/lib.rs | 0 {fluent => fluent-bundle}/benches/menubar.ftl | 0 {fluent => fluent-bundle}/benches/simple.ftl | 0 {fluent => fluent-bundle}/examples/README.md | 0 .../examples/external_arguments.rs | 6 +- .../examples/functions.rs | 8 +- {fluent => fluent-bundle}/examples/hello.rs | 4 +- .../examples/message_reference.rs | 4 +- .../examples/resources/en-US/simple.ftl | 5 + .../examples/resources/pl/simple.ftl | 2 + .../examples/selector.rs | 6 +- .../examples/simple-app.rs | 6 +- {fluent => fluent-bundle}/src/bundle.rs | 74 +++------ {fluent => fluent-bundle}/src/entry.rs | 0 {fluent => fluent-bundle}/src/errors.rs | 3 +- fluent-bundle/src/lib.rs | 48 ++++++ {fluent => fluent-bundle}/src/resolve.rs | 0 {fluent => fluent-bundle}/src/resource.rs | 0 {fluent => fluent-bundle}/src/types.rs | 0 {fluent => fluent-bundle}/tests/bundle.rs | 28 ++-- {fluent => fluent-bundle}/tests/format.rs | 4 +- .../tests/format_message.rs | 6 +- .../tests/helpers/mod.rs | 4 +- .../tests/resolve_attribute_expression.rs | 4 +- .../tests/resolve_external_argument.rs | 4 +- .../tests/resolve_message_reference.rs | 2 +- .../tests/resolve_plural_rule.rs | 8 +- .../tests/resolve_select_expression.rs | 46 +++--- .../tests/resolve_value.rs | 6 +- .../tests/resolve_variant_expression.rs | 4 +- .../examples/resources/en-US/common.ftl | 1 - .../examples/resources/en-US/simple.ftl | 5 - fluent-res/examples/resources/pl/common.ftl | 1 - fluent-res/examples/resources/pl/errors.ftl | 2 - fluent-res/examples/simple.rs | 154 ------------------ fluent-res/src/lib.rs | 1 - fluent/Cargo.toml | 7 +- fluent/examples/simple.rs | 2 +- fluent/src/lib.rs | 41 ----- fluent/src/resource_manager.rs | 27 +-- 44 files changed, 166 insertions(+), 368 deletions(-) rename {fluent => fluent-bundle}/CHANGELOG.md (100%) rename {fluent-res => fluent-bundle}/Cargo.toml (81%) rename {fluent => fluent-bundle}/README.md (100%) rename {fluent => fluent-bundle}/benches/lib.rs (100%) rename {fluent => fluent-bundle}/benches/menubar.ftl (100%) rename {fluent => fluent-bundle}/benches/simple.ftl (100%) rename {fluent => fluent-bundle}/examples/README.md (100%) rename {fluent => fluent-bundle}/examples/external_arguments.rs (88%) rename {fluent => fluent-bundle}/examples/functions.rs (92%) rename {fluent => fluent-bundle}/examples/hello.rs (66%) rename {fluent => fluent-bundle}/examples/message_reference.rs (80%) rename fluent-res/examples/resources/en-US/errors.ftl => fluent-bundle/examples/resources/en-US/simple.ftl (50%) rename {fluent-res => fluent-bundle}/examples/resources/pl/simple.ftl (56%) rename {fluent => fluent-bundle}/examples/selector.rs (82%) rename {fluent => fluent-bundle}/examples/simple-app.rs (97%) rename {fluent => fluent-bundle}/src/bundle.rs (79%) rename {fluent => fluent-bundle}/src/entry.rs (100%) rename {fluent => fluent-bundle}/src/errors.rs (91%) create mode 100644 fluent-bundle/src/lib.rs rename {fluent => fluent-bundle}/src/resolve.rs (100%) rename {fluent => fluent-bundle}/src/resource.rs (100%) rename {fluent => fluent-bundle}/src/types.rs (100%) rename {fluent => fluent-bundle}/tests/bundle.rs (53%) rename {fluent => fluent-bundle}/tests/format.rs (88%) rename {fluent => fluent-bundle}/tests/format_message.rs (82%) rename {fluent => fluent-bundle}/tests/helpers/mod.rs (89%) rename {fluent => fluent-bundle}/tests/resolve_attribute_expression.rs (90%) rename {fluent => fluent-bundle}/tests/resolve_external_argument.rs (95%) rename {fluent => fluent-bundle}/tests/resolve_message_reference.rs (98%) rename {fluent => fluent-bundle}/tests/resolve_plural_rule.rs (91%) rename {fluent => fluent-bundle}/tests/resolve_select_expression.rs (86%) rename {fluent => fluent-bundle}/tests/resolve_value.rs (75%) rename {fluent => fluent-bundle}/tests/resolve_variant_expression.rs (92%) delete mode 100644 fluent-res/examples/resources/en-US/common.ftl delete mode 100644 fluent-res/examples/resources/en-US/simple.ftl delete mode 100644 fluent-res/examples/resources/pl/common.ftl delete mode 100644 fluent-res/examples/resources/pl/errors.ftl delete mode 100644 fluent-res/examples/simple.rs delete mode 100644 fluent-res/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index dd3d5a8a..29cf9892 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "fluent-syntax", + "fluent-bundle", "fluent-cli", "fluent", ] diff --git a/fluent/CHANGELOG.md b/fluent-bundle/CHANGELOG.md similarity index 100% rename from fluent/CHANGELOG.md rename to fluent-bundle/CHANGELOG.md diff --git a/fluent-res/Cargo.toml b/fluent-bundle/Cargo.toml similarity index 81% rename from fluent-res/Cargo.toml rename to fluent-bundle/Cargo.toml index 277631f4..59f91b86 100644 --- a/fluent-res/Cargo.toml +++ b/fluent-bundle/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "fluent" +name = "fluent-bundle" description = """ A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" +edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " ] -edition = "2018" homepage = "http://www.projectfluent.org" license = "Apache-2.0/MIT" repository = "https://github.com/projectfluent/fluent-rs" @@ -18,6 +18,8 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] -fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" -elsa = "^0.1.2" +fluent-syntax = { path = "../fluent-syntax" } +failure = "0.1" +failure_derive = "0.1" +intl_pluralrules = "1.0" diff --git a/fluent/README.md b/fluent-bundle/README.md similarity index 100% rename from fluent/README.md rename to fluent-bundle/README.md diff --git a/fluent/benches/lib.rs b/fluent-bundle/benches/lib.rs similarity index 100% rename from fluent/benches/lib.rs rename to fluent-bundle/benches/lib.rs diff --git a/fluent/benches/menubar.ftl b/fluent-bundle/benches/menubar.ftl similarity index 100% rename from fluent/benches/menubar.ftl rename to fluent-bundle/benches/menubar.ftl diff --git a/fluent/benches/simple.ftl b/fluent-bundle/benches/simple.ftl similarity index 100% rename from fluent/benches/simple.ftl rename to fluent-bundle/benches/simple.ftl diff --git a/fluent/examples/README.md b/fluent-bundle/examples/README.md similarity index 100% rename from fluent/examples/README.md rename to fluent-bundle/examples/README.md diff --git a/fluent/examples/external_arguments.rs b/fluent-bundle/examples/external_arguments.rs similarity index 88% rename from fluent/examples/external_arguments.rs rename to fluent-bundle/examples/external_arguments.rs index e8b4b2e8..0b817451 100644 --- a/fluent/examples/external_arguments.rs +++ b/fluent-bundle/examples/external_arguments.rs @@ -1,9 +1,9 @@ -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["en"], None); + let mut bundle = FluentBundle::new(&["en"]); bundle .add_messages( " diff --git a/fluent/examples/functions.rs b/fluent-bundle/examples/functions.rs similarity index 92% rename from fluent/examples/functions.rs rename to fluent-bundle/examples/functions.rs index f80c7535..a0955b40 100644 --- a/fluent/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -1,8 +1,8 @@ -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; fn main() { - let mut bundle = FluentBundle::new(&["en-US"], None); + let mut bundle = FluentBundle::new(&["en-US"]); // Test for a simple function that returns a string bundle @@ -45,7 +45,7 @@ fn main() { .add_messages("meaning-of-life = { MEANING_OF_LIFE(42) }") .unwrap(); bundle - .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: "us") }") + .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") .unwrap(); let value = bundle.format("hello-world", None); diff --git a/fluent/examples/hello.rs b/fluent-bundle/examples/hello.rs similarity index 66% rename from fluent/examples/hello.rs rename to fluent-bundle/examples/hello.rs index 5bfe083b..2828b333 100644 --- a/fluent/examples/hello.rs +++ b/fluent-bundle/examples/hello.rs @@ -1,7 +1,7 @@ -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; fn main() { - let mut bundle = FluentBundle::new(&["en-US"], None); + let mut bundle = FluentBundle::new(&["en-US"]); bundle.add_messages("hello-world = Hello, world!").unwrap(); let (value, _) = bundle.format("hello-world", None).unwrap(); assert_eq!(&value, "Hello, world!"); diff --git a/fluent/examples/message_reference.rs b/fluent-bundle/examples/message_reference.rs similarity index 80% rename from fluent/examples/message_reference.rs rename to fluent-bundle/examples/message_reference.rs index da1cf50b..bfc512e5 100644 --- a/fluent/examples/message_reference.rs +++ b/fluent-bundle/examples/message_reference.rs @@ -1,7 +1,7 @@ -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); bundle .add_messages( " diff --git a/fluent-res/examples/resources/en-US/errors.ftl b/fluent-bundle/examples/resources/en-US/simple.ftl similarity index 50% rename from fluent-res/examples/resources/en-US/errors.ftl rename to fluent-bundle/examples/resources/en-US/simple.ftl index 6a228bbe..99f0a6bb 100644 --- a/fluent-res/examples/resources/en-US/errors.ftl +++ b/fluent-bundle/examples/resources/en-US/simple.ftl @@ -1,2 +1,7 @@ missing-arg-error = Error: Please provide a number as argument. input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } +response-msg = + { $value -> + [one] "{ $input }" has one Collatz step. + *[other] "{ $input }" has { $value } Collatz steps. + } diff --git a/fluent-res/examples/resources/pl/simple.ftl b/fluent-bundle/examples/resources/pl/simple.ftl similarity index 56% rename from fluent-res/examples/resources/pl/simple.ftl rename to fluent-bundle/examples/resources/pl/simple.ftl index 7a17d125..16173dd9 100644 --- a/fluent-res/examples/resources/pl/simple.ftl +++ b/fluent-bundle/examples/resources/pl/simple.ftl @@ -1,3 +1,5 @@ +missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. +input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } response-msg = { $value -> [one] "{ $input }" ma jeden krok Collatza. diff --git a/fluent/examples/selector.rs b/fluent-bundle/examples/selector.rs similarity index 82% rename from fluent/examples/selector.rs rename to fluent-bundle/examples/selector.rs index 01b8012d..c7e3672b 100644 --- a/fluent/examples/selector.rs +++ b/fluent-bundle/examples/selector.rs @@ -1,9 +1,9 @@ -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); bundle .add_messages( " diff --git a/fluent/examples/simple-app.rs b/fluent-bundle/examples/simple-app.rs similarity index 97% rename from fluent/examples/simple-app.rs rename to fluent-bundle/examples/simple-app.rs index f08d6e7e..bf7c8854 100644 --- a/fluent/examples/simple-app.rs +++ b/fluent-bundle/examples/simple-app.rs @@ -17,8 +17,8 @@ //! //! If the second argument is omitted, `en-US` locale is used as the //! default one. -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; @@ -107,7 +107,7 @@ fn main() { // 4. Create a new Fluent FluentBundle using the // resolved locales. - let mut bundle = FluentBundle::new(&locales, None); + let mut bundle = FluentBundle::new(&locales); // 5. Load the localization resource for path in L10N_RESOURCES { diff --git a/fluent/src/bundle.rs b/fluent-bundle/src/bundle.rs similarity index 79% rename from fluent/src/bundle.rs rename to fluent-bundle/src/bundle.rs index e593fe99..6c096aad 100644 --- a/fluent/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -6,7 +6,6 @@ use std::cell::RefCell; use std::collections::hash_map::{Entry as HashEntry, HashMap}; -use std::borrow::Cow; use super::entry::{Entry, GetEntry}; pub use super::errors::FluentError; @@ -15,7 +14,6 @@ use super::resource::FluentResource; use super::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use fluent_syntax::ast; -use crate::resource_manager::ResourceManager; use intl_pluralrules::{IntlPluralRules, PluralRuleType}; #[derive(Debug, PartialEq)] @@ -50,15 +48,15 @@ pub struct Message { /// purpose of language negotiation with i18n formatters. For instance, if date and time formatting /// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain /// to negotiate a sensible fallback for date and time formatting. +#[allow(dead_code)] pub struct FluentBundle<'bundle> { pub locales: Vec, pub entries: HashMap>, pub plural_rules: IntlPluralRules, - pub resource_manager: Cow<'bundle, ResourceManager<'bundle>>, } impl<'bundle> FluentBundle<'bundle> { - pub fn new<'a, S: ToString>(locales: &'a [S], res_mgr: Option<&'bundle ResourceManager<'bundle>>) -> FluentBundle<'bundle> { + pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { let locales = locales.iter().map(|s| s.to_string()).collect::>(); let pr_locale = negotiate_languages( &locales, @@ -69,15 +67,10 @@ impl<'bundle> FluentBundle<'bundle> { .to_owned(); let pr = IntlPluralRules::create(&pr_locale, PluralRuleType::CARDINAL).unwrap(); - let res_mgr = match res_mgr { - Some(res_mgr) => Cow::Borrowed(res_mgr), - None => Cow::Owned(ResourceManager::new()) - }; FluentBundle { locales, entries: HashMap::new(), plural_rules: pr, - resource_manager: res_mgr } } @@ -104,45 +97,22 @@ impl<'bundle> FluentBundle<'bundle> { } } - pub fn add_messages(&'bundle mut self, source: &str) -> Result<(), Vec> { - let res_mgr = &self.resource_manager; - let res = res_mgr.get_resource_for_string(source); - let mut errors = vec![]; - - for entry in &res.ast.body { - let id = match entry { - ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { ref id, .. })) - | ast::ResourceEntry::Entry(ast::Entry::Term(ast::Term { ref id, .. })) => id.name, - _ => continue, - }; - - let (entry, kind) = match entry { - ast::ResourceEntry::Entry(ast::Entry::Message(message)) => { - (Entry::Message(message), "message") - } - ast::ResourceEntry::Entry(ast::Entry::Term(term)) => (Entry::Term(term), "term"), - _ => continue, - }; - - match self.entries.entry(id.to_string()) { - HashEntry::Vacant(empty) => { - empty.insert(entry); - } - HashEntry::Occupied(_) => { - errors.push(FluentError::Overriding { - kind, - id: id.to_string(), - }); - } - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(errors) - } - } + //pub fn add_messages(&mut self, source: &'bundle str) -> Result<(), Vec> { + //match FluentResource::from_string(source) { + //Ok(res) => self.add_resource(&res), + //Err((res, err)) => { + //let mut errors: Vec = + //err.into_iter().map(FluentError::ParserError).collect(); + + //self.add_resource(&res).map_err(|err| { + //for e in err { + //errors.push(e); + //} + //errors + //}) + //} + //} + //} pub fn add_resource( &mut self, @@ -186,9 +156,9 @@ impl<'bundle> FluentBundle<'bundle> { } pub fn format( - &'bundle self, + &self, path: &str, - args: Option<&'bundle HashMap<&str, FluentValue>>, + args: Option<&HashMap<&str, FluentValue>>, ) -> Option<(String, Vec)> { let env = Env { bundle: self, @@ -237,9 +207,9 @@ impl<'bundle> FluentBundle<'bundle> { } pub fn format_message( - &'bundle self, + &self, message_id: &str, - args: Option<&'bundle HashMap<&str, FluentValue>>, + args: Option<&HashMap<&str, FluentValue>>, ) -> Option<(Message, Vec)> { let mut errors = vec![]; diff --git a/fluent/src/entry.rs b/fluent-bundle/src/entry.rs similarity index 100% rename from fluent/src/entry.rs rename to fluent-bundle/src/entry.rs diff --git a/fluent/src/errors.rs b/fluent-bundle/src/errors.rs similarity index 91% rename from fluent/src/errors.rs rename to fluent-bundle/src/errors.rs index 9c430c3d..9b7cf418 100644 --- a/fluent/src/errors.rs +++ b/fluent-bundle/src/errors.rs @@ -1,6 +1,5 @@ -use crate::resolve::ResolverError; +use super::resolve::ResolverError; use fluent_syntax::parser::ParserError; -use failure_derive::Fail; #[derive(Debug, Fail, PartialEq)] pub enum FluentError { diff --git a/fluent-bundle/src/lib.rs b/fluent-bundle/src/lib.rs new file mode 100644 index 00000000..b516ac9c --- /dev/null +++ b/fluent-bundle/src/lib.rs @@ -0,0 +1,48 @@ +//! Fluent is a localization system designed to improve how software is translated. +//! +//! The Rust implementation provides the low level components for syntax operations, like parser +//! and AST, and the core localization struct - `FluentBundle`. +//! +//! `FluentBundle` is the low level container for storing and formatting localization messages. It +//! is expected that implementations will build on top of it by providing language negotiation +//! between user requested languages and available resources and I/O for loading selected +//! resources. +//! +//! # Example +//! +//! ``` +//! use fluent_bundle::bundle::FluentBundle; +//! use fluent_bundle::types::FluentValue; +//! use std::collections::HashMap; +//! +//! let mut bundle = FluentBundle::new(&["en-US"]); +//! bundle.add_messages( +//! " +//! hello-world = Hello, world! +//! intro = Welcome, { $name }. +//! " +//! ); +//! +//! let value = bundle.format("hello-world", None); +//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); +//! +//! let mut args = HashMap::new(); +//! args.insert("name", FluentValue::from("John")); +//! +//! let value = bundle.format("intro", Some(&args)); +//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); +//! ``` + +extern crate failure; +#[macro_use] +extern crate failure_derive; +extern crate fluent_locale; +extern crate fluent_syntax; +extern crate intl_pluralrules; + +pub mod bundle; +pub mod entry; +pub mod errors; +pub mod resolve; +pub mod resource; +pub mod types; diff --git a/fluent/src/resolve.rs b/fluent-bundle/src/resolve.rs similarity index 100% rename from fluent/src/resolve.rs rename to fluent-bundle/src/resolve.rs diff --git a/fluent/src/resource.rs b/fluent-bundle/src/resource.rs similarity index 100% rename from fluent/src/resource.rs rename to fluent-bundle/src/resource.rs diff --git a/fluent/src/types.rs b/fluent-bundle/src/types.rs similarity index 100% rename from fluent/src/types.rs rename to fluent-bundle/src/types.rs diff --git a/fluent/tests/bundle.rs b/fluent-bundle/tests/bundle.rs similarity index 53% rename from fluent/tests/bundle.rs rename to fluent-bundle/tests/bundle.rs index 71b5e935..2dd4ed3a 100644 --- a/fluent/tests/bundle.rs +++ b/fluent-bundle/tests/bundle.rs @@ -1,19 +1,19 @@ -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn bundle_new_from_str() { let arr_of_str = ["x-testing"]; - let _ = FluentBundle::new(&arr_of_str, None); - let _ = FluentBundle::new(&arr_of_str[..], None); + let _ = FluentBundle::new(&arr_of_str); + let _ = FluentBundle::new(&arr_of_str[..]); let vec_of_str = vec!["x-testing"]; - let _ = FluentBundle::new(&vec_of_str, None); - let _ = FluentBundle::new(&vec_of_str[..], None); + let _ = FluentBundle::new(&vec_of_str); + let _ = FluentBundle::new(&vec_of_str[..]); let iter_of_str = ["x-testing"].iter(); let vec_from_iter = iter_of_str.cloned().collect::>(); - let _ = FluentBundle::new(&vec_from_iter, None); - let _ = FluentBundle::new(&vec_from_iter[..], None); + let _ = FluentBundle::new(&vec_from_iter); + let _ = FluentBundle::new(&vec_from_iter[..]); } #[test] @@ -21,25 +21,25 @@ fn bundle_new_from_strings() { let arr_of_strings = ["x-testing".to_string()]; let arr_of_str = [arr_of_strings[0].as_str()]; - let _ = FluentBundle::new(&arr_of_str, None); - let _ = FluentBundle::new(&arr_of_str[..], None); + let _ = FluentBundle::new(&arr_of_str); + let _ = FluentBundle::new(&arr_of_str[..]); let vec_of_strings = ["x-testing".to_string()]; let vec_of_str = [vec_of_strings[0].as_str()]; - let _ = FluentBundle::new(&vec_of_str, None); - let _ = FluentBundle::new(&vec_of_str[..], None); + let _ = FluentBundle::new(&vec_of_str); + let _ = FluentBundle::new(&vec_of_str[..]); let iter_of_strings = arr_of_strings.iter(); let vec_from_iter = iter_of_strings .map(|elem| elem.as_str()) .collect::>(); - let _ = FluentBundle::new(&vec_from_iter, None); - let _ = FluentBundle::new(&vec_from_iter[..], None); + let _ = FluentBundle::new(&vec_from_iter); + let _ = FluentBundle::new(&vec_from_iter[..]); } fn create_bundle<'a, 'b>(locales: &'b Vec<&'b str>) -> FluentBundle<'a> { - FluentBundle::new(locales, None) + FluentBundle::new(locales) } #[test] diff --git a/fluent/tests/format.rs b/fluent-bundle/tests/format.rs similarity index 88% rename from fluent/tests/format.rs rename to fluent-bundle/tests/format.rs index acaa08b2..f6ee28dc 100644 --- a/fluent/tests/format.rs +++ b/fluent-bundle/tests/format.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent/tests/format_message.rs b/fluent-bundle/tests/format_message.rs similarity index 82% rename from fluent/tests/format_message.rs rename to fluent-bundle/tests/format_message.rs index fd7b4919..28394855 100644 --- a/fluent/tests/format_message.rs +++ b/fluent-bundle/tests/format_message.rs @@ -1,13 +1,13 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; -use fluent::bundle::FluentBundle; -use fluent::bundle::Message; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::bundle::Message; use std::collections::HashMap; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent/tests/helpers/mod.rs b/fluent-bundle/tests/helpers/mod.rs similarity index 89% rename from fluent/tests/helpers/mod.rs rename to fluent-bundle/tests/helpers/mod.rs index 6733bb9e..7b39cf4d 100644 --- a/fluent/tests/helpers/mod.rs +++ b/fluent-bundle/tests/helpers/mod.rs @@ -1,5 +1,5 @@ -use fluent::bundle::FluentError; -use fluent::bundle::Message; +use fluent_bundle::bundle::FluentError; +use fluent_bundle::bundle::Message; #[allow(dead_code)] pub fn assert_format_none(result: Option<(String, Vec)>) { diff --git a/fluent/tests/resolve_attribute_expression.rs b/fluent-bundle/tests/resolve_attribute_expression.rs similarity index 90% rename from fluent/tests/resolve_attribute_expression.rs rename to fluent-bundle/tests/resolve_attribute_expression.rs index d1be881b..80c97385 100644 --- a/fluent/tests/resolve_attribute_expression.rs +++ b/fluent-bundle/tests/resolve_attribute_expression.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn attribute_expression() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " diff --git a/fluent/tests/resolve_external_argument.rs b/fluent-bundle/tests/resolve_external_argument.rs similarity index 95% rename from fluent/tests/resolve_external_argument.rs rename to fluent-bundle/tests/resolve_external_argument.rs index 56c38948..ade852d5 100644 --- a/fluent/tests/resolve_external_argument.rs +++ b/fluent-bundle/tests/resolve_external_argument.rs @@ -3,8 +3,8 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_string() { diff --git a/fluent/tests/resolve_message_reference.rs b/fluent-bundle/tests/resolve_message_reference.rs similarity index 98% rename from fluent/tests/resolve_message_reference.rs rename to fluent-bundle/tests/resolve_message_reference.rs index 3166c0e9..3ad695ce 100644 --- a/fluent/tests/resolve_message_reference.rs +++ b/fluent-bundle/tests/resolve_message_reference.rs @@ -1,7 +1,7 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn message_reference() { diff --git a/fluent/tests/resolve_plural_rule.rs b/fluent-bundle/tests/resolve_plural_rule.rs similarity index 91% rename from fluent/tests/resolve_plural_rule.rs rename to fluent-bundle/tests/resolve_plural_rule.rs index 36058826..7c655b1e 100644 --- a/fluent/tests/resolve_plural_rule.rs +++ b/fluent-bundle/tests/resolve_plural_rule.rs @@ -3,12 +3,12 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["en"], None); + let mut bundle = FluentBundle::new(&["en"]); assert_add_messages_no_errors(bundle.add_messages( " unread-emails = @@ -43,7 +43,7 @@ unread-emails-dec = #[test] fn exact_match() { - let mut bundle = FluentBundle::new(&["en"], None); + let mut bundle = FluentBundle::new(&["en"]); assert_add_messages_no_errors(bundle.add_messages( " unread-emails = diff --git a/fluent/tests/resolve_select_expression.rs b/fluent-bundle/tests/resolve_select_expression.rs similarity index 86% rename from fluent/tests/resolve_select_expression.rs rename to fluent-bundle/tests/resolve_select_expression.rs index 9b8bd35d..57a36b3d 100644 --- a/fluent/tests/resolve_select_expression.rs +++ b/fluent-bundle/tests/resolve_select_expression.rs @@ -3,26 +3,26 @@ mod helpers; use std::collections::HashMap; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; -use fluent::types::FluentValue; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::types::FluentValue; #[test] fn select_expression_string_selector() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " foo = - { "genitive" -> + { \"genitive\" -> *[nominative] Foo [genitive] Foo's } bar = - { "missing" -> + { \"missing\" -> *[nominative] Bar [genitive] Bar's } -"#, +", )); assert_format_no_errors(bundle.format("foo", None), "Foo's"); @@ -32,9 +32,9 @@ bar = #[test] fn select_expression_number_selector() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " foo = { 3 -> *[1] Foo 1 @@ -53,7 +53,7 @@ baz = [3] Baz 3 [3.14] Baz Pi } -"#, +", )); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -65,9 +65,9 @@ baz = #[test] fn select_expression_plurals() { - let mut bundle = FluentBundle::new(&["en"], None); + let mut bundle = FluentBundle::new(&["en"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " foo = { 3 -> [one] Foo One @@ -83,12 +83,12 @@ bar = } baz = - { "one" -> + { \"one\" -> [1] Bar One [3] Bar 3 *[other] Bar Other } -"#, +", )); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -100,9 +100,9 @@ baz = #[test] fn select_expression_external_argument_selector() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " foo-hit = { $str -> *[foo] Foo @@ -156,7 +156,7 @@ baz-unknown = *[1] Baz 1 [2] Baz 2 } -"#, +", )); let mut args = HashMap::new(); @@ -185,9 +185,9 @@ baz-unknown = #[test] fn select_expression_message_selector() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " -bar = Bar .attr = attr_val @@ -196,7 +196,7 @@ use-bar = [attr_val] Bar *[other] Other } -"#, +", )); assert_format_no_errors(bundle.format("use-bar", None), "Bar"); @@ -204,9 +204,9 @@ use-bar = #[test] fn select_expression_attribute_selector() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( - r#" + " -foo = Foo .attr = FooAttr @@ -215,7 +215,7 @@ use-foo = [FooAttr] Foo *[other] Other } -"#, +", )); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent/tests/resolve_value.rs b/fluent-bundle/tests/resolve_value.rs similarity index 75% rename from fluent/tests/resolve_value.rs rename to fluent-bundle/tests/resolve_value.rs index 4e6259af..76f5601c 100644 --- a/fluent/tests/resolve_value.rs +++ b/fluent-bundle/tests/resolve_value.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn format_message() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo @@ -17,7 +17,7 @@ foo = Foo #[test] fn format_attribute() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " foo = Foo diff --git a/fluent/tests/resolve_variant_expression.rs b/fluent-bundle/tests/resolve_variant_expression.rs similarity index 92% rename from fluent/tests/resolve_variant_expression.rs rename to fluent-bundle/tests/resolve_variant_expression.rs index 26b2b7c5..922e1831 100644 --- a/fluent/tests/resolve_variant_expression.rs +++ b/fluent-bundle/tests/resolve_variant_expression.rs @@ -1,11 +1,11 @@ mod helpers; use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent::bundle::FluentBundle; +use fluent_bundle::bundle::FluentBundle; #[test] fn variant_expression() { - let mut bundle = FluentBundle::new(&["x-testing"], None); + let mut bundle = FluentBundle::new(&["x-testing"]); assert_add_messages_no_errors(bundle.add_messages( " -foo = Foo diff --git a/fluent-res/examples/resources/en-US/common.ftl b/fluent-res/examples/resources/en-US/common.ftl deleted file mode 100644 index 09e3bdaa..00000000 --- a/fluent-res/examples/resources/en-US/common.ftl +++ /dev/null @@ -1 +0,0 @@ -hello-world = Hello World! diff --git a/fluent-res/examples/resources/en-US/simple.ftl b/fluent-res/examples/resources/en-US/simple.ftl deleted file mode 100644 index 104d3e30..00000000 --- a/fluent-res/examples/resources/en-US/simple.ftl +++ /dev/null @@ -1,5 +0,0 @@ -response-msg = - { $value -> - [one] "{ $input }" has one Collatz step. - *[other] "{ $input }" has { $value } Collatz steps. - } diff --git a/fluent-res/examples/resources/pl/common.ftl b/fluent-res/examples/resources/pl/common.ftl deleted file mode 100644 index 5f6a3d9f..00000000 --- a/fluent-res/examples/resources/pl/common.ftl +++ /dev/null @@ -1 +0,0 @@ -hello-world = Witaj, Świecie! diff --git a/fluent-res/examples/resources/pl/errors.ftl b/fluent-res/examples/resources/pl/errors.ftl deleted file mode 100644 index f27124a3..00000000 --- a/fluent-res/examples/resources/pl/errors.ftl +++ /dev/null @@ -1,2 +0,0 @@ -missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. -input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } diff --git a/fluent-res/examples/simple.rs b/fluent-res/examples/simple.rs deleted file mode 100644 index 0b0d8806..00000000 --- a/fluent-res/examples/simple.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! This is an example of a simple application -//! which calculates the Collatz conjecture. -//! -//! The function itself is trivial on purpose, -//! so that we can focus on understanding how -//! the application can be made localizable -//! via Fluent. -//! -//! To try the app launch `cargo run --example simple NUM (LOCALES)` -//! -//! NUM is a number to be calculated, and LOCALES is an optional -//! parameter with a comma-separated list of locales requested by the user. -//! -//! Example: -//! -//! caron run --example simple 123 de,pl -//! -//! If the second argument is omitted, `en-US` locale is used as the -//! default one. -use fluent::resource_manager::ResourceManager; -use fluent_bundle::types::FluentValue; -use fluent_locale::{negotiate_languages, NegotiationStrategy}; -use std::collections::HashMap; -use std::env; -use std::fs; -use std::io; -use std::str::FromStr; - -/// This helper function allows us to read the list -/// of available locales by reading the list of -/// directories in `./examples/resources`. -/// -/// It is expected that every directory inside it -/// has a name that is a valid BCP47 language tag. -fn get_available_locales() -> Result, io::Error> { - let mut locales = vec![]; - - let res_dir = fs::read_dir("./examples/resources/")?; - for entry in res_dir { - if let Ok(entry) = entry { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name() { - if let Some(name) = name.to_str() { - locales.push(String::from(name)); - } - } - } - } - } - return Ok(locales); -} - -/// This function negotiates the locales between available -/// and requested by the user. -/// -/// It uses `fluent-locale` library but one could -/// use any other that will resolve the list of -/// available locales based on the list of -/// requested locales. -fn get_app_locales(requested: &[&str]) -> Result, io::Error> { - let available = get_available_locales()?; - let resolved_locales = negotiate_languages( - requested, - &available, - Some("en-US"), - &NegotiationStrategy::Filtering, - ); - return Ok(resolved_locales - .into_iter() - .map(|s| String::from(s)) - .collect::>()); -} - -static L10N_RESOURCES: &[&str] = &["simple.ftl", "errors.ftl"]; - -fn main() { - // 1. Get the command line arguments. - let args: Vec = env::args().collect(); - - let mgr = ResourceManager::new(); - - // 2. If the argument length is more than 1, - // take the second argument as a comma-separated - // list of requested locales. - // - // Otherwise, take ["en-US"] as the default. - let requested = args - .get(2) - .map_or(vec!["en-US"], |arg| arg.split(",").collect()); - - // 3. Negotiate it against the avialable ones - let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); - - // 4. Create a new Fluent FluentBundle using the - // resolved locales. - let paths = L10N_RESOURCES - .iter() - .map(|path| { - format!( - "./examples/resources/{locale}/{path}", - locale = locales[0], - path = path - ) - }) - .collect(); - - // 5. Get a bundle for given paths and locales. - let bundle = mgr.get_bundle(&locales, &paths); - - // 6. Check if the input is provided. - match args.get(1) { - Some(input) => { - // 6.1. Cast it to a number. - match isize::from_str(&input) { - Ok(i) => { - // 6.2. Construct a map of arguments - // to format the message. - let mut args = HashMap::new(); - args.insert("input", FluentValue::from(i)); - args.insert("value", FluentValue::from(collatz(i))); - // 6.3. Format the message. - println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); - } - Err(err) => { - let mut args = HashMap::new(); - args.insert("input", FluentValue::from(input.to_string())); - args.insert("reason", FluentValue::from(err.to_string())); - println!( - "{}", - bundle - .format("input-parse-error-msg", Some(&args)) - .unwrap() - .0 - ); - } - } - } - None => { - println!("{}", bundle.format("missing-arg-error", None).unwrap().0); - } - } -} - -/// Collatz conjecture calculating function. -fn collatz(n: isize) -> isize { - match n { - 1 => 0, - _ => match n % 2 { - 0 => 1 + collatz(n / 2), - _ => 1 + collatz(n * 3 + 1), - }, - } -} diff --git a/fluent-res/src/lib.rs b/fluent-res/src/lib.rs deleted file mode 100644 index 4f362414..00000000 --- a/fluent-res/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod resource_manager; diff --git a/fluent/Cargo.toml b/fluent/Cargo.toml index b8e1fae3..277631f4 100644 --- a/fluent/Cargo.toml +++ b/fluent/Cargo.toml @@ -5,11 +5,11 @@ A localization system designed to unleash the entire expressive power of natural language translations. """ version = "0.4.3" -edition = "2018" authors = [ "Zibi Braniecki ", "Staś Małolepszy " ] +edition = "2018" homepage = "http://www.projectfluent.org" license = "Apache-2.0/MIT" repository = "https://github.com/projectfluent/fluent-rs" @@ -18,9 +18,6 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] +fluent-bundle = { path = "../fluent-bundle" } fluent-locale = "^0.4.1" -fluent-syntax = { path = "../fluent-syntax" } -failure = "0.1" -failure_derive = "0.1" -intl_pluralrules = "^1.0" elsa = "^0.1.2" diff --git a/fluent/examples/simple.rs b/fluent/examples/simple.rs index 3def15ab..0b0d8806 100644 --- a/fluent/examples/simple.rs +++ b/fluent/examples/simple.rs @@ -18,7 +18,7 @@ //! If the second argument is omitted, `en-US` locale is used as the //! default one. use fluent::resource_manager::ResourceManager; -use fluent::types::FluentValue; +use fluent_bundle::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; use std::env; diff --git a/fluent/src/lib.rs b/fluent/src/lib.rs index 885a4f97..4f362414 100644 --- a/fluent/src/lib.rs +++ b/fluent/src/lib.rs @@ -1,42 +1 @@ -//! Fluent is a localization system designed to improve how software is translated. -//! -//! The Rust implementation provides the low level components for syntax operations, like parser -//! and AST, and the core localization struct - `FluentBundle`. -//! -//! `FluentBundle` is the low level container for storing and formatting localization messages. It -//! is expected that implementations will build on top of it by providing language negotiation -//! between user requested languages and available resources and I/O for loading selected -//! resources. -//! -//! # Example -//! -//! ``` -//! use fluent_bundle::bundle::FluentBundle; -//! use fluent_bundle::types::FluentValue; -//! use std::collections::HashMap; -//! -//! let mut bundle = FluentBundle::new(&["en-US"]); -//! bundle.add_messages( -//! " -//! hello-world = Hello, world! -//! intro = Welcome, { $name }. -//! " -//! ); -//! -//! let value = bundle.format("hello-world", None); -//! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); -//! -//! let mut args = HashMap::new(); -//! args.insert("name", FluentValue::from("John")); -//! -//! let value = bundle.format("intro", Some(&args)); -//! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); -//! ``` - -pub mod bundle; -pub mod entry; -pub mod errors; -pub mod resolve; -pub mod resource; -pub mod types; pub mod resource_manager; diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs index e81dc09a..4798aedd 100644 --- a/fluent/src/resource_manager.rs +++ b/fluent/src/resource_manager.rs @@ -1,6 +1,6 @@ use elsa::FrozenMap; -use crate::bundle::FluentBundle; -use crate::resource::FluentResource; +use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; use std::fs::File; use std::io; use std::io::prelude::*; @@ -17,12 +17,6 @@ pub struct ResourceManager<'mgr> { resources: FrozenMap>>, } -impl<'mgr> Clone for ResourceManager<'mgr> { - fn clone(&self) -> ResourceManager<'mgr> { - unimplemented!() - } -} - impl<'mgr> ResourceManager<'mgr> { pub fn new() -> Self { ResourceManager { @@ -47,27 +41,12 @@ impl<'mgr> ResourceManager<'mgr> { } } - pub fn get_resource_for_string(&'mgr self, string: &str) -> &'mgr FluentResource<'mgr> { - let strings = &self.strings; - - if strings.get(string).is_some() { - return self.resources.get(string).unwrap(); - } else { - let val = self.strings.insert(string.to_string(), string.to_owned()); - let res = match FluentResource::from_string(val) { - Ok(res) => res, - Err((res, _err)) => res, - }; - self.resources.insert(string.to_string(), Box::new(res)) - } - } - pub fn get_bundle( &'mgr self, locales: &Vec, paths: &Vec, ) -> FluentBundle<'mgr> { - let mut bundle = FluentBundle::new(locales, Some(&self)); + let mut bundle = FluentBundle::new(locales); for path in paths { let res = self.get_resource(path); bundle.add_resource(res).unwrap(); From 9a9c3ab8adf293ab275c478ccf59d6da12db2cad Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 31 Dec 2018 16:57:38 -0800 Subject: [PATCH 06/36] Drop FluentBundle::add_messages --- fluent-bundle/examples/functions.rs | 19 ++++++++++--------- fluent-bundle/src/bundle.rs | 17 ----------------- fluent-bundle/src/resource.rs | 1 + .../tests/resolve_external_argument.rs | 4 +++- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/fluent-bundle/examples/functions.rs b/fluent-bundle/examples/functions.rs index a0955b40..6ecc4bc8 100644 --- a/fluent-bundle/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -1,7 +1,14 @@ use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; use fluent_bundle::types::FluentValue; fn main() { + let res1 = FluentResource::from_string("hello-world = Hey there! { HELLO() }").unwrap(); + let res2 = FluentResource::from_string("meaning-of-life = { MEANING_OF_LIFE(42) }").unwrap(); + let res3 = + FluentResource::from_string("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") + .unwrap(); + let mut bundle = FluentBundle::new(&["en-US"]); // Test for a simple function that returns a string @@ -38,15 +45,9 @@ fn main() { }) .unwrap(); - bundle - .add_messages("hello-world = Hey there! { HELLO() }") - .unwrap(); - bundle - .add_messages("meaning-of-life = { MEANING_OF_LIFE(42) }") - .unwrap(); - bundle - .add_messages("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") - .unwrap(); + bundle.add_resource(&res1).unwrap(); + bundle.add_resource(&res2).unwrap(); + bundle.add_resource(&res3).unwrap(); let value = bundle.format("hello-world", None); assert_eq!( diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 6c096aad..e2755d5d 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -97,23 +97,6 @@ impl<'bundle> FluentBundle<'bundle> { } } - //pub fn add_messages(&mut self, source: &'bundle str) -> Result<(), Vec> { - //match FluentResource::from_string(source) { - //Ok(res) => self.add_resource(&res), - //Err((res, err)) => { - //let mut errors: Vec = - //err.into_iter().map(FluentError::ParserError).collect(); - - //self.add_resource(&res).map_err(|err| { - //for e in err { - //errors.push(e); - //} - //errors - //}) - //} - //} - //} - pub fn add_resource( &mut self, res: &'bundle FluentResource<'bundle>, diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs index d68dfbc9..70e5d2d2 100644 --- a/fluent-bundle/src/resource.rs +++ b/fluent-bundle/src/resource.rs @@ -2,6 +2,7 @@ use fluent_syntax::ast; use fluent_syntax::parser::parse; use fluent_syntax::parser::ParserError; +#[derive(Debug)] pub struct FluentResource<'resource> { pub ast: ast::Resource<'resource>, } diff --git a/fluent-bundle/tests/resolve_external_argument.rs b/fluent-bundle/tests/resolve_external_argument.rs index ade852d5..9c5b0157 100644 --- a/fluent-bundle/tests/resolve_external_argument.rs +++ b/fluent-bundle/tests/resolve_external_argument.rs @@ -10,7 +10,9 @@ use fluent_bundle::types::FluentValue; fn external_argument_string() { let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages("hello-world = Hello { $name }")); + assert_add_messages_no_errors( + bundle.add_resource(FluentResource::from_string("hello-world = Hello { $name }")), + ); let mut args = HashMap::new(); args.insert("name", FluentValue::from("John")); From 957942ed7c8288200c7ac3ab287819a3f7d15c46 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 2 Jan 2019 13:41:42 -0800 Subject: [PATCH 07/36] Convert simple-app example to use add_resource --- fluent-bundle/examples/functions.rs | 14 ++++++++----- fluent-bundle/examples/simple-app.rs | 30 ++++++++++++++++++---------- fluent-bundle/src/resource.rs | 2 +- fluent/src/resource_manager.rs | 12 ++++------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/fluent-bundle/examples/functions.rs b/fluent-bundle/examples/functions.rs index 6ecc4bc8..abb2f29d 100644 --- a/fluent-bundle/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -3,11 +3,11 @@ use fluent_bundle::resource::FluentResource; use fluent_bundle::types::FluentValue; fn main() { - let res1 = FluentResource::from_string("hello-world = Hey there! { HELLO() }").unwrap(); - let res2 = FluentResource::from_string("meaning-of-life = { MEANING_OF_LIFE(42) }").unwrap(); - let res3 = - FluentResource::from_string("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") - .unwrap(); + // We define the resources here so that they outlive + // the bundle. + let res1; + let res2; + let res3; let mut bundle = FluentBundle::new(&["en-US"]); @@ -45,8 +45,12 @@ fn main() { }) .unwrap(); + res1 = FluentResource::from_str("hello-world = Hey there! { HELLO() }").unwrap(); bundle.add_resource(&res1).unwrap(); + res2 = FluentResource::from_str("meaning-of-life = { MEANING_OF_LIFE(42) }").unwrap(); bundle.add_resource(&res2).unwrap(); + res3 = FluentResource::from_str("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") + .unwrap(); bundle.add_resource(&res3).unwrap(); let value = bundle.format("hello-world", None); diff --git a/fluent-bundle/examples/simple-app.rs b/fluent-bundle/examples/simple-app.rs index bf7c8854..aaa9e519 100644 --- a/fluent-bundle/examples/simple-app.rs +++ b/fluent-bundle/examples/simple-app.rs @@ -18,6 +18,7 @@ //! If the second argument is omitted, `en-US` locale is used as the //! default one. use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; use fluent_bundle::types::FluentValue; use fluent_locale::{negotiate_languages, NegotiationStrategy}; use std::collections::HashMap; @@ -91,9 +92,12 @@ static L10N_RESOURCES: &[&str] = &["simple.ftl"]; fn main() { // 1. Get the command line arguments. let args: Vec = env::args().collect(); + + // 2. Allocate strings and their resources. let mut sources: Vec = vec![]; + let mut resources: Vec = vec![]; - // 2. If the argument length is more than 1, + // 3. If the argument length is more than 1, // take the second argument as a comma-separated // list of requested locales. // @@ -102,40 +106,44 @@ fn main() { .get(2) .map_or(vec!["en-US"], |arg| arg.split(",").collect()); - // 3. Negotiate it against the avialable ones + // 4. Negotiate it against the avialable ones let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); - // 4. Create a new Fluent FluentBundle using the + // 5. Create a new Fluent FluentBundle using the // resolved locales. let mut bundle = FluentBundle::new(&locales); - // 5. Load the localization resource + // 6. Load the localization resource for path in L10N_RESOURCES { let full_path = format!( "./examples/resources/{locale}/{path}", locale = locales[0], path = path ); - let source = read_file(&full_path).unwrap(); - sources.push(source); + sources.push(read_file(&full_path).unwrap()); } for source in &sources { - bundle.add_messages(&source).unwrap(); + let resource = FluentResource::from_str(source).unwrap(); + resources.push(resource); + } + + for res in &resources { + bundle.add_resource(res).unwrap(); } - // 6. Check if the input is provided. + // 7. Check if the input is provided. match args.get(1) { Some(input) => { - // 6.1. Cast it to a number. + // 7.1. Cast it to a number. match isize::from_str(&input) { Ok(i) => { - // 6.2. Construct a map of arguments + // 7.2. Construct a map of arguments // to format the message. let mut args = HashMap::new(); args.insert("input", FluentValue::from(i)); args.insert("value", FluentValue::from(collatz(i))); - // 6.3. Format the message. + // 7.3. Format the message. println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); } Err(err) => { diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs index 70e5d2d2..6b82d006 100644 --- a/fluent-bundle/src/resource.rs +++ b/fluent-bundle/src/resource.rs @@ -8,7 +8,7 @@ pub struct FluentResource<'resource> { } impl<'resource> FluentResource<'resource> { - pub fn from_string(source: &'resource str) -> Result)> { + pub fn from_str(source: &'resource str) -> Result)> { match parse(&source) { Ok(ast) => Ok(FluentResource { ast }), Err((ast, errors)) => Err((FluentResource { ast }, errors)), diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs index 4798aedd..813ff81f 100644 --- a/fluent/src/resource_manager.rs +++ b/fluent/src/resource_manager.rs @@ -29,11 +29,11 @@ impl<'mgr> ResourceManager<'mgr> { let strings = &self.strings; if strings.get(path).is_some() { - return self.resources.get(path).unwrap(); + self.resources.get(path).unwrap() } else { let string = read_file(path).unwrap(); let val = self.strings.insert(path.to_string(), string); - let res = match FluentResource::from_string(val) { + let res = match FluentResource::from_str(val) { Ok(res) => res, Err((res, _err)) => res, }; @@ -41,16 +41,12 @@ impl<'mgr> ResourceManager<'mgr> { } } - pub fn get_bundle( - &'mgr self, - locales: &Vec, - paths: &Vec, - ) -> FluentBundle<'mgr> { + pub fn get_bundle(&'mgr self, locales: &[String], paths: &[String]) -> FluentBundle<'mgr> { let mut bundle = FluentBundle::new(locales); for path in paths { let res = self.get_resource(path); bundle.add_resource(res).unwrap(); } - return bundle; + bundle } } From 377ea56f755b9be8f7942d480b329dc33c774bd0 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Fri, 4 Jan 2019 17:27:17 -0800 Subject: [PATCH 08/36] Switch to use rental crate to store source string on FluentResource --- fluent-bundle/Cargo.toml | 7 ++- fluent-bundle/examples/external_arguments.rs | 16 ++--- fluent-bundle/examples/functions.rs | 14 ++--- fluent-bundle/examples/hello.rs | 4 +- fluent-bundle/examples/message_reference.rs | 17 +++--- fluent-bundle/examples/selector.rs | 16 ++--- fluent-bundle/examples/simple-app.rs | 10 +--- fluent-bundle/src/bundle.rs | 7 +-- fluent-bundle/src/lib.rs | 14 +++-- fluent-bundle/src/resource.rs | 43 +++++++++++--- fluent-bundle/tests/format.rs | 14 +++-- fluent-bundle/tests/format_message.rs | 14 +++-- fluent-bundle/tests/helpers/mod.rs | 17 +++++- .../tests/resolve_attribute_expression.rs | 14 ++--- .../tests/resolve_external_argument.rs | 36 ++++++------ .../tests/resolve_message_reference.rs | 58 +++++++++---------- fluent-bundle/tests/resolve_plural_rule.rs | 23 ++++---- .../tests/resolve_select_expression.rs | 53 ++++++++--------- fluent-bundle/tests/resolve_value.rs | 21 +++---- .../tests/resolve_variant_expression.rs | 13 +++-- fluent-syntax/src/parser/mod.rs | 12 ++-- fluent/src/resource_manager.rs | 19 +++--- 22 files changed, 245 insertions(+), 197 deletions(-) diff --git a/fluent-bundle/Cargo.toml b/fluent-bundle/Cargo.toml index 59f91b86..66c40972 100644 --- a/fluent-bundle/Cargo.toml +++ b/fluent-bundle/Cargo.toml @@ -20,6 +20,7 @@ categories = ["localization", "internationalization"] [dependencies] fluent-locale = "^0.4.1" fluent-syntax = { path = "../fluent-syntax" } -failure = "0.1" -failure_derive = "0.1" -intl_pluralrules = "1.0" +failure = "^0.1" +failure_derive = "^0.1" +intl_pluralrules = "^1.0" +rental = "^0.5.2" diff --git a/fluent-bundle/examples/external_arguments.rs b/fluent-bundle/examples/external_arguments.rs index 0b817451..8e09e91d 100644 --- a/fluent-bundle/examples/external_arguments.rs +++ b/fluent-bundle/examples/external_arguments.rs @@ -1,12 +1,11 @@ use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["en"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " hello-world = Hello { $name } ref = The previous message says { hello-world } unread-emails = @@ -14,9 +13,12 @@ unread-emails = [one] You have { $emailCount } unread email *[other] You have { $emailCount } unread emails } -", - ) - .unwrap(); +" + .to_owned(), + ) + .unwrap(); + let mut bundle = FluentBundle::new(&["en"]); + bundle.add_resource(&res).unwrap(); let mut args = HashMap::new(); args.insert("name", FluentValue::from("John")); diff --git a/fluent-bundle/examples/functions.rs b/fluent-bundle/examples/functions.rs index abb2f29d..ba50dda6 100644 --- a/fluent-bundle/examples/functions.rs +++ b/fluent-bundle/examples/functions.rs @@ -5,9 +5,13 @@ use fluent_bundle::types::FluentValue; fn main() { // We define the resources here so that they outlive // the bundle. - let res1; - let res2; - let res3; + let res1 = FluentResource::try_new("hello-world = Hey there! { HELLO() }".to_owned()).unwrap(); + let res2 = + FluentResource::try_new("meaning-of-life = { MEANING_OF_LIFE(42) }".to_owned()).unwrap(); + let res3 = FluentResource::try_new( + "all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }".to_owned(), + ) + .unwrap(); let mut bundle = FluentBundle::new(&["en-US"]); @@ -45,12 +49,8 @@ fn main() { }) .unwrap(); - res1 = FluentResource::from_str("hello-world = Hey there! { HELLO() }").unwrap(); bundle.add_resource(&res1).unwrap(); - res2 = FluentResource::from_str("meaning-of-life = { MEANING_OF_LIFE(42) }").unwrap(); bundle.add_resource(&res2).unwrap(); - res3 = FluentResource::from_str("all-your-base = { BASE_OWNERSHIP(hello, ownership: \"us\") }") - .unwrap(); bundle.add_resource(&res3).unwrap(); let value = bundle.format("hello-world", None); diff --git a/fluent-bundle/examples/hello.rs b/fluent-bundle/examples/hello.rs index 2828b333..c800a571 100644 --- a/fluent-bundle/examples/hello.rs +++ b/fluent-bundle/examples/hello.rs @@ -1,8 +1,10 @@ use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; fn main() { + let res = FluentResource::try_new("hello-world = Hello, world!".to_owned()).unwrap(); let mut bundle = FluentBundle::new(&["en-US"]); - bundle.add_messages("hello-world = Hello, world!").unwrap(); + bundle.add_resource(&res).unwrap(); let (value, _) = bundle.format("hello-world", None).unwrap(); assert_eq!(&value, "Hello, world!"); } diff --git a/fluent-bundle/examples/message_reference.rs b/fluent-bundle/examples/message_reference.rs index bfc512e5..6b0438ab 100644 --- a/fluent-bundle/examples/message_reference.rs +++ b/fluent-bundle/examples/message_reference.rs @@ -1,16 +1,19 @@ use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " foo = Foo foobar = { foo } Bar bazbar = { baz } Bar -", - ) - .unwrap(); +" + .to_owned(), + ) + .unwrap(); + + let mut bundle = FluentBundle::new(&["x-testing"]); + bundle.add_resource(&res).unwrap(); match bundle.format("foobar", None) { Some((value, _)) => println!("{}", value), diff --git a/fluent-bundle/examples/selector.rs b/fluent-bundle/examples/selector.rs index c7e3672b..c2e83c58 100644 --- a/fluent-bundle/examples/selector.rs +++ b/fluent-bundle/examples/selector.rs @@ -1,12 +1,11 @@ use fluent_bundle::bundle::FluentBundle; +use fluent_bundle::resource::FluentResource; use fluent_bundle::types::FluentValue; use std::collections::HashMap; fn main() { - let mut bundle = FluentBundle::new(&["x-testing"]); - bundle - .add_messages( - " + let res = FluentResource::try_new( + " hello-world = Hello { *[one] World [two] Moon @@ -16,9 +15,12 @@ hello-world2 = Hello { $name -> *[world] World [moon] Moon } -", - ) - .unwrap(); + " + .to_owned(), + ) + .unwrap(); + let mut bundle = FluentBundle::new(&["x-testing"]); + bundle.add_resource(&res).unwrap(); match bundle.format("hello-world", None) { Some((value, _)) => println!("{}", value), diff --git a/fluent-bundle/examples/simple-app.rs b/fluent-bundle/examples/simple-app.rs index aaa9e519..79f2803f 100644 --- a/fluent-bundle/examples/simple-app.rs +++ b/fluent-bundle/examples/simple-app.rs @@ -93,8 +93,7 @@ fn main() { // 1. Get the command line arguments. let args: Vec = env::args().collect(); - // 2. Allocate strings and their resources. - let mut sources: Vec = vec![]; + // 2. Allocate resources. let mut resources: Vec = vec![]; // 3. If the argument length is more than 1, @@ -120,11 +119,8 @@ fn main() { locale = locales[0], path = path ); - sources.push(read_file(&full_path).unwrap()); - } - - for source in &sources { - let resource = FluentResource::from_str(source).unwrap(); + let source = read_file(&full_path).unwrap(); + let resource = FluentResource::try_new(source).unwrap(); resources.push(resource); } diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index e2755d5d..07d09071 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -97,13 +97,10 @@ impl<'bundle> FluentBundle<'bundle> { } } - pub fn add_resource( - &mut self, - res: &'bundle FluentResource<'bundle>, - ) -> Result<(), Vec> { + pub fn add_resource(&mut self, res: &'bundle FluentResource) -> Result<(), Vec> { let mut errors = vec![]; - for entry in &res.ast.body { + for entry in &res.ast().body { let id = match entry { ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { ref id, .. })) | ast::ResourceEntry::Entry(ast::Entry::Term(ast::Term { ref id, .. })) => id.name, diff --git a/fluent-bundle/src/lib.rs b/fluent-bundle/src/lib.rs index b516ac9c..17228b3e 100644 --- a/fluent-bundle/src/lib.rs +++ b/fluent-bundle/src/lib.rs @@ -13,15 +13,17 @@ //! ``` //! use fluent_bundle::bundle::FluentBundle; //! use fluent_bundle::types::FluentValue; +//! use fluent_bundle::resource::FluentResource; //! use std::collections::HashMap; //! -//! let mut bundle = FluentBundle::new(&["en-US"]); -//! bundle.add_messages( -//! " +//! let res = FluentResource::try_new(" //! hello-world = Hello, world! //! intro = Welcome, { $name }. -//! " -//! ); +//! ".to_owned()).expect("Failed to parse FTL."); +//! +//! let mut bundle = FluentBundle::new(&["en-US"]); +//! +//! bundle.add_resource(&res).expect("Failed to add FluentResource to Bundle."); //! //! let value = bundle.format("hello-world", None); //! assert_eq!(value, Some(("Hello, world!".to_string(), vec![]))); @@ -33,6 +35,8 @@ //! assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); //! ``` +#[macro_use] +extern crate rental; extern crate failure; #[macro_use] extern crate failure_derive; diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs index 6b82d006..4fef0555 100644 --- a/fluent-bundle/src/resource.rs +++ b/fluent-bundle/src/resource.rs @@ -2,16 +2,43 @@ use fluent_syntax::ast; use fluent_syntax::parser::parse; use fluent_syntax::parser::ParserError; -#[derive(Debug)] -pub struct FluentResource<'resource> { - pub ast: ast::Resource<'resource>, +rental! { + mod rentals { + use super::*; + #[rental(covariant, debug)] + pub struct FluentResource { + string: String, + ast: ast::Resource<'string>, + } + } } -impl<'resource> FluentResource<'resource> { - pub fn from_str(source: &'resource str) -> Result)> { - match parse(&source) { - Ok(ast) => Ok(FluentResource { ast }), - Err((ast, errors)) => Err((FluentResource { ast }, errors)), +#[derive(Debug)] +pub struct FluentResource(rentals::FluentResource); + +impl FluentResource { + pub fn try_new(source: String) -> Result)> { + let mut errors = None; + let res = rentals::FluentResource::new(source, |s| match parse(s) { + Ok(ast) => ast, + Err((ast, err)) => { + errors = Some(err); + ast + } + }); + + if let Some(errors) = errors { + return Err((Self(res), errors)); + } else { + return Ok(Self(res)); } } + + pub fn ast<'a>(&'a self) -> &ast::Resource<'a> { + self.0.all().ast + } + + pub fn string<'a>(&'a self) -> &'a str { + self.0.all().string + } } diff --git a/fluent-bundle/tests/format.rs b/fluent-bundle/tests/format.rs index f6ee28dc..a51a5152 100644 --- a/fluent-bundle/tests/format.rs +++ b/fluent-bundle/tests/format.rs @@ -1,18 +1,20 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors, assert_format_none}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_format_none, assert_get_bundle_no_errors, + assert_get_resource_from_str_no_errors, +}; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Attribute -term = Term -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo"); diff --git a/fluent-bundle/tests/format_message.rs b/fluent-bundle/tests/format_message.rs index 28394855..f427b22c 100644 --- a/fluent-bundle/tests/format_message.rs +++ b/fluent-bundle/tests/format_message.rs @@ -1,20 +1,22 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_message_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_message_no_errors, assert_get_bundle_no_errors, + assert_get_resource_from_str_no_errors, +}; use fluent_bundle::bundle::Message; use std::collections::HashMap; #[test] fn format() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Attribute .attr2 = Attribute 2 -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut attrs = HashMap::new(); attrs.insert("attr".to_string(), "Attribute".to_string()); diff --git a/fluent-bundle/tests/helpers/mod.rs b/fluent-bundle/tests/helpers/mod.rs index 7b39cf4d..90d7b547 100644 --- a/fluent-bundle/tests/helpers/mod.rs +++ b/fluent-bundle/tests/helpers/mod.rs @@ -1,5 +1,7 @@ +use fluent_bundle::bundle::FluentBundle; use fluent_bundle::bundle::FluentError; use fluent_bundle::bundle::Message; +use fluent_bundle::resource::FluentResource; #[allow(dead_code)] pub fn assert_format_none(result: Option<(String, Vec)>) { @@ -20,6 +22,17 @@ pub fn assert_format_message_no_errors( assert_eq!(result, Some((expected, vec![]))); } -pub fn assert_add_messages_no_errors(result: Result<(), Vec>) { - assert!(result.is_ok()); +pub fn assert_get_resource_from_str_no_errors(s: &str) -> FluentResource { + FluentResource::try_new(s.to_owned()).unwrap() +} + +pub fn assert_get_bundle_no_errors<'a>( + res: &'a FluentResource, + locale: Option<&str>, +) -> FluentBundle<'a> { + let mut bundle = FluentBundle::new(&[locale.unwrap_or("x-testing")]); + bundle + .add_resource(res) + .expect("Failed to add FluentResource to FluentBundle."); + bundle } diff --git a/fluent-bundle/tests/resolve_attribute_expression.rs b/fluent-bundle/tests/resolve_attribute_expression.rs index 80c97385..e8c58631 100644 --- a/fluent-bundle/tests/resolve_attribute_expression.rs +++ b/fluent-bundle/tests/resolve_attribute_expression.rs @@ -1,13 +1,12 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn attribute_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Foo Attr @@ -21,8 +20,9 @@ use-bar-attr = { bar.attr } missing-attr = { foo.missing } missing-missing = { missing.missing } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); diff --git a/fluent-bundle/tests/resolve_external_argument.rs b/fluent-bundle/tests/resolve_external_argument.rs index 9c5b0157..854cbf22 100644 --- a/fluent-bundle/tests/resolve_external_argument.rs +++ b/fluent-bundle/tests/resolve_external_argument.rs @@ -2,17 +2,15 @@ mod helpers; use std::collections::HashMap; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; use fluent_bundle::types::FluentValue; #[test] fn external_argument_string() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors( - bundle.add_resource(FluentResource::from_string("hello-world = Hello { $name }")), - ); + let res = assert_get_resource_from_str_no_errors("hello-world = Hello { $name }"); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("name", FluentValue::from("John")); @@ -22,14 +20,13 @@ fn external_argument_string() { #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors( - bundle.add_messages("unread-emails = You have { $emailsCount } unread emails."), - ); - assert_add_messages_no_errors( - bundle.add_messages("unread-emails-dec = You have { $emailsCountDec } unread emails."), + let res = assert_get_resource_from_str_no_errors( + " +unread-emails = You have { $emailsCount } unread emails. +unread-emails-dec = You have { $emailsCountDec } unread emails. + ", ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(5)); @@ -48,12 +45,13 @@ fn external_argument_number() { #[test] fn reference_message_with_external_argument() { - let mut bundle = FluentBundle::new(&["x-testing"]); - - assert_add_messages_no_errors(bundle.add_messages("greetings = Hello, { $userName }")); - assert_add_messages_no_errors( - bundle.add_messages("click-on = Click on the `{ greetings }` label."), + let res = assert_get_resource_from_str_no_errors( + " +greetings = Hello, { $userName } +click-on = Click on the `{ greetings }` label. + ", ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("userName", FluentValue::from("Mary")); diff --git a/fluent-bundle/tests/resolve_message_reference.rs b/fluent-bundle/tests/resolve_message_reference.rs index 3ad695ce..0bcda5b1 100644 --- a/fluent-bundle/tests/resolve_message_reference.rs +++ b/fluent-bundle/tests/resolve_message_reference.rs @@ -1,79 +1,79 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn message_reference() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar"); } #[test] fn term_reference() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo bar = { -foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar"); } #[test] fn message_reference_nested() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar baz = { bar } Baz -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("baz", None), "Foo Bar Baz"); } #[test] fn message_reference_missing() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages("bar = { foo } Bar")); - + let res = assert_get_resource_from_str_no_errors("bar = { foo } Bar"); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "___ Bar"); } #[test] fn message_reference_cyclic() { { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo { bar } bar = { foo } Bar -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo ___"); assert_format_no_errors(bundle.format("bar", None), "___ Bar"); } { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { bar } bar = { foo } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "___"); assert_format_no_errors(bundle.format("bar", None), "___"); @@ -82,13 +82,13 @@ bar = { foo } #[test] fn message_reference_multiple() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo bar = { foo } Bar { foo } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("bar", None), "Foo Bar Foo"); } diff --git a/fluent-bundle/tests/resolve_plural_rule.rs b/fluent-bundle/tests/resolve_plural_rule.rs index 7c655b1e..a0caa7fd 100644 --- a/fluent-bundle/tests/resolve_plural_rule.rs +++ b/fluent-bundle/tests/resolve_plural_rule.rs @@ -2,14 +2,14 @@ mod helpers; use std::collections::HashMap; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; use fluent_bundle::types::FluentValue; #[test] fn external_argument_number() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " unread-emails = { $emailsCount -> @@ -22,9 +22,9 @@ unread-emails-dec = [one] You have { $emailsCountDec } unread email. *[other] You have { $emailsCountDec } unread emails. } - -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(1)); @@ -43,8 +43,7 @@ unread-emails-dec = #[test] fn exact_match() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " unread-emails = { $emailsCount -> @@ -59,9 +58,9 @@ unread-emails-dec = [one] You have { $emailsCountDec } unread email. *[other] You have { $emailsCountDec } unread emails. } - -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); let mut args = HashMap::new(); args.insert("emailsCount", FluentValue::from(1)); diff --git a/fluent-bundle/tests/resolve_select_expression.rs b/fluent-bundle/tests/resolve_select_expression.rs index 57a36b3d..7cda0a73 100644 --- a/fluent-bundle/tests/resolve_select_expression.rs +++ b/fluent-bundle/tests/resolve_select_expression.rs @@ -2,14 +2,14 @@ mod helpers; use std::collections::HashMap; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; use fluent_bundle::types::FluentValue; #[test] fn select_expression_string_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { \"genitive\" -> @@ -22,8 +22,9 @@ bar = *[nominative] Bar [genitive] Bar's } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo's"); @@ -32,8 +33,7 @@ bar = #[test] fn select_expression_number_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { 3 -> @@ -53,8 +53,9 @@ baz = [3] Baz 3 [3.14] Baz Pi } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -65,8 +66,7 @@ baz = #[test] fn select_expression_plurals() { - let mut bundle = FluentBundle::new(&["en"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = { 3 -> @@ -88,8 +88,9 @@ baz = [3] Bar 3 *[other] Bar Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, Some("en")); assert_format_no_errors(bundle.format("foo", None), "Foo 3"); @@ -100,8 +101,7 @@ baz = #[test] fn select_expression_external_argument_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo-hit = { $str -> @@ -156,8 +156,9 @@ baz-unknown = *[1] Baz 1 [2] Baz 2 } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); let mut args = HashMap::new(); args.insert("str", FluentValue::from("qux")); @@ -185,8 +186,7 @@ baz-unknown = #[test] fn select_expression_message_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -bar = Bar .attr = attr_val @@ -196,16 +196,16 @@ use-bar = [attr_val] Bar *[other] Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-bar", None), "Bar"); } #[test] fn select_expression_attribute_selector() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo .attr = FooAttr @@ -215,8 +215,9 @@ use-foo = [FooAttr] Foo *[other] Other } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("use-foo", None), "Foo"); } diff --git a/fluent-bundle/tests/resolve_value.rs b/fluent-bundle/tests/resolve_value.rs index 76f5601c..e0b1c04d 100644 --- a/fluent-bundle/tests/resolve_value.rs +++ b/fluent-bundle/tests/resolve_value.rs @@ -1,29 +1,30 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn format_message() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo", None), "Foo"); } #[test] fn format_attribute() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " foo = Foo .attr = Foo Attr -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("foo.attr", None), "Foo Attr"); } diff --git a/fluent-bundle/tests/resolve_variant_expression.rs b/fluent-bundle/tests/resolve_variant_expression.rs index 922e1831..978c0ec0 100644 --- a/fluent-bundle/tests/resolve_variant_expression.rs +++ b/fluent-bundle/tests/resolve_variant_expression.rs @@ -1,12 +1,12 @@ mod helpers; -use self::helpers::{assert_add_messages_no_errors, assert_format_no_errors}; -use fluent_bundle::bundle::FluentBundle; +use self::helpers::{ + assert_format_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, +}; #[test] fn variant_expression() { - let mut bundle = FluentBundle::new(&["x-testing"]); - assert_add_messages_no_errors(bundle.add_messages( + let res = assert_get_resource_from_str_no_errors( " -foo = Foo -bar = @@ -25,8 +25,9 @@ use-bar-genitive = { -bar[genitive] } use-bar-missing = { -bar[missing] } missing-missing = { -missing[missing] } -", - )); + ", + ); + let bundle = assert_get_bundle_no_errors(&res, None); assert_format_no_errors(bundle.format("baz", None), "Bar"); diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 56635aa7..e95f0be3 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -357,15 +357,15 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> let mut indent = 0; if text_element_role == TextElementPosition::LineStart { indent = ps.skip_blank_inline(); + if ps.ptr >= ps.length { + break; + } + let b = ps.source[ps.ptr]; if indent == 0 { - if ps.source[ps.ptr] != b'\n' && ps.source[ps.ptr] != b'\r' { + if b != b'\n' && b != b'\r' { break; } - } else if ps.source[ps.ptr] == b'.' - || ps.source[ps.ptr] == b'}' - || ps.source[ps.ptr] == b'*' - || ps.source[ps.ptr] == b'[' - { + } else if b == b'.' || b == b'}' || b == b'*' || b == b'[' { ps.ptr = slice_start; break; } diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs index 813ff81f..0f871a52 100644 --- a/fluent/src/resource_manager.rs +++ b/fluent/src/resource_manager.rs @@ -12,28 +12,25 @@ fn read_file(path: &str) -> Result { Ok(s) } -pub struct ResourceManager<'mgr> { - strings: FrozenMap, - resources: FrozenMap>>, +pub struct ResourceManager { + resources: FrozenMap>, } -impl<'mgr> ResourceManager<'mgr> { +impl ResourceManager { pub fn new() -> Self { ResourceManager { - strings: FrozenMap::new(), resources: FrozenMap::new(), } } - pub fn get_resource(&'mgr self, path: &str) -> &'mgr FluentResource<'mgr> { - let strings = &self.strings; + pub fn get_resource(&self, path: &str) -> &FluentResource { + let resources = &self.resources; - if strings.get(path).is_some() { + if resources.get(path).is_some() { self.resources.get(path).unwrap() } else { let string = read_file(path).unwrap(); - let val = self.strings.insert(path.to_string(), string); - let res = match FluentResource::from_str(val) { + let res = match FluentResource::try_new(string) { Ok(res) => res, Err((res, _err)) => res, }; @@ -41,7 +38,7 @@ impl<'mgr> ResourceManager<'mgr> { } } - pub fn get_bundle(&'mgr self, locales: &[String], paths: &[String]) -> FluentBundle<'mgr> { + pub fn get_bundle(&self, locales: &[String], paths: &Vec) -> FluentBundle { let mut bundle = FluentBundle::new(locales); for path in paths { let res = self.get_resource(path); From d50fc3eccf804ea18b6ab7e94d482c3afbe851ff Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Fri, 4 Jan 2019 17:31:07 -0800 Subject: [PATCH 09/36] Fix benches for fluent-bundle --- fluent-bundle/benches/lib.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/fluent-bundle/benches/lib.rs b/fluent-bundle/benches/lib.rs index abcb53ce..4dddc0c6 100644 --- a/fluent-bundle/benches/lib.rs +++ b/fluent-bundle/benches/lib.rs @@ -3,7 +3,8 @@ extern crate test; use fluent_bundle::bundle::FluentBundle; -use fluent_syntax::{ast, parser::parse}; +use fluent_bundle::resource::FluentResource; +use fluent_syntax::ast; use std::fs::File; use std::io; use std::io::Read; @@ -19,11 +20,11 @@ fn read_file(path: &str) -> Result { #[bench] fn bench_simple_format(b: &mut Bencher) { let source = read_file("./benches/simple.ftl").expect("Couldn't load file"); - let resource = parse(&source).unwrap(); + let res = FluentResource::try_new(source).expect("Couldn't parse an FTL source"); let mut ids = Vec::new(); - for entry in resource.body { + for entry in &res.ast().body { match entry { ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { ids.push(id.name) @@ -33,7 +34,9 @@ fn bench_simple_format(b: &mut Bencher) { } let mut bundle = FluentBundle::new(&["x-testing"]); - bundle.add_messages(&source).unwrap(); + bundle + .add_resource(&res) + .expect("Couldn't add FluentResource to the FluentBundle"); b.iter(|| { for id in &ids { @@ -45,11 +48,11 @@ fn bench_simple_format(b: &mut Bencher) { #[bench] fn bench_menubar_format(b: &mut Bencher) { let source = read_file("./benches/menubar.ftl").expect("Couldn't load file"); - let resource = parse(&source).unwrap(); + let res = FluentResource::try_new(source).expect("Couldn't parse an FTL source"); let mut ids = Vec::new(); - for entry in resource.body { + for entry in &res.ast().body { match entry { ast::ResourceEntry::Entry(ast::Entry::Message(ast::Message { id, .. })) => { ids.push(id.name) @@ -59,7 +62,9 @@ fn bench_menubar_format(b: &mut Bencher) { } let mut bundle = FluentBundle::new(&["x-testing"]); - bundle.add_messages(&source).unwrap(); + bundle + .add_resource(&res) + .expect("Couldn't add FluentResource to the FluentBundle"); b.iter(|| { for id in &ids { From e26dd251504b95959102540e7953fe861afbad71 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Fri, 4 Jan 2019 17:49:16 -0800 Subject: [PATCH 10/36] Fix the crate for stable rust --- fluent-bundle/src/resource.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs index 4fef0555..f6a34328 100644 --- a/fluent-bundle/src/resource.rs +++ b/fluent-bundle/src/resource.rs @@ -28,17 +28,13 @@ impl FluentResource { }); if let Some(errors) = errors { - return Err((Self(res), errors)); + return Err((FluentResource(res), errors)); } else { - return Ok(Self(res)); + return Ok(FluentResource(res)); } } - pub fn ast<'a>(&'a self) -> &ast::Resource<'a> { + pub fn ast(&self) -> &ast::Resource { self.0.all().ast } - - pub fn string<'a>(&'a self) -> &'a str { - self.0.all().string - } } From bbac43576b8edca6ffd8481ef46a0b60e3c69fb3 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Sat, 5 Jan 2019 13:53:41 -0800 Subject: [PATCH 11/36] Use annotate-snippets 0.5 --- fluent-cli/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent-cli/Cargo.toml b/fluent-cli/Cargo.toml index 0d19b704..db47bd62 100644 --- a/fluent-cli/Cargo.toml +++ b/fluent-cli/Cargo.toml @@ -18,6 +18,6 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dependencies] -annotate-snippets = {version = "0.1", features = ["color"]} -clap = "2.32" +annotate-snippets = {version = "^0.5", features = ["color"]} +clap = "^2.32" fluent-syntax = { path = "../fluent-syntax" } From bc57ec60edfce88800b4f6ad05f35ac5a9712f71 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Sun, 6 Jan 2019 20:34:13 -0800 Subject: [PATCH 12/36] Use newer serde to simplify the json ast serialization --- fluent-syntax/Cargo.toml | 10 +- fluent-syntax/tests/ast/mod.rs | 186 ++++++++++++--------------------- 2 files changed, 72 insertions(+), 124 deletions(-) diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml index 5850be5c..9b8bd953 100644 --- a/fluent-syntax/Cargo.toml +++ b/fluent-syntax/Cargo.toml @@ -17,8 +17,8 @@ keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] categories = ["localization", "internationalization"] [dev-dependencies] -serde = "1.0" -serde_derive = "1.0" -serde_json = "1.0" -glob = "0.2" -assert-json-diff = "0.2.0" +serde = "^1.0" +serde_derive = "^1.0" +serde_json = "^1.0" +glob = "^0.2" +assert-json-diff = "^0.2.1" diff --git a/fluent-syntax/tests/ast/mod.rs b/fluent-syntax/tests/ast/mod.rs index 65a67f45..4d4d13f3 100644 --- a/fluent-syntax/tests/ast/mod.rs +++ b/fluent-syntax/tests/ast/mod.rs @@ -9,13 +9,13 @@ use std::error::Error; pub fn serialize<'s>(res: &'s ast::Resource) -> Result> { #[derive(Serialize)] - struct Helper<'ast>(#[serde(serialize_with = "serialize_resource")] &'ast ast::Resource<'ast>); + struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); Ok(serde_json::to_string(&Helper(res)).unwrap()) } pub fn _serialize_to_pretty_json<'s>(res: &'s ast::Resource) -> Result> { #[derive(Serialize)] - struct Helper<'ast>(#[serde(serialize_with = "serialize_resource")] &'ast ast::Resource<'ast>); + struct Helper<'ast>(#[serde(with = "ResourceDef")] &'ast ast::Resource<'ast>); let buf = Vec::new(); let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); @@ -24,20 +24,13 @@ pub fn _serialize_to_pretty_json<'s>(res: &'s ast::Resource) -> Result(res: &'se ast::Resource, serializer: S) -> Result -where - S: Serializer, -{ - #[derive(Serialize)] - struct Helper<'ast>( - #[serde(serialize_with = "serialize_resource_entry_vec")] - &'ast Vec>, - ); - - let mut map = serializer.serialize_map(Some(2))?; - map.serialize_entry("type", "Resource")?; - map.serialize_entry("body", &Helper(&res.body))?; - map.end() +#[derive(Serialize)] +#[serde(remote = "ast::Resource")] +#[serde(tag = "type")] +#[serde(rename = "Resource")] +pub struct ResourceDef<'ast> { + #[serde(serialize_with = "serialize_resource_entry_vec")] + pub body: Vec>, } fn serialize_resource_entry_vec<'se, S>( @@ -104,7 +97,7 @@ where #[derive(Serialize)] #[serde(remote = "ast::Message")] pub struct MessageDef<'ast> { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] pub id: ast::Identifier<'ast>, #[serde(serialize_with = "serialize_pattern_option")] pub value: Option>, @@ -114,10 +107,30 @@ pub struct MessageDef<'ast> { pub comment: Option>, } +#[derive(Serialize)] +#[serde(remote = "ast::Identifier")] +#[serde(tag = "type")] +#[serde(rename = "Identifier")] +pub struct IdentifierDef<'ast> { + pub name: &'ast str, +} + +#[derive(Serialize)] +#[serde(remote = "ast::Variant")] +#[serde(tag = "type")] +#[serde(rename = "Variant")] +pub struct VariantDef<'ast> { + #[serde(with = "VariantKeyDef")] + pub key: ast::VariantKey<'ast>, + #[serde(with = "ValueDef")] + pub value: ast::Value<'ast>, + pub default: bool, +} + #[derive(Serialize)] #[serde(remote = "ast::Term")] pub struct TermDef<'ast> { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] pub id: ast::Identifier<'ast>, #[serde(with = "ValueDef")] pub value: ast::Value<'ast>, @@ -135,7 +148,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(serialize_with = "serialize_pattern")] &'ast ast::Pattern<'ast>); + struct Helper<'ast>(#[serde(with = "PatternDef")] &'ast ast::Pattern<'ast>); v.as_ref().map(Helper).serialize(serializer) } @@ -147,9 +160,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>( - #[serde(serialize_with = "serialize_attribute")] &'ast ast::Attribute<'ast>, - ); + struct Helper<'ast>(#[serde(with = "AttributeDef")] &'ast ast::Attribute<'ast>); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; @@ -173,7 +184,7 @@ where #[serde(remote = "ast::Value")] #[serde(tag = "type")] pub enum ValueDef<'ast> { - #[serde(serialize_with = "serialize_pattern")] + #[serde(with = "PatternDef")] Pattern(ast::Pattern<'ast>), VariantList { #[serde(serialize_with = "serialize_variants")] @@ -181,20 +192,13 @@ pub enum ValueDef<'ast> { }, } -fn serialize_pattern<'se, S>(pattern: &'se ast::Pattern, serializer: S) -> Result -where - S: Serializer, -{ - #[derive(Serialize)] - struct Helper<'ast>( - #[serde(serialize_with = "serialize_pattern_elements")] - &'ast Vec>, - ); - - let mut map = serializer.serialize_map(Some(2))?; - map.serialize_entry("type", "Pattern")?; - map.serialize_entry("elements", &Helper(&pattern.elements))?; - map.end() +#[derive(Serialize)] +#[serde(remote = "ast::Pattern")] +#[serde(tag = "type")] +#[serde(rename = "Pattern")] +pub struct PatternDef<'ast> { + #[serde(serialize_with = "serialize_pattern_elements")] + pub elements: Vec>, } fn serialize_pattern_elements<'se, S>( @@ -261,58 +265,6 @@ where map.end() } -fn serialize_attribute<'se, S>( - attribute: &'se ast::Attribute, - serializer: S, -) -> Result -where - S: Serializer, -{ - #[derive(Serialize)] - struct IdHelper<'ast>( - #[serde(serialize_with = "serialize_identifier")] &'ast ast::Identifier<'ast>, - ); - - #[derive(Serialize)] - struct ValueHelper<'ast>( - #[serde(serialize_with = "serialize_pattern")] &'ast ast::Pattern<'ast>, - ); - - let mut map = serializer.serialize_map(Some(3))?; - map.serialize_entry("type", "Attribute")?; - map.serialize_entry("id", &IdHelper(&attribute.id))?; - map.serialize_entry("value", &ValueHelper(&attribute.value))?; - map.end() -} - -fn serialize_identifier<'se, S>(id: &'se ast::Identifier, serializer: S) -> Result -where - S: Serializer, -{ - let mut map = serializer.serialize_map(Some(2))?; - map.serialize_entry("type", "Identifier")?; - map.serialize_entry("name", id.name)?; - map.end() -} - -fn serialize_variant<'se, S>(variant: &'se ast::Variant, serializer: S) -> Result -where - S: Serializer, -{ - #[derive(Serialize)] - struct KeyHelper<'ast>(#[serde(with = "VariantKeyDef")] &'ast ast::VariantKey<'ast>); - - #[derive(Serialize)] - struct ValueHelper<'ast>(#[serde(with = "ValueDef")] &'ast ast::Value<'ast>); - - let mut map = serializer.serialize_map(Some(4))?; - map.serialize_entry("type", "Variant")?; - map.serialize_entry("key", &KeyHelper(&variant.key))?; - map.serialize_entry("value", &ValueHelper(&variant.value))?; - map.serialize_entry("default", &variant.default)?; - map.end() -} - #[derive(Serialize, Debug)] #[serde(remote = "ast::VariantKey")] #[serde(tag = "type")] @@ -358,7 +310,7 @@ pub enum InlineExpressionDef<'ast> { value: &'ast str, }, VariableReference { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] id: ast::Identifier<'ast>, }, CallExpression { @@ -373,7 +325,7 @@ pub enum InlineExpressionDef<'ast> { #[serde(with = "InlineExpressionDef")] #[serde(rename = "ref")] reference: ast::InlineExpression<'ast>, - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] name: ast::Identifier<'ast>, }, VariantExpression { @@ -384,15 +336,15 @@ pub enum InlineExpressionDef<'ast> { key: ast::VariantKey<'ast>, }, MessageReference { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] id: ast::Identifier<'ast>, }, TermReference { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] id: ast::Identifier<'ast>, }, FunctionReference { - #[serde(serialize_with = "serialize_identifier")] + #[serde(with = "IdentifierDef")] id: ast::Identifier<'ast>, }, Placeable { @@ -412,28 +364,26 @@ where map.end() } -fn serialize_named_argument<'se, S>( - arg: &'se ast::NamedArgument, - serializer: S, -) -> Result -where - S: Serializer, -{ - #[derive(Serialize)] - struct IdentifierHelper<'ast>( - #[serde(serialize_with = "serialize_identifier")] &'ast ast::Identifier<'ast>, - ); - - #[derive(Serialize)] - struct InlineExpressionHelper<'ast>( - #[serde(with = "InlineExpressionDef")] &'ast ast::InlineExpression<'ast>, - ); +#[derive(Serialize)] +#[serde(remote = "ast::Attribute")] +#[serde(tag = "type")] +#[serde(rename = "Attribute")] +pub struct AttributeDef<'ast> { + #[serde(with = "IdentifierDef")] + pub id: ast::Identifier<'ast>, + #[serde(with = "PatternDef")] + pub value: ast::Pattern<'ast>, +} - let mut map = serializer.serialize_map(Some(3))?; - map.serialize_entry("type", "NamedArgument")?; - map.serialize_entry("name", &IdentifierHelper(&arg.name))?; - map.serialize_entry("value", &InlineExpressionHelper(&arg.value))?; - map.end() +#[derive(Serialize)] +#[serde(remote = "ast::NamedArgument")] +#[serde(tag = "type")] +#[serde(rename = "NamedArgument")] +pub struct NamedArgumentDef<'ast> { + #[serde(with = "IdentifierDef")] + pub name: ast::Identifier<'ast>, + #[serde(with = "InlineExpressionDef")] + pub value: ast::InlineExpression<'ast>, } #[derive(Serialize)] @@ -455,7 +405,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>(#[serde(serialize_with = "serialize_variant")] &'ast ast::Variant<'ast>); + struct Helper<'ast>(#[serde(with = "VariantDef")] &'ast ast::Variant<'ast>); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; @@ -487,9 +437,7 @@ where S: Serializer, { #[derive(Serialize)] - struct Helper<'ast>( - #[serde(serialize_with = "serialize_named_argument")] &'ast ast::NamedArgument<'ast>, - ); + struct Helper<'ast>(#[serde(with = "NamedArgumentDef")] &'ast ast::NamedArgument<'ast>); let mut seq = serializer.serialize_seq(Some(v.len()))?; for e in v { seq.serialize_element(&Helper(e))?; From 2242b296a50c392d0bb4d0a2e8190ca3f5ab2463 Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Tue, 11 Dec 2018 16:59:53 -0800 Subject: [PATCH 13/36] Add docs for FluentBundle Added docs for FluentBundle. Also, corrected some errors in the pre-existing docs. --- fluent-bundle/src/bundle.rs | 163 ++++++++++++++++++++++++++++++++++-- 1 file changed, 156 insertions(+), 7 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 07d09071..54d99ec5 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -22,21 +22,40 @@ pub struct Message { pub attributes: HashMap, } -/// `FluentBundle` is a collection of localization messages which are meant to be used together -/// in a single view, widget or any other UI abstraction. +/// A collection of localization messages for a single locale, which are meant +/// to be used together in a single view, widget or any other UI abstraction. /// -/// # `FluentBundle` Life-cycle +/// # Examples /// -/// To create a bundle, call `FluentBundle::new` with a locale list that represents the best -/// possible fallback chain for a given locale. The simplest case is a one-locale list. +/// ``` +/// use fluent::bundle::FluentBundle; +/// use fluent::types::FluentValue; +/// use std::collections::HashMap; /// -/// Next, call `add_messages` one or more times, supplying translations in the FTL syntax. The +/// let mut bundle = FluentBundle::new(&["en-US"]); +/// bundle.add_messages("intro = Welcome, { $name }."); +/// +/// let mut args = HashMap::new(); +/// args.insert("name", FluentValue::from("John")); +/// +/// let value = bundle.format("intro", Some(&args)); +/// assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); +/// +/// ``` +/// +/// # `FluentBundle` Life Cycle +/// +/// To create a bundle, call [`FluentBundle::new`] with a locale list that represents the best +/// possible fallback chain for a given locale. The simplest case is a one-locale list. +/// +/// Next, call [`add_resource`] one or more times, supplying translations in the FTL syntax. The /// `FluentBundle` instance is now ready to be used for localization. /// /// To format a translation, call `get_message` to retrieve a `fluent_bundle::bundle::Message` structure /// and then `format` it within the bundle. /// -/// The result is an Option wrapping a single string that should be displayed in the UI. It is +/// The result is an [`Option`] wrapping a `(String, Vec)`. On success, the string +/// is a formatted value that should be displayed in the UI. It is /// recommended to treat the result as opaque from the perspective of the program and use it only /// to display localized messages. Do not examine it or alter in any way before displaying. This /// is a general good practice as far as all internationalization operations are concerned. @@ -48,6 +67,13 @@ pub struct Message { /// purpose of language negotiation with i18n formatters. For instance, if date and time formatting /// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain /// to negotiate a sensible fallback for date and time formatting. +/// +/// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource +/// [`FluentBundle::new`]: ./struct.FluentBundle.html#method.new +/// [`fluent::bundle::Message`]: ./struct.FluentBundle.html#method.new +/// [`format`]: ./struct.FluentBundle.html#method.format +/// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource +/// [`Option`]: http://doc.rust-lang.org/std/option/enum.Option.html #[allow(dead_code)] pub struct FluentBundle<'bundle> { pub locales: Vec, @@ -56,6 +82,21 @@ pub struct FluentBundle<'bundle> { } impl<'bundle> FluentBundle<'bundle> { + /// Constructs a FluentBundle. `locales` is the fallback chain of locales + /// to use for formatters like date and time. `locales` does not influence + /// message selection. + /// + /// # Examples + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// ``` + /// + /// # Errors + /// + /// This will panic if no formatters can be found for the locales. pub fn new<'a, S: ToString>(locales: &'a [S]) -> FluentBundle<'bundle> { let locales = locales.iter().map(|s| s.to_string()).collect::>(); let pr_locale = negotiate_languages( @@ -74,10 +115,48 @@ impl<'bundle> FluentBundle<'bundle> { } } + /// Returns true if this bundle contains a message with the given id. + /// + /// # Examples + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_messages("hello = Hi!"); + /// assert_eq!(true, bundle.has_message("hello")); + /// ``` pub fn has_message(&self, id: &str) -> bool { self.entries.get_message(id).is_some() } + /// Makes the provided rust function available to messages with the name `id`. See + /// the [FTL syntax guide] to learn how these are used in messages. + /// + /// FTL functions accept both positional and named args. The rust function you + /// provide therefore has two parameters: a slice of values for the positional + /// args, and a HashMap of values for named args. + /// + /// # Examples + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// use fluent::types::FluentValue; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// + /// // Register a fn that maps from string to string length + /// bundle.add_function("STRLEN", |positional, named| match positional { + /// [Some(FluentValue::String(str))] => Some(FluentValue::Number(str.len().to_string())), + /// _ => None, + /// }).unwrap(); + /// + /// bundle.add_messages("length = { STRLEN(\"12345\") }").unwrap(); + /// let (value, _) = bundle.format("length", None).unwrap(); + /// assert_eq!(&value, "5"); + /// ``` + /// + /// [FTL syntax guide]: https://projectfluent.org/fluent/guide/functions.html pub fn add_function(&mut self, id: &str, func: F) -> Result<(), FluentError> where F: 'bundle @@ -97,6 +176,33 @@ impl<'bundle> FluentBundle<'bundle> { } } + /// Adds the message or messages, in [FTL syntax], to the bundle, returning an + /// empty [`Result`] on success. + /// + /// # Examples + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_messages(" + /// hello = Hi! + /// goodbye = Bye! + /// "); + /// assert_eq!(true, bundle.has_message("hello")); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result`]: https://doc.rust-lang.org/std/result/enum.Result.html pub fn add_resource(&mut self, res: &'bundle FluentResource) -> Result<(), Vec> { let mut errors = vec![]; @@ -135,6 +241,48 @@ impl<'bundle> FluentBundle<'bundle> { } } + /// Formats the message value identified by `path` using `args`. + /// `path` is either a message id ("hello"), or message id plus + /// attribute ("hello.tooltip"). + /// + /// # Examples + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// use fluent::types::FluentValue; + /// use std::collections::HashMap; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_messages("intro = Welcome, { $name }."); + /// + /// let mut args = HashMap::new(); + /// args.insert("name", FluentValue::from("John")); + /// + /// let value = bundle.format("intro", Some(&args)); + /// assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); + /// + /// ``` + /// + /// An example with attributes and no args: + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_messages(" + /// hello = + /// .title = Hi! + /// .tooltip = This says 'Hi!' + /// "); + /// + /// let value = bundle.format("hello.title", None); + /// assert_eq!(value, Some(("Hi!".to_string(), vec![]))); + /// ``` + /// + /// # Errors + /// + /// On error, the string returned will be the path. This allows + /// for slightly graceful fallback during errors. pub fn format( &self, path: &str, @@ -186,6 +334,7 @@ impl<'bundle> FluentBundle<'bundle> { None } + /// Use [`format`](./struct.FluentBundle.html#method.format) instead. pub fn format_message( &self, message_id: &str, From bf6a75ec22b713d7ac94e8ff54fa135302556bdd Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Tue, 11 Dec 2018 17:14:46 -0800 Subject: [PATCH 14/36] Anonymized example. Tiny style tweak. --- fluent-bundle/src/bundle.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 54d99ec5..78d3d5e9 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -36,10 +36,10 @@ pub struct Message { /// bundle.add_messages("intro = Welcome, { $name }."); /// /// let mut args = HashMap::new(); -/// args.insert("name", FluentValue::from("John")); +/// args.insert("name", FluentValue::from("Rustacean")); /// /// let value = bundle.format("intro", Some(&args)); -/// assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); +/// assert_eq!(value, Some(("Welcome, Rustacean.".to_string(), vec![]))); /// /// ``` /// @@ -146,7 +146,7 @@ impl<'bundle> FluentBundle<'bundle> { /// let mut bundle = FluentBundle::new(&["en-US"]); /// /// // Register a fn that maps from string to string length - /// bundle.add_function("STRLEN", |positional, named| match positional { + /// bundle.add_function("STRLEN", |positional, _named| match positional { /// [Some(FluentValue::String(str))] => Some(FluentValue::Number(str.len().to_string())), /// _ => None, /// }).unwrap(); @@ -256,10 +256,10 @@ impl<'bundle> FluentBundle<'bundle> { /// bundle.add_messages("intro = Welcome, { $name }."); /// /// let mut args = HashMap::new(); - /// args.insert("name", FluentValue::from("John")); + /// args.insert("name", FluentValue::from("Rustacean")); /// /// let value = bundle.format("intro", Some(&args)); - /// assert_eq!(value, Some(("Welcome, John.".to_string(), vec![]))); + /// assert_eq!(value, Some(("Welcome, Rustacean.".to_string(), vec![]))); /// /// ``` /// From bd74a4a1814ddd4d61bb5a25eafe5a9f755c41d1 Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Wed, 12 Dec 2018 15:41:48 -0800 Subject: [PATCH 15/36] Corrected error description for FluentBundle.format --- fluent-bundle/src/bundle.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 78d3d5e9..2e2b0d4e 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -281,8 +281,10 @@ impl<'bundle> FluentBundle<'bundle> { /// /// # Errors /// - /// On error, the string returned will be the path. This allows - /// for slightly graceful fallback during errors. + /// If the message id or path is not found in the bundle, `format` + /// returns None. On Fluent processing errors after initial lookup + /// `format` returns `Some((path, errors)`. `path` is the path you + /// originally provided, and `errors` explains what went wrong. pub fn format( &self, path: &str, From b9f0886c341e845c646894ac6628a2b1518d35fe Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Wed, 12 Dec 2018 18:11:20 -0800 Subject: [PATCH 16/36] More clarification on the various flavors of format error. --- fluent-bundle/src/bundle.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 2e2b0d4e..25baf482 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -281,10 +281,28 @@ impl<'bundle> FluentBundle<'bundle> { /// /// # Errors /// - /// If the message id or path is not found in the bundle, `format` - /// returns None. On Fluent processing errors after initial lookup - /// `format` returns `Some((path, errors)`. `path` is the path you - /// originally provided, and `errors` explains what went wrong. + /// If no message is found at `path`, then `format` returns `None`. + /// + /// In all other cases, `format` returns a string even if it + /// encountered errors. `format` uses two fallback techniques to + /// create the fallback string. If there are bad references in the + /// message, then they will be substituted with `'___'`. If there + /// are more extensive errors, then `format` will fall back to using + /// `path` itself as the formatted string. Sometimes, but not always, + /// these partial failures will emit extra error information in the + /// second term of the return tuple. + /// + /// ``` + /// use fluent::bundle::FluentBundle; + /// + /// // Create a message with bad cyclic reference + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_messages("foo = a { foo } b"); + /// + /// // The result falls back to "___" + /// let value = bundle.format("foo", None); + /// assert_eq!(value, Some(("___".to_string(), vec![]))); + /// ``` pub fn format( &self, path: &str, From c555be57cb20102bfbd0976d43ef7dc8e6cf2c61 Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Mon, 7 Jan 2019 14:42:39 -0800 Subject: [PATCH 17/36] Merge fixes --- fluent-bundle/src/bundle.rs | 66 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 25baf482..3dcdcbb2 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -28,12 +28,14 @@ pub struct Message { /// # Examples /// /// ``` -/// use fluent::bundle::FluentBundle; -/// use fluent::types::FluentValue; +/// use fluent_bundle::bundle::FluentBundle; +/// use fluent_bundle::resource::FluentResource; +/// use fluent_bundle::types::FluentValue; /// use std::collections::HashMap; /// +/// let resource = FluentResource::try_new("intro = Welcome, { $name }.".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); -/// bundle.add_messages("intro = Welcome, { $name }."); +/// bundle.add_resource(&resource); /// /// let mut args = HashMap::new(); /// args.insert("name", FluentValue::from("Rustacean")); @@ -89,7 +91,7 @@ impl<'bundle> FluentBundle<'bundle> { /// # Examples /// /// ``` - /// use fluent::bundle::FluentBundle; + /// use fluent_bundle::bundle::FluentBundle; /// /// let mut bundle = FluentBundle::new(&["en-US"]); /// ``` @@ -120,11 +122,14 @@ impl<'bundle> FluentBundle<'bundle> { /// # Examples /// /// ``` - /// use fluent::bundle::FluentBundle; + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; /// + /// let resource = FluentResource::try_new("hello = Hi!".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); - /// bundle.add_messages("hello = Hi!"); + /// bundle.add_resource(&resource); /// assert_eq!(true, bundle.has_message("hello")); + /// /// ``` pub fn has_message(&self, id: &str) -> bool { self.entries.get_message(id).is_some() @@ -140,10 +145,13 @@ impl<'bundle> FluentBundle<'bundle> { /// # Examples /// /// ``` - /// use fluent::bundle::FluentBundle; - /// use fluent::types::FluentValue; + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; /// + /// let resource = FluentResource::try_new("length = { STRLEN(\"12345\") }".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); /// /// // Register a fn that maps from string to string length /// bundle.add_function("STRLEN", |positional, _named| match positional { @@ -151,7 +159,6 @@ impl<'bundle> FluentBundle<'bundle> { /// _ => None, /// }).unwrap(); /// - /// bundle.add_messages("length = { STRLEN(\"12345\") }").unwrap(); /// let (value, _) = bundle.format("length", None).unwrap(); /// assert_eq!(&value, "5"); /// ``` @@ -182,13 +189,15 @@ impl<'bundle> FluentBundle<'bundle> { /// # Examples /// /// ``` - /// use fluent::bundle::FluentBundle; + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; /// - /// let mut bundle = FluentBundle::new(&["en-US"]); - /// bundle.add_messages(" + /// let resource = FluentResource::try_new(" /// hello = Hi! /// goodbye = Bye! - /// "); + /// ".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); /// assert_eq!(true, bundle.has_message("hello")); /// ``` /// @@ -248,12 +257,14 @@ impl<'bundle> FluentBundle<'bundle> { /// # Examples /// /// ``` - /// use fluent::bundle::FluentBundle; - /// use fluent::types::FluentValue; + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; /// use std::collections::HashMap; /// + /// let resource = FluentResource::try_new("intro = Welcome, { $name }.".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); - /// bundle.add_messages("intro = Welcome, { $name }."); + /// bundle.add_resource(&resource); /// /// let mut args = HashMap::new(); /// args.insert("name", FluentValue::from("Rustacean")); @@ -266,14 +277,16 @@ impl<'bundle> FluentBundle<'bundle> { /// An example with attributes and no args: /// /// ``` - /// use fluent::bundle::FluentBundle; + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; /// - /// let mut bundle = FluentBundle::new(&["en-US"]); - /// bundle.add_messages(" + /// let resource = FluentResource::try_new(" /// hello = /// .title = Hi! /// .tooltip = This says 'Hi!' - /// "); + /// ".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&resource); /// /// let value = bundle.format("hello.title", None); /// assert_eq!(value, Some(("Hi!".to_string(), vec![]))); @@ -291,18 +304,6 @@ impl<'bundle> FluentBundle<'bundle> { /// `path` itself as the formatted string. Sometimes, but not always, /// these partial failures will emit extra error information in the /// second term of the return tuple. - /// - /// ``` - /// use fluent::bundle::FluentBundle; - /// - /// // Create a message with bad cyclic reference - /// let mut bundle = FluentBundle::new(&["en-US"]); - /// bundle.add_messages("foo = a { foo } b"); - /// - /// // The result falls back to "___" - /// let value = bundle.format("foo", None); - /// assert_eq!(value, Some(("___".to_string(), vec![]))); - /// ``` pub fn format( &self, path: &str, @@ -354,7 +355,6 @@ impl<'bundle> FluentBundle<'bundle> { None } - /// Use [`format`](./struct.FluentBundle.html#method.format) instead. pub fn format_message( &self, message_id: &str, From 7aef4d1b8b96faacc8946ffc2b9fea0004bc3d27 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 7 Jan 2019 15:45:40 -0800 Subject: [PATCH 18/36] Minor cleanups in the parser --- fluent-syntax/src/parser/mod.rs | 68 ++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index e95f0be3..452aaafc 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -332,7 +332,7 @@ enum TextElementType { fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result>> { let mut elements = vec![]; let mut last_non_blank = None; - let mut common_indent = 10; + let mut common_indent = None; ps.skip_blank_inline(); @@ -346,7 +346,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> while ps.ptr < ps.length { if ps.source[ps.ptr] == b'{' { if text_element_role == TextElementPosition::LineStart { - common_indent = 0; + common_indent = Some(0); } let exp = get_placeable(ps)?; last_non_blank = Some(elements.len()); @@ -374,9 +374,14 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> if start != end { if text_element_role == TextElementPosition::LineStart && text_element_type == TextElementType::NonBlank - && indent < common_indent { - common_indent = indent; + if let Some(common) = common_indent { + if indent < common { + common_indent = Some(indent); + } + } else { + common_indent = Some(indent); + } } if text_element_role != TextElementPosition::LineStart || text_element_type == TextElementType::NonBlank @@ -419,7 +424,11 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> PatternElementPointers::Placeable(exp) => Some(ast::PatternElement::Placeable(exp)), PatternElementPointers::TextElement(start, end, indent, role) => { let start = if role == TextElementPosition::LineStart { - start + cmp::min(indent, common_indent) + if let Some(common_indent) = common_indent { + start + cmp::min(indent, common_indent) + } else { + start + indent + } } else { start }; @@ -448,24 +457,26 @@ fn get_text_slice<'p>( let mut text_element_type = TextElementType::Blank; while ps.ptr < ps.length { - if ps.source[ps.ptr] == b'\n' { - ps.ptr += 1; - return Ok(( - start_pos, - ps.ptr, - text_element_type, - TextElementTermination::LineFeed, - )); - } else if ps.source[ps.ptr] == b'\r' && ps.is_byte_at(b'\n', ps.ptr + 1) { - ps.ptr += 1; - return Ok(( - start_pos, - ps.ptr - 1, - text_element_type, - TextElementTermination::CarriageReturn, - )); - } match ps.source[ps.ptr] { + b' ' => ps.ptr += 1, + b'\n' => { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr, + text_element_type, + TextElementTermination::LineFeed, + )); + } + b'\r' if ps.is_byte_at(b'\n', ps.ptr + 1) => { + ps.ptr += 1; + return Ok(( + start_pos, + ps.ptr - 1, + text_element_type, + TextElementTermination::CarriageReturn, + )); + } b'{' => { return Ok(( start_pos, @@ -489,7 +500,6 @@ fn get_text_slice<'p>( _ => {} } } - b' ' => ps.ptr += 1, _ => { text_element_type = TextElementType::NonBlank; ps.ptr += 1 @@ -530,12 +540,10 @@ fn get_comment<'p>(ps: &mut ParserStream<'p>) -> Result> { ps.skip_eol(); } - let comment = if level == Some(3) { - ast::Comment::ResourceComment { content } - } else if level == Some(2) { - ast::Comment::GroupComment { content } - } else { - ast::Comment::Comment { content } + let comment = match level { + Some(3) => ast::Comment::ResourceComment { content }, + Some(2) => ast::Comment::GroupComment { content }, + _ => ast::Comment::Comment { content }, }; Ok(comment) } @@ -557,7 +565,7 @@ fn get_comment_line<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { ps.ptr += 1; } - Ok(str::from_utf8(&ps.source[start_pos..ps.ptr]).unwrap()) + Ok(ps.get_slice(start_pos, ps.ptr)) } fn get_placeable<'p>(ps: &mut ParserStream<'p>) -> Result> { From 5191b37cea036646742915a8a54fee6b8069164d Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Mon, 7 Jan 2019 15:54:45 -0800 Subject: [PATCH 19/36] Speed up the parser by reducing the Pattern logic --- fluent-syntax/src/parser/mod.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 452aaafc..273b5d71 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -409,16 +409,9 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> } if let Some(last_non_blank) = last_non_blank { - let mut collected_elems = 0; - let elements = elements.into_iter().filter(|_| { - if collected_elems > last_non_blank { - return false; - } - collected_elems += 1; - true - }); - let elements = elements + .into_iter() + .take(last_non_blank + 1) .enumerate() .filter_map(|(i, elem)| match elem { PatternElementPointers::Placeable(exp) => Some(ast::PatternElement::Placeable(exp)), From 37201d9c5cb58c0536b88bbc7cce3a83bdd3a9b4 Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Mon, 7 Jan 2019 19:31:56 -0800 Subject: [PATCH 20/36] Improve FluentBundle docs * Added error example to format() * Documented format_message() --- fluent-bundle/src/bundle.rs | 57 +++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 3dcdcbb2..688eb081 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -250,9 +250,9 @@ impl<'bundle> FluentBundle<'bundle> { } } - /// Formats the message value identified by `path` using `args`. - /// `path` is either a message id ("hello"), or message id plus - /// attribute ("hello.tooltip"). + /// Formats the message value identified by `path` using `args` to + /// provide variables. `path` is either a message id ("hello"), or + /// message id plus attribute ("hello.tooltip"). /// /// # Examples /// @@ -304,6 +304,20 @@ impl<'bundle> FluentBundle<'bundle> { /// `path` itself as the formatted string. Sometimes, but not always, /// these partial failures will emit extra error information in the /// second term of the return tuple. + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// + /// // Create a message with bad cyclic reference + /// let mut res = FluentResource::try_new("foo = a { foo } b".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&res); + /// + /// // The result falls back to "___" + /// let value = bundle.format("foo", None); + /// assert_eq!(value, Some(("___".to_string(), vec![]))); + /// ``` pub fn format( &self, path: &str, @@ -355,6 +369,43 @@ impl<'bundle> FluentBundle<'bundle> { None } + /// Formats both the message value and attributes identified by `message_id` + /// using `args` to provide variables. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::resource::FluentResource; + /// use fluent_bundle::types::FluentValue; + /// use std::collections::HashMap; + /// + /// let mut res = FluentResource::try_new(" + /// login-input = Predefined value + /// .placeholder = example@email.com + /// .aria-label = Login input value + /// .title = Type your login email".to_string()).unwrap(); + /// let mut bundle = FluentBundle::new(&["en-US"]); + /// bundle.add_resource(&res); + /// + /// let (message, _) = bundle.format_message("login-input", None).unwrap(); + /// assert_eq!(message.value, Some("Predefined value".to_string())); + /// assert_eq!(message.attributes.get("title"), Some(&"Type your login email".to_string())); + /// ``` + /// + /// # Errors + /// + /// If no message is found at `message_id`, then `format_message` + /// returns `None`. + /// + /// In all other cases, `format_message` returns a `Message` even if it + /// encountered errors. `format_message` uses two fallback techniques to + /// create the fallback string. If there are bad references in the + /// message, then they will be substituted with `'___'`. If there + /// are more extensive errors, then `format_message` will fall back to using + /// `path` itself as the formatted string. Sometimes, but not always, + /// these partial failures will emit extra error information in the + /// second term of the return tuple. pub fn format_message( &self, message_id: &str, From bd3319d9f9ea4708c18886b99a4ef790bf100694 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 09:45:25 -0800 Subject: [PATCH 21/36] A bunch of small cleanups to prepare for fuzzer --- fluent-bundle/src/bundle.rs | 1 - fluent-syntax/src/parser/mod.rs | 66 ++++++++++++++++----------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 07d09071..7840fc19 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -48,7 +48,6 @@ pub struct Message { /// purpose of language negotiation with i18n formatters. For instance, if date and time formatting /// are not available in the first locale, `FluentBundle` will use its `locales` fallback chain /// to negotiate a sensible fallback for date and time formatting. -#[allow(dead_code)] pub struct FluentBundle<'bundle> { pub locales: Vec, pub entries: HashMap>, diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 273b5d71..21ce1893 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -89,13 +89,10 @@ fn get_message<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result attrs, - Err(_err) => { - ps.ptr = ptr; - vec![] - } - }; + let attributes = get_attributes(ps).unwrap_or_else(|_| { + ps.ptr = ptr; + vec![] + }); if pattern.is_none() && attributes.is_empty() { return error!( @@ -126,13 +123,10 @@ fn get_term<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result attrs, - Err(_err) => { - ps.ptr = ptr; - vec![] - } - }; + let attributes = get_attributes(ps).unwrap_or_else(|_| { + ps.ptr = ptr; + vec![] + }); if let Some(value) = value { Ok(ast::Term { @@ -344,7 +338,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> }; while ps.ptr < ps.length { - if ps.source[ps.ptr] == b'{' { + if ps.is_current_byte(b'{') { if text_element_role == TextElementPosition::LineStart { common_indent = Some(0); } @@ -483,10 +477,9 @@ fn get_text_slice<'p>( } b'\\' => { text_element_type = TextElementType::NonBlank; - ps.ptr += 1; - match ps.source[ps.ptr] { - b'\\' => ps.ptr += 1, - b'u' => { + match ps.source.get(ps.ptr) { + Some(b'\\') => ps.ptr += 1, + Some(b'u') => { ps.ptr += 1; ps.skip_unicode_escape_sequence(4)?; } @@ -715,15 +708,15 @@ fn get_literal<'p>(ps: &mut ParserStream<'p>) -> Result match ps.source[ps.ptr + 1] { - b'\\' => ps.ptr += 2, - b'{' => ps.ptr += 2, - b'"' => ps.ptr += 2, - b'u' => { + b'\\' => match ps.source.get(ps.ptr + 1) { + Some(b'\\') => ps.ptr += 2, + Some(b'{') => ps.ptr += 2, + Some(b'"') => ps.ptr += 2, + Some(b'u') => { ps.ptr += 2; ps.skip_unicode_escape_sequence(4)?; } - b'U' => { + Some(b'U') => { ps.ptr += 2; ps.skip_unicode_escape_sequence(6)?; } @@ -751,8 +744,8 @@ fn get_literal<'p>(ps: &mut ParserStream<'p>) -> Result { + match ps.source.get(ps.ptr) { + Some(b'[') => { let key = get_variant_key(ps)?; Ok(ast::InlineExpression::VariantExpression { reference: Box::new(ast::InlineExpression::TermReference { id }), @@ -797,8 +790,7 @@ fn get_call_args<'p>( ps.skip_blank(); while ps.ptr < ps.length { - let b = ps.source[ps.ptr]; - if b == b')' { + if ps.is_current_byte(b')') { break; } @@ -849,12 +841,20 @@ fn get_call_args<'p>( fn get_number_literal<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { let start = ps.ptr; ps.take_if(b'-'); - while ps.source[ps.ptr] >= b'0' && ps.source[ps.ptr] <= b'9' { - ps.ptr += 1; + while let Some(b) = ps.source.get(ps.ptr) { + if b >= &b'0' && b <= &b'9' { + ps.ptr += 1; + } else { + break; + } } ps.take_if(b'.'); - while ps.source[ps.ptr] >= b'0' && ps.source[ps.ptr] <= b'9' { - ps.ptr += 1; + while let Some(b) = ps.source.get(ps.ptr) { + if b >= &b'0' && b <= &b'9' { + ps.ptr += 1; + } else { + break; + } } Ok(ps.get_slice(start, ps.ptr)) From 79410547a02880ceb3ff641288ad978074656c28 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 11:42:12 -0800 Subject: [PATCH 22/36] Move ast::Variant::value to be a Pattern --- fluent-syntax/src/ast.rs | 2 +- fluent-syntax/src/parser/mod.rs | 2 +- fluent-syntax/tests/ast/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fluent-syntax/src/ast.rs b/fluent-syntax/src/ast.rs index 2959e1d1..f52e1719 100644 --- a/fluent-syntax/src/ast.rs +++ b/fluent-syntax/src/ast.rs @@ -63,7 +63,7 @@ pub struct Identifier<'ast> { #[derive(Debug, PartialEq)] pub struct Variant<'ast> { pub key: VariantKey<'ast>, - pub value: Value<'ast>, + pub value: Pattern<'ast>, pub default: bool, } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 21ce1893..d06b91f5 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -275,7 +275,7 @@ fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> let key = get_variant_key(ps)?; - let value = get_pattern(ps)?.map(ast::Value::Pattern); + let value = get_pattern(ps)?; if let Some(value) = value { variants.push(ast::Variant { diff --git a/fluent-syntax/tests/ast/mod.rs b/fluent-syntax/tests/ast/mod.rs index 4d4d13f3..807b850b 100644 --- a/fluent-syntax/tests/ast/mod.rs +++ b/fluent-syntax/tests/ast/mod.rs @@ -122,8 +122,8 @@ pub struct IdentifierDef<'ast> { pub struct VariantDef<'ast> { #[serde(with = "VariantKeyDef")] pub key: ast::VariantKey<'ast>, - #[serde(with = "ValueDef")] - pub value: ast::Value<'ast>, + #[serde(with = "PatternDef")] + pub value: ast::Pattern<'ast>, pub default: bool, } From 8a64bbe4aba6fa8b60b688f3d58939544c629dca Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 11:58:59 -0800 Subject: [PATCH 23/36] Add a note explaining the deviation from the EBNF --- fluent-syntax/src/ast.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fluent-syntax/src/ast.rs b/fluent-syntax/src/ast.rs index f52e1719..30dae45f 100644 --- a/fluent-syntax/src/ast.rs +++ b/fluent-syntax/src/ast.rs @@ -110,6 +110,9 @@ pub enum InlineExpression<'ast> { TermReference { id: Identifier<'ast>, }, + // This node is standalone in EBNF, but it + // is more convinient for us to store it a + // variant of the InlineExpression in Rust. FunctionReference { id: Identifier<'ast>, }, From fb569af4c99be579a2e06f85c09b9b2d3aea155f Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 12:01:16 -0800 Subject: [PATCH 24/36] Remove the unnecessary externs --- fluent-bundle/README.md | 2 -- fluent-bundle/src/lib.rs | 4 ---- 2 files changed, 6 deletions(-) diff --git a/fluent-bundle/README.md b/fluent-bundle/README.md index f620b43e..200f7f5d 100644 --- a/fluent-bundle/README.md +++ b/fluent-bundle/README.md @@ -22,8 +22,6 @@ Usage ----- ```rust -extern crate fluent; - use fluent_bundle::FluentBundle; fn main() { diff --git a/fluent-bundle/src/lib.rs b/fluent-bundle/src/lib.rs index 17228b3e..c36d5f58 100644 --- a/fluent-bundle/src/lib.rs +++ b/fluent-bundle/src/lib.rs @@ -37,12 +37,8 @@ #[macro_use] extern crate rental; -extern crate failure; #[macro_use] extern crate failure_derive; -extern crate fluent_locale; -extern crate fluent_syntax; -extern crate intl_pluralrules; pub mod bundle; pub mod entry; From a2fa84396ec20d290880d08e0ee448b6bd3bf22f Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 12:54:49 -0800 Subject: [PATCH 25/36] Improve get_number_literal to report errors on broken numbers --- fluent-syntax/src/parser/ftlstream.rs | 34 +++++++++++++++++---------- fluent-syntax/src/parser/mod.rs | 17 +++----------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index 4205a3eb..28824120 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -24,10 +24,6 @@ impl<'p> ParserStream<'p> { self.source[self.ptr] == b } - pub fn _get_current_byte(&self) -> String { - str::from_utf8(&[self.source[self.ptr]]).unwrap().to_owned() - } - pub fn is_byte_at(&self, b: u8, pos: usize) -> bool { if pos >= self.length { return false; @@ -125,14 +121,6 @@ impl<'p> ParserStream<'p> { false } - pub fn _is_entry_start(&self) -> bool { - if self.ptr >= self.length { - return false; - } - let b = self.source[self.ptr]; - (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') || b == b'-' - } - pub fn skip_to_value_start(&mut self) -> Option { self.skip_blank_inline(); @@ -145,7 +133,6 @@ impl<'p> ParserStream<'p> { let inline = self.skip_blank_inline(); if self.is_current_byte(b'{') { - //self.ptr -= inline; return Some(true); } @@ -217,4 +204,25 @@ impl<'p> ParserStream<'p> { pub fn get_slice(&self, start: usize, end: usize) -> &'p str { unsafe { str::from_utf8_unchecked(&self.source[start..end]) } } + + pub fn skip_digits(&mut self) -> Result<()> { + let start = self.ptr; + while let Some(b) = self.source.get(self.ptr) { + if b >= &b'0' && b <= &b'9' { + self.ptr += 1; + } else { + break; + } + } + if start == self.ptr { + error!( + ErrorKind::ExpectedCharRange { + range: "0-9".to_string() + }, + self.ptr + ) + } else { + Ok(()) + } + } } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index d06b91f5..bb75d7e1 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -841,20 +841,9 @@ fn get_call_args<'p>( fn get_number_literal<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { let start = ps.ptr; ps.take_if(b'-'); - while let Some(b) = ps.source.get(ps.ptr) { - if b >= &b'0' && b <= &b'9' { - ps.ptr += 1; - } else { - break; - } - } - ps.take_if(b'.'); - while let Some(b) = ps.source.get(ps.ptr) { - if b >= &b'0' && b <= &b'9' { - ps.ptr += 1; - } else { - break; - } + ps.skip_digits()?; + if ps.take_if(b'.') { + ps.skip_digits()?; } Ok(ps.get_slice(start, ps.ptr)) From d629ae2d2a23c2771ff86396ed5ed788eee08971 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Tue, 8 Jan 2019 19:15:28 -0800 Subject: [PATCH 26/36] Various small perf optimizations and pointer overflow prevention --- fluent-syntax/src/parser/ftlstream.rs | 97 ++++++++++----------------- fluent-syntax/src/parser/mod.rs | 2 +- 2 files changed, 38 insertions(+), 61 deletions(-) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index 28824120..e116d640 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -18,17 +18,11 @@ impl<'p> ParserStream<'p> { } pub fn is_current_byte(&self, b: u8) -> bool { - if self.ptr >= self.length { - return false; - } - self.source[self.ptr] == b + self.source.get(self.ptr) == Some(&b) } pub fn is_byte_at(&self, b: u8, pos: usize) -> bool { - if pos >= self.length { - return false; - } - self.source[pos] == b + self.source.get(pos) == Some(&b) } pub fn expect_byte(&mut self, b: u8) -> Result<()> { @@ -63,24 +57,20 @@ impl<'p> ParserStream<'p> { } pub fn skip_blank(&mut self) { - while self.ptr < self.length { - let b = self.source[self.ptr]; - if b == b' ' || b == b'\n' { - self.ptr += 1; - } else { - break; + loop { + match self.source.get(self.ptr) { + Some(b' ') | Some(b'\n') => self.ptr += 1, + _ => break, } } } pub fn skip_blank_inline(&mut self) -> usize { let start = self.ptr; - while self.ptr < self.length { - let b = self.source[self.ptr]; - if b == b' ' { - self.ptr += 1; - } else { - break; + loop { + match self.source.get(self.ptr) { + Some(b' ') => self.ptr += 1, + _ => break, } } self.ptr - start @@ -105,20 +95,17 @@ impl<'p> ParserStream<'p> { } pub fn skip_eol(&mut self) -> bool { - if self.ptr >= self.length { - return false; - } - - if self.is_current_byte(b'\n') { - self.ptr += 1; - return true; - } - - if self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) { - self.ptr += 2; - return true; + match self.source.get(self.ptr) { + Some(b'\n') => { + self.ptr += 1; + true + } + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => { + self.ptr += 2; + true + } + _ => false, } - false } pub fn skip_to_value_start(&mut self) -> Option { @@ -150,10 +137,8 @@ impl<'p> ParserStream<'p> { pub fn skip_unicode_escape_sequence(&mut self, length: usize) -> Result<()> { let start = self.ptr; for _ in 0..length { - match self.source[self.ptr] { - b'0'...b'9' => self.ptr += 1, - b'a'...b'f' => self.ptr += 1, - b'A'...b'F' => self.ptr += 1, + match self.source.get(self.ptr) { + Some(b'0'...b'9') | Some(b'a'...b'f') | Some(b'A'...b'F') => self.ptr += 1, _ => break, } } @@ -169,36 +154,29 @@ impl<'p> ParserStream<'p> { } pub fn is_char_pattern_continuation(&self) -> bool { - if self.ptr >= self.length { - return false; + match self.source.get(self.ptr) { + Some(b) if b == &b'}' || b == &b'.' || b == &b'[' || b == &b'*' => false, + _ => true, } - - let b = self.source[self.ptr]; - b != b'}' && b != b'.' && b != b'[' && b != b'*' } pub fn is_identifier_start(&self) -> bool { - if self.ptr >= self.length { - return false; + match self.source.get(self.ptr) { + Some(b) if (b >= &b'a' && b <= &b'z') || (b >= &b'A' && b <= &b'Z') => true, + _ => false, } - let b = self.source[self.ptr]; - (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') } pub fn is_number_start(&self) -> bool { - if self.ptr >= self.length { - return false; + match self.source.get(self.ptr) { + Some(b) if (b == &b'-') || (b >= &b'0' && b <= &b'9') => true, + _ => false, } - let b = self.source[self.ptr]; - b == b'-' || (b >= b'0' && b <= b'9') } pub fn is_eol(&self) -> bool { - if self.is_current_byte(b'\n') { - return true; - } - - self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) + self.is_current_byte(b'\n') + || self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) } pub fn get_slice(&self, start: usize, end: usize) -> &'p str { @@ -207,11 +185,10 @@ impl<'p> ParserStream<'p> { pub fn skip_digits(&mut self) -> Result<()> { let start = self.ptr; - while let Some(b) = self.source.get(self.ptr) { - if b >= &b'0' && b <= &b'9' { - self.ptr += 1; - } else { - break; + loop { + match self.source.get(self.ptr) { + Some(b) if b >= &b'0' && b <= &b'9' => self.ptr += 1, + _ => break, } } if start == self.ptr { diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index bb75d7e1..08b43f9c 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -356,7 +356,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> } let b = ps.source[ps.ptr]; if indent == 0 { - if b != b'\n' && b != b'\r' { + if b != b'\n' { break; } } else if b == b'.' || b == b'}' || b == b'*' || b == b'[' { From aebb85d47e74bad82382c0cb9e35336124509003 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 9 Jan 2019 12:52:40 -0800 Subject: [PATCH 27/36] Some more parser cleanups. --- fluent-syntax/src/parser/ftlstream.rs | 54 +++++++++++++++------------ fluent-syntax/src/parser/mod.rs | 46 ++++++++++------------- 2 files changed, 49 insertions(+), 51 deletions(-) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index e116d640..a4b14c00 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -59,7 +59,7 @@ impl<'p> ParserStream<'p> { pub fn skip_blank(&mut self) { loop { match self.source.get(self.ptr) { - Some(b' ') | Some(b'\n') => self.ptr += 1, + Some(b) if [b' ', b'\n'].contains(b) => self.ptr += 1, _ => break, } } @@ -67,30 +67,21 @@ impl<'p> ParserStream<'p> { pub fn skip_blank_inline(&mut self) -> usize { let start = self.ptr; - loop { - match self.source.get(self.ptr) { - Some(b' ') => self.ptr += 1, - _ => break, - } + while let Some(b' ') = self.source.get(self.ptr) { + self.ptr += 1; } self.ptr - start } pub fn skip_to_next_entry_start(&mut self) { - while self.ptr < self.length { - if (self.ptr == 0 || self.is_byte_at(b'\n', self.ptr - 1)) - && (self.is_identifier_start() - || self.is_current_byte(b'-') - || self.is_current_byte(b'#')) - { + while let Some(b) = self.source.get(self.ptr) { + let new_line = self.ptr == 0 || self.source.get(self.ptr - 1) == Some(&b'\n'); + + if new_line && (self.is_byte_alphabetic(*b) || [b'-', b'#'].contains(b)) { break; } self.ptr += 1; - - while self.ptr < self.length && !self.is_byte_at(b'\n', self.ptr - 1) { - self.ptr += 1; - } } } @@ -138,7 +129,7 @@ impl<'p> ParserStream<'p> { let start = self.ptr; for _ in 0..length { match self.source.get(self.ptr) { - Some(b'0'...b'9') | Some(b'a'...b'f') | Some(b'A'...b'F') => self.ptr += 1, + Some(b) if b.is_ascii_hexdigit() => self.ptr += 1, _ => break, } } @@ -155,28 +146,43 @@ impl<'p> ParserStream<'p> { pub fn is_char_pattern_continuation(&self) -> bool { match self.source.get(self.ptr) { - Some(b) if b == &b'}' || b == &b'.' || b == &b'[' || b == &b'*' => false, - _ => true, + Some(b) => self.is_byte_pattern_continuation(*b), + _ => false, } } pub fn is_identifier_start(&self) -> bool { match self.source.get(self.ptr) { - Some(b) if (b >= &b'a' && b <= &b'z') || (b >= &b'A' && b <= &b'Z') => true, + Some(b) if self.is_byte_alphabetic(*b) => true, _ => false, } } + pub fn is_byte_alphabetic(&self, b: u8) -> bool { + (b >= b'a' && b <= b'z') || (b >= b'A' && b <= b'Z') + } + + pub fn is_byte_digit(&self, b: u8) -> bool { + b >= b'0' && b <= b'9' + } + + pub fn is_byte_pattern_continuation(&self, b: u8) -> bool { + ![b'}', b'.', b'[', b'*'].contains(&b) + } + pub fn is_number_start(&self) -> bool { match self.source.get(self.ptr) { - Some(b) if (b == &b'-') || (b >= &b'0' && b <= &b'9') => true, + Some(b) if (b == &b'-') || self.is_byte_digit(*b) => true, _ => false, } } pub fn is_eol(&self) -> bool { - self.is_current_byte(b'\n') - || self.is_current_byte(b'\r') && self.is_byte_at(b'\n', self.ptr + 1) + match self.source.get(self.ptr) { + Some(b'\n') => true, + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => true, + _ => false, + } } pub fn get_slice(&self, start: usize, end: usize) -> &'p str { @@ -187,7 +193,7 @@ impl<'p> ParserStream<'p> { let start = self.ptr; loop { match self.source.get(self.ptr) { - Some(b) if b >= &b'0' && b <= &b'9' => self.ptr += 1, + Some(b) if self.is_byte_digit(*b) => self.ptr += 1, _ => break, } } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 08b43f9c..bc113e37 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -185,7 +185,6 @@ fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Result(ps: &mut ParserStream<'p>) -> Result> { } fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> { - let start_pos = ps.ptr; - - while ps.ptr < ps.length { - let b = ps.source[ps.ptr]; - if start_pos == ps.ptr { - if ps.is_identifier_start() { - ps.ptr += 1; - } else { - return error!( - ErrorKind::ExpectedCharRange { - range: "a-zA-Z".to_string() - }, - ps.ptr - ); - } - } else if (b >= b'a' && b <= b'z') - || (b >= b'A' && b <= b'Z') - || (b >= b'0' && b <= b'9') - || b == b'_' - || b == b'-' - { - ps.ptr += 1; + let mut ptr = ps.ptr; + + while let Some(b) = ps.source.get(ptr) { + if ps.is_byte_alphabetic(*b) { + ptr += 1; + } else if ptr == ps.ptr { + return error!( + ErrorKind::ExpectedCharRange { + range: "a-zA-Z".to_string() + }, + ptr + ); + } else if ps.is_byte_digit(*b) || [b'_', b'-'].contains(&b) { + ptr += 1; } else { break; } } - let name = ps.get_slice(start_pos, ps.ptr); + let name = ps.get_slice(ps.ptr, ptr); + ps.ptr = ptr; Ok(ast::Identifier { name }) } @@ -359,7 +351,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> if b != b'\n' { break; } - } else if b == b'.' || b == b'}' || b == b'*' || b == b'[' { + } else if !ps.is_byte_pattern_continuation(b) { ps.ptr = slice_start; break; } @@ -736,7 +728,7 @@ fn get_literal<'p>(ps: &mut ParserStream<'p>) -> Result { + Some(b) if ps.is_byte_digit(*b) => { let num = get_number_literal(ps)?; Ok(ast::InlineExpression::NumberLiteral { value: num }) } @@ -765,7 +757,7 @@ fn get_literal<'p>(ps: &mut ParserStream<'p>) -> Result { + Some(b) if ps.is_byte_alphabetic(*b) => { let id = get_identifier(ps)?; Ok(ast::InlineExpression::MessageReference { id }) } From 4956bebfe4a894cb447dbb1a452e88f68f056bae Mon Sep 17 00:00:00 2001 From: John Heitmann Date: Fri, 11 Jan 2019 01:25:22 -0800 Subject: [PATCH 28/36] Extra docs for format_message. Clarified parts of error handling, blurred others. --- fluent-bundle/src/bundle.rs | 53 ++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 688eb081..93264328 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -53,11 +53,12 @@ pub struct Message { /// Next, call [`add_resource`] one or more times, supplying translations in the FTL syntax. The /// `FluentBundle` instance is now ready to be used for localization. /// -/// To format a translation, call `get_message` to retrieve a `fluent_bundle::bundle::Message` structure -/// and then `format` it within the bundle. +/// To format a translation, call [`format`] with the path of a message or attribute in order to +/// retrieve the translated string. Alternately, [`format_message`] provides a convenient way of +/// formatting all attributes of a message at once. /// -/// The result is an [`Option`] wrapping a `(String, Vec)`. On success, the string -/// is a formatted value that should be displayed in the UI. It is +/// The result of `format` is an [`Option`] wrapping a `(String, Vec)`. On success, +/// the string is a formatted value that should be displayed in the UI. It is /// recommended to treat the result as opaque from the perspective of the program and use it only /// to display localized messages. Do not examine it or alter in any way before displaying. This /// is a general good practice as far as all internationalization operations are concerned. @@ -74,6 +75,7 @@ pub struct Message { /// [`FluentBundle::new`]: ./struct.FluentBundle.html#method.new /// [`fluent::bundle::Message`]: ./struct.FluentBundle.html#method.new /// [`format`]: ./struct.FluentBundle.html#method.format +/// [`format_message`]: ./struct.FluentBundle.html#method.format_message /// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource /// [`Option`]: http://doc.rust-lang.org/std/option/enum.Option.html #[allow(dead_code)] @@ -294,16 +296,17 @@ impl<'bundle> FluentBundle<'bundle> { /// /// # Errors /// - /// If no message is found at `path`, then `format` returns `None`. + /// For some cases where `format` can't find a message it will return `None`. /// - /// In all other cases, `format` returns a string even if it - /// encountered errors. `format` uses two fallback techniques to - /// create the fallback string. If there are bad references in the - /// message, then they will be substituted with `'___'`. If there - /// are more extensive errors, then `format` will fall back to using - /// `path` itself as the formatted string. Sometimes, but not always, - /// these partial failures will emit extra error information in the - /// second term of the return tuple. + /// In all other cases `format` returns a string even if it + /// encountered errors. Generally, during partial errors `format` will + /// use `'___'` to replace parts of the formatted message that it could + /// not successfuly build. For more fundamental errors `format` will return + /// the path itself as the translation. + /// + /// The second term of the tuple will contain any extra error information + /// gathered during formatting. A caller may safely ignore the extra errors + /// if the fallback formatting policies are acceptable. /// /// ``` /// use fluent_bundle::bundle::FluentBundle; @@ -370,7 +373,9 @@ impl<'bundle> FluentBundle<'bundle> { } /// Formats both the message value and attributes identified by `message_id` - /// using `args` to provide variables. + /// using `args` to provide variables. This is useful for cases where a UI + /// element requires multiple related text fields, such as a button that has + /// both display text and assistive text. /// /// # Examples /// @@ -395,17 +400,17 @@ impl<'bundle> FluentBundle<'bundle> { /// /// # Errors /// - /// If no message is found at `message_id`, then `format_message` - /// returns `None`. + /// For some cases where `format_message` can't find a message it will return `None`. + /// + /// In all other cases `format_message` returns a message even if it + /// encountered errors. Generally, during partial errors `format_message` will + /// use `'___'` to replace parts of the formatted message that it could + /// not successfuly build. For more fundamental errors `format_message` will return + /// the path itself as the translation. /// - /// In all other cases, `format_message` returns a `Message` even if it - /// encountered errors. `format_message` uses two fallback techniques to - /// create the fallback string. If there are bad references in the - /// message, then they will be substituted with `'___'`. If there - /// are more extensive errors, then `format_message` will fall back to using - /// `path` itself as the formatted string. Sometimes, but not always, - /// these partial failures will emit extra error information in the - /// second term of the return tuple. + /// The second term of the tuple will contain any extra error information + /// gathered during formatting. A caller may safely ignore the extra errors + /// if the fallback formatting policies are acceptable. pub fn format_message( &self, message_id: &str, From be4b1eeb6c1c1341f7931d304fb76495c1e4a003 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 16 Jan 2019 12:08:45 -0800 Subject: [PATCH 29/36] Rename format_message to compound --- fluent-bundle/benches/lib.rs | 2 +- fluent-bundle/src/bundle.rs | 34 +++++++++---------- .../tests/{format_message.rs => compound.rs} | 7 ++-- fluent-bundle/tests/helpers/mod.rs | 5 +-- 4 files changed, 22 insertions(+), 26 deletions(-) rename fluent-bundle/tests/{format_message.rs => compound.rs} (76%) diff --git a/fluent-bundle/benches/lib.rs b/fluent-bundle/benches/lib.rs index 4dddc0c6..d7512af8 100644 --- a/fluent-bundle/benches/lib.rs +++ b/fluent-bundle/benches/lib.rs @@ -73,7 +73,7 @@ fn bench_menubar_format(b: &mut Bencher) { // widgets may only expect attributes and they shouldn't be forced to display a value. // Here however it doesn't matter because we know for certain that the message for `id` // exists. - bundle.format_message(id, None); + bundle.compound(id, None); } }); } diff --git a/fluent-bundle/src/bundle.rs b/fluent-bundle/src/bundle.rs index 60bbe866..ebf0b8df 100644 --- a/fluent-bundle/src/bundle.rs +++ b/fluent-bundle/src/bundle.rs @@ -54,7 +54,7 @@ pub struct Message { /// `FluentBundle` instance is now ready to be used for localization. /// /// To format a translation, call [`format`] with the path of a message or attribute in order to -/// retrieve the translated string. Alternately, [`format_message`] provides a convenient way of +/// retrieve the translated string. Alternately, [`compound`] provides a convenient way of /// formatting all attributes of a message at once. /// /// The result of `format` is an [`Option`] wrapping a `(String, Vec)`. On success, @@ -75,7 +75,7 @@ pub struct Message { /// [`FluentBundle::new`]: ./struct.FluentBundle.html#method.new /// [`fluent::bundle::Message`]: ./struct.FluentBundle.html#method.new /// [`format`]: ./struct.FluentBundle.html#method.format -/// [`format_message`]: ./struct.FluentBundle.html#method.format_message +/// [`compound`]: ./struct.FluentBundle.html#method.compound /// [`add_resource`]: ./struct.FluentBundle.html#method.add_resource /// [`Option`]: http://doc.rust-lang.org/std/option/enum.Option.html pub struct FluentBundle<'bundle> { @@ -130,7 +130,7 @@ impl<'bundle> FluentBundle<'bundle> { /// let mut bundle = FluentBundle::new(&["en-US"]); /// bundle.add_resource(&resource); /// assert_eq!(true, bundle.has_message("hello")); - /// + /// /// ``` pub fn has_message(&self, id: &str) -> bool { self.entries.get_message(id).is_some() @@ -296,26 +296,26 @@ impl<'bundle> FluentBundle<'bundle> { /// # Errors /// /// For some cases where `format` can't find a message it will return `None`. - /// + /// /// In all other cases `format` returns a string even if it /// encountered errors. Generally, during partial errors `format` will /// use `'___'` to replace parts of the formatted message that it could /// not successfuly build. For more fundamental errors `format` will return /// the path itself as the translation. - /// + /// /// The second term of the tuple will contain any extra error information /// gathered during formatting. A caller may safely ignore the extra errors /// if the fallback formatting policies are acceptable. - /// + /// /// ``` /// use fluent_bundle::bundle::FluentBundle; /// use fluent_bundle::resource::FluentResource; - /// + /// /// // Create a message with bad cyclic reference /// let mut res = FluentResource::try_new("foo = a { foo } b".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); /// bundle.add_resource(&res); - /// + /// /// // The result falls back to "___" /// let value = bundle.format("foo", None); /// assert_eq!(value, Some(("___".to_string(), vec![]))); @@ -391,26 +391,26 @@ impl<'bundle> FluentBundle<'bundle> { /// .title = Type your login email".to_string()).unwrap(); /// let mut bundle = FluentBundle::new(&["en-US"]); /// bundle.add_resource(&res); - /// - /// let (message, _) = bundle.format_message("login-input", None).unwrap(); + /// + /// let (message, _) = bundle.compound("login-input", None).unwrap(); /// assert_eq!(message.value, Some("Predefined value".to_string())); /// assert_eq!(message.attributes.get("title"), Some(&"Type your login email".to_string())); /// ``` /// /// # Errors /// - /// For some cases where `format_message` can't find a message it will return `None`. - /// - /// In all other cases `format_message` returns a message even if it - /// encountered errors. Generally, during partial errors `format_message` will + /// For some cases where `compound` can't find a message it will return `None`. + /// + /// In all other cases `compound` returns a message even if it + /// encountered errors. Generally, during partial errors `compound` will /// use `'___'` to replace parts of the formatted message that it could - /// not successfuly build. For more fundamental errors `format_message` will return + /// not successfuly build. For more fundamental errors `compound` will return /// the path itself as the translation. - /// + /// /// The second term of the tuple will contain any extra error information /// gathered during formatting. A caller may safely ignore the extra errors /// if the fallback formatting policies are acceptable. - pub fn format_message( + pub fn compound( &self, message_id: &str, args: Option<&HashMap<&str, FluentValue>>, diff --git a/fluent-bundle/tests/format_message.rs b/fluent-bundle/tests/compound.rs similarity index 76% rename from fluent-bundle/tests/format_message.rs rename to fluent-bundle/tests/compound.rs index f427b22c..7b40a786 100644 --- a/fluent-bundle/tests/format_message.rs +++ b/fluent-bundle/tests/compound.rs @@ -1,8 +1,7 @@ mod helpers; use self::helpers::{ - assert_format_message_no_errors, assert_get_bundle_no_errors, - assert_get_resource_from_str_no_errors, + assert_compound_no_errors, assert_get_bundle_no_errors, assert_get_resource_from_str_no_errors, }; use fluent_bundle::bundle::Message; use std::collections::HashMap; @@ -22,8 +21,8 @@ foo = Foo attrs.insert("attr".to_string(), "Attribute".to_string()); attrs.insert("attr2".to_string(), "Attribute 2".to_string()); - assert_format_message_no_errors( - bundle.format_message("foo", None), + assert_compound_no_errors( + bundle.compound("foo", None), Message { value: Some("Foo".to_string()), attributes: attrs, diff --git a/fluent-bundle/tests/helpers/mod.rs b/fluent-bundle/tests/helpers/mod.rs index 90d7b547..595170d2 100644 --- a/fluent-bundle/tests/helpers/mod.rs +++ b/fluent-bundle/tests/helpers/mod.rs @@ -15,10 +15,7 @@ pub fn assert_format_no_errors(result: Option<(String, Vec)>, expec } #[allow(dead_code)] -pub fn assert_format_message_no_errors( - result: Option<(Message, Vec)>, - expected: Message, -) { +pub fn assert_compound_no_errors(result: Option<(Message, Vec)>, expected: Message) { assert_eq!(result, Some((expected, vec![]))); } From 37695b7e479d56343be83b779d71f3b32103c123 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 16 Jan 2019 12:11:52 -0800 Subject: [PATCH 30/36] Remove ResourceManager to a separate PR --- Cargo.toml | 3 +- fluent/Cargo.toml | 23 --- fluent/examples/resources/en-US/common.ftl | 1 - fluent/examples/resources/en-US/errors.ftl | 2 - fluent/examples/resources/en-US/simple.ftl | 5 - fluent/examples/resources/pl/common.ftl | 1 - fluent/examples/resources/pl/errors.ftl | 2 - fluent/examples/resources/pl/simple.ftl | 6 - fluent/examples/simple.rs | 154 --------------------- fluent/src/lib.rs | 1 - fluent/src/resource_manager.rs | 49 ------- 11 files changed, 1 insertion(+), 246 deletions(-) delete mode 100644 fluent/Cargo.toml delete mode 100644 fluent/examples/resources/en-US/common.ftl delete mode 100644 fluent/examples/resources/en-US/errors.ftl delete mode 100644 fluent/examples/resources/en-US/simple.ftl delete mode 100644 fluent/examples/resources/pl/common.ftl delete mode 100644 fluent/examples/resources/pl/errors.ftl delete mode 100644 fluent/examples/resources/pl/simple.ftl delete mode 100644 fluent/examples/simple.rs delete mode 100644 fluent/src/lib.rs delete mode 100644 fluent/src/resource_manager.rs diff --git a/Cargo.toml b/Cargo.toml index 29cf9892..d897918e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,5 @@ members = [ "fluent-syntax", "fluent-bundle", - "fluent-cli", - "fluent", + "fluent-cli" ] diff --git a/fluent/Cargo.toml b/fluent/Cargo.toml deleted file mode 100644 index 277631f4..00000000 --- a/fluent/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "fluent" -description = """ -A localization system designed to unleash the entire expressive power of -natural language translations. -""" -version = "0.4.3" -authors = [ - "Zibi Braniecki ", - "Staś Małolepszy " -] -edition = "2018" -homepage = "http://www.projectfluent.org" -license = "Apache-2.0/MIT" -repository = "https://github.com/projectfluent/fluent-rs" -readme = "README.md" -keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] -categories = ["localization", "internationalization"] - -[dependencies] -fluent-bundle = { path = "../fluent-bundle" } -fluent-locale = "^0.4.1" -elsa = "^0.1.2" diff --git a/fluent/examples/resources/en-US/common.ftl b/fluent/examples/resources/en-US/common.ftl deleted file mode 100644 index 09e3bdaa..00000000 --- a/fluent/examples/resources/en-US/common.ftl +++ /dev/null @@ -1 +0,0 @@ -hello-world = Hello World! diff --git a/fluent/examples/resources/en-US/errors.ftl b/fluent/examples/resources/en-US/errors.ftl deleted file mode 100644 index 6a228bbe..00000000 --- a/fluent/examples/resources/en-US/errors.ftl +++ /dev/null @@ -1,2 +0,0 @@ -missing-arg-error = Error: Please provide a number as argument. -input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } diff --git a/fluent/examples/resources/en-US/simple.ftl b/fluent/examples/resources/en-US/simple.ftl deleted file mode 100644 index 104d3e30..00000000 --- a/fluent/examples/resources/en-US/simple.ftl +++ /dev/null @@ -1,5 +0,0 @@ -response-msg = - { $value -> - [one] "{ $input }" has one Collatz step. - *[other] "{ $input }" has { $value } Collatz steps. - } diff --git a/fluent/examples/resources/pl/common.ftl b/fluent/examples/resources/pl/common.ftl deleted file mode 100644 index 5f6a3d9f..00000000 --- a/fluent/examples/resources/pl/common.ftl +++ /dev/null @@ -1 +0,0 @@ -hello-world = Witaj, Świecie! diff --git a/fluent/examples/resources/pl/errors.ftl b/fluent/examples/resources/pl/errors.ftl deleted file mode 100644 index f27124a3..00000000 --- a/fluent/examples/resources/pl/errors.ftl +++ /dev/null @@ -1,2 +0,0 @@ -missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. -input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } diff --git a/fluent/examples/resources/pl/simple.ftl b/fluent/examples/resources/pl/simple.ftl deleted file mode 100644 index 7a17d125..00000000 --- a/fluent/examples/resources/pl/simple.ftl +++ /dev/null @@ -1,6 +0,0 @@ -response-msg = - { $value -> - [one] "{ $input }" ma jeden krok Collatza. - [few] "{ $input }" ma { $value } kroki Collatza. - *[many] "{ $input }" ma { $value } kroków Collatza. - } diff --git a/fluent/examples/simple.rs b/fluent/examples/simple.rs deleted file mode 100644 index 0b0d8806..00000000 --- a/fluent/examples/simple.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! This is an example of a simple application -//! which calculates the Collatz conjecture. -//! -//! The function itself is trivial on purpose, -//! so that we can focus on understanding how -//! the application can be made localizable -//! via Fluent. -//! -//! To try the app launch `cargo run --example simple NUM (LOCALES)` -//! -//! NUM is a number to be calculated, and LOCALES is an optional -//! parameter with a comma-separated list of locales requested by the user. -//! -//! Example: -//! -//! caron run --example simple 123 de,pl -//! -//! If the second argument is omitted, `en-US` locale is used as the -//! default one. -use fluent::resource_manager::ResourceManager; -use fluent_bundle::types::FluentValue; -use fluent_locale::{negotiate_languages, NegotiationStrategy}; -use std::collections::HashMap; -use std::env; -use std::fs; -use std::io; -use std::str::FromStr; - -/// This helper function allows us to read the list -/// of available locales by reading the list of -/// directories in `./examples/resources`. -/// -/// It is expected that every directory inside it -/// has a name that is a valid BCP47 language tag. -fn get_available_locales() -> Result, io::Error> { - let mut locales = vec![]; - - let res_dir = fs::read_dir("./examples/resources/")?; - for entry in res_dir { - if let Ok(entry) = entry { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name() { - if let Some(name) = name.to_str() { - locales.push(String::from(name)); - } - } - } - } - } - return Ok(locales); -} - -/// This function negotiates the locales between available -/// and requested by the user. -/// -/// It uses `fluent-locale` library but one could -/// use any other that will resolve the list of -/// available locales based on the list of -/// requested locales. -fn get_app_locales(requested: &[&str]) -> Result, io::Error> { - let available = get_available_locales()?; - let resolved_locales = negotiate_languages( - requested, - &available, - Some("en-US"), - &NegotiationStrategy::Filtering, - ); - return Ok(resolved_locales - .into_iter() - .map(|s| String::from(s)) - .collect::>()); -} - -static L10N_RESOURCES: &[&str] = &["simple.ftl", "errors.ftl"]; - -fn main() { - // 1. Get the command line arguments. - let args: Vec = env::args().collect(); - - let mgr = ResourceManager::new(); - - // 2. If the argument length is more than 1, - // take the second argument as a comma-separated - // list of requested locales. - // - // Otherwise, take ["en-US"] as the default. - let requested = args - .get(2) - .map_or(vec!["en-US"], |arg| arg.split(",").collect()); - - // 3. Negotiate it against the avialable ones - let locales = get_app_locales(&requested).expect("Failed to retrieve available locales"); - - // 4. Create a new Fluent FluentBundle using the - // resolved locales. - let paths = L10N_RESOURCES - .iter() - .map(|path| { - format!( - "./examples/resources/{locale}/{path}", - locale = locales[0], - path = path - ) - }) - .collect(); - - // 5. Get a bundle for given paths and locales. - let bundle = mgr.get_bundle(&locales, &paths); - - // 6. Check if the input is provided. - match args.get(1) { - Some(input) => { - // 6.1. Cast it to a number. - match isize::from_str(&input) { - Ok(i) => { - // 6.2. Construct a map of arguments - // to format the message. - let mut args = HashMap::new(); - args.insert("input", FluentValue::from(i)); - args.insert("value", FluentValue::from(collatz(i))); - // 6.3. Format the message. - println!("{}", bundle.format("response-msg", Some(&args)).unwrap().0); - } - Err(err) => { - let mut args = HashMap::new(); - args.insert("input", FluentValue::from(input.to_string())); - args.insert("reason", FluentValue::from(err.to_string())); - println!( - "{}", - bundle - .format("input-parse-error-msg", Some(&args)) - .unwrap() - .0 - ); - } - } - } - None => { - println!("{}", bundle.format("missing-arg-error", None).unwrap().0); - } - } -} - -/// Collatz conjecture calculating function. -fn collatz(n: isize) -> isize { - match n { - 1 => 0, - _ => match n % 2 { - 0 => 1 + collatz(n / 2), - _ => 1 + collatz(n * 3 + 1), - }, - } -} diff --git a/fluent/src/lib.rs b/fluent/src/lib.rs deleted file mode 100644 index 4f362414..00000000 --- a/fluent/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod resource_manager; diff --git a/fluent/src/resource_manager.rs b/fluent/src/resource_manager.rs deleted file mode 100644 index 0f871a52..00000000 --- a/fluent/src/resource_manager.rs +++ /dev/null @@ -1,49 +0,0 @@ -use elsa::FrozenMap; -use fluent_bundle::bundle::FluentBundle; -use fluent_bundle::resource::FluentResource; -use std::fs::File; -use std::io; -use std::io::prelude::*; - -fn read_file(path: &str) -> Result { - let mut f = File::open(path)?; - let mut s = String::new(); - f.read_to_string(&mut s)?; - Ok(s) -} - -pub struct ResourceManager { - resources: FrozenMap>, -} - -impl ResourceManager { - pub fn new() -> Self { - ResourceManager { - resources: FrozenMap::new(), - } - } - - pub fn get_resource(&self, path: &str) -> &FluentResource { - let resources = &self.resources; - - if resources.get(path).is_some() { - self.resources.get(path).unwrap() - } else { - let string = read_file(path).unwrap(); - let res = match FluentResource::try_new(string) { - Ok(res) => res, - Err((res, _err)) => res, - }; - self.resources.insert(path.to_string(), Box::new(res)) - } - } - - pub fn get_bundle(&self, locales: &[String], paths: &Vec) -> FluentBundle { - let mut bundle = FluentBundle::new(locales); - for path in paths { - let res = self.get_resource(path); - bundle.add_resource(res).unwrap(); - } - bundle - } -} From 96e4b91f89e887f98b56e142bfd0bea4b535b963 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 16 Jan 2019 12:15:25 -0800 Subject: [PATCH 31/36] Rename lifetime in resolver --- fluent-bundle/src/resolve.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fluent-bundle/src/resolve.rs b/fluent-bundle/src/resolve.rs index 89375951..bbcfef23 100644 --- a/fluent-bundle/src/resolve.rs +++ b/fluent-bundle/src/resolve.rs @@ -62,7 +62,7 @@ pub trait ResolveValue { fn to_value(&self, env: &Env) -> Result; } -impl<'resolver> ResolveValue for ast::Message<'resolver> { +impl<'source> ResolveValue for ast::Message<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || { self.value @@ -73,19 +73,19 @@ impl<'resolver> ResolveValue for ast::Message<'resolver> { } } -impl<'resolver> ResolveValue for ast::Term<'resolver> { +impl<'source> ResolveValue for ast::Term<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl<'resolver> ResolveValue for ast::Attribute<'resolver> { +impl<'source> ResolveValue for ast::Attribute<'source> { fn to_value(&self, env: &Env) -> Result { env.track(&self.id.name, || self.value.to_value(env)) } } -impl<'resolver> ResolveValue for ast::Value<'resolver> { +impl<'source> ResolveValue for ast::Value<'source> { fn to_value(&self, env: &Env) -> Result { match self { ast::Value::Pattern(p) => p.to_value(env), @@ -97,7 +97,7 @@ impl<'resolver> ResolveValue for ast::Value<'resolver> { } } -impl<'resolver> ResolveValue for ast::Pattern<'resolver> { +impl<'source> ResolveValue for ast::Pattern<'source> { fn to_value(&self, env: &Env) -> Result { let mut string = String::with_capacity(128); for elem in &self.elements { @@ -119,7 +119,7 @@ impl<'resolver> ResolveValue for ast::Pattern<'resolver> { } } -impl<'resolver> ResolveValue for ast::PatternElement<'resolver> { +impl<'source> ResolveValue for ast::PatternElement<'source> { fn to_value(&self, env: &Env) -> Result { match self { ast::PatternElement::TextElement(s) => Ok(FluentValue::from(*s)), @@ -128,7 +128,7 @@ impl<'resolver> ResolveValue for ast::PatternElement<'resolver> { } } -impl<'resolver> ResolveValue for ast::VariantKey<'resolver> { +impl<'source> ResolveValue for ast::VariantKey<'source> { fn to_value(&self, _env: &Env) -> Result { match self { ast::VariantKey::Identifier { name } => Ok(FluentValue::from(*name)), @@ -139,7 +139,7 @@ impl<'resolver> ResolveValue for ast::VariantKey<'resolver> { } } -impl<'resolver> ResolveValue for ast::Expression<'resolver> { +impl<'source> ResolveValue for ast::Expression<'source> { fn to_value(&self, env: &Env) -> Result { match self { ast::Expression::InlineExpression(exp) => exp.to_value(env), @@ -172,7 +172,7 @@ impl<'resolver> ResolveValue for ast::Expression<'resolver> { } } -impl<'resolver> ResolveValue for ast::InlineExpression<'resolver> { +impl<'source> ResolveValue for ast::InlineExpression<'source> { fn to_value(&self, env: &Env) -> Result { match self { ast::InlineExpression::StringLiteral { raw } => { @@ -289,9 +289,9 @@ impl<'resolver> ResolveValue for ast::InlineExpression<'resolver> { } } -fn select_default<'resolver>( - variants: &'resolver [ast::Variant<'resolver>], -) -> Option<&ast::Variant<'resolver>> { +fn select_default<'source>( + variants: &'source [ast::Variant<'source>], +) -> Option<&ast::Variant<'source>> { for variant in variants { if variant.default { return Some(variant); From 4083bba660c34a52219f16ea34e126cb804ad959 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 16 Jan 2019 12:17:53 -0800 Subject: [PATCH 32/36] Various minor reviewer feedback --- fluent-cli/Cargo.toml | 2 +- fluent-syntax/src/parser/{errors/mod.rs => errors.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename fluent-syntax/src/parser/{errors/mod.rs => errors.rs} (100%) diff --git a/fluent-cli/Cargo.toml b/fluent-cli/Cargo.toml index db47bd62..0751bbee 100644 --- a/fluent-cli/Cargo.toml +++ b/fluent-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "fluent-cli" description = """ A collection of command line interface programs -used for Fluent Localization System. +for Fluent Localization System. """ version = "0.0.1" edition = "2018" diff --git a/fluent-syntax/src/parser/errors/mod.rs b/fluent-syntax/src/parser/errors.rs similarity index 100% rename from fluent-syntax/src/parser/errors/mod.rs rename to fluent-syntax/src/parser/errors.rs From 60357e68eebe3dc9db9e2d97734fa66cebff22c7 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 23 Jan 2019 13:37:35 -0800 Subject: [PATCH 33/36] Fix the only case where we attempted to get slice with a pointer incremented potentially beyond the slice range. --- fluent-syntax/src/parser/ftlstream.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index a4b14c00..56e30d34 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -134,10 +134,13 @@ impl<'p> ParserStream<'p> { } } if self.ptr - start != length { + let end = if self.ptr >= self.length { + self.ptr + } else { + self.ptr + 1 + }; return error!( - ErrorKind::InvalidUnicodeEscapeSequence( - self.get_slice(start, self.ptr + 1).to_owned() - ), + ErrorKind::InvalidUnicodeEscapeSequence(self.get_slice(start, end).to_owned()), self.ptr ); } From 6ce04db0b752b8573f5d32d92b0977c0bea0d97a Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 23 Jan 2019 14:24:05 -0800 Subject: [PATCH 34/36] Apply reviewers feedback --- fluent-bundle/src/resolve.rs | 6 +- fluent-cli/src/main.rs | 13 +-- fluent-syntax/src/parser/ftlstream.rs | 2 +- fluent-syntax/src/parser/mod.rs | 111 +++++++++++++++----------- fluent-syntax/tests/fixtures/Makefile | 11 --- 5 files changed, 76 insertions(+), 67 deletions(-) delete mode 100644 fluent-syntax/tests/fixtures/Makefile diff --git a/fluent-bundle/src/resolve.rs b/fluent-bundle/src/resolve.rs index bbcfef23..a04e4806 100644 --- a/fluent-bundle/src/resolve.rs +++ b/fluent-bundle/src/resolve.rs @@ -192,11 +192,11 @@ impl<'source> ResolveValue for ast::InlineExpression<'source> { ref positional, ref named, } => { - let mut resolved_unnamed_args = Vec::new(); + let mut resolved_positional_args = Vec::new(); let mut resolved_named_args = HashMap::new(); for expression in positional { - resolved_unnamed_args.push(expression.to_value(env).ok()); + resolved_positional_args.push(expression.to_value(env).ok()); } for arg in named { @@ -212,7 +212,7 @@ impl<'source> ResolveValue for ast::InlineExpression<'source> { }; func.ok_or(ResolverError::None).and_then(|func| { - func(resolved_unnamed_args.as_slice(), &resolved_named_args) + func(resolved_positional_args.as_slice(), &resolved_named_args) .ok_or(ResolverError::None) }) } diff --git a/fluent-cli/src/main.rs b/fluent-cli/src/main.rs index 34013b48..dcfb4980 100644 --- a/fluent-cli/src/main.rs +++ b/fluent-cli/src/main.rs @@ -28,8 +28,8 @@ fn main() { .version("1.0") .about("Parses FTL file into an AST") .args_from_usage( - "-s, --silence 'disable output' - 'Sets the input file to use'", + "-s, --silent 'Disables error reporting' + 'FTL file to parse'", ) .get_matches(); @@ -39,14 +39,15 @@ fn main() { let res = parse(&source); - if matches.is_present("silence") { - return; - }; - match res { Ok(res) => print_entries_resource(&res), Err((res, errors)) => { print_entries_resource(&res); + + if matches.is_present("silence") { + return; + }; + println!("==============================\n"); if errors.len() == 1 { println!("Parser encountered one error:"); diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index 56e30d34..da1e7ea7 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -33,7 +33,7 @@ impl<'p> ParserStream<'p> { Ok(()) } - pub fn take_if(&mut self, b: u8) -> bool { + pub fn take_byte_if(&mut self, b: u8) -> bool { if self.is_current_byte(b) { self.ptr += 1; true diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index bc113e37..67331d74 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -88,11 +88,7 @@ fn get_message<'p>(ps: &mut ParserStream<'p>, entry_start: usize) -> Result(ps: &mut ParserStream<'p>, entry_start: usize) -> Result(ps: &mut ParserStream<'p>) -> Result>> { Ok(pattern.map(ast::Value::Pattern)) } -fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Result>> { +fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Vec> { let mut attributes = vec![]; loop { @@ -186,7 +178,7 @@ fn get_attributes<'p>(ps: &mut ParserStream<'p>) -> Result(ps: &mut ParserStream<'p>) -> Result> { @@ -205,22 +197,28 @@ fn get_attribute<'p>(ps: &mut ParserStream<'p>) -> Result> { fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> { let mut ptr = ps.ptr; - while let Some(b) = ps.source.get(ptr) { - if ps.is_byte_alphabetic(*b) { + match ps.source.get(ptr) { + Some(b) if ps.is_byte_alphabetic(*b) => { ptr += 1; - } else if ptr == ps.ptr { + } + _ => { return error!( ErrorKind::ExpectedCharRange { range: "a-zA-Z".to_string() }, ptr ); - } else if ps.is_byte_digit(*b) || [b'_', b'-'].contains(&b) { + } + } + + while let Some(b) = ps.source.get(ptr) { + if ps.is_byte_alphabetic(*b) || ps.is_byte_digit(*b) || [b'_', b'-'].contains(b) { ptr += 1; } else { break; } } + let name = ps.get_slice(ps.ptr, ptr); ps.ptr = ptr; @@ -228,7 +226,7 @@ fn get_identifier<'p>(ps: &mut ParserStream<'p>) -> Result> } fn get_variant_key<'p>(ps: &mut ParserStream<'p>) -> Result> { - if !ps.take_if(b'[') { + if !ps.take_byte_if(b'[') { return error!(ErrorKind::ExpectedToken('['), ps.ptr); } ps.skip_blank(); @@ -255,7 +253,7 @@ fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> let mut has_default = false; while ps.is_current_byte(b'*') || ps.is_current_byte(b'[') { - let default = ps.take_if(b'*'); + let default = ps.take_byte_if(b'*'); if default { if has_default { @@ -288,6 +286,10 @@ fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> } } +// This enum tracks the reason for which +// a text slice ended. +// It is used by the pattern to set the +// proper state for the next line. #[derive(Debug, PartialEq)] enum TextElementTermination { LineFeed, @@ -296,12 +298,9 @@ enum TextElementTermination { EOF, } -#[derive(Debug)] -enum PatternElementPointers<'a> { - Placeable(ast::Expression<'a>), - TextElement(usize, usize, usize, TextElementPosition), -} - +// This enum tracks the placement of the text +// element in the pattern, which is needed for +// dedentation logic. #[derive(Debug, PartialEq)] enum TextElementPosition { InitialLineStart, @@ -309,6 +308,24 @@ enum TextElementPosition { Continuation, } +// This enum allows us to mark pointers in the +// source which will later become text elements +// but without slicing them out of the source string. +// This makes the indentation adjustments cheaper +// since they'll happen on the pointers, rather than +// extracted slices. +#[derive(Debug)] +enum PatternElementPlaceholders<'a> { + Placeable(ast::Expression<'a>), + // (start, end, indent, position) + TextElement(usize, usize, usize, TextElementPosition), +} + +// This enum tracks whether the text element +// is blank or not. +// This is important to identify text elements +// which should not be taken into account +// when calculating common indent. #[derive(Debug, PartialEq)] enum TextElementType { Blank, @@ -336,7 +353,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> } let exp = get_placeable(ps)?; last_non_blank = Some(elements.len()); - elements.push(PatternElementPointers::Placeable(exp)); + elements.push(PatternElementPlaceholders::Placeable(exp)); text_element_role = TextElementPosition::Continuation; } else { let slice_start = ps.ptr; @@ -376,7 +393,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> if text_element_type == TextElementType::NonBlank { last_non_blank = Some(elements.len()); } - elements.push(PatternElementPointers::TextElement( + elements.push(PatternElementPlaceholders::TextElement( slice_start, end, indent, @@ -400,8 +417,10 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> .take(last_non_blank + 1) .enumerate() .filter_map(|(i, elem)| match elem { - PatternElementPointers::Placeable(exp) => Some(ast::PatternElement::Placeable(exp)), - PatternElementPointers::TextElement(start, end, indent, role) => { + PatternElementPlaceholders::Placeable(exp) => { + Some(ast::PatternElement::Placeable(exp)) + } + PatternElementPlaceholders::TextElement(start, end, indent, role) => { let start = if role == TextElementPosition::LineStart { if let Some(common_indent) = common_indent { start + cmp::min(indent, common_indent) @@ -529,7 +548,7 @@ fn get_comment<'p>(ps: &mut ParserStream<'p>) -> Result> { fn get_comment_level<'p>(ps: &mut ParserStream<'p>) -> usize { let mut chars = 0; - while ps.take_if(b'#') { + while ps.take_byte_if(b'#') { chars += 1; } @@ -615,7 +634,7 @@ fn get_expression<'p>(ps: &mut ParserStream<'p>) -> Result> }; if !is_valid { - //XXX: Generalize error type + //XXX: Give more specific error type return error!(ErrorKind::MessageReferenceAsSelector, ps.ptr); } @@ -652,19 +671,19 @@ fn get_call_expression<'p>(ps: &mut ParserStream<'p>) -> Result false, }; - if is_valid { - if let ast::InlineExpression::MessageReference { id } = callee { - callee = ast::InlineExpression::FunctionReference { id }; - } - let (positional, named) = get_call_args(ps)?; - ast::InlineExpression::CallExpression { - callee: Box::new(callee), - positional, - named, - } - } else { + if !is_valid { return error!(ErrorKind::ForbiddenCallee, ps.ptr); } + + if let ast::InlineExpression::MessageReference { id } = callee { + callee = ast::InlineExpression::FunctionReference { id }; + } + let (positional, named) = get_call_args(ps)?; + ast::InlineExpression::CallExpression { + callee: Box::new(callee), + positional, + named, + } } else { callee }; @@ -673,7 +692,7 @@ fn get_call_expression<'p>(ps: &mut ParserStream<'p>) -> Result(ps: &mut ParserStream<'p>) -> Result> { - let reference = get_literal(ps)?; + let reference = get_simple_expression(ps)?; match reference { ast::InlineExpression::MessageReference { .. } @@ -693,7 +712,7 @@ fn get_attribute_expression<'p>(ps: &mut ParserStream<'p>) -> Result(ps: &mut ParserStream<'p>) -> Result> { +fn get_simple_expression<'p>(ps: &mut ParserStream<'p>) -> Result> { match ps.source.get(ps.ptr) { Some(b'"') => { ps.ptr += 1; // " @@ -822,7 +841,7 @@ fn get_call_args<'p>( } ps.skip_blank(); - ps.take_if(b','); + ps.take_byte_if(b','); ps.skip_blank(); } @@ -832,9 +851,9 @@ fn get_call_args<'p>( fn get_number_literal<'p>(ps: &mut ParserStream<'p>) -> Result<&'p str> { let start = ps.ptr; - ps.take_if(b'-'); + ps.take_byte_if(b'-'); ps.skip_digits()?; - if ps.take_if(b'.') { + if ps.take_byte_if(b'.') { ps.skip_digits()?; } diff --git a/fluent-syntax/tests/fixtures/Makefile b/fluent-syntax/tests/fixtures/Makefile deleted file mode 100644 index 49c98e72..00000000 --- a/fluent-syntax/tests/fixtures/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -FTL_FIXTURES := $(wildcard *.ftl) -AST_FIXTURES := $(FTL_FIXTURES:%.ftl=%.json) - -all: $(AST_FIXTURES) - -.PHONY: $(AST_FIXTURES) -$(AST_FIXTURES): %.json: %.ftl - @node --experimental-modules ../../bin/parse.mjs $< \ - 2> /dev/null \ - 1> $@; - @echo "$< → $@" From 793b01992e08effbf6bca15c6f7b3a18ee88dcc9 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 23 Jan 2019 14:34:29 -0800 Subject: [PATCH 35/36] Another round of feedback --- fluent-bundle/src/resource.rs | 4 ++-- fluent-syntax/src/parser/mod.rs | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fluent-bundle/src/resource.rs b/fluent-bundle/src/resource.rs index f6a34328..cb6d167d 100644 --- a/fluent-bundle/src/resource.rs +++ b/fluent-bundle/src/resource.rs @@ -28,9 +28,9 @@ impl FluentResource { }); if let Some(errors) = errors { - return Err((FluentResource(res), errors)); + return Err((Self(res), errors)); } else { - return Ok(FluentResource(res)); + return Ok(Self(res)); } } diff --git a/fluent-syntax/src/parser/mod.rs b/fluent-syntax/src/parser/mod.rs index 67331d74..cabba3e6 100644 --- a/fluent-syntax/src/parser/mod.rs +++ b/fluent-syntax/src/parser/mod.rs @@ -290,10 +290,15 @@ fn get_variants<'p>(ps: &mut ParserStream<'p>) -> Result>> // a text slice ended. // It is used by the pattern to set the // proper state for the next line. +// +// CRLF variant is specific because we want +// to skip it in text elements production. +// For example `a\r\nb` will produce +// (`a`, `\n` and `b`) TextElements. #[derive(Debug, PartialEq)] enum TextElementTermination { LineFeed, - CarriageReturn, + CRLF, PlaceableStart, EOF, } @@ -404,7 +409,7 @@ fn get_pattern<'p>(ps: &mut ParserStream<'p>) -> Result> text_element_role = match termination_reason { TextElementTermination::LineFeed => TextElementPosition::LineStart, - TextElementTermination::CarriageReturn => TextElementPosition::Continuation, + TextElementTermination::CRLF => TextElementPosition::Continuation, TextElementTermination::PlaceableStart => TextElementPosition::Continuation, TextElementTermination::EOF => TextElementPosition::Continuation, }; @@ -472,7 +477,7 @@ fn get_text_slice<'p>( start_pos, ps.ptr - 1, text_element_type, - TextElementTermination::CarriageReturn, + TextElementTermination::CRLF, )); } b'{' => { From a4af23aced09e2a9caf9bac239e9d8272ba55012 Mon Sep 17 00:00:00 2001 From: Zibi Braniecki Date: Wed, 23 Jan 2019 14:36:50 -0800 Subject: [PATCH 36/36] Move get_slice to use safe code --- fluent-syntax/src/parser/ftlstream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-syntax/src/parser/ftlstream.rs b/fluent-syntax/src/parser/ftlstream.rs index da1e7ea7..d7338153 100644 --- a/fluent-syntax/src/parser/ftlstream.rs +++ b/fluent-syntax/src/parser/ftlstream.rs @@ -189,7 +189,7 @@ impl<'p> ParserStream<'p> { } pub fn get_slice(&self, start: usize, end: usize) -> &'p str { - unsafe { str::from_utf8_unchecked(&self.source[start..end]) } + str::from_utf8(&self.source[start..end]).expect("Slicing the source failed") } pub fn skip_digits(&mut self) -> Result<()> {