From 521a9a6e83c17722948d71e4e52a6434ae8c74c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 15 May 2017 19:46:43 +0200 Subject: [PATCH 1/2] Structure tests --- fluent/syntax/ast.py | 26 +++++-- fluent/syntax/errors.py | 41 ++++++++++- fluent/syntax/ftlstream.py | 71 ++++++++++++++++--- fluent/syntax/parser.py | 46 ++++++------ .../message_with_empty_pattern.ftl | 1 + .../message_with_empty_pattern.json | 26 +++++++ .../fixtures_structure/placeable_at_eol.ftl | 4 ++ .../fixtures_structure/placeable_at_eol.json | 42 +++++++++++ .../fixtures_structure/resource_comment.ftl | 2 + .../fixtures_structure/resource_comment.json | 14 ++++ .../resource_comment_trailing_line.ftl | 3 + .../resource_comment_trailing_line.json | 14 ++++ tests/syntax/fixtures_structure/section.ftl | 2 + tests/syntax/fixtures_structure/section.json | 20 ++++++ .../fixtures_structure/simple_message.ftl | 1 + .../fixtures_structure/simple_message.json | 31 ++++++++ .../fixtures_structure/standalone_comment.ftl | 3 + .../standalone_comment.json | 41 +++++++++++ tests/syntax/test_serializer.py | 15 ---- tests/syntax/test_structure.py | 54 ++++++++++++++ 20 files changed, 401 insertions(+), 56 deletions(-) create mode 100644 tests/syntax/fixtures_structure/message_with_empty_pattern.ftl create mode 100644 tests/syntax/fixtures_structure/message_with_empty_pattern.json create mode 100644 tests/syntax/fixtures_structure/placeable_at_eol.ftl create mode 100644 tests/syntax/fixtures_structure/placeable_at_eol.json create mode 100644 tests/syntax/fixtures_structure/resource_comment.ftl create mode 100644 tests/syntax/fixtures_structure/resource_comment.json create mode 100644 tests/syntax/fixtures_structure/resource_comment_trailing_line.ftl create mode 100644 tests/syntax/fixtures_structure/resource_comment_trailing_line.json create mode 100644 tests/syntax/fixtures_structure/section.ftl create mode 100644 tests/syntax/fixtures_structure/section.json create mode 100644 tests/syntax/fixtures_structure/simple_message.ftl create mode 100644 tests/syntax/fixtures_structure/simple_message.json create mode 100644 tests/syntax/fixtures_structure/standalone_comment.ftl create mode 100644 tests/syntax/fixtures_structure/standalone_comment.json create mode 100644 tests/syntax/test_structure.py diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index 205d58cf..23ad36e8 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -14,12 +14,23 @@ def to_json(value): def from_json(value): if isinstance(value, dict): - cls = getattr(sys.modules[__name__], value["type"]) + cls = getattr(sys.modules[__name__], value['type']) args = { k: from_json(v) - for k, v in value.items() if k != "type" + for k, v in value.items() + if k != 'type' + if k != 'span' } - return cls(**args) + node = cls(**args) + + # Spans need to be added via add_span, not __init__. + if 'span' in value: + span = value['span'] + # Message and section comments don't have their own spans. + if span is not None: + node.add_span(span['start'], span['end']) + + return node if isinstance(value, list): return list(map(from_json, value)) else: @@ -219,8 +230,11 @@ def __init__(self, start, end): class Annotation(Node): - def __init__(self, name, message, pos): + def __init__(self, code, args=None, message=None): super(Annotation, self).__init__() - self.name = name + self.code = code + self.args = args or [] self.message = message - self.pos = pos + + def add_span(self, start, end): + self.span = Span(start, end) diff --git a/fluent/syntax/errors.py b/fluent/syntax/errors.py index 1840f85e..3c9ca309 100644 --- a/fluent/syntax/errors.py +++ b/fluent/syntax/errors.py @@ -1,3 +1,40 @@ class ParseError(Exception): - def __init__(self, message): - self.message = message + def __init__(self, code, *args): + self.code = code + self.args = args + self.message = get_error_message(code, args) + + +def get_error_message(code, args): + if code == 'E00001': + return 'Generic error' + if code == 'E0002': + return 'Expected an entry start' + if code == 'E0003': + return 'Expected token: "{}"'.format(args[0]) + if code == 'E0004': + return 'Expected a character from range: "{}"'.format(args[0]) + if code == 'E0005': + msg = 'Expected entry "{}" to have a value, attributes or tags' + return msg.format(args[0]) + if code == 'E0006': + return 'Expected field: "{}"'.format(args[0]) + if code == 'E0007': + return 'Keyword cannot end with a whitespace' + if code == 'E0008': + return 'Callee has to be a simple identifier' + if code == 'E0009': + return 'Key has to be a simple identifier' + if code == 'E0010': + return 'Expected one of the variants to be marked as default (*)' + if code == 'E0011': + return 'Expected at least one variant after "->"' + if code == 'E0012': + return 'Tags cannot be added to messages with attributes' + if code == 'E0013': + return 'Expected variant key' + if code == 'E0014': + return 'Expected literal' + if code == 'E0015': + return 'Only one variant can be marked as default (*)' + return code diff --git a/fluent/syntax/ftlstream.py b/fluent/syntax/ftlstream.py index 83d436ad..871849db 100644 --- a/fluent/syntax/ftlstream.py +++ b/fluent/syntax/ftlstream.py @@ -32,7 +32,11 @@ def expect_char(self, ch): self.next() return True - raise ParseError('Expected token "{}"'.format(ch)) + if ch == '\n': + # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) + raise ParseError('E0003', '\u2424') + + raise ParseError('E0003', ch) def take_char_if(self, ch): if self.ch == ch: @@ -42,12 +46,15 @@ def take_char_if(self, ch): def take_char(self, f): ch = self.ch - if f(ch): + if ch is not None and f(ch): self.next() return ch return None def is_id_start(self): + if self.ch is None: + return False + cc = ord(self.ch) return (cc >= 97 and cc <= 122) or \ @@ -62,7 +69,7 @@ def is_number_start(self): def is_peek_next_line_indented(self): if not self.current_peek_is('\n'): return False - + self.peek() if self.current_peek_is(' '): @@ -75,10 +82,17 @@ def is_peek_next_line_indented(self): def is_peek_next_line_variant_start(self): if not self.current_peek_is('\n'): return False - + self.peek() + ptr = self.get_peek_index() + self.peek_line_ws() + + if (self.get_peek_index() - ptr == 0): + self.reset_peek() + return False + if self.current_peek_is('*'): self.peek() @@ -92,11 +106,17 @@ def is_peek_next_line_variant_start(self): def is_peek_next_line_attribute_start(self): if not self.current_peek_is('\n'): return False - + self.peek() + ptr = self.get_peek_index() + self.peek_line_ws() + if (self.get_peek_index() - ptr == 0): + self.reset_peek() + return False + if self.current_peek_is('.'): self.reset_peek() return True @@ -104,6 +124,31 @@ def is_peek_next_line_attribute_start(self): self.reset_peek() return False + def is_peek_next_line_pattern(self): + if not self.current_peek_is('\n'): + return False + + self.peek() + + ptr = self.get_peek_index() + + self.peek_line_ws() + + if (self.get_peek_index() - ptr == 0): + self.reset_peek() + return False + + if (self.current_peek_is('}') or + self.current_peek_is('.') or + self.current_peek_is('#') or + self.current_peek_is('[') or + self.current_peek_is('*')): + self.reset_peek() + return False + + self.reset_peek() + return True + def is_peek_next_line_tag_start(self): if not self.current_peek_is('\n'): @@ -111,8 +156,14 @@ def is_peek_next_line_tag_start(self): self.peek() + ptr = self.get_peek_index() + self.peek_line_ws() + if (self.get_peek_index() - ptr == 0): + self.reset_peek() + return False + if self.current_peek_is('#'): self.reset_peek() return True @@ -136,15 +187,15 @@ def take_id_start(self): self.next() return ret - raise ParseError('Expected char range') + raise ParseError('E0004', 'a-zA-Z') def take_id_char(self): def closure(ch): cc = ord(ch) - return (cc >= 97 and cc <= 122) or \ - (cc >= 65 and cc <= 90) or \ - (cc >= 48 and cc <= 57) or \ - cc == 95 or cc == 45 + return ((cc >= 97 and cc <= 122) or + (cc >= 65 and cc <= 90) or + (cc >= 48 and cc <= 57) or + cc == 95 or cc == 45) return self.take_char(closure) def take_symb_char(self): diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index 6c127d7a..5a293eea 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -39,7 +39,8 @@ def get_entry_or_junk(ps): entry.add_span(entry_start_pos, ps.get_index()) return entry except ParseError as err: - annot = ast.Annotation("ParseError", err.message, ps.get_index()) + annot = ast.Annotation(err.code, err.args, err.message) + annot.add_span(ps.get_index(), ps.get_index()) ps.skip_to_next_entry_start() next_entry_start = ps.get_index() @@ -66,7 +67,8 @@ def get_entry(ps): if comment: return comment - raise ParseError('Expected entry') + + raise ParseError('E0002') def get_comment(ps): ps.expect_char('/') @@ -134,12 +136,11 @@ def get_message(ps, comment): if ps.is_peek_next_line_tag_start(): if attrs is not None: - raise ParseError( - 'Tags cannot be added to messages with attributes') + raise ParseError('E0012') tags = get_tags(ps) if pattern is None and attrs is None and tags is None: - raise ParseError('Missing field') + raise ParseError('E0005', id.name) return ast.Message(id, pattern, attrs, tags, comment) @@ -161,7 +162,7 @@ def get_attributes(ps): value = get_pattern(ps) if value is None: - raise ParseError('Expected field') + raise ParseError('E0006', 'value') attrs.append(ast.Attribute(key, value)) @@ -202,7 +203,7 @@ def get_variant_key(ps): ch = ps.current() if ch is None: - raise ParseError('Expected variant key') + raise ParseError('E0013') if ps.is_number_start(): return get_number(ps) @@ -220,6 +221,8 @@ def get_variants(ps): ps.skip_line_ws() if ps.current_is('*'): + if has_default: + raise ParseError('E0015') ps.next() default_index = True has_default = True @@ -235,7 +238,7 @@ def get_variants(ps): value = get_pattern(ps) if value is None: - raise ParseError('Expected field') + raise ParseError('E0006', 'value') variants.append(ast.Variant(key, value, default_index)) @@ -243,7 +246,7 @@ def get_variants(ps): break if not has_default: - raise ParseError('Missing default variant') + raise ParseError('E0010') return variants @@ -270,7 +273,7 @@ def get_digits(ps): ch = ps.take_digit() if len(num) == 0: - raise ParseError('Expected char range') + raise ParseError('E0004', '0-9') return num @@ -301,19 +304,16 @@ def get_pattern(ps): if first_line and len(buffer) != 0: break - ps.peek() - - if not ps.current_peek_is(' '): - ps.reset_peek() + if not ps.is_peek_next_line_pattern(): break - ps.peek_line_ws() - ps.skip_to_peek() - - first_line = False + ps.next() + ps.skip_line_ws() - if len(buffer) != 0: + if not first_line: buffer += ch + + first_line = False continue elif ch == '\\': ch2 = ps.peek() @@ -374,7 +374,7 @@ def get_expression(ps): variants = get_variants(ps) if len(variants) == 0: - raise ParseError('Missing variables') + raise ParseError('E0011') ps.expect_char('\n') ps.expect_char(' ') @@ -429,7 +429,7 @@ def get_call_args(ps): if ps.current_is(':'): if not isinstance(exp, ast.MessageReference): - raise ParseError('Forbidden key') + raise ParseError('E0009') ps.next() ps.skip_line_ws() @@ -456,7 +456,7 @@ def get_arg_val(ps): return get_number(ps) elif ps.current_is('"'): return get_string(ps) - raise ParseError('Expected field') + raise ParseError('E0006', 'value') def get_string(ps): val = '' @@ -476,7 +476,7 @@ def get_literal(ps): ch = ps.current() if ch is None: - raise ParseError('Expected literal') + raise ParseError('E0014') if ps.is_number_start(): return get_number(ps) diff --git a/tests/syntax/fixtures_structure/message_with_empty_pattern.ftl b/tests/syntax/fixtures_structure/message_with_empty_pattern.ftl new file mode 100644 index 00000000..179dbacc --- /dev/null +++ b/tests/syntax/fixtures_structure/message_with_empty_pattern.ftl @@ -0,0 +1 @@ +foo = diff --git a/tests/syntax/fixtures_structure/message_with_empty_pattern.json b/tests/syntax/fixtures_structure/message_with_empty_pattern.json new file mode 100644 index 00000000..e4ee002e --- /dev/null +++ b/tests/syntax/fixtures_structure/message_with_empty_pattern.json @@ -0,0 +1,26 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 6 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [] + }, + "attributes": null, + "tags": null, + "comment": null + } + ], + "comment": null +} diff --git a/tests/syntax/fixtures_structure/placeable_at_eol.ftl b/tests/syntax/fixtures_structure/placeable_at_eol.ftl new file mode 100644 index 00000000..37665cc2 --- /dev/null +++ b/tests/syntax/fixtures_structure/placeable_at_eol.ftl @@ -0,0 +1,4 @@ +key = + A multiline message with a { placeable } + at the end of line. The message should + consist of three lines of text. diff --git a/tests/syntax/fixtures_structure/placeable_at_eol.json b/tests/syntax/fixtures_structure/placeable_at_eol.json new file mode 100644 index 00000000..5da1469a --- /dev/null +++ b/tests/syntax/fixtures_structure/placeable_at_eol.json @@ -0,0 +1,42 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 130 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline message with a " + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + }, + { + "type": "TextElement", + "value": "\nat the end of line. The message should\nconsist of three lines of text." + } + ] + }, + "attributes": null, + "tags": null, + "comment": null + } + ], + "comment": null +} diff --git a/tests/syntax/fixtures_structure/resource_comment.ftl b/tests/syntax/fixtures_structure/resource_comment.ftl new file mode 100644 index 00000000..4c7a33c9 --- /dev/null +++ b/tests/syntax/fixtures_structure/resource_comment.ftl @@ -0,0 +1,2 @@ +// This is a resource wide comment +// It's multiline diff --git a/tests/syntax/fixtures_structure/resource_comment.json b/tests/syntax/fixtures_structure/resource_comment.json new file mode 100644 index 00000000..d8a529e0 --- /dev/null +++ b/tests/syntax/fixtures_structure/resource_comment.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [], + "comment": { + "type": "Comment", + "span": { + "type": "Span", + "start": 0, + "end": 53 + }, + "annotations": [], + "content": "This is a resource wide comment\nIt's multiline" + } +} diff --git a/tests/syntax/fixtures_structure/resource_comment_trailing_line.ftl b/tests/syntax/fixtures_structure/resource_comment_trailing_line.ftl new file mode 100644 index 00000000..14b5cced --- /dev/null +++ b/tests/syntax/fixtures_structure/resource_comment_trailing_line.ftl @@ -0,0 +1,3 @@ +// This is a comment +// This comment is multiline +// diff --git a/tests/syntax/fixtures_structure/resource_comment_trailing_line.json b/tests/syntax/fixtures_structure/resource_comment_trailing_line.json new file mode 100644 index 00000000..49c8c167 --- /dev/null +++ b/tests/syntax/fixtures_structure/resource_comment_trailing_line.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [], + "comment": { + "type": "Comment", + "span": { + "type": "Span", + "start": 0, + "end": 53 + }, + "annotations": [], + "content": "This is a comment\nThis comment is multiline\n" + } +} diff --git a/tests/syntax/fixtures_structure/section.ftl b/tests/syntax/fixtures_structure/section.ftl new file mode 100644 index 00000000..60a26d32 --- /dev/null +++ b/tests/syntax/fixtures_structure/section.ftl @@ -0,0 +1,2 @@ + +[[ This is a section ]] diff --git a/tests/syntax/fixtures_structure/section.json b/tests/syntax/fixtures_structure/section.json new file mode 100644 index 00000000..657f491b --- /dev/null +++ b/tests/syntax/fixtures_structure/section.json @@ -0,0 +1,20 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Section", + "span": { + "type": "Span", + "start": 1, + "end": 25 + }, + "annotations": [], + "name": { + "type": "Symbol", + "name": "This is a section" + }, + "comment": null + } + ], + "comment": null +} diff --git a/tests/syntax/fixtures_structure/simple_message.ftl b/tests/syntax/fixtures_structure/simple_message.ftl new file mode 100644 index 00000000..3036ce1c --- /dev/null +++ b/tests/syntax/fixtures_structure/simple_message.ftl @@ -0,0 +1 @@ +foo = Foo diff --git a/tests/syntax/fixtures_structure/simple_message.json b/tests/syntax/fixtures_structure/simple_message.json new file mode 100644 index 00000000..6a8b54a3 --- /dev/null +++ b/tests/syntax/fixtures_structure/simple_message.json @@ -0,0 +1,31 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 9 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Foo" + } + ] + }, + "attributes": null, + "tags": null, + "comment": null + } + ], + "comment": null +} diff --git a/tests/syntax/fixtures_structure/standalone_comment.ftl b/tests/syntax/fixtures_structure/standalone_comment.ftl new file mode 100644 index 00000000..49fce24e --- /dev/null +++ b/tests/syntax/fixtures_structure/standalone_comment.ftl @@ -0,0 +1,3 @@ +foo = Value + +// This is a standalone comment diff --git a/tests/syntax/fixtures_structure/standalone_comment.json b/tests/syntax/fixtures_structure/standalone_comment.json new file mode 100644 index 00000000..0ae86257 --- /dev/null +++ b/tests/syntax/fixtures_structure/standalone_comment.json @@ -0,0 +1,41 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "span": { + "type": "Span", + "start": 0, + "end": 11 + }, + "annotations": [], + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": null, + "tags": null, + "comment": null + }, + { + "type": "Comment", + "span": { + "type": "Span", + "start": 13, + "end": 45 + }, + "annotations": [], + "content": "This is a standalone comment" + } + ], + "comment": null +} diff --git a/tests/syntax/test_serializer.py b/tests/syntax/test_serializer.py index 88aaedfb..51874133 100644 --- a/tests/syntax/test_serializer.py +++ b/tests/syntax/test_serializer.py @@ -113,7 +113,6 @@ def test_comment_section(self): """ self.assertEqual(pretty_ftl(input), dedent_ftl(input)) - @unittest.skip("The parser ignores the new-line after }.") def test_multiline_with_placeable(self): input = """\ foo = @@ -122,20 +121,6 @@ def test_multiline_with_placeable(self): """ self.assertEqual(pretty_ftl(input), dedent_ftl(input)) - # The parser ignores the new-line after { bar }. Consequently, none of the - # Text elements composing the Pattern contain a new-line and the serializer - # output a single line. There's also no space between { bar } and Baz. - def test_multiline_with_placeable_current(self): - input = """\ - foo = - Foo { bar } - Baz - """ - output = """\ - foo = Foo { bar }Baz - """ - self.assertEqual(pretty_ftl(input), dedent_ftl(output)) - def test_tag(self): input = """\ foo = Foo diff --git a/tests/syntax/test_structure.py b/tests/syntax/test_structure.py new file mode 100644 index 00000000..af2f2f65 --- /dev/null +++ b/tests/syntax/test_structure.py @@ -0,0 +1,54 @@ +from __future__ import unicode_literals +import os +import sys +import json +import codecs +import unittest + +sys.path.append('.') + +from fluent.syntax.parser import parse + + +def read_file(path): + with codecs.open(path, 'r', encoding='utf-8') as file: + text = file.read() + return text + + +fixtures = os.path.join( + os.path.dirname(__file__), 'fixtures_structure' +) + + +class TestStructureMeta(type): + def __new__(mcs, name, bases, attrs): + + def gen_test(file_name): + def test(self): + ftl_path = os.path.join(fixtures, file_name + '.ftl') + ast_path = os.path.join(fixtures, file_name + '.json') + + source = read_file(ftl_path) + expected = read_file(ast_path) + + ast = parse(source) + + self.assertEqual(ast.to_json(), json.loads(expected)) + + return test + + for f in os.listdir(fixtures): + file_name, ext = os.path.splitext(f) + + if ext != '.ftl': + continue + + test_name = 'test_{}'.format(file_name) + attrs[test_name] = gen_test(file_name) + + return type.__new__(mcs, name, bases, attrs) + + +class TestStructure(unittest.TestCase): + __metaclass__ = TestStructureMeta From 98360cdfc11aac42012cdc9301e9d92dd951cf97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 15 May 2017 22:24:47 +0200 Subject: [PATCH 2/2] Behavior tests --- fluent/syntax/errors.py | 3 + .../attribute_expression_with_wrong_attr.ftl | 7 ++ .../attribute_starts_from_nl.ftl | 3 + .../attribute_with_empty_pattern.ftl | 2 + .../attribute_without_equal_sign.ftl | 3 + .../fixtures_behavior/broken_number.ftl | 14 +++ .../call_expression_with_trailing_comma.ftl | 1 + .../call_expression_with_wrong_kwarg_name.ftl | 2 + .../call_expression_with_wrong_value_type.ftl | 2 + .../comment_continues_with_one_slash.ftl | 3 + .../fixtures_behavior/comment_with_eof.ftl | 1 + .../fixtures_behavior/empty_resource.ftl | 0 .../empty_resource_with_ws.ftl | 2 + .../entry_start_with_one_slash.ftl | 3 + .../fixtures_behavior/leading_empty_lines.ftl | 3 + .../leading_empty_lines_with_ws.ftl | 5 + .../fixtures_behavior/multiline_string.ftl | 3 + .../multiline_with_non_empty_first_line.ftl | 3 + .../non_id_attribute_name.ftl | 3 + .../placeable_at_line_extremes.ftl | 8 ++ .../placeable_without_close_bracket.ftl | 3 + .../second_attribute_starts_from_nl.ftl | 4 + .../second_tag_starts_from_nl.ftl | 4 + .../section_starts_with_one_bracket.ftl | 2 + .../section_with_nl_in_the_middle.ftl | 3 + .../section_with_no_nl_after_it.ftl | 1 + .../section_with_one_bracket_at_the_end.ftl | 2 + .../select_expression_with_two_selectors.ftl | 2 + .../select_expression_without_arrow.ftl | 2 + .../select_expression_without_variants.ftl | 6 + .../selector_expression_ends_abruptly.ftl | 2 + .../fixtures_behavior/simple_message.ftl | 1 + .../fixtures_behavior/single_char_id.ftl | 2 + .../standalone_identifier.ftl | 2 + .../tag_and_attribute_together.ftl | 4 + .../fixtures_behavior/tag_starts_from_nl.ftl | 3 + .../unclosed_empty_placeable_error.ftl | 2 + .../fixtures_behavior/unknown_entry_start.ftl | 3 + .../variant_ends_abruptly.ftl | 3 + .../variant_expression_empty_key.ftl | 2 + .../variant_starts_from_nl.ftl | 4 + .../variant_with_digit_key.ftl | 3 + .../variant_with_empty_pattern.ftl | 3 + .../variant_with_leading_space_in_name.ftl | 4 + .../variant_with_symbol_with_space.ftl | 3 + .../variants_with_two_defaults.ftl | 5 + tests/syntax/test_behavior.py | 103 ++++++++++++++++++ 47 files changed, 249 insertions(+) create mode 100644 tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl create mode 100644 tests/syntax/fixtures_behavior/attribute_starts_from_nl.ftl create mode 100644 tests/syntax/fixtures_behavior/attribute_with_empty_pattern.ftl create mode 100644 tests/syntax/fixtures_behavior/attribute_without_equal_sign.ftl create mode 100644 tests/syntax/fixtures_behavior/broken_number.ftl create mode 100644 tests/syntax/fixtures_behavior/call_expression_with_trailing_comma.ftl create mode 100644 tests/syntax/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl create mode 100644 tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl create mode 100644 tests/syntax/fixtures_behavior/comment_continues_with_one_slash.ftl create mode 100644 tests/syntax/fixtures_behavior/comment_with_eof.ftl create mode 100644 tests/syntax/fixtures_behavior/empty_resource.ftl create mode 100644 tests/syntax/fixtures_behavior/empty_resource_with_ws.ftl create mode 100644 tests/syntax/fixtures_behavior/entry_start_with_one_slash.ftl create mode 100644 tests/syntax/fixtures_behavior/leading_empty_lines.ftl create mode 100644 tests/syntax/fixtures_behavior/leading_empty_lines_with_ws.ftl create mode 100644 tests/syntax/fixtures_behavior/multiline_string.ftl create mode 100644 tests/syntax/fixtures_behavior/multiline_with_non_empty_first_line.ftl create mode 100644 tests/syntax/fixtures_behavior/non_id_attribute_name.ftl create mode 100644 tests/syntax/fixtures_behavior/placeable_at_line_extremes.ftl create mode 100644 tests/syntax/fixtures_behavior/placeable_without_close_bracket.ftl create mode 100644 tests/syntax/fixtures_behavior/second_attribute_starts_from_nl.ftl create mode 100644 tests/syntax/fixtures_behavior/second_tag_starts_from_nl.ftl create mode 100644 tests/syntax/fixtures_behavior/section_starts_with_one_bracket.ftl create mode 100644 tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl create mode 100644 tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl create mode 100644 tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl create mode 100644 tests/syntax/fixtures_behavior/select_expression_with_two_selectors.ftl create mode 100644 tests/syntax/fixtures_behavior/select_expression_without_arrow.ftl create mode 100644 tests/syntax/fixtures_behavior/select_expression_without_variants.ftl create mode 100644 tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl create mode 100644 tests/syntax/fixtures_behavior/simple_message.ftl create mode 100644 tests/syntax/fixtures_behavior/single_char_id.ftl create mode 100644 tests/syntax/fixtures_behavior/standalone_identifier.ftl create mode 100644 tests/syntax/fixtures_behavior/tag_and_attribute_together.ftl create mode 100644 tests/syntax/fixtures_behavior/tag_starts_from_nl.ftl create mode 100644 tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl create mode 100644 tests/syntax/fixtures_behavior/unknown_entry_start.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_ends_abruptly.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_expression_empty_key.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_starts_from_nl.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_with_digit_key.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_with_empty_pattern.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_with_leading_space_in_name.ftl create mode 100644 tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl create mode 100644 tests/syntax/fixtures_behavior/variants_with_two_defaults.ftl create mode 100644 tests/syntax/test_behavior.py diff --git a/fluent/syntax/errors.py b/fluent/syntax/errors.py index 3c9ca309..fd3fa00a 100644 --- a/fluent/syntax/errors.py +++ b/fluent/syntax/errors.py @@ -1,3 +1,6 @@ +from __future__ import unicode_literals + + class ParseError(Exception): def __init__(self, code, *args): self.code = code diff --git a/tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl b/tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl new file mode 100644 index 00000000..5fbf7b77 --- /dev/null +++ b/tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl @@ -0,0 +1,7 @@ +key = { foo.23 } + +//~ ERROR E0004, pos 12, args "a-zA-Z" + +key = { foo. } + +//~ ERROR E0004, pos 31, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/attribute_starts_from_nl.ftl b/tests/syntax/fixtures_behavior/attribute_starts_from_nl.ftl new file mode 100644 index 00000000..380b08cc --- /dev/null +++ b/tests/syntax/fixtures_behavior/attribute_starts_from_nl.ftl @@ -0,0 +1,3 @@ +foo = Value +.attr = Value 2 +//~ ERROR E0002, pos 12 diff --git a/tests/syntax/fixtures_behavior/attribute_with_empty_pattern.ftl b/tests/syntax/fixtures_behavior/attribute_with_empty_pattern.ftl new file mode 100644 index 00000000..66e4df3e --- /dev/null +++ b/tests/syntax/fixtures_behavior/attribute_with_empty_pattern.ftl @@ -0,0 +1,2 @@ +key = Value + .label = diff --git a/tests/syntax/fixtures_behavior/attribute_without_equal_sign.ftl b/tests/syntax/fixtures_behavior/attribute_without_equal_sign.ftl new file mode 100644 index 00000000..f412f83c --- /dev/null +++ b/tests/syntax/fixtures_behavior/attribute_without_equal_sign.ftl @@ -0,0 +1,3 @@ +key = Value + .label +//~ ERROR E0003, pos 22, args "=" diff --git a/tests/syntax/fixtures_behavior/broken_number.ftl b/tests/syntax/fixtures_behavior/broken_number.ftl new file mode 100644 index 00000000..46cce00c --- /dev/null +++ b/tests/syntax/fixtures_behavior/broken_number.ftl @@ -0,0 +1,14 @@ +key = { -2.4.5 } +//~ ERROR E0003, pos 12, args "}" + +key = { -2.4. } +//~ ERROR E0003, pos 30, args "}" + +key = { -.4 } +//~ ERROR E0004, pos 44, args "0-9" + +key = { -2..4 } +//~ ERROR E0004, pos 61, args "0-9" + +key = { 24d } +//~ ERROR E0003, pos 77, args "}" diff --git a/tests/syntax/fixtures_behavior/call_expression_with_trailing_comma.ftl b/tests/syntax/fixtures_behavior/call_expression_with_trailing_comma.ftl new file mode 100644 index 00000000..974cf99c --- /dev/null +++ b/tests/syntax/fixtures_behavior/call_expression_with_trailing_comma.ftl @@ -0,0 +1 @@ +key = { BUILTIN(23, ) } diff --git a/tests/syntax/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl b/tests/syntax/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl new file mode 100644 index 00000000..b0bd8500 --- /dev/null +++ b/tests/syntax/fixtures_behavior/call_expression_with_wrong_kwarg_name.ftl @@ -0,0 +1,2 @@ +key = { BUILTIN(2: "foo") } +//~ ERROR E0009, pos 17 diff --git a/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl b/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl new file mode 100644 index 00000000..26ac1398 --- /dev/null +++ b/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl @@ -0,0 +1,2 @@ +key = { BUILTIN(key: foo) } +//~ ERROR E0006, pos 21, args "value" diff --git a/tests/syntax/fixtures_behavior/comment_continues_with_one_slash.ftl b/tests/syntax/fixtures_behavior/comment_continues_with_one_slash.ftl new file mode 100644 index 00000000..5dd830cd --- /dev/null +++ b/tests/syntax/fixtures_behavior/comment_continues_with_one_slash.ftl @@ -0,0 +1,3 @@ +// This is a normal comment +/ but this is not +//~ ERROR E0003, pos 29, args "/" diff --git a/tests/syntax/fixtures_behavior/comment_with_eof.ftl b/tests/syntax/fixtures_behavior/comment_with_eof.ftl new file mode 100644 index 00000000..0ae10470 --- /dev/null +++ b/tests/syntax/fixtures_behavior/comment_with_eof.ftl @@ -0,0 +1 @@ +// This is a comment with no new line diff --git a/tests/syntax/fixtures_behavior/empty_resource.ftl b/tests/syntax/fixtures_behavior/empty_resource.ftl new file mode 100644 index 00000000..e69de29b diff --git a/tests/syntax/fixtures_behavior/empty_resource_with_ws.ftl b/tests/syntax/fixtures_behavior/empty_resource_with_ws.ftl new file mode 100644 index 00000000..4e435443 --- /dev/null +++ b/tests/syntax/fixtures_behavior/empty_resource_with_ws.ftl @@ -0,0 +1,2 @@ + + diff --git a/tests/syntax/fixtures_behavior/entry_start_with_one_slash.ftl b/tests/syntax/fixtures_behavior/entry_start_with_one_slash.ftl new file mode 100644 index 00000000..440b3ffc --- /dev/null +++ b/tests/syntax/fixtures_behavior/entry_start_with_one_slash.ftl @@ -0,0 +1,3 @@ + +/ Test +//~ ERROR E0003, pos 2, args "/" diff --git a/tests/syntax/fixtures_behavior/leading_empty_lines.ftl b/tests/syntax/fixtures_behavior/leading_empty_lines.ftl new file mode 100644 index 00000000..60e34ee6 --- /dev/null +++ b/tests/syntax/fixtures_behavior/leading_empty_lines.ftl @@ -0,0 +1,3 @@ + + +foo = Value diff --git a/tests/syntax/fixtures_behavior/leading_empty_lines_with_ws.ftl b/tests/syntax/fixtures_behavior/leading_empty_lines_with_ws.ftl new file mode 100644 index 00000000..cd626c8d --- /dev/null +++ b/tests/syntax/fixtures_behavior/leading_empty_lines_with_ws.ftl @@ -0,0 +1,5 @@ + + + + +foo = Value diff --git a/tests/syntax/fixtures_behavior/multiline_string.ftl b/tests/syntax/fixtures_behavior/multiline_string.ftl new file mode 100644 index 00000000..9f3cdba0 --- /dev/null +++ b/tests/syntax/fixtures_behavior/multiline_string.ftl @@ -0,0 +1,3 @@ +key = { BUILTIN(key: " + text + ") } diff --git a/tests/syntax/fixtures_behavior/multiline_with_non_empty_first_line.ftl b/tests/syntax/fixtures_behavior/multiline_with_non_empty_first_line.ftl new file mode 100644 index 00000000..70417cc5 --- /dev/null +++ b/tests/syntax/fixtures_behavior/multiline_with_non_empty_first_line.ftl @@ -0,0 +1,3 @@ +key = Value + Value 2 +//~ ERROR E0002, pos 12 diff --git a/tests/syntax/fixtures_behavior/non_id_attribute_name.ftl b/tests/syntax/fixtures_behavior/non_id_attribute_name.ftl new file mode 100644 index 00000000..1cf580b2 --- /dev/null +++ b/tests/syntax/fixtures_behavior/non_id_attribute_name.ftl @@ -0,0 +1,3 @@ +key = Value + .2 = Foo +//~ ERROR E0004, pos 17, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/placeable_at_line_extremes.ftl b/tests/syntax/fixtures_behavior/placeable_at_line_extremes.ftl new file mode 100644 index 00000000..64389db4 --- /dev/null +++ b/tests/syntax/fixtures_behavior/placeable_at_line_extremes.ftl @@ -0,0 +1,8 @@ +key1 = + { foo } + +key2 = + Foo { foo } + +key3 = + { foo } Foo diff --git a/tests/syntax/fixtures_behavior/placeable_without_close_bracket.ftl b/tests/syntax/fixtures_behavior/placeable_without_close_bracket.ftl new file mode 100644 index 00000000..282cf2b6 --- /dev/null +++ b/tests/syntax/fixtures_behavior/placeable_without_close_bracket.ftl @@ -0,0 +1,3 @@ +key = { $num + +//~ ERROR E0003, pos 12, args "}" diff --git a/tests/syntax/fixtures_behavior/second_attribute_starts_from_nl.ftl b/tests/syntax/fixtures_behavior/second_attribute_starts_from_nl.ftl new file mode 100644 index 00000000..e87c2b98 --- /dev/null +++ b/tests/syntax/fixtures_behavior/second_attribute_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = Value + .label = Value +.accesskey = K +//~ ERROR E0002, pos 31 diff --git a/tests/syntax/fixtures_behavior/second_tag_starts_from_nl.ftl b/tests/syntax/fixtures_behavior/second_tag_starts_from_nl.ftl new file mode 100644 index 00000000..00c9dcd9 --- /dev/null +++ b/tests/syntax/fixtures_behavior/second_tag_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = Value + #tag +#tag2 +//~ ERROR E0002, pos 21 diff --git a/tests/syntax/fixtures_behavior/section_starts_with_one_bracket.ftl b/tests/syntax/fixtures_behavior/section_starts_with_one_bracket.ftl new file mode 100644 index 00000000..eb423561 --- /dev/null +++ b/tests/syntax/fixtures_behavior/section_starts_with_one_bracket.ftl @@ -0,0 +1,2 @@ +[ This is a broken section ]] +//~ ERROR E0003, pos 1, args "[" diff --git a/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl b/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl new file mode 100644 index 00000000..d4731b4a --- /dev/null +++ b/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl @@ -0,0 +1,3 @@ +[[ This is +a broken section]] +//~ ERROR E0003, pos 10, args "]" diff --git a/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl b/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl new file mode 100644 index 00000000..f11f4c06 --- /dev/null +++ b/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl @@ -0,0 +1 @@ +[[ This is a correct section ]] diff --git a/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl b/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl new file mode 100644 index 00000000..0464afea --- /dev/null +++ b/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl @@ -0,0 +1,2 @@ +[[ This is a broken section ] +//~ ERROR E0003, pos 29, args "]" diff --git a/tests/syntax/fixtures_behavior/select_expression_with_two_selectors.ftl b/tests/syntax/fixtures_behavior/select_expression_with_two_selectors.ftl new file mode 100644 index 00000000..0a05e57d --- /dev/null +++ b/tests/syntax/fixtures_behavior/select_expression_with_two_selectors.ftl @@ -0,0 +1,2 @@ +key = { $foo $faa } +//~ ERROR E0003, pos 13, args "}" diff --git a/tests/syntax/fixtures_behavior/select_expression_without_arrow.ftl b/tests/syntax/fixtures_behavior/select_expression_without_arrow.ftl new file mode 100644 index 00000000..b89618dd --- /dev/null +++ b/tests/syntax/fixtures_behavior/select_expression_without_arrow.ftl @@ -0,0 +1,2 @@ +key = { $foo - } +//~ ERROR E0003, pos 13, args "}" diff --git a/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl b/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl new file mode 100644 index 00000000..198654bb --- /dev/null +++ b/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl @@ -0,0 +1,6 @@ +key = { $foo -> } +//~ ERROR E0003, pos 16, args "␤" + +key = { $foo -> + } +//~ ERROR E0003, pos 39, args "[" diff --git a/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl b/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl new file mode 100644 index 00000000..a2ad26be --- /dev/null +++ b/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -0,0 +1,2 @@ +key = { $foo -> +//~ ERROR E0003, pos 16, args "[" diff --git a/tests/syntax/fixtures_behavior/simple_message.ftl b/tests/syntax/fixtures_behavior/simple_message.ftl new file mode 100644 index 00000000..3036ce1c --- /dev/null +++ b/tests/syntax/fixtures_behavior/simple_message.ftl @@ -0,0 +1 @@ +foo = Foo diff --git a/tests/syntax/fixtures_behavior/single_char_id.ftl b/tests/syntax/fixtures_behavior/single_char_id.ftl new file mode 100644 index 00000000..b4d3b479 --- /dev/null +++ b/tests/syntax/fixtures_behavior/single_char_id.ftl @@ -0,0 +1,2 @@ +k = Value + .l = Foo diff --git a/tests/syntax/fixtures_behavior/standalone_identifier.ftl b/tests/syntax/fixtures_behavior/standalone_identifier.ftl new file mode 100644 index 00000000..d3ee59c7 --- /dev/null +++ b/tests/syntax/fixtures_behavior/standalone_identifier.ftl @@ -0,0 +1,2 @@ +foo +//~ ERROR E0005, pos 3, args "foo" diff --git a/tests/syntax/fixtures_behavior/tag_and_attribute_together.ftl b/tests/syntax/fixtures_behavior/tag_and_attribute_together.ftl new file mode 100644 index 00000000..2adfb21a --- /dev/null +++ b/tests/syntax/fixtures_behavior/tag_and_attribute_together.ftl @@ -0,0 +1,4 @@ +key = Value + .label = Foo + #masculine +//~ ERROR E0012, pos 28 diff --git a/tests/syntax/fixtures_behavior/tag_starts_from_nl.ftl b/tests/syntax/fixtures_behavior/tag_starts_from_nl.ftl new file mode 100644 index 00000000..797ab57a --- /dev/null +++ b/tests/syntax/fixtures_behavior/tag_starts_from_nl.ftl @@ -0,0 +1,3 @@ +key = Value +#tag +//~ ERROR E0002, pos 12 diff --git a/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl b/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl new file mode 100644 index 00000000..0c86cfb1 --- /dev/null +++ b/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl @@ -0,0 +1,2 @@ +bar = Bar { +//~ ERROR E0004, pos 11, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/unknown_entry_start.ftl b/tests/syntax/fixtures_behavior/unknown_entry_start.ftl new file mode 100644 index 00000000..357d3b46 --- /dev/null +++ b/tests/syntax/fixtures_behavior/unknown_entry_start.ftl @@ -0,0 +1,3 @@ + +8Foo = Foo +//~ ERROR E0002, pos 1 diff --git a/tests/syntax/fixtures_behavior/variant_ends_abruptly.ftl b/tests/syntax/fixtures_behavior/variant_ends_abruptly.ftl new file mode 100644 index 00000000..94dfcd4b --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_ends_abruptly.ftl @@ -0,0 +1,3 @@ +key = { $foo -> + *[ +//~ ERROR E0004, pos 22, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/variant_expression_empty_key.ftl b/tests/syntax/fixtures_behavior/variant_expression_empty_key.ftl new file mode 100644 index 00000000..c6fd0b8d --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_expression_empty_key.ftl @@ -0,0 +1,2 @@ +key = { foo[] } +//~ ERROR E0004, pos 12, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/variant_starts_from_nl.ftl b/tests/syntax/fixtures_behavior/variant_starts_from_nl.ftl new file mode 100644 index 00000000..139f4f10 --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_starts_from_nl.ftl @@ -0,0 +1,4 @@ +key = { +*[one] Value + } +//~ ERROR E0004, pos 7, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/variant_with_digit_key.ftl b/tests/syntax/fixtures_behavior/variant_with_digit_key.ftl new file mode 100644 index 00000000..3ab304d6 --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_with_digit_key.ftl @@ -0,0 +1,3 @@ +key = { + *[-2] Foo + } diff --git a/tests/syntax/fixtures_behavior/variant_with_empty_pattern.ftl b/tests/syntax/fixtures_behavior/variant_with_empty_pattern.ftl new file mode 100644 index 00000000..8dfb5bc1 --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_with_empty_pattern.ftl @@ -0,0 +1,3 @@ +key = { + *[one] + } diff --git a/tests/syntax/fixtures_behavior/variant_with_leading_space_in_name.ftl b/tests/syntax/fixtures_behavior/variant_with_leading_space_in_name.ftl new file mode 100644 index 00000000..0b8f9c95 --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_with_leading_space_in_name.ftl @@ -0,0 +1,4 @@ +key = { + *[ one] Foo + } +//~ ERROR E0004, pos 18, args "a-zA-Z" diff --git a/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl b/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl new file mode 100644 index 00000000..68a60c6e --- /dev/null +++ b/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl @@ -0,0 +1,3 @@ +key = { + *[New York] Nowy Jork + } diff --git a/tests/syntax/fixtures_behavior/variants_with_two_defaults.ftl b/tests/syntax/fixtures_behavior/variants_with_two_defaults.ftl new file mode 100644 index 00000000..f49c2998 --- /dev/null +++ b/tests/syntax/fixtures_behavior/variants_with_two_defaults.ftl @@ -0,0 +1,5 @@ +key = { + *[one] Foo + *[two] Two + } +//~ ERROR E0015, pos 27 diff --git a/tests/syntax/test_behavior.py b/tests/syntax/test_behavior.py new file mode 100644 index 00000000..49fa8705 --- /dev/null +++ b/tests/syntax/test_behavior.py @@ -0,0 +1,103 @@ +from __future__ import unicode_literals +import os +import re +import sys +import codecs +import unittest + +sys.path.append('.') + +from fluent.syntax.parser import parse + + +sigil = r'^\/\/~ ' +re_directive = re.compile(r'{}(.*)[\n$]'.format(sigil), re.MULTILINE) + + +def preprocess(source): + return [ + re.findall(re_directive, source), + re.sub(re_directive, '', source) + ] + + +def get_code_name(code): + first = code[0] + if first == 'E': + return 'ERROR {}'.format(code) + if first == 'W': + return 'ERROR {}'.format(code) + if first == 'H': + return 'ERROR {}'.format(code) + raise Exception('Unknown Annotation code') + + +def serialize_annotation(annot): + parts = [get_code_name(annot.code)] + span = annot.span + + if (span.start == span.end): + parts.append('pos {}'.format(span.start)) + else: + parts.append( + 'start {}'.format(span.start), + 'end {}'.format(span.end) + ) + + if len(annot.args): + pretty_args = ' '.join([ + '"{}"'.format(arg) + for arg in annot.args + ]) + parts.append('args {}'.format(pretty_args)) + + return ', '.join(parts) + + +def read_file(path): + with codecs.open(path, 'r', encoding='utf-8') as file: + text = file.read() + return text + + +fixtures = os.path.join( + os.path.dirname(__file__), 'fixtures_behavior' +) + + +class TestBehaviorMeta(type): + def __new__(mcs, name, bases, attrs): + + def gen_test(file_name): + def test(self): + ftl_path = os.path.join(fixtures, file_name + '.ftl') + ftl_file = read_file(ftl_path) + + [expected_directives, source] = preprocess(ftl_file) + expected = '{}\n'.format('\n'.join(expected_directives)) + ast = parse(source) + actual_directives = [ + serialize_annotation(annot) + for entry in ast.body + for annot in entry.annotations + ] + actual = '{}\n'.format('\n'.join(actual_directives)) + + self.assertEqual(actual, expected) + + return test + + for f in os.listdir(fixtures): + file_name, ext = os.path.splitext(f) + + if ext != '.ftl': + continue + + test_name = 'test_{}'.format(file_name) + attrs[test_name] = gen_test(file_name) + + return type.__new__(mcs, name, bases, attrs) + + +class TestBehavior(unittest.TestCase): + __metaclass__ = TestBehaviorMeta