Structure and behavior testing

fluent/syntax/ast.py

@@ -14,12 +14,23 @@ def to_json(value):
 
 def from_json(value):
     if isinstance(value, dict):
-        cls = getattr(sys.modules[__name__], value["type"])
+        cls = getattr(sys.modules[__name__], value['type'])
         args = {
             k: from_json(v)
-            for k, v in value.items() if k != "type"
+            for k, v in value.items()
+            if k != 'type'
+            if k != 'span'
         }
-        return cls(**args)
+        node = cls(**args)
+
+        # Spans need to be added via add_span, not __init__.
+        if 'span' in value:
+            span = value['span']
+            # Message and section comments don't have their own spans.
+            if span is not None:
+                node.add_span(span['start'], span['end'])
+
+        return node
     if isinstance(value, list):
         return list(map(from_json, value))
     else:
@@ -219,8 +230,11 @@ def __init__(self, start, end):
 
 
 class Annotation(Node):
-    def __init__(self, name, message, pos):
+    def __init__(self, code, args=None, message=None):
         super(Annotation, self).__init__()
-        self.name = name
+        self.code = code
+        self.args = args or []
         self.message = message
-        self.pos = pos
+
+    def add_span(self, start, end):
+        self.span = Span(start, end)

fluent/syntax/errors.py

@@ -1,3 +1,43 @@
+from __future__ import unicode_literals
+
+
 class ParseError(Exception):
-    def __init__(self, message):
-        self.message = message
+    def __init__(self, code, *args):
+        self.code = code
+        self.args = args
+        self.message = get_error_message(code, args)
+
+
+def get_error_message(code, args):
+    if code == 'E00001':
+        return 'Generic error'
+    if code == 'E0002':
+        return 'Expected an entry start'
+    if code == 'E0003':
+        return 'Expected token: "{}"'.format(args[0])
+    if code == 'E0004':
+        return 'Expected a character from range: "{}"'.format(args[0])
+    if code == 'E0005':
+        msg = 'Expected entry "{}" to have a value, attributes or tags'
+        return msg.format(args[0])
+    if code == 'E0006':
+        return 'Expected field: "{}"'.format(args[0])
+    if code == 'E0007':
+        return 'Keyword cannot end with a whitespace'
+    if code == 'E0008':
+        return 'Callee has to be a simple identifier'
+    if code == 'E0009':
+        return 'Key has to be a simple identifier'
+    if code == 'E0010':
+        return 'Expected one of the variants to be marked as default (*)'
+    if code == 'E0011':
+        return 'Expected at least one variant after "->"'
+    if code == 'E0012':
+        return 'Tags cannot be added to messages with attributes'
+    if code == 'E0013':
+        return 'Expected variant key'
+    if code == 'E0014':
+        return 'Expected literal'
+    if code == 'E0015':
+        return 'Only one variant can be marked as default (*)'
+    return code

fluent/syntax/ftlstream.py

@@ -32,7 +32,11 @@ def expect_char(self, ch):
             self.next()
             return True
 
-        raise ParseError('Expected token "{}"'.format(ch))
+        if ch == '\n':
+            # Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
+            raise ParseError('E0003', '\u2424')
+
+        raise ParseError('E0003', ch)
 
     def take_char_if(self, ch):
         if self.ch == ch:
@@ -42,12 +46,15 @@ def take_char_if(self, ch):
 
     def take_char(self, f):
         ch = self.ch
-        if f(ch):
+        if ch is not None and f(ch):
             self.next()
             return ch
         return None
 
     def is_id_start(self):
+        if self.ch is None:
+            return False
+
         cc = ord(self.ch)
 
         return (cc >= 97 and cc <= 122) or \
@@ -62,7 +69,7 @@ def is_number_start(self):
     def is_peek_next_line_indented(self):
         if not self.current_peek_is('\n'):
             return False
-        
+
         self.peek()
 
         if self.current_peek_is(' '):
@@ -75,10 +82,17 @@ def is_peek_next_line_indented(self):
     def is_peek_next_line_variant_start(self):
         if not self.current_peek_is('\n'):
             return False
-        
+
         self.peek()
 
+        ptr = self.get_peek_index()
+
         self.peek_line_ws()
+
+        if (self.get_peek_index() - ptr == 0):
+            self.reset_peek()
+            return False
+
         if self.current_peek_is('*'):
             self.peek()
 
@@ -92,27 +106,64 @@ def is_peek_next_line_variant_start(self):
     def is_peek_next_line_attribute_start(self):
         if not self.current_peek_is('\n'):
             return False
-        
+
         self.peek()
 
+        ptr = self.get_peek_index()
+
         self.peek_line_ws()
 
+        if (self.get_peek_index() - ptr == 0):
+            self.reset_peek()
+            return False
+
         if self.current_peek_is('.'):
             self.reset_peek()
             return True
 
         self.reset_peek()
         return False
 
+    def is_peek_next_line_pattern(self):
+        if not self.current_peek_is('\n'):
+            return False
+
+        self.peek()
+
+        ptr = self.get_peek_index()
+
+        self.peek_line_ws()
+
+        if (self.get_peek_index() - ptr == 0):
+            self.reset_peek()
+            return False
+
+        if (self.current_peek_is('}') or
+                self.current_peek_is('.') or
+                self.current_peek_is('#') or
+                self.current_peek_is('[') or
+                self.current_peek_is('*')):
+            self.reset_peek()
+            return False
+
+        self.reset_peek()
+        return True
+
     def is_peek_next_line_tag_start(self):
 
         if not self.current_peek_is('\n'):
             return False
 
         self.peek()
 
+        ptr = self.get_peek_index()
+
         self.peek_line_ws()
 
+        if (self.get_peek_index() - ptr == 0):
+            self.reset_peek()
+            return False
+
         if self.current_peek_is('#'):
             self.reset_peek()
             return True
@@ -136,15 +187,15 @@ def take_id_start(self):
             self.next()
             return ret
 
-        raise ParseError('Expected char range')
+        raise ParseError('E0004', 'a-zA-Z')
 
     def take_id_char(self):
         def closure(ch):
             cc = ord(ch)
-            return (cc >= 97 and cc <= 122) or \
-                   (cc >= 65 and cc <= 90) or \
-                   (cc >= 48 and cc <= 57) or \
-                    cc == 95 or cc == 45
+            return ((cc >= 97 and cc <= 122) or
+                    (cc >= 65 and cc <= 90) or
+                    (cc >= 48 and cc <= 57) or
+                    cc == 95 or cc == 45)
         return self.take_char(closure)
 
     def take_symb_char(self):

fluent/syntax/parser.py

@@ -39,7 +39,8 @@ def get_entry_or_junk(ps):
         entry.add_span(entry_start_pos, ps.get_index())
         return entry
     except ParseError as err:
-        annot = ast.Annotation("ParseError", err.message, ps.get_index())
+        annot = ast.Annotation(err.code, err.args, err.message)
+        annot.add_span(ps.get_index(), ps.get_index())
 
         ps.skip_to_next_entry_start()
         next_entry_start = ps.get_index()
@@ -66,7 +67,8 @@ def get_entry(ps):
 
     if comment:
         return comment
-    raise ParseError('Expected entry')
+
+    raise ParseError('E0002')
 
 def get_comment(ps):
     ps.expect_char('/')
@@ -134,12 +136,11 @@ def get_message(ps, comment):
 
     if ps.is_peek_next_line_tag_start():
         if attrs is not None:
-            raise ParseError(
-                'Tags cannot be added to messages with attributes')
+            raise ParseError('E0012')
         tags = get_tags(ps)
 
     if pattern is None and attrs is None and tags is None:
-        raise ParseError('Missing field')
+        raise ParseError('E0005', id.name)
 
     return ast.Message(id, pattern, attrs, tags, comment)
 
@@ -161,7 +162,7 @@ def get_attributes(ps):
         value = get_pattern(ps)
 
         if value is None:
-            raise ParseError('Expected field')
+            raise ParseError('E0006', 'value')
 
         attrs.append(ast.Attribute(key, value))
 
@@ -202,7 +203,7 @@ def get_variant_key(ps):
     ch = ps.current()
 
     if ch is None:
-        raise ParseError('Expected variant key')
+        raise ParseError('E0013')
 
     if ps.is_number_start():
         return get_number(ps)
@@ -220,6 +221,8 @@ def get_variants(ps):
         ps.skip_line_ws()
 
         if ps.current_is('*'):
+            if has_default:
+                raise ParseError('E0015')
             ps.next()
             default_index = True
             has_default = True
@@ -235,15 +238,15 @@ def get_variants(ps):
         value = get_pattern(ps)
 
         if value is None:
-            raise ParseError('Expected field')
+            raise ParseError('E0006', 'value')
 
         variants.append(ast.Variant(key, value, default_index))
 
         if not ps.is_peek_next_line_variant_start():
             break
 
     if not has_default:
-        raise ParseError('Missing default variant')
+        raise ParseError('E0010')
 
     return variants
 
@@ -270,7 +273,7 @@ def get_digits(ps):
         ch = ps.take_digit()
 
     if len(num) == 0:
-        raise ParseError('Expected char range')
+        raise ParseError('E0004', '0-9')
 
     return num
 
@@ -301,19 +304,16 @@ def get_pattern(ps):
             if first_line and len(buffer) != 0:
                 break
 
-            ps.peek()
-
-            if not ps.current_peek_is(' '):
-                ps.reset_peek()
+            if not ps.is_peek_next_line_pattern():
                 break
 
-            ps.peek_line_ws()
-            ps.skip_to_peek()
-
-            first_line = False
+            ps.next()
+            ps.skip_line_ws()
 
-            if len(buffer) != 0:
+            if not first_line:
                 buffer += ch
+
+            first_line = False
             continue
         elif ch == '\\':
             ch2 = ps.peek()
@@ -374,7 +374,7 @@ def get_expression(ps):
             variants = get_variants(ps)
 
             if len(variants) == 0:
-                raise ParseError('Missing variables')
+                raise ParseError('E0011')
 
             ps.expect_char('\n')
             ps.expect_char(' ')
@@ -429,7 +429,7 @@ def get_call_args(ps):
 
         if ps.current_is(':'):
             if not isinstance(exp, ast.MessageReference):
-                raise ParseError('Forbidden key')
+                raise ParseError('E0009')
 
             ps.next()
             ps.skip_line_ws()
@@ -456,7 +456,7 @@ def get_arg_val(ps):
         return get_number(ps)
     elif ps.current_is('"'):
         return get_string(ps)
-    raise ParseError('Expected field')
+    raise ParseError('E0006', 'value')
 
 def get_string(ps):
     val = ''
@@ -476,7 +476,7 @@ def get_literal(ps):
     ch = ps.current()
 
     if ch is None:
-        raise ParseError('Expected literal')
+        raise ParseError('E0014')
 
     if ps.is_number_start():
         return get_number(ps)

tests/syntax/fixtures_behavior/attribute_expression_with_wrong_attr.ftl

@@ -0,0 +1,7 @@
+key = { foo.23 }
+
+//~ ERROR E0004, pos 12, args "a-zA-Z"
+
+key = { foo. }
+
+//~ ERROR E0004, pos 31, args "a-zA-Z"

tests/syntax/fixtures_behavior/attribute_starts_from_nl.ftl

@@ -0,0 +1,3 @@
+foo = Value
+.attr = Value 2
+//~ ERROR E0002, pos 12

tests/syntax/fixtures_behavior/attribute_with_empty_pattern.ftl

@@ -0,0 +1,2 @@
+key = Value
+    .label =