Multiline @note, #4

dial/interpreter.py

@@ -83,20 +83,28 @@ def parse(self, string):
     def states(self):
         raise NotImplementedError()
 
+    def _action(self, action):
+        newstate = action(self)
+
+        if newstate is not None:
+            return self.states[newstate]
+        return None
+
     def perform(self, token):
         backup = self.state
         try:
             self.state = self.state[token.type]
         except KeyError:
             raise BadSyntax(self, token)
 
-        if token.type == NAME:
+        if token.type in (NAME, MULTILINE):
             self.tokenstack.append(token.string)
 
+        newstate = None
         if callable(self.state):
-            newstate = self.state(self)
+            newstate = self._action(self.state)
 
-            if newstate is not None:
-                self.state = self.states[newstate]
+            if newstate:
+                self.state = newstate
             else:
                 self.state = backup

dial/sequence.py

@@ -192,6 +192,7 @@ def _eat_keyword_value(self, *values):
         },
         '@...:': {
             NAME: Goto('@...:'),
-            NEWLINE: Hook(_eat_keyword_value, 'start')
+            MULTILINE: Hook(_eat_keyword_value, 'start'),
+            NEWLINE: Hook(_eat_keyword_value, 'start'),
         },
     }

dial/token.py

@@ -19,6 +19,7 @@
 BACKSLASH = 12
 SLASH = 13
 PIPE = 14
+MULTILINE = 15
 
 TOKEN_NAMES = {
     value: name for name, value in globals().items()

dial/tokenizer.py

@@ -22,6 +22,10 @@ def __init__(self):
         self.indent = 0
         self.escape = False
         self.newline = True
+        self._mlmatch = []
+        self._ml = False
+        self._mlindent = False
+        self._mllastlen = 0
 
     def _token(self, type_, string, start, end, line):
         return Token(
@@ -61,9 +65,34 @@ def _newlinetoken(self, token, start, end, line):
             line
         )
 
-    def tokenizeline(self, line):
+    def _tokenizeline(self, line):
         self.lineno += 1
 
+        if self._ml:
+            m = re.match(WHITESPACE_RE, line)
+            end = m.span()[1] if m else 0
+            if self._mlindent == 0 and end > 0:
+                self._mlindent = end
+                self._mltoken = line[self._mlindent:]
+                self._mllastlen = len(line)
+                return
+            elif end < self._mlindent:
+                sl = self._mlmatch[-1].start[0]
+                self._mlmatch.clear()
+                yield Token(
+                    MULTILINE,
+                    self._mltoken[:-1],
+                    (sl + 1, self._mlindent),
+                    (self.lineno - 1, self._mllastlen - 1),
+                    line
+                )
+                self._ml = False
+                self._mlindent = 0
+            elif end > 0:
+                self._mltoken += line[self._mlindent:]
+                self._mllastlen = len(line)
+                return
+
         if line == '':
             yield self._eoftoken(line)
             return
@@ -127,6 +156,20 @@ def tokenizeline(self, line):
                 line
             )
 
+    def tokenizeline(self, line):
+        for token in self._tokenizeline(line):
+            if token.type == PIPE:
+                self._mlmatch.append(token)
+                continue
+            elif token.type == NEWLINE and len(self._mlmatch) == 1:
+                self._mlmatch.append(token)
+                self._ml = True
+                continue
+            else:
+                while self._mlmatch:
+                    yield self._mlmatch.pop(0)
+                yield token
+
     def tokenize(self, readline):
         eof = False
 

tests/test_interpreter.py

@@ -13,13 +13,17 @@ def test_interpreter_sequencediagram_note():
                 @note: bar
                 @note left: baz
                 @note right: foo \
-                    bar \\n \
-                    baz
+                    bar
+                @note: |
+                    foo
+                    bar
+                    !@#$%^&*+_)(
             ''')
     assert repr(d[0]) == '@note left: foo'
     assert repr(d[1]) == '@note left of foo: bar'
     assert repr(d[2]) == '@note left of foo: baz'
-    assert repr(d[3]) == '@note right of foo: foo bar n baz'
+    assert repr(d[3]) == '@note right of foo: foo bar'
+    assert repr(d[4]) == '@note left of foo: foo\nbar\n!@#$%^&*+_)('
 
 
 def test_interpreter_sequencediagram_indent():

tests/test_tokenizer.py

@@ -9,6 +9,36 @@ def tokenizes(string):
         yield t.type, t.string, t.start, t.end
 
 
+def test_tokenizer_multiline():
+    gen = tokenizes('''
+    |
+        foo
+        bar.
+    foo: bar
+    ''')
+    exp = 'foo\nbar.'
+    assert next(gen) == (NEWLINE,   '\n',  (1,  0), (1,  1))
+    assert next(gen) == (MULTILINE, exp,   (3,  8), (4, 12))
+    assert next(gen) == (NAME,      'foo', (5,  4), (5,  7))
+    assert next(gen) == (COLON,     ':',   (5,  7), (5,  8))
+    assert next(gen) == (NAME,      'bar', (5,  9), (5, 12))
+    assert next(gen) == (NEWLINE,   '\n',  (5, 12), (5, 13))
+    assert next(gen) == (EOF,       '',    (7,  0), (7,  0))
+    with pytest.raises(StopIteration):
+        next(gen)
+
+    gen = tokenizes('''
+    | foo
+    ''')
+    assert next(gen) == (NEWLINE,   '\n',  (1,  0), (1,  1))
+    assert next(gen) == (PIPE,      '|',   (2,  4), (2,  5))
+    assert next(gen) == (NAME,      'foo', (2,  6), (2,  9))
+    assert next(gen) == (NEWLINE,   '\n',  (2,  9), (2, 10))
+    assert next(gen) == (EOF,       '',    (4,  0), (4,  0))
+    with pytest.raises(StopIteration):
+        next(gen)
+
+
 def test_tokenizer_emptyinput():
     gen = tokenizes('')
     assert next(gen) == (EOF, '', (1, 0), (1, 0))