fix: swallow syntax errors when checkpoint is not committed yet

tests/test_stream.py

@@ -1,3 +1,5 @@
+from typing import Tuple, Union
+
 import pytest
 
 from tokenstream import TokenStream, UnexpectedToken
@@ -138,3 +140,26 @@ def test_checkpoint_commit():
             assert next(stream).value == "world"
             commit()
         assert [token.value for token in stream] == []
+
+
+def test_checkpoint_error():
+    stream = TokenStream("hello world 1 2 3 thing")
+
+    def argument(stream: TokenStream) -> Union[str, Tuple[int, int, int]]:
+        with stream.checkpoint() as commit:
+            triplet = (
+                int(stream.expect("number").value),
+                int(stream.expect("number").value),
+                int(stream.expect("number").value),
+            )
+            commit()
+            return triplet
+        return stream.expect("word").value  # type: ignore
+
+    with stream.syntax(number=r"\d+", word=r"\w+"):
+        assert [argument(stream) for _ in stream.peek_until()] == [
+            "hello",
+            "world",
+            (1, 2, 3),
+            "thing",
+        ]

tokenstream/stream.py

@@ -838,7 +838,8 @@ def checkpoint(self) -> Iterator[CheckpointCommit]:
         'hello'
         'hello'
 
-        You can use the returned ``commit()`` function to keep the state of the stream at the end of the ``with`` statement.
+        You can use the returned ``commit()`` function to keep the state of the stream
+        at the end of the ``with`` statement.
 
         >>> stream = TokenStream("hello world")
         >>> with stream.syntax(word=r"[a-z]+"):
@@ -848,11 +849,16 @@ def checkpoint(self) -> Iterator[CheckpointCommit]:
         ...     stream.expect("word").value
         'hello'
         'world'
+
+        Checkpoints will swallow syntax errors until the ``commit()`` function is called.
         """
         previous_index = [self.index]
 
         try:
             yield lambda: previous_index.clear()
+        except InvalidSyntax:
+            if not previous_index:
+                raise
         finally:
             if previous_index:
                 self.index = previous_index[0]