Fix set position and added tests

flexparser/flexparser.py

@@ -443,7 +443,9 @@ def set_delimiters(self, delimiters: DelimiterDictT):
         self._spliter.set_delimiters(delimiters)
         if self._cache:
             value = self.peek()
-            self._spliter.set_position(value.start_line, value.start_col)
+            # Elements are 1 based indexing, while splitter is 0 based.
+            self._spliter.set_position(value.start_line - 1, value.start_col)
+            self._cache.clear()
 
     def _get_next_strip(self) -> Statement:
         part = ""
@@ -623,7 +625,7 @@ def get_position(self):
 
     @property
     def format_position(self):
-        if self.start_line == -1:
+        if self.start_line is None:
             return "N/A"
         return "%d,%d-%d,%d" % self.get_position()
 
@@ -981,7 +983,7 @@ def parse(self, source_location: SourceLocationT) -> ParsedSource[RBT, CT]:
 
     def parse_bytes(self, b: bytes, bos: BOS = None) -> ParsedSource[RBT, CT]:
         if bos is None:
-            bos = BOS(Hash.from_bytes(self._hasher, b))
+            bos = BOS(Hash.from_bytes(self._hasher, b)).set_simple_position(0, 0, 0)
 
         sic = self._statement_iterator_class(
             b.decode(self._encoding), self._delimiters, self._strip_spaces

flexparser/testsuite/test_iterators.py

@@ -54,6 +54,15 @@ def test_split_single_line(delimiters, content, expected):
             ((0, 0, 0, 8, "Testing "),),
         ),
         # ### 2
+        (
+            {"#": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.STOP_PARSING_LINE)},
+            "Testing # 123\nCaption # 456",
+            (
+                (0, 0, 0, 8, "Testing "),
+                (1, 0, 1, 8, "Caption "),
+            ),
+        ),
+        # ### 3
         (
             {"#": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CAPTURE_NEXT_TIL_EOL)},
             "Testing # 123\nCaption # 456",
@@ -63,7 +72,7 @@ def test_split_single_line(delimiters, content, expected):
                 (1, 9, 1, 13, " 456"),
             ),
         ),
-        # ### 3
+        # ### 4
         (
             {
                 "#": (
@@ -82,7 +91,7 @@ def test_split_single_line(delimiters, content, expected):
                 (1, 8, 1, 13, "# 456"),
             ),
         ),
-        # ### 4
+        # ### 5
         (
             {
                 "#": (
@@ -101,7 +110,7 @@ def test_split_single_line(delimiters, content, expected):
                 (1, 8, 1, 14, "## 456"),
             ),
         ),
-        # ### 5
+        # ### 6
         (
             {
                 "#": (fp.DelimiterInclude.SPLIT_BEFORE, fp.DelimiterAction.CONTINUE),
@@ -119,7 +128,7 @@ def test_split_single_line(delimiters, content, expected):
                 (1, 9, 1, 14, "# 456"),
             ),
         ),
-        # ### 6
+        # ### 7
         (
             {
                 "#": (
@@ -179,3 +188,72 @@ def test_statement():
         bi.peek()
     with pytest.raises(StopIteration):
         next(bi)
+
+
+def test_statement2():
+    dlm = {
+        "#": (
+            fp.DelimiterInclude.SPLIT_BEFORE,
+            fp.DelimiterAction.CAPTURE_NEXT_TIL_EOL,
+        ),
+        "\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+    }
+    content = "Testing ## 123\nCaption ## 456"
+    bi = fp.StatementIterator(content, dlm)
+    assert bi.peek().raw_strip == "Testing"
+    assert next(bi).raw_strip == "Testing"
+    assert bi.peek().raw_strip == "## 123"
+    assert next(bi).raw_strip == "## 123"
+
+    el = next(bi)
+    # strip spaces now changes the element
+    # not the parser.
+    assert el.raw == "Caption"
+    assert el.raw_strip == "Caption"
+    assert el.start_line == 2
+    assert el.start_col == 0
+    assert el.end_line == 2
+    assert el.end_col == 7
+
+    assert next(bi).raw_strip == "## 456"
+    assert bi.peek("blip") == "blip"
+    with pytest.raises(StopIteration):
+        bi.peek()
+    with pytest.raises(StopIteration):
+        next(bi)
+
+
+def test_statement_change_dlm():
+    dlm = {
+        "#": (
+            fp.DelimiterInclude.SPLIT_BEFORE,
+            fp.DelimiterAction.CAPTURE_NEXT_TIL_EOL,
+        ),
+        "\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+    }
+
+    dlm_new = {
+        "!": (
+            fp.DelimiterInclude.SPLIT_BEFORE,
+            fp.DelimiterAction.CAPTURE_NEXT_TIL_EOL,
+        ),
+        "\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+        "\r\n": (fp.DelimiterInclude.SPLIT, fp.DelimiterAction.CONTINUE),
+    }
+    content = "Testing ## 123\nCaption !! 456"
+    bi = fp.StatementIterator(content, dlm)
+    assert bi.peek().raw_strip == "Testing"
+    assert next(bi).raw_strip == "Testing"
+    assert bi.peek().raw_strip == "## 123"
+    assert next(bi).raw_strip == "## 123"
+
+    assert bi.peek().raw_strip == "Caption !! 456"
+    bi.set_delimiters(dlm_new)
+    assert bi.peek().raw_strip == "Caption"
+    assert next(bi).raw_strip == "Caption"
+    assert next(bi).raw_strip == "!! 456"

flexparser/testsuite/test_parser.py

@@ -42,6 +42,11 @@ def test_consume(content):
     myparser = MyParser(None)
 
     pf = myparser.parse_bytes(content).parsed_source
+    assert pf.start_line == 0
+    assert pf.start_col == 0
+    assert pf.end_line == 3
+    assert pf.end_col == 0
+    assert pf.format_position == "0,0-3,0"
     assert isinstance(pf.opening, fp.BOS)
     assert isinstance(pf.closing, fp.EOS)
     body = tuple(pf.body)

flexparser/testsuite/test_project.py

@@ -195,6 +195,13 @@ def test_parse3(tmp_path, definition):
     body = tuple(mb.body)
     assert len(body) == 1
     mb = body[0]
+
+    assert mb.start_line == 1
+    assert mb.start_col == 0
+    assert mb.end_line == 4
+    assert mb.end_col == 4
+    assert mb.format_position == "1,0-4,4"
+
     assert mb.opening == Open().set_simple_position(FIRST_NUMBER + 0, 0, 6).set_raw(
         "@begin"
     )