feat(backend/sdoc_source_code): Allow empty lines in source node fields

.gitattributes

@@ -1,2 +1 @@
 tests/** text=auto eol=lf
-

strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py

@@ -22,12 +22,11 @@ class GrammarTemplate(Template):
 NODE_GRAMMAR_EXTENSION = GrammarTemplate("""
 node_field: node_name ":" node_multiline_value
 node_name: /##CUSTOM_TAGS/
-node_multiline_value: (_WS_INLINE | _NL) (NODE_FIRST_STRING_VALUE _NL) (NODE_STRING_VALUE _NL)*
+node_multiline_value: (_WS_INLINE? | (_WS_INLINE NODE_STRING_VALUE)) NEWLINE (NODE_STRING_VALUE NEWLINE)*
 
-NODE_FIRST_STRING_VALUE.2: /\\s*[^\n\r]+/x
-NODE_STRING_VALUE.2: /(?![ ]*##RELATION_MARKER_START)(?!\\s*(##CUSTOM_TAGS): )[^\n\r]+/x
+NODE_STRING_VALUE.2: /(?![ ]*##RELATION_MARKER_START)(?!\\s*(##CUSTOM_TAGS):\\s)(?!\\s*##NODE_FIELD_END_MARKER)[^\n\r]+/x
 
-_NORMAL_STRING_NO_MARKER_NO_NODE: /(?!\\s*##RELATION_MARKER_START)((?!\\s*(##CUSTOM_TAGS): )|(##RESERVED_KEYWORDS)).+/
+_NORMAL_STRING_NO_MARKER_NO_NODE: /(?!\\s*##RELATION_MARKER_START)((?!\\s*(##CUSTOM_TAGS):\\s)|(##RESERVED_KEYWORDS)).+/
 """)
 
 GRAMMAR = GrammarTemplate("""
@@ -74,6 +73,7 @@ def parse(
                 CUSTOM_TAGS="|".join(f"{tag}(?=:)" for tag in custom_tags),
                 RESERVED_KEYWORDS=RESERVED_KEYWORDS,
                 RELATION_MARKER_START=RELATION_MARKER_START,
+                NODE_FIELD_END_MARKER="SPDX-Req-End",
             )
             start = "(relation_marker | node_field | _NORMAL_STRING_NO_MARKER_NO_NODE | _WS)*"
         else:

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -227,17 +227,36 @@ def _parse_node_field(
         assert isinstance(node_value_node, Tree)
         assert node_value_node.data == "node_multiline_value"
 
-        processed_node_values = []
-        for node_value_component_ in node_value_node.children:
+        # Find minimal indent in lines 1..n. It will be used to dedent the block.
+        dedent = None
+        if len(node_value_node.children) > 1:
+            for node_value_component_ in node_value_node.children[1:]:
+                assert isinstance(node_value_component_, Token)
+                if node_value_component_.type == "NEWLINE":
+                    continue
+                line_value = node_value_component_.value
+                non_ws_len = len(line_value.lstrip(" "))
+                this_dedent = len(line_value) - non_ws_len
+                if dedent is None:
+                    dedent = this_dedent
+                elif non_ws_len > 0:
+                    dedent = min(this_dedent, dedent)
+        if dedent is None:
+            dedent = 0
+
+        # Join and dedent.
+        node_value = ""
+        for i, node_value_component_ in enumerate(node_value_node.children):
             assert isinstance(node_value_component_, Token)
-            processed_node_value = node_value_component_.value.strip()
-            if "\\n\\n" in processed_node_value:
-                processed_node_value = processed_node_value.replace(
-                    "\\n\\n", ""
-                )
-
-            processed_node_values.append(processed_node_value)
-
-        node_value = "\n".join(processed_node_values)
+            line_value = node_value_component_.value
+            if (
+                i > 0
+                and node_value_component_.type != "NEWLINE"
+                and dedent is not None
+            ):
+                line_value = line_value[min(dedent, len(line_value)) :]
+            node_value += line_value
+
+        node_value = node_value.rstrip()
 
         return node_name, node_value

tests/integration/features/source_code_traceability/_source_nodes/_auto_uuid/10_linux_spdx_req_id_autogen/src/example/example.c

@@ -12,7 +12,7 @@
  * SPDX-Text: This
  *            is
  *            a statement
- *            \n\n
+ *
  *            And this is the same statement's another paragraph.
  */
 void example_1(void) {

tests/unit/strictdoc/backend/sdoc_source_code/test_marker_lexer.py

@@ -44,7 +44,8 @@ def assert_node_field(
 
     values = list(
         node_value.scan_values(
-            lambda t: t.type in ("NODE_FIRST_STRING_VALUE", "NODE_STRING_VALUE")
+            lambda t: t.type
+            in ("NODE_FIRST_STRING_VALUE", "NODE_STRING_VALUE", "NEWLINE")
         )
     )
     for idx in range(len(expected_field_value)):
@@ -232,7 +233,7 @@ def test_30_relation_and_field() -> None:
 
 STATEMENT: When 1, The system 2 shall do 3
 
-FOOBAR
+Additionally, the system 3 shall do 4.
 """
 
     tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
@@ -252,13 +253,13 @@ def test_30_relation_and_field() -> None:
     assert_node_field(
         node_fields[0],
         "STATEMENT",
-        ["When C,", "           The system A shall do B"],
+        ["When C,", "\n", "           The system A shall do B"],
     )
 
     assert_node_field(
         node_fields[1],
         "STATEMENT",
-        ["When Z,", "           The system X shall do Y"],
+        ["When Z,", "\n", "           The system X shall do Y"],
     )
 
     assert_node_field(
@@ -270,7 +271,11 @@ def test_30_relation_and_field() -> None:
     assert_node_field(
         node_fields[3],
         "STATEMENT",
-        ["When 1, The system 2 shall do 3"],
+        [
+            "When 1, The system 2 shall do 3",
+            "\n\n",
+            "Additionally, the system 3 shall do 4.",
+        ],
     )
 
 
@@ -433,12 +438,10 @@ def test_33_multiline_and_multiparagraph_fields() -> None:
 FOOBAR
 
 STATEMENT: This
-           \\n\\n
+
            is
-           \\n\\n
-           how we do paragraphs.
 
-FOOBAR
+           how we do paragraphs.
 """
 
     tree = MarkerLexer.parse(input_string, custom_tags={"STATEMENT"})
@@ -451,14 +454,41 @@ def test_33_multiline_and_multiparagraph_fields() -> None:
         "STATEMENT",
         [
             "This",
-            "           \\n\\n",
+            "\n\n",
             "           is",
-            "           \\n\\n",
+            "\n\n",
             "           how we do paragraphs.",
         ],
     )
 
 
+def test_34_node_text_starting_below() -> None:
+    """
+    Ensure that first text can start somewhere in the next lines after a field,
+    and that the result keeps starting newlines as separate NEWLINE-symbol.
+    """
+    input_string = """\
+FIELD1:
+
+   Text starting far off from tag.
+"""
+    tree = MarkerLexer.parse(input_string, custom_tags={"FIELD1"})
+    assert tree.data == "start"
+
+    node_fields = list(tree.find_data("node_field"))
+    assert len(node_fields) == 1
+
+    assert_node_field(
+        node_fields[0],
+        "FIELD1",
+        [
+            "\n\n",
+            "   Text starting far off from tag.",
+            "\n",
+        ],
+    )
+
+
 def test_60_exclude_reserved_keywords() -> None:
     input_string = """
         FIXME: This can likely replace _weak below with no problem.
@@ -510,7 +540,7 @@ def test_80_linux_spdx_like_identifiers() -> None:
 SPDX-Text: This
            is
            a statement
-           \\n\\n
+
            And this is the same statement's another paragraph.
 """
 
@@ -530,9 +560,11 @@ def test_80_linux_spdx_like_identifiers() -> None:
         "SPDX-Text",
         [
             "This",
+            "\n",
             "           is",
+            "\n",
             "           a statement",
-            "           \\n\\n",
+            "\n\n",
             "           And this is the same statement's another paragraph.",
         ],
     )

tests/unit/strictdoc/backend/sdoc_source_code/test_marker_parser.py

@@ -267,6 +267,52 @@ def test_24_parses_multiline_marker():
     assert function_range.reqs_objs[2].ng_source_column == 8
 
 
+def test_30_parser_dedents_field_lines():
+    """
+    Since source code fields will likely opt for text rendering (instead of RST),
+    ensure that ASCII formating is preserved reasonably.
+    """
+    input_string = """\
+    /**
+     * FIELD1: Nothing to dedent here.
+     *
+     * FIELD2: Nothing to
+     * dedent here.
+     *
+     * FIELD3: Dedent
+     *         - this list
+     *           - but keep
+     *             - inner indent
+     *
+     * FIELD4:
+     *        ASCII art
+     *    ___           ___
+     *  ( foo ) <---> ( bar )
+     *    ‾‾‾           ‾‾‾
+     */"""
+
+    source_node = MarkerParser.parse(
+        input_string=input_string,
+        line_start=1,
+        line_end=7,
+        comment_line_start=1,
+        comment_byte_range=None,
+        custom_tags=["FIELD1", "FIELD2", "FIELD3", "FIELD4"],
+    )
+    assert source_node.fields["FIELD1"] == "Nothing to dedent here."
+    assert source_node.fields["FIELD2"] == "Nothing to\ndedent here."
+    assert source_node.fields["FIELD3"] == (
+        "Dedent\n- this list\n  - but keep\n    - inner indent"
+    )
+    assert source_node.fields["FIELD4"] == (
+        "\n"
+        "      ASCII art\n"
+        "  ___           ___\n"
+        "( foo ) <---> ( bar )\n"
+        "  ‾‾‾           ‾‾‾"
+    )
+
+
 def test_80_linux_spdx_example():
     input_string = """\
 /**
@@ -281,7 +327,7 @@ def test_80_linux_spdx_example():
  * SPDX-Text: This
  *            is
  *            a statement
- *            \\n\\n
+ *
  *            And this is the same statement's another paragraph.
  */
 """