feat(backend/sdoc_source_code): Linux Kernel Template proposal: auto-generate SPDX-Req-ID and SPDX-Req-HKey

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -20,6 +20,7 @@
     RangeMarker,
 )
 from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
+from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
 from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
 
 
@@ -31,6 +32,7 @@ def parse(
         line_start: int,
         line_end: int,
         comment_line_start: int,
+        comment_byte_range: Optional[ByteRange],
         entity_name: Optional[str] = None,
         col_offset: int = 0,
         custom_tags: Optional[set[str]] = None,
@@ -50,8 +52,11 @@ def parse(
         """
 
         node_fields: Dict[str, str] = {}
-        source_node: SourceNode = SourceNode(entity_name)
 
+        source_node: SourceNode = SourceNode(
+            entity_name=entity_name,
+            comment_byte_range=comment_byte_range,
+        )
         input_string = preprocess_source_code_comment(input_string)
 
         tree: ParseTree = MarkerLexer.parse(
@@ -78,6 +83,11 @@ def parse(
                     element_,
                 )
                 node_fields[node_name] = node_value
+
+                source_node.fields_locations[node_name] = (
+                    element_.meta.start_pos,
+                    element_.meta.end_pos - 1,
+                )
             else:
                 raise AssertionError
 

strictdoc/backend/sdoc_source_code/marker_writer.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
+
+
+class MarkerWriter:
+    def write(
+        self,
+        source_node: SourceNode,
+        rewrites: dict[Any, bytes],
+        comment_file_bytes: bytes,
+    ) -> bytes:
+        output = bytearray()
+        prev_end = 0
+
+        for field_name_ in source_node.fields.keys():
+            if field_name_ not in rewrites:
+                continue
+
+            rewrite = rewrites[field_name_]
+            location = source_node.fields_locations[field_name_]
+
+            output += comment_file_bytes[prev_end : location[0]]
+
+            output += bytes(field_name_, encoding="utf8") + b": "
+            output += rewrite
+
+            prev_end = location[1]
+
+        # Possible trailing whitespace after last token.
+        if prev_end < len(comment_file_bytes):
+            output += comment_file_bytes[prev_end:]
+
+        return bytes(output)

strictdoc/backend/sdoc_source_code/models/function.py

@@ -2,12 +2,13 @@
 @relation(SDOC-SRS-142, scope=file)
 """
 
-from typing import Any, List, Set
+from typing import Any, List, Optional, Set
 
 from strictdoc.backend.sdoc_source_code.constants import FunctionAttribute
 from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
     FunctionRangeMarker,
 )
+from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
 from strictdoc.helpers.auto_described import auto_described
 
 
@@ -21,6 +22,7 @@ def __init__(
         display_name: str,
         line_begin: int,
         line_end: int,
+        code_byte_range: Optional[ByteRange],
         child_functions: List[Any],
         markers: List[FunctionRangeMarker],
         attributes: Set[FunctionAttribute],
@@ -36,6 +38,11 @@ def __init__(
         self.markers: List[FunctionRangeMarker] = markers
         self.line_begin = line_begin
         self.line_end = line_end
+
+        # Not all source code functions have ranges.
+        # Example: Robot framework files.
+        self.code_byte_range: Optional[ByteRange] = code_byte_range
+
         self.attributes: Set[FunctionAttribute] = attributes
 
     def is_declaration(self) -> bool:

strictdoc/backend/sdoc_source_code/models/source_location.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+
+from tree_sitter import Node
+
+
+@dataclass
+class ByteRange:
+    start: int
+    end: int
+
+    @classmethod
+    def create_from_ts_node(cls, node: Node) -> "ByteRange":
+        return cls(
+            start=node.start_byte,
+            end=node.end_byte,
+        )

strictdoc/backend/sdoc_source_code/models/source_node.py

@@ -13,16 +13,27 @@
 from strictdoc.backend.sdoc_source_code.models.range_marker import (
     RangeMarker,
 )
+from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
 from strictdoc.core.project_config import SourceNodesEntry
 
 
-@dataclass
+@dataclass(eq=False)
 class SourceNode:
+    """
+    NOTE: eq=False is needed to make this dataclass support being a dictionary key.
+
+    eq=False means that dictionaries will index by object identity. Copied
+    SourceNode objects will appear as two different SourceNodes. An alternative
+    could be to implement __eq__ and __hash__ so that they target byte_range.
+    """
+
     entity_name: Optional[str]
+    comment_byte_range: Optional[ByteRange]
     markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = field(
         default_factory=list
     )
     fields: dict[str, str] = field(default_factory=dict)
+    fields_locations: dict[str, tuple[int, int]] = field(default_factory=dict)
     function: Optional[Function] = None
 
     def get_sdoc_field(

strictdoc/backend/sdoc_source_code/reader_c.py

@@ -21,6 +21,7 @@
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
     SourceFileTraceabilityInfo,
 )
+from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
 from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
 from strictdoc.backend.sdoc_source_code.parse_context import ParseContext
 from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import (
@@ -50,13 +51,6 @@ def read(
     ) -> SourceFileTraceabilityInfo:
         assert isinstance(input_buffer, bytes)
 
-        file_size = len(input_buffer)
-
-        traceability_info = SourceFileTraceabilityInfo([])
-
-        if file_size == 0:
-            return traceability_info
-
         file_stats = SourceFileStats.create(input_buffer)
         parse_context = ParseContext(file_path, file_stats)
 
@@ -69,6 +63,8 @@ def read(
 
         tree = parser.parse(input_buffer)
 
+        traceability_info = SourceFileTraceabilityInfo([])
+
         nodes = traverse_tree(tree)
 
         source_node: Optional[SourceNode]
@@ -93,6 +89,9 @@ def read(
                             if input_buffer[-1] == 10
                             else node_.end_point[0] + 1,
                             comment_line_start=node_.start_point[0] + 1,
+                            comment_byte_range=ByteRange.create_from_ts_node(
+                                comment_node
+                            ),
                             custom_tags=self.custom_tags,
                         )
                         for marker_ in source_node.markers:
@@ -192,6 +191,9 @@ def read(
                         line_end=function_last_line,
                         comment_line_start=function_comment_node.start_point[0]
                         + 1,
+                        comment_byte_range=ByteRange.create_from_ts_node(
+                            function_comment_node
+                        ),
                         entity_name=function_display_name,
                         custom_tags=self.custom_tags,
                     )
@@ -216,6 +218,7 @@ def read(
                     if function_comment_node is not None
                     else node_.range.start_point[0] + 1,
                     line_end=node_.range.end_point[0] + 1,
+                    code_byte_range=ByteRange.create_from_ts_node(node_),
                     child_functions=[],
                     markers=function_markers,
                     attributes=function_attributes,
@@ -301,9 +304,13 @@ def read(
                         line_end=function_last_line,
                         comment_line_start=function_comment_node.start_point[0]
                         + 1,
+                        comment_byte_range=ByteRange.create_from_ts_node(
+                            function_comment_node
+                        ),
                         entity_name=function_display_name,
                         custom_tags=self.custom_tags,
                     )
+
                     traceability_info.source_nodes.append(source_node)
                     for marker_ in source_node.markers:
                         if isinstance(marker_, FunctionRangeMarker):
@@ -322,6 +329,7 @@ def read(
                     if function_comment_node is not None
                     else node_.range.start_point[0] + 1,
                     line_end=node_.range.end_point[0] + 1,
+                    code_byte_range=ByteRange.create_from_ts_node(node_),
                     child_functions=[],
                     markers=function_markers,
                     attributes={FunctionAttribute.DEFINITION},
@@ -358,6 +366,7 @@ def read(
                     line_start=node_.start_point[0] + 1,
                     line_end=node_.end_point[0] + 1,
                     comment_line_start=node_.start_point[0] + 1,
+                    comment_byte_range=ByteRange.create_from_ts_node(node_),
                     custom_tags=None,
                 )
 

strictdoc/backend/sdoc_source_code/reader_python.py

@@ -21,6 +21,7 @@
     RelationMarkerType,
     SourceFileTraceabilityInfo,
 )
+from strictdoc.backend.sdoc_source_code.models.source_location import ByteRange
 from strictdoc.backend.sdoc_source_code.parse_context import ParseContext
 from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import (
     function_range_marker_processor,
@@ -72,6 +73,7 @@ def read(
                     display_name="module",
                     line_begin=node_.start_point[0] + 1,
                     line_end=node_.end_point[0] + 1,
+                    code_byte_range=ByteRange.create_from_ts_node(node_),
                     child_functions=[],
                     markers=[],
                     attributes=set(),
@@ -111,6 +113,9 @@ def read(
                             else node_.end_point[0] + 1,
                             comment_line_start=string_content.start_point[0]
                             + 1,
+                            comment_byte_range=ByteRange.create_from_ts_node(
+                                string_content
+                            ),
                         )
                         for marker_ in source_node.markers:
                             if isinstance(marker_, FunctionRangeMarker) and (
@@ -170,6 +175,9 @@ def read(
                                 line_end=node_.end_point[0] + 1,
                                 comment_line_start=string_content.start_point[0]
                                 + 1,
+                                comment_byte_range=ByteRange.create_from_ts_node(
+                                    string_content
+                                ),
                                 entity_name=function_name,
                             )
                             for marker_ in source_node.markers:
@@ -201,6 +209,7 @@ def read(
                     display_name=function_name,
                     line_begin=node_.range.start_point[0] + 1,
                     line_end=node_.range.end_point[0] + 1,
+                    code_byte_range=ByteRange.create_from_ts_node(node_),
                     child_functions=[],
                     # Python functions do not need to track markers.
                     markers=[],
@@ -247,6 +256,9 @@ def read(
                     line_start=node_.start_point[0] + 1,
                     line_end=last_comment.end_point[0] + 1,
                     comment_line_start=node_.start_point[0] + 1,
+                    comment_byte_range=ByteRange(
+                        node_.start_byte, last_comment.end_byte
+                    ),
                     entity_name=None,
                 )
                 for marker_ in source_node.markers:

strictdoc/backend/sdoc_source_code/reader_robot.py

@@ -98,6 +98,8 @@ def visit_TestCase(self, node: TestCase) -> None:
                         input_string=token.value,
                         line_start=token.lineno,
                         line_end=token.lineno,
+                        # FIXME: Byte range is currently not used for Robot framework.
+                        comment_byte_range=None,
                         comment_line_start=token.lineno,
                         entity_name=node.name,
                         col_offset=token.col_offset,
@@ -125,6 +127,8 @@ def visit_TestCase(self, node: TestCase) -> None:
             display_name=node.name,
             line_begin=node.lineno,
             line_end=node.end_lineno - trailing_empty_lines,
+            # FIXME: Byte range is currently not used for Robot framework.
+            code_byte_range=None,
             child_functions=[],
             markers=function_markers,
             attributes={FunctionAttribute.DEFINITION},
@@ -139,6 +143,8 @@ def _visit_possibly_marked_node(
                 input_string=token.value,
                 line_start=node.lineno,
                 line_end=node.lineno,
+                # FIXME: Byte range is currently not used for Robot framework.
+                comment_byte_range=None,
                 comment_line_start=node.lineno,
                 entity_name=None,
                 col_offset=token.col_offset,

strictdoc/backend/sdoc_source_code/source_writer.py

@@ -0,0 +1,37 @@
+from strictdoc.backend.sdoc_source_code.models.source_file_info import (
+    SourceFileTraceabilityInfo,
+)
+from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
+
+
+class SourceWriter:
+    def write(
+        self,
+        trace_info: SourceFileTraceabilityInfo,
+        rewrites: dict[SourceNode, bytes],
+        file_bytes: bytes,
+    ) -> bytes:
+        output = bytearray()
+        prev_end = 0
+
+        for source_node_ in trace_info.source_nodes:
+            if source_node_.comment_byte_range is None:
+                continue
+
+            if source_node_ not in rewrites:
+                continue
+
+            rewrite = rewrites[source_node_]
+            output += file_bytes[
+                prev_end : source_node_.comment_byte_range.start
+            ]
+
+            output += rewrite
+
+            prev_end = source_node_.comment_byte_range.end
+
+        # Possible trailing whitespace after last token.
+        if prev_end < len(file_bytes):
+            output += file_bytes[prev_end:]
+
+        return bytes(output)

strictdoc/cli/main.py

@@ -6,6 +6,7 @@
 import multiprocessing
 import os
 import sys
+from pathlib import Path
 from typing import Optional
 
 strictdoc_root_path = os.path.abspath(
@@ -115,9 +116,13 @@ def _main_internal(parallelizer: Parallelizer, parser: SDocArgsParser) -> None:
         project_config = ProjectConfigLoader.load_from_path_or_get_default(
             path_to_config=manage_config.get_path_to_config(),
         )
-        # FIXME: This must be improved.
+
+        # FIXME: Encapsulate all this in project_config.integrate_manage_autouid_config(),
+        #        following the example of integrate_export_config().
         project_config.input_paths = [manage_config.input_path]
-        # FIXME: This must be improved.
+        project_config.source_root_path = str(
+            Path(manage_config.input_path).resolve()
+        )
         project_config.auto_uid_mode = True
         project_config.autouuid_include_sections = (
             manage_config.include_sections