🔧 Centralise need parts creation and strongly type needs (#1129)

sphinx_needs/api/need.py

@@ -119,7 +119,7 @@ def run():
     :param tags: Tags as single string.
     :param constraints: Constraints as single, comma separated, string.
     :param constraints_passed: Contains bool describing if all constraints have passed
-    :param links_string: Links as single string.
+    :param links_string: Links as single string. (Not used)
     :param delete: boolean value (Remove the complete need).
     :param hide: boolean value.
     :param hide_tags: boolean value. (Not used with Sphinx-Needs >0.5.0)
@@ -325,14 +325,13 @@ def run():
         doctype = ".rst"
 
     # Add the need and all needed information
-    needs_info: NeedsInfoType = {  # type: ignore[typeddict-item]
-        "docname": docname,
+    needs_info: NeedsInfoType = {
+        "docname": docname,  # type: ignore[typeddict-item]
+        "lineno": lineno,  # type: ignore[typeddict-item]
         "doctype": doctype,
-        "lineno": lineno,
         "target_id": need_id,
-        "external_url": external_url if is_external else None,
-        "content_node": None,  # gets set after rst parsing
-        "content_id": None,  # gets set after rst parsing
+        "content_node": None,
+        "content_id": None,
         "type": need_type,
         "type_name": type_name,
         "type_prefix": type_prefix,
@@ -360,17 +359,18 @@ def run():
         "parts": {},
         "is_part": False,
         "is_need": True,
+        "id_parent": need_id,
+        "id_complete": need_id,
         "is_external": is_external or False,
+        "external_url": external_url if is_external else None,
         "external_css": external_css or "external_link",
-        "is_modified": False,  # needed by needextend
-        "modifications": 0,  # needed by needextend
+        "is_modified": False,
+        "modifications": 0,
         "has_dead_links": False,
         "has_forbidden_dead_links": False,
-        # these are set later in the analyse_need_locations transform
         "sections": [],
         "section_name": "",
         "signature": "",
-        "parent_needs": [],
         "parent_need": "",
     }
     needs_extra_option_names = list(NEEDS_CONFIG.extra_options)
@@ -404,12 +404,10 @@ def run():
             or len(str(kwargs[link_type["option"]])) == 0
         ):
             # If it is in global option, value got already set during prior handling of them
-            links_string = needs_info[link_type["option"]]
-            links = _read_in_links(links_string)
+            links = _read_in_links(needs_info[link_type["option"]])
         else:
             # if it is set in kwargs, take this value and maybe override set value from global_options
-            links_string = kwargs[link_type["option"]]
-            links = _read_in_links(links_string)
+            links = _read_in_links(kwargs[link_type["option"]])
 
         needs_info[link_type["option"]] = links
         needs_info["{}_back".format(link_type["option"])] = []

sphinx_needs/data.py

@@ -10,6 +10,7 @@
     from docutils.nodes import Element, Text
     from sphinx.application import Sphinx
     from sphinx.environment import BuildEnvironment
+    from typing_extensions import Required
 
     from sphinx_needs.services.manager import ServiceManager
 
@@ -31,139 +32,144 @@ class NeedsPartType(TypedDict):
 
     id: str
     """ID of the part"""
-
-    is_part: bool
-    is_need: bool
-
     content: str
     """Content of the part."""
-    document: str
-    """docname where the part is defined."""
     links: list[str]
     """List of need IDs, which are referenced by this part."""
     links_back: list[str]
     """List of need IDs, which are referencing this part."""
 
 
-class NeedsInfoType(TypedDict):
+class NeedsInfoType(TypedDict, total=False):
     """Data for a single need."""
 
-    target_id: str
+    target_id: Required[str]
     """ID of the data."""
-    id: str
+    id: Required[str]
     """ID of the data (same as target_id)"""
 
     # TODO docname and lineno can be None, if the need is external,
     # but currently this raises mypy errors for other parts of the code base
-    docname: str
+    docname: Required[str]
     """Name of the document where the need is defined."""
-    lineno: int
+    lineno: Required[int]
     """Line number where the need is defined."""
 
     # meta information
-    full_title: str
+    full_title: Required[str]
     """Title of the need, of unlimited length."""
-    title: str
+    title: Required[str]
     """Title of the need, trimmed to a maximum length."""
-    status: None | str
-    tags: list[str]
+    status: Required[None | str]
+    tags: Required[list[str]]
 
     # rendering information
-    collapse: None | bool
+    collapse: Required[None | bool]
     """hide the meta-data information of the need."""
-    hide: bool
+    hide: Required[bool]
     """If true, the need is not rendered."""
-    delete: bool
+    delete: Required[bool]
     """If true, the need is deleted entirely."""
-    layout: None | str
+    layout: Required[None | str]
     """Key of the layout, which is used to render the need."""
-    style: None | str
+    style: Required[None | str]
     """Comma-separated list of CSS classes (all appended by `needs_style_`)."""
 
     # TODO why is it called arch?
-    arch: dict[str, str]
+    arch: Required[dict[str, str]]
     """Mapping of uml key to uml content."""
 
     # external reference information
-    is_external: bool
+    is_external: Required[bool]
     """If true, no node is created and need is referencing external url"""
-    external_url: None | str
+    external_url: Required[None | str]
     """URL of the need, if it is an external need."""
-    external_css: str
+    external_css: Required[str]
     """CSS class name, added to the external reference."""
 
     # type information (based on needs_types config)
-    type: str
-    type_name: str
-    type_prefix: str
-    type_color: str
+    type: Required[str]
+    type_name: Required[str]
+    type_prefix: Required[str]
+    type_color: Required[str]
     """Hexadecimal color code of the type."""
-    type_style: str
+    type_style: Required[str]
 
-    is_modified: bool
+    is_modified: Required[bool]
     """Whether the need was modified by needextend."""
-    modifications: int
+    modifications: Required[int]
     """Number of modifications by needextend."""
 
-    # parts information
-    parts: dict[str, NeedsPartType]
-    is_need: bool
-    is_part: bool
+    # used to distinguish a part from a need
+    is_need: Required[bool]
+    is_part: Required[bool]
+    # Mapping of parts, a.k.a. sub-needs, IDs to data that overrides the need's data
+    parts: Required[dict[str, NeedsPartType]]
+    # additional information required for compatibility with parts
+    id_parent: Required[str]
+    """ID of the parent need, or self ID if not a part"""
+    id_complete: Required[str]
+    """ID of the parent need, followed by the part ID, 
+    delimited by a dot: ``<id_parent>.<id>``,
+    or self ID if not a part
+    """
 
     # content creation information
-    jinja_content: bool
-    template: None | str
-    pre_template: None | str
-    post_template: None | str
-    content: str
+    jinja_content: Required[bool]
+    template: Required[None | str]
+    pre_template: Required[None | str]
+    post_template: Required[None | str]
+    content: Required[str]
     pre_content: str
     post_content: str
-    content_id: None | str
-    """ID of the content node."""
-    content_node: None | Element
-    """deep copy of the content node."""
-
-    # link information
-    links: list[str]
-    """List of need IDs, which are referenced by this need."""
-    links_back: list[str]
-    """List of need IDs, which are referencing this need."""
-    # TODO there is more dynamically added link information;
-    # for each item in needs_extra_links config
-    # (and in prepare_env 'links' and 'parent_needs' are added if not present),
-    # you end up with a key named by the "option" field,
-    # and then another key named by the "option" field + "_back"
-    # these all have value type `list[str]`
-    # back links are all set in process_need_nodes (-> create_back_links) transform
+    content_id: Required[None | str]
+    """ID of the content node (set after parsing)."""
+    content_node: Required[None | Element]
+    """deep copy of the content node (set after parsing)."""
 
     # these default to False and are updated in check_links post-process
-    has_dead_links: bool
+    has_dead_links: Required[bool]
     """True if any links reference need ids that are not found in the need list."""
-    has_forbidden_dead_links: bool
+    has_forbidden_dead_links: Required[bool]
     """True if any links reference need ids that are not found in the need list,
     and the link type does not allow dead links.
     """
 
     # constraints information
-    constraints: list[str]
+    constraints: Required[list[str]]
     """List of constraint names, which are defined for this need."""
     # set in process_need_nodes (-> process_constraints) transform
-    constraints_results: dict[str, dict[str, bool]]
+    constraints_results: Required[dict[str, dict[str, bool]]]
     """Mapping of constraint name, to check name, to result."""
-    constraints_passed: None | bool
+    constraints_passed: Required[None | bool]
     """True if all constraints passed, False if any failed, None if not yet checked."""
     constraints_error: str
     """An error message set if any constraint failed, and `error_message` field is set in config."""
 
     # additional source information
-    doctype: str
+    doctype: Required[str]
     """Type of the document where the need is defined, e.g. '.rst'"""
     # set in analyse_need_locations transform
-    sections: list[str]
-    section_name: str
+    sections: Required[list[str]]
+    section_name: Required[str]
     """Simply the first section"""
-    signature: str | Text
+    signature: Required[str | Text]
     """Derived from a docutils desc_name node"""
+    parent_need: Required[str]
+    """Simply the first parent id"""
+
+    # link information
+    # Note, there is more dynamically added link information;
+    # for each item in needs_extra_links config
+    # (and in prepare_env 'links' and 'parent_needs' are added if not present),
+    # you end up with a key named by the "option" field,
+    # and then another key named by the "option" field + "_back"
+    # these all have value type `list[str]`
+    # back links are all set in process_need_nodes (-> create_back_links) transform
+    links: list[str]
+    """List of need IDs, which are referenced by this need."""
+    links_back: list[str]
+    """List of need IDs, which are referencing this need."""
     parent_needs: list[str]
     """List of parents of the this need (by id),
     i.e. if this need is nested in another
@@ -172,8 +178,6 @@ class NeedsInfoType(TypedDict):
     """List of children of this need (by id),
     i.e. if needs are nested within this one
     """
-    parent_need: str
-    """Simply the first parent id"""
 
     # Fields added dynamically by services:
     # options from ``BaseService.options`` get added to ``NEEDS_CONFIG.extra_options``,
@@ -206,15 +210,6 @@ class NeedsInfoType(TypedDict):
     # - keys in ``needs_global_options`` config are added to every need via ``add_need``
 
 
-class NeedsPartsInfoType(NeedsInfoType):
-    """Generated by prepare_need_list"""
-
-    document: str
-    """docname where the part is defined."""
-    id_parent: str
-    id_complete: str
-
-
 class NeedsBaseDataType(TypedDict):
     """A base type for data items collected from directives."""
 

sphinx_needs/diagrams_common.py

@@ -17,7 +17,7 @@
 from sphinx.util.docutils import SphinxDirective
 
 from sphinx_needs.config import NeedsSphinxConfig
-from sphinx_needs.data import NeedsFilteredBaseType, NeedsInfoType, NeedsPartsInfoType
+from sphinx_needs.data import NeedsFilteredBaseType, NeedsInfoType
 from sphinx_needs.errors import NoUri
 from sphinx_needs.logging import get_logger
 from sphinx_needs.utils import get_scale, split_link_types
@@ -169,7 +169,7 @@ def get_debug_container(puml_node: nodes.Element) -> nodes.container:
 
 
 def calculate_link(
-    app: Sphinx, need_info: NeedsInfoType | NeedsPartsInfoType, _fromdocname: None | str
+    app: Sphinx, need_info: NeedsInfoType, _fromdocname: None | str
 ) -> str:
     """
     Link calculation

sphinx_needs/directives/needflow.py

@@ -16,7 +16,6 @@
 from sphinx_needs.data import (
     NeedsFlowType,
     NeedsInfoType,
-    NeedsPartsInfoType,
     SphinxNeedsData,
 )
 from sphinx_needs.debug import measure_time
@@ -125,7 +124,7 @@ def get_need_node_rep_for_plantuml(
     fromdocname: str,
     current_needflow: NeedsFlowType,
     all_needs: Iterable[NeedsInfoType],
-    need_info: NeedsPartsInfoType,
+    need_info: NeedsInfoType,
 ) -> str:
     """Calculate need node representation for plantuml."""
     needs_config = NeedsSphinxConfig(app.config)
@@ -167,8 +166,8 @@ def walk_curr_need_tree(
     fromdocname: str,
     current_needflow: NeedsFlowType,
     all_needs: Iterable[NeedsInfoType],
-    found_needs: list[NeedsPartsInfoType],
-    need: NeedsPartsInfoType,
+    found_needs: list[NeedsInfoType],
+    need: NeedsInfoType,
 ) -> str:
     """
     Walk through each need to find all its child needs and need parts recursively and wrap them together in nested structure.
@@ -239,7 +238,7 @@ def walk_curr_need_tree(
     return curr_need_tree
 
 
-def get_root_needs(found_needs: list[NeedsPartsInfoType]) -> list[NeedsPartsInfoType]:
+def get_root_needs(found_needs: list[NeedsInfoType]) -> list[NeedsInfoType]:
     return_list = []
     for current_need in found_needs:
         if current_need["is_need"]:
@@ -262,7 +261,7 @@ def cal_needs_node(
     fromdocname: str,
     current_needflow: NeedsFlowType,
     all_needs: Iterable[NeedsInfoType],
-    found_needs: list[NeedsPartsInfoType],
+    found_needs: list[NeedsInfoType],
 ) -> str:
     """Calculate and get needs node representaion for plantuml including all child needs and need parts."""
     top_needs = get_root_needs(found_needs)

sphinx_needs/directives/needtable.py

@@ -19,6 +19,7 @@
 )
 from sphinx_needs.filter_common import FilterBase, process_filters
 from sphinx_needs.functions.functions import check_and_get_content
+from sphinx_needs.roles.need_part import iter_need_parts
 from sphinx_needs.utils import add_doc, profile, remove_node_from_tree, row_col_maker
 
 
@@ -303,15 +304,7 @@ def sort(need: NeedsInfoType) -> Any:
 
             # Need part rows
             if current_needtable["show_parts"] and need_info["is_need"]:
-                for part in need_info["parts"].values():
-                    # update the part with all information from its parent
-                    # this is required to make ID links work
-                    # The dict has to be manipulated, so that row_col_maker() can be used
-                    temp_part: NeedsInfoType = {**need_info, **part.copy()}  # type: ignore[typeddict-unknown-key]
-                    temp_part["id_complete"] = f"{need_info['id']}.{temp_part['id']}"  # type: ignore[typeddict-unknown-key]
-                    temp_part["id_parent"] = need_info["id"]  # type: ignore[typeddict-unknown-key]
-                    temp_part["docname"] = need_info["docname"]
-
+                for temp_part in iter_need_parts(need_info):
                     row = nodes.row(classes=["need_part"])
 
                     for option, _title in current_needtable["columns"]: