Linking requirements to Jira issues

[mk-xw]

Hi,

We are currently looking into linking sdoc requirements to jira issues and epics for project management. It is obvously desirable that this is, as much as possible, an automated process. I've created a small python script that can read .sdoc files and then edit or create jira issues with data from requirements in the .sdoc file.

Details

import json
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from sys import exit as sys_exit

from dataclasses_json import dataclass_json
from jira import JIRA, Issue
from loguru import logger
from strictdoc.backend.sdoc.models.node import SDocNode, SDocNodeField
from strictdoc.backend.sdoc.reader import ProjectConfig, SDReader
from strictdoc.backend.sdoc.writer import ParentReqReference, SDWriter


class IssueType(Enum):
    EPIC = "Epic"
    REQUIREMENT = "Requirement"
    TASK = "Task"


@dataclass_json
@dataclass
class JiraConfig:
    url: str
    project: str

    epics: list[str]  # list of sdoc files that contain nodes to be mapped to jira epics
    requirements: list[str]  # list of files that contain nodes to be mapped to jira requirement tickets

    node_to_issue: dict[str, str]  # values to write from sdoc node to jira issue
    issue_to_node: dict[str, str]  # values to write from jira issue to sdoc node


@dataclass_json
@dataclass
class JiraAuth:
    email: str
    token: str


class JiraExporter:
    JIRA_CONFIG_PATH = Path(__file__).parent.parent / "jira_config.json"
    JIRA_AUTH_PATH = Path(__file__).parent.parent / "jira_auth.json"

    def __init__(self) -> None:
        with self.JIRA_CONFIG_PATH.open() as f_json:
            self.config: JiraConfig = JiraConfig.from_dict(json.load(f_json))
            logger.success("Loaded jira_config.json")

        if self.JIRA_AUTH_PATH.exists():
            with self.JIRA_AUTH_PATH.open() as f_json:
                self.auth: JiraAuth = JiraAuth.from_dict(json.load(f_json))
                logger.success("Loaded jira_auth.json")
        else:
            logger.error(
                f"""

                    Could not find "{self.JIRA_AUTH_PATH}"

                    This file is not version controlled as it contains an api key tied to your personal account.
                    Create {self.JIRA_AUTH_PATH.name} at the root of the project and populate with the template below:
                    {{
                        "email": "<your email>",
                        "token": "<your token>"
                    }}

                    To get an api token, follow the instructions on this page:
                        https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/#Create-an-API-token

                    Do not share you api token with anyone, and do not add it to any version control
                """.replace("                 ", "")
            )
            sys_exit()

        logger.info(f"Connecting to Jira: {self.config.url} as user {self.auth.email}")

        # Connect
        try:
            self.jira = JIRA(server=self.config.url, basic_auth=(self.auth.email, self.auth.token))
        except:
            logger.exception("Jira connection failed!")
            sys_exit()

        logger.success("Connected to Jira!")

        # Get field metadata
        self.field_map = self.get_field_map()

        self.sdoc_config = ProjectConfig.default_config()

    def get_issue_from_sd_id(self, sd_id: str) -> Issue | None:
        jql = f'project = {self.config.project} AND "SD ID" = "{sd_id}"'
        issues = self.jira.search_issues(jql)
        return issues[0] if len(issues) > 0 else None

    def get_field_map(self) -> dict[str, dict[str, str]]:
        fields = self.jira.fields()
        field_map = {}

        for f in fields:
            field_map[f["name"]] = {"id": f["id"], "schema": f.get("schema", {})}

        return field_map

    def get_issue_field_by_name(self, name: str) -> str | None:
        if name in self.field_map:
            return self.field_map[name]["id"]
        else:
            return None

    def diff_sdoc_jira(self, node: SDocNode, issue: Issue) -> bool:
        sd_id = self.get_issue_field_by_name("SD ID")
        if sd_id:
            issue_value = issue.get_field(sd_id)
            node_value = node.get_field_by_name("UID").get_text_value()
            if issue_value != node_value:
                logger.debug(f"SDOC node and Jira Issue SD ID different {node_value} != {issue_value}")
                return True

        sd_statement = self.get_issue_field_by_name("SD Statement")
        if sd_statement:
            issue_value = issue.get_field(sd_statement)
            node_value = node.get_field_by_name("STATEMENT").get_text_value()
            if issue_value != node_value:
                logger.debug(f"SDOC and Jira SD Statement different {node_value} != {issue_value}")
                return True

        sd_title = self.get_issue_field_by_name("Summary")
        if sd_title:
            issue_value = issue.get_field(sd_title)
            node_value = node.get_field_by_name("TITLE").get_text_value()
            if issue_value != node_value:
                logger.debug(f"SDOC and Jira Title different {node_value} != {issue_value}")
                return True

        return False

    def parse_epics(self) -> None:
        for epic_file in self.config.epics:
            logger.info(f"Parsing {epic_file} for sdoc requirements")
            reader = SDReader()
            sdoc = reader.read_from_file(epic_file, self.sdoc_config)
            for node in sdoc.iterate_nodes():
                assert isinstance(node, SDocNode)
                self.node_to_issue(node, IssueType.EPIC)

    def parse_requirements(self) -> None:
        for requirement_file in self.config.requirements:
            logger.info(f"Parsing {requirement_file} for sdoc requirements")
            reader = SDReader()
            sdoc = reader.read_from_file(requirement_file, self.sdoc_config)
            for node in sdoc.iterate_nodes():
                assert isinstance(node, SDocNode)
                self.node_to_issue(node, IssueType.REQUIREMENT)

    def node_to_issue(self, node: SDocNode, issue_type: IssueType) -> None:
        if node.node_type == "REQUIREMENT":
            uid = node.get_field_by_name("UID").get_text_value()

            # find any parent links for node, and see if any of those parent links match a jira epic
            # if it does match a jira epic, store it for use later once we have linked this node to a jira issue
            epic = None
            if issue_type == IssueType.REQUIREMENT:
                for relation in node.relations:
                    if isinstance(relation, ParentReqReference):
                        logger.info(f"Found parent relation {relation.ref_uid} for node {uid}")
                        epic = self.get_issue_from_sd_id(relation.ref_uid)
                        if epic:
                            break

            if uid:
                logger.success(f"sdoc requirement found with UID: {uid}")
                # attempt to find a jira issue already linked to node
                issue = self.get_issue_from_sd_id(uid)

                #  if there is no linked issue then we create one!
                if not issue:
                    fields = {
                        "project": self.config.project,
                        "issuetype": issue_type.value,
                    }
                    for sdoc_field, jira_field in self.config.node_to_issue.items():
                        issue_field = self.get_issue_field_by_name(jira_field)
                        if issue_field:
                            fields[issue_field] = node.get_field_by_name(sdoc_field).get_text_value()
                        else:
                            logger.warning(f"Jira issue field {jira_field} not found in field map")

                    issue = self.jira.create_issue(fields=fields)

                    logger.warning(f"jira requirement created with ID: {issue.key} from sdoc UID: {uid}")

                # if there is an existing issue, then check if anything needs updating from the node to the issue
                elif self.diff_sdoc_jira(node, issue):
                    fields = {}

                    for sdoc_field, jira_field in self.config.node_to_issue.items():
                        fields[self.get_issue_field_by_name(jira_field)] = node.get_field_by_name(sdoc_field).get_text_value()

                    issue.update(fields=fields)

                    logger.warning(f"jira issue {issue.key} updated from sdoc UID: {uid}")
                    # node.set_field_value(
                    #     field_name="JIRA_ID", form_field_index=0, value=issue.key
                    # )
                else:
                    logger.info("Sdoc node and Jira Issue in Sync, no changes required")

                if epic:
                    logger.info(f"Linking issue {issue.key} to epic {epic.key} ")
                    issue.update({"parent": {"key": epic.key}})


exporter = JiraExporter()

exporter.parse_epics()
exporter.parse_requirements()

I've been struggling to then write data from jira to the sdoc requirement (issue key, status etc.). I've had a handful of errors some which make sense and others which I haven't been able to decipher.

Before going into detail on the issues encountered, is there a way to open an entire strictdoc project in python that can the be used to read/write files with the context of the project ( document fragments, grammar aliases etc.) available?

I know that there isn't a python API available at this point in time, but any pointers or help on where to look in the source code would be appreciated!

Thanks!

[stanislaw]

Hey @xw-mk,

This combination should help:

• https://strictdoc.readthedocs.io/en/stable/stable/docs/strictdoc_01_user_guide.html#14-Python-API
• https://github.com/strictdoc-project/strictdoc/tree/main/tests/integration/examples/python_api
• strictdoc/backend/reqif (examples of how SDoc <-> ReqIF bi-directional conversion is set up)

Ideally, your IDE should be able to browse through the Python modules of StrictDoc. Then you should be able to explore how the APIs are currently used.