Fix generate-docs Command Sections Parsing

.changelog/4303.yml

@@ -0,0 +1,8 @@
+changes:
+- description: Fixed an issue where **generate-docs** command couldn't find the sections within the integration README.md when updating documentation for modified commands.
+  type: fix
+- description: Added the `--force` flag to the **generate-docs** command to force the README.md generation instead of use version control to update the doc.
+  type: feature
+- description: Fixed an issue where modified command names were treated as a new command.
+  type: fix
+pr_number: 4303

demisto_sdk/__main__.py

@@ -28,6 +28,7 @@
 from demisto_sdk.commands.common.constants import (
     DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV,
     ENV_DEMISTO_SDK_MARKETPLACE,
+    INTEGRATIONS_README_FILE_NAME,
     FileType,
     MarketplaceVersions,
 )
@@ -2231,6 +2232,13 @@ def init(ctx, **kwargs):
     is_flag=True,
     default=True,
 )
+@click.option(
+    "-f",
+    "--force",
+    help="Whether to force the generation of documentation (rather than update when it exists in version control)",
+    is_flag=True,
+    default=False,
+)
 @click.pass_context
 @logging_setup_decorator
 def generate_docs(ctx, **kwargs):
@@ -2302,21 +2310,23 @@ def _generate_docs_for_file(kwargs: Dict[str, Any]):
     custom_image_path: str = kwargs.get("custom_image_path", "")
     readme_template: str = kwargs.get("readme_template", "")
     use_graph = kwargs.get("graph", True)
+    force = kwargs.get("force", False)
 
     try:
         if command:
             if (
                 output_path
-                and (not Path(output_path, "README.md").is_file())
+                and (not Path(output_path, INTEGRATIONS_README_FILE_NAME).is_file())
                 or (not output_path)
                 and (
                     not Path(
-                        os.path.dirname(os.path.realpath(input_path)), "README.md"
+                        os.path.dirname(os.path.realpath(input_path)),
+                        INTEGRATIONS_README_FILE_NAME,
                     ).is_file()
                 )
             ):
                 raise Exception(
-                    "[red]The `command` argument must be presented with existing `README.md` docs."
+                    f"[red]The `command` argument must be presented with existing `{INTEGRATIONS_README_FILE_NAME}` docs."
                 )
 
         file_type = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
@@ -2356,6 +2366,7 @@ def _generate_docs_for_file(kwargs: Dict[str, Any]):
                 command=command,
                 old_version=old_version,
                 skip_breaking_changes=skip_breaking_changes,
+                force=force,
             )
         elif file_type == FileType.SCRIPT:
             logger.info(f"Generating {file_type.value.lower()} documentation")

demisto_sdk/commands/generate_docs/README.md

@@ -1,11 +1,12 @@
-## generate-docs
+# `generate-docs`
 
-Generate a README file for an Integration, Script or a Playbook.
+Generate a `README.md` file for an Integration, Script or a Playbook.
+
+## Use Cases
 
-**Use-Cases**
 This command is used to create a documentation file for Cortex XSOAR content files.
 
-**Arguments**:
+## Arguments
 
 * **-i, --input**
 Path of the yml file.
@@ -35,32 +36,40 @@ Path of the old integration version yml file.
 Skip generating of breaking changes section.
 * **-ngr, --no-graph**
 Whether to use the content graph or not.
+* **-f, --force**
+Whether to force the generation of documentation (rather than update when it exists in version control).
 
-**Notes**
-
-* If `command_permissions` wil not be given, a generic message regarding the need of permissions will be given.
-* If no `output` given, the README.md file will be generated in the `input` file repository.
-* If no `additionalinfo` is provided for a commonly-used parameter (for example, `API Key`), a matching default value
+> [!NOTE]
+>
+> * If `command_permissions` wil not be given, a generic message regarding the need of permissions will be given.
+>
+> * If no `output` given, the `README.md` file will be generated in the `input` file repository.
+>
+> * If no `additionalinfo` is provided for a commonly-used parameter (for example, `API Key`), a matching default value
   will be used, see the parameters and defaults in `default_additional_information.json`.
-* In order to generate an **incident mirroring** section, make sure that the *isremotesyncin* and/or *isremotesyncout* parameters are set to true in the YML file. In addition, the following configuration parameters (if used) should be named as stated:
-  * incidents_fetch_query
-  * Mirroring tags - the available names are 'comment_tag', 'work_notes_tag' and 'file_tag'
-  * mirror_direction
-  * close_incident
-  * close_out - (opposite to close_incident)
+>
+> In order to generate an **incident mirroring** section, make sure that the `isremotesyncin` and/or `isremotesyncout` parameters are set to `true` in the YML file. In addition, the following configuration parameters (if used) should be named as stated:
+>
+> * `incidents_fetch_query`
+> * Mirroring tags - the available names are `comment_tag`, `work_notes_tag` and `file_tag`
+> * `mirror_direction`
+> * `close_incident`
+> * `close_out` - (opposite to `close_incident`)
+>
+> * If the Integration/Script/Playbook exists in version control, the version from the main branch (e.g. `master`) will be used to only render the modified sections (e.g. configuration, command) unless the `--force` flag is specified.
 
 ### Examples
 
 ```bash
 demisto-sdk generate-docs -i Packs/MyPack/Integrations/MyInt/MyInt.yml -e Packs/MyPack/Integrations/MyInt/command_example.txt
 ```
 
-This will generate a documentation for the MyInt integration using the command examples found in the .txt file in the MyInt integration.
+This will generate a documentation for the `MyInt` integration using the command examples found in the .txt file in the `MyInt` integration.
 
 ```bash
 demisto-sdk generate-docs -i Packs/MyPack/Integrations/MyInt/MyInt_v2.yml --old-version Packs/MyPack/Integrations/MyInt/MyInt.yml
 ```
 
-This will generate a documentation for MyInt_v2 integration including a section about changes compared the MyInt integration.
+This will generate a documentation for `MyInt_v2` integration including a section about changes compared the `MyInt` integration.
 The command will automatically detect if the given integration is a v2 using the integration's display name and create the changes section.
-If no '--old-version' is supplied a prompt will appear asking for the path to the old integration.
+If no `--old-version` is supplied a prompt will appear asking for the path to the old integration.

demisto_sdk/commands/generate_docs/generate_integration_doc.py

@@ -14,6 +14,7 @@
     DOCS_COMMAND_SECTION_REGEX,
     INTEGRATIONS_DIR,
     INTEGRATIONS_README_FILE_NAME,
+    SCRIPT,
 )
 from demisto_sdk.commands.common.default_additional_info_loader import (
     load_default_additional_info_dict,
@@ -30,6 +31,7 @@
 from demisto_sdk.commands.generate_docs.common import (
     CONFIGURATION_SECTION_STEPS,
     DEFAULT_ARG_DESCRIPTION,
+    HEADER_TYPE,
     add_lines,
     build_example_dict,
     generate_numbered_section,
@@ -217,9 +219,6 @@ def _update_conf_section(self):
 
             doc_text_lines = self.output_doc.splitlines()
 
-            # We take the first and the second-to-last index of the old section
-            # and use the section range to replace it with the new section.
-            # Second-to-last index because the last element is an empty string
             old_config_start_line = doc_text_lines.index(
                 CONFIGURATION_SECTION_STEPS.STEP_1.value
             )
@@ -245,16 +244,12 @@ def _update_commands_section(self):
         README.
         """
 
-        for i, modified_command in enumerate(
-            self.integration_diff.get_modified_commands()
-        ):
+        command_sections = get_commands_sections(self.output_doc)
+
+        for modified_command in self.integration_diff.get_modified_commands():
             try:
-                old_command_section, _ = generate_commands_section(
-                    self.integration_diff.old_yaml_data,
-                    {},
-                    {},
-                    modified_command,
-                )
+
+                start_line, end_line = command_sections[modified_command]
 
                 (
                     new_command_section,
@@ -281,35 +276,15 @@ def _update_commands_section(self):
 
                 doc_text_lines = self.output_doc.splitlines()
 
-                # We take the first and the second-to-last index of the old section
-                # and use the section range to replace it with the new section.
-                # Second-to-last index because the last element is an empty string
-                old_cmd_start_line = doc_text_lines.index(old_command_section[0])
-
-                # In cases when there are multiple identical context outputs
-                # in the second-to-last line, we need to find the relevant
-                # second-to-last line for the specific command we're replacing.
-                indices = [
-                    i
-                    for i, doc_line in enumerate(doc_text_lines)
-                    if doc_line == old_command_section[-2]
-                ]
-
-                if indices and len(indices) > 1:
-                    old_cmd_end_line = doc_text_lines.index(
-                        old_command_section[-2], indices[i]
-                    )
-                else:
-                    old_cmd_end_line = doc_text_lines.index(old_command_section[-2])
-
-                doc_text_lines[
-                    old_cmd_start_line : old_cmd_end_line + 1
-                ] = new_command_section
+                doc_text_lines[start_line:end_line] = new_command_section
 
                 self.output_doc = "\n".join(doc_text_lines)
             except (ValueError, IndexError) as e:
                 error = f"Unable to replace '{modified_command}' section in README: {str(e)}"
                 self.update_errors.append(error)
+            except KeyError:
+                error = f"Unable to find '{modified_command}' in the README. The command was likely renamed."
+                self.update_errors.append(error)
 
     def _get_sections_to_update(self) -> Tuple[bool, List[str], List[str]]:
 
@@ -319,11 +294,6 @@ def _get_sections_to_update(self) -> Tuple[bool, List[str], List[str]]:
             self.integration_diff.get_added_commands(),
         )
 
-    def _get_resource_path(self) -> str:
-        """
-        Helper function to resolve the resource path.
-        """
-
     def _write_resource_to_tmp(self, resource_path: Path, content: str) -> Path:
         """
         Helper function to write
@@ -372,7 +342,25 @@ def update_docs(self) -> Tuple[str, List[str]]:
         if added_commands:
             self.output_doc += "\n"
 
+            renamed_commands = self.integration_diff.get_renamed_commands()
+
             for cmd in added_commands:
+
+                skip = False
+                # We don't want to add renamed commands
+                if renamed_commands:
+                    for (
+                        renamed_command_original_name,
+                        renamed_command_changed_name,
+                    ) in renamed_commands:
+                        if cmd == renamed_command_changed_name:
+                            error = f"Skipping adding command '{cmd}' as it was detected as a renamed from '{renamed_command_original_name}'"
+                            self.update_errors.append(error)
+                            skip = True
+                            break
+                if skip:
+                    continue
+
                 logger.info(f"\t\u2699 Generating docs for command `{cmd}`...")
                 (
                     command_section,
@@ -456,6 +444,7 @@ def generate_integration_doc(
     old_version: str = "",
     skip_breaking_changes: bool = False,
     is_contribution: bool = False,
+    force: bool = False,
 ):
     """
     Generate integration documentation.
@@ -474,6 +463,7 @@ def generate_integration_doc(
         insecure: should use insecure
         command: specific command to generate docs for
         is_contribution: Check if the content item is a new integration contribution or not.
+        force: `bool` whether to force create a new integration doc even if it exists in version control.
 
     """
     try:
@@ -537,7 +527,7 @@ def generate_integration_doc(
         # in source control:
         # - An integration YAML.
         # - An integration README.
-        elif update_mgr.can_update_docs():
+        elif not force and update_mgr.can_update_docs():
             logger.info("Found existing integration, updating documentation...")
             doc_text, update_errors = update_mgr.update_docs()
 
@@ -851,9 +841,7 @@ def generate_commands_section(
         "After you successfully execute a command, a DBot message appears in the War Room with the command details.",
         "",
     ]
-    commands = filter(
-        lambda cmd: not cmd.get("deprecated", False), yaml_data["script"]["commands"]
-    )
+    commands = get_integration_commands(yaml_data)
     command_sections: list = []
     if command:
         # for specific command, return it only.
@@ -906,7 +894,7 @@ def generate_single_command_section(
         cmd_permission_example = []
 
     section = [
-        "### {}".format(cmd["name"]),
+        f"{HEADER_TYPE.H3} {cmd['name']}",
         "",
         "***",
     ]
@@ -1298,3 +1286,65 @@ def add_access_data_of_type_credentials(
             "Required": credentials_conf.get("required", ""),
         }
     )
+
+
+def get_integration_commands(yaml_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Helper function to return a list of integration commands.
+    Integration commands that are marked as deprecated will not be
+    returned.
+
+    Args:
+    - `yml_data` (``Dict[str, Any]``): The integration YAML as a dictionary.
+
+    Returns:
+    - `List[Dict[str, Any]]` of integration commands.
+    """
+
+    return list(
+        filter(
+            lambda cmd: not cmd.get("deprecated", False), yaml_data[SCRIPT]["commands"]
+        )
+    )
+
+
+def get_commands_sections(doc_text: str) -> Dict[str, Tuple[int, int]]:
+    """
+    Helper function that takes the integration README text
+    and returns a map of the commands and the start, end lines.
+
+    Args:
+    - `doc_text` (``str``): The integration README.
+
+    Returns:
+    - `dict[str, tuple]` with the name of the command and the
+    start and end line of the command section within the README.
+    """
+
+    command_start_section_pattern = rf"^{HEADER_TYPE.H3}\s+([a-z0-9]+(-[a-z0-9]+)*$)"
+
+    out = {}
+
+    # Here we iterate over the README line by line
+    # and find what lines the command sections are defined
+    for line_nr, line_text in enumerate(doc_text.splitlines()):
+        cmd_search = re.search(command_start_section_pattern, line_text)
+
+        if cmd_search:
+            out[cmd_search.group(1)] = line_nr
+
+    # We then transform the structure
+    # to include the end line as well
+    keys = list(out.keys())
+    values = list(out.values())
+
+    transformed = {}
+
+    # Iterate over the keys and values
+    for i in range(len(keys)):
+        if i < len(keys) - 1:
+            transformed[keys[i]] = (values[i], values[i + 1])
+        else:
+            transformed[keys[i]] = (values[i], len(doc_text.splitlines()))
+
+    return transformed