Merge bc69fd3 into adf7fdc

.changelog/4303.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Improved implementation of finding the commands' sections within the integration README.md
+  type: fix
+pr_number: 4303

demisto_sdk/__main__.py

@@ -28,6 +28,7 @@
 from demisto_sdk.commands.common.constants import (
     DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV,
     ENV_DEMISTO_SDK_MARKETPLACE,
+    INTEGRATIONS_README_FILE_NAME,
     FileType,
     MarketplaceVersions,
 )
@@ -2231,6 +2232,13 @@ def init(ctx, **kwargs):
     is_flag=True,
     default=True,
 )
+@click.option(
+    "-f",
+    "--force",
+    help="Whether to force the generation of documentation (rather than update when it exists in version control)",
+    is_flag=True,
+    default=False,
+)
 @click.pass_context
 @logging_setup_decorator
 def generate_docs(ctx, **kwargs):
@@ -2302,21 +2310,23 @@ def _generate_docs_for_file(kwargs: Dict[str, Any]):
     custom_image_path: str = kwargs.get("custom_image_path", "")
     readme_template: str = kwargs.get("readme_template", "")
     use_graph = kwargs.get("graph", True)
+    force = kwargs.get("force", False)
 
     try:
         if command:
             if (
                 output_path
-                and (not Path(output_path, "README.md").is_file())
+                and (not Path(output_path, INTEGRATIONS_README_FILE_NAME).is_file())
                 or (not output_path)
                 and (
                     not Path(
-                        os.path.dirname(os.path.realpath(input_path)), "README.md"
+                        os.path.dirname(os.path.realpath(input_path)),
+                        INTEGRATIONS_README_FILE_NAME,
                     ).is_file()
                 )
             ):
                 raise Exception(
-                    "[red]The `command` argument must be presented with existing `README.md` docs."
+                    f"[red]The `command` argument must be presented with existing `{INTEGRATIONS_README_FILE_NAME}` docs."
                 )
 
         file_type = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
@@ -2356,6 +2366,7 @@ def _generate_docs_for_file(kwargs: Dict[str, Any]):
                 command=command,
                 old_version=old_version,
                 skip_breaking_changes=skip_breaking_changes,
+                force=force,
             )
         elif file_type == FileType.SCRIPT:
             logger.info(f"Generating {file_type.value.lower()} documentation")

demisto_sdk/commands/generate_docs/README.md

@@ -48,6 +48,8 @@ Whether to use the content graph or not.
   * mirror_direction
   * close_incident
   * close_out - (opposite to close_incident)
+* If the Integration/Script/Playbook exists in version control, the version
+from the main branch (e.g. `master`) will be used to only render the modified sections (e.g. configuration, command).
 
 ### Examples
 

demisto_sdk/commands/generate_docs/generate_integration_doc.py

@@ -14,6 +14,7 @@
     DOCS_COMMAND_SECTION_REGEX,
     INTEGRATIONS_DIR,
     INTEGRATIONS_README_FILE_NAME,
+    SCRIPT,
 )
 from demisto_sdk.commands.common.default_additional_info_loader import (
     load_default_additional_info_dict,
@@ -216,9 +217,6 @@ def _update_conf_section(self):
 
             doc_text_lines = self.output_doc.splitlines()
 
-            # We take the first and the second-to-last index of the old section
-            # and use the section range to replace it with the new section.
-            # Second-to-last index because the last element is an empty string
             old_config_start_line = doc_text_lines.index(
                 CONFIGURATION_SECTION_STEPS.STEP_1.value
             )
@@ -244,16 +242,12 @@ def _update_commands_section(self):
         README.
         """
 
-        for i, modified_command in enumerate(
-            self.integration_diff.get_modified_commands()
-        ):
+        command_sections = get_commands_sections(self.output_doc)
+
+        for modified_command in self.integration_diff.get_modified_commands():
             try:
-                old_command_section, _ = generate_commands_section(
-                    self.integration_diff.old_yaml_data,
-                    {},
-                    {},
-                    modified_command,
-                )
+
+                start_line, end_line = command_sections[modified_command]
 
                 (
                     new_command_section,
@@ -280,30 +274,7 @@ def _update_commands_section(self):
 
                 doc_text_lines = self.output_doc.splitlines()
 
-                # We take the first and the second-to-last index of the old section
-                # and use the section range to replace it with the new section.
-                # Second-to-last index because the last element is an empty string
-                old_cmd_start_line = doc_text_lines.index(old_command_section[0])
-
-                # In cases when there are multiple identical context outputs
-                # in the second-to-last line, we need to find the relevant
-                # second-to-last line for the specific command we're replacing.
-                indices = [
-                    i
-                    for i, doc_line in enumerate(doc_text_lines)
-                    if doc_line == old_command_section[-2]
-                ]
-
-                if indices and len(indices) > 1:
-                    old_cmd_end_line = doc_text_lines.index(
-                        old_command_section[-2], indices[i]
-                    )
-                else:
-                    old_cmd_end_line = doc_text_lines.index(old_command_section[-2])
-
-                doc_text_lines[
-                    old_cmd_start_line : old_cmd_end_line + 1
-                ] = new_command_section
+                doc_text_lines[start_line:end_line] = new_command_section
 
                 self.output_doc = "\n".join(doc_text_lines)
             except (ValueError, IndexError) as e:
@@ -318,11 +289,6 @@ def _get_sections_to_update(self) -> Tuple[bool, List[str], List[str]]:
             self.integration_diff.get_added_commands(),
         )
 
-    def _get_resource_path(self) -> str:
-        """
-        Helper function to resolve the resource path.
-        """
-
     def _write_resource_to_tmp(self, resource_path: Path, content: str) -> Path:
         """
         Helper function to write
@@ -455,6 +421,7 @@ def generate_integration_doc(
     old_version: str = "",
     skip_breaking_changes: bool = False,
     is_contribution: bool = False,
+    force: bool = False,
 ):
     """
     Generate integration documentation.
@@ -536,7 +503,7 @@ def generate_integration_doc(
         # in source control:
         # - An integration YAML.
         # - An integration README.
-        elif update_mgr.can_update_docs():
+        elif not force and update_mgr.can_update_docs():
             logger.info("Found existing integration, updating documentation...")
             doc_text, update_errors = update_mgr.update_docs()
 
@@ -851,9 +818,7 @@ def generate_commands_section(
         "After you successfully execute a command, a DBot message appears in the War Room with the command details.",
         "",
     ]
-    commands = filter(
-        lambda cmd: not cmd.get("deprecated", False), yaml_data["script"]["commands"]
-    )
+    commands = get_integration_commands(yaml_data)
     command_sections: list = []
     if command:
         # for specific command, return it only.
@@ -1298,3 +1263,65 @@ def add_access_data_of_type_credentials(
             "Required": credentials_conf.get("required", ""),
         }
     )
+
+
+def get_integration_commands(yaml_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Helper function to return a list of integration commands.
+    Integration commands that are marked as deprecated will not be
+    returned.
+
+    Args:
+    - `yml_data` (``Dict[str, Any]``): The integration YAML as a dictionary.
+
+    Returns:
+    - `List[Dict[str, Any]]` of integration commands.
+    """
+
+    return list(
+        filter(
+            lambda cmd: not cmd.get("deprecated", False), yaml_data[SCRIPT]["commands"]
+        )
+    )
+
+
+def get_commands_sections(doc_text: str) -> Dict[str, Tuple[int, int]]:
+    """
+    Helper function that takes the integration README text
+    and returns a map of the commands and the start, end lines.
+
+    Args:
+    - `doc_text` (``str``): The integration README.
+
+    Returns:
+    - `dict[str, tuple]` with the name of the command and the
+    start and end line of the command section within the README.
+    """
+
+    command_start_section_pattern = r"^###\s+([a-z]+(-[a-z]+)*$)"
+
+    out = {}
+
+    # Here we iterate over the README line by line
+    # and find what lines the command sections are defined
+    for line_nr, line_text in enumerate(doc_text.splitlines()):
+        cmd_search = re.search(command_start_section_pattern, line_text)
+
+        if cmd_search:
+            out[cmd_search.group(1)] = line_nr
+
+    # We then transform the structure
+    # to include the end line as well
+    keys = list(out.keys())
+    values = list(out.values())
+
+    transformed = {}
+
+    # Iterate over the keys and values
+    for i in range(len(keys)):
+        if i < len(keys) - 1:
+            transformed[keys[i]] = (values[i], values[i + 1])
+        else:
+            transformed[keys[i]] = (values[i], len(doc_text.splitlines()))
+
+    return transformed