Xsiam validate format changes (#2677)

CHANGELOG.md

@@ -3,6 +3,8 @@
 * Fixed an issue in **prepare-content** command where large code lines were broken.
 * Fixed an issue where git-*renamed_files* were not retrieved properly.
 * Fixed an issue where test dependencies were calculated in all level dependencies calculation.
+* Added formatting and validation to XSIAM content types.
+* Fixed an issue where several XSIAM content types were not validated when passing the `-a` flag.
 * Added a UUID to name mapper for **download** it replaces UUIDs with names on all downloaded files.
 * Improved the message shown when using **upload** and overwriting packs.
 

TestSuite/json_based.py

@@ -1,5 +1,7 @@
 import os
+import re
 from pathlib import Path
+from typing import Union
 
 from demisto_sdk.commands.common.constants import PACKS_DIR
 from demisto_sdk.commands.common.handlers import JSON_Handler
@@ -52,3 +54,57 @@ def remove(self, key: str):
         file_content = self.read_json_as_dict()
         file_content.pop(key, None)
         self.write_json(file_content)
+
+    def _set_field_by_path(self, path_to_field: str, new_val: str = None):
+        """Inner method to remove, add or update a given field.
+
+        Args:
+            path_to_field (str): The path to field to remove.
+                Ex: alerts_filter.filter.AND.[0].SEARCH_FIELD
+
+            new_val (str): The field's new value. If not provided, removes the field.
+        """
+
+        def get_index_or_key(k) -> Union[str, int]:
+            list_index_pattern = r"\[([0-9]+)\]"
+            if idx_match := re.match(list_index_pattern, k):
+                return int(idx_match[1])
+            return k
+
+        splitted_path = path_to_field.split(".")
+        data = pointer = self.read_json_as_dict()
+        try:
+            for idx, k in enumerate(splitted_path):
+                if idx == len(splitted_path) - 1:
+                    if not new_val:
+                        del pointer[get_index_or_key(k)]
+                    else:
+                        pointer[get_index_or_key(k)] = new_val
+                    self.update(data)
+                else:
+                    pointer = pointer[get_index_or_key(k)]
+        except Exception:
+            path = ".".join(splitted_path[: idx - 1])
+            raise Exception(f"Invalid path: {k} does not exist under {path}")
+
+    def remove_field_by_path(self, path_to_field: str):
+        """Removes an inner field by dot notation.
+        If the path to field does not exist, raises an error.
+
+        Args:
+            path_to_field (str): The path to field to remove.
+                Ex: alerts_filter.filter.AND.[0].SEARCH_FIELD
+        """
+        self._set_field_by_path(path_to_field)
+
+    def add_or_update_field_by_path(self, path_to_field: str, val: str):
+        """Adds or updates an inner field by dot notation.
+        If the path to field does not exist, raises an error.
+
+        Args:
+            path_to_field (str): The path to field to set.
+                Ex: alerts_filter.filter.AND.[0].SEARCH_FIELD
+
+            val (str): The field's expected value.
+        """
+        self._set_field_by_path(path_to_field, val)

TestSuite/pack.py

@@ -570,7 +570,7 @@ def create_parsing_rule(
                 "id": "parsing-rule",
                 "name": "Parsing Rule",
                 "fromversion": "6.8.0",
-                "tags": "tag",
+                "tags": ["tag"],
                 "rules": "",
                 "samples": "",
             }
@@ -648,8 +648,10 @@ def create_trigger(self, name, content: dict = None) -> Trigger:
     def create_xdrc_template(
         self, name, json_content: dict = None, yaml_content: dict = None
     ) -> XDRCTemplate:
+        xdrc_template_dir: Path = self._xdrc_templates_path / f"{self.name}_{name}"
+        xdrc_template_dir.mkdir()
         xdrc_template = XDRCTemplate(
-            name, self._xdrc_templates_path, json_content, yaml_content
+            name, xdrc_template_dir, json_content, yaml_content
         )
         self.xdrc_templates.append(xdrc_template)
         return xdrc_template

TestSuite/trigger.py

@@ -19,6 +19,20 @@ def create_default_trigger(self):
         self.write_json(
             {
                 "trigger_id": self.name,
+                "playbook_id": "mock playbook",
+                "suggestion_reason": "mock reason",
+                "description": "desc",
                 "trigger_name": self.name,
+                "alerts_filter": {
+                    "filter": {
+                        "AND": [
+                            {
+                                "SEARCH_FIELD": "alert_name",
+                                "SEARCH_TYPE": "EQ",
+                                "SEARCH_VALUE": "multiple unauthorized action attempts detected by a user",
+                            }
+                        ]
+                    }
+                },
             }
         )

TestSuite/xdrc_template.py

@@ -39,6 +39,7 @@ def create_default_xdrc_template_json(self):
                 "name": self.name,
                 "os_type": "os_type_test",
                 "profile_type": "profile_type_test",
+                "from_xdr_version": "3.5.0",
                 "yaml_template": "",
             }
         )

TestSuite/xsiam_dashboard.py

@@ -19,5 +19,73 @@ def __init__(
 
     def create_default_xsiam_dashboard(self):
         self.write_json(
-            {"dashboards_data": [{"global_id": self.name, "name": self.name}]}
+            {
+                "dashboards_data": [
+                    {
+                        "name": self.name,
+                        "description": "mock dashboard desc",
+                        "status": "ENABLED",
+                        "layout": [
+                            {
+                                "id": "row-1768",
+                                "data": [
+                                    {
+                                        "key": "xql_1668676732415",
+                                        "data": {
+                                            "type": "Custom XQL",
+                                            "width": 50,
+                                            "height": 434,
+                                            "phrase": 'datamodel \r\n|filter xdm.observer.vendor="mock vendor"',
+                                            "time_frame": {"relativeTime": 2592000000},
+                                            "viewOptions": {
+                                                "type": "map",
+                                                "commands": [
+                                                    {
+                                                        "command": {
+                                                            "op": "=",
+                                                            "name": "header",
+                                                            "value": '"Cloud Regions"',
+                                                        }
+                                                    }
+                                                ],
+                                            },
+                                        },
+                                    },
+                                ],
+                            },
+                        ],
+                        "default_dashboard_id": 1,
+                        "global_id": "f9c52470483a41e4a6afa65c93f70a4b",
+                    }
+                ],
+                "widgets_data": [
+                    {
+                        "widget_key": "xql_1668676732415",
+                        "title": "mock widget",
+                        "creation_time": 1668676732415,
+                        "description": "mock widget desc",
+                        "data": {
+                            "phrase": 'datamodel \r\n|filter xdm.observer.vendor="mock vendor"',
+                            "time_frame": {"relativeTime": 2592000000},
+                            "viewOptions": {
+                                "type": "map",
+                                "commands": [
+                                    {
+                                        "command": {
+                                            "op": "=",
+                                            "name": "header",
+                                            "value": '"Cloud Regions"',
+                                        }
+                                    }
+                                ],
+                            },
+                        },
+                        "support_time_range": True,
+                        "additional_info": {
+                            "query_tables": [],
+                            "query_uses_library": False,
+                        },
+                    },
+                ],
+            }
         )

TestSuite/xsiam_report.py

@@ -19,5 +19,5 @@ def __init__(
 
     def create_default_xsiam_report(self):
         self.write_json(
-            {"templates_data": [{"global_id": self.name, "name": self.name}]}
+            {"templates_data": [{"global_id": self.name, "report_name": self.name}]}
         )

demisto_sdk/commands/common/constants.py

@@ -145,6 +145,7 @@ class FileType(str, Enum):
     JOB = "job"
     BUILD_CONFIG_FILE = "build-config-file"
     PARSING_RULE = "parsingrule"
+    PARSING_RULE_XIF = "parsingrulexif"
     MODELING_RULE = "modelingrule"
     MODELING_RULE_TEST_DATA = "modelingruletestdata"
     MODELING_RULE_XIF = "modelingrulexif"
@@ -296,6 +297,11 @@ class FileType(str, Enum):
     WIZARDS_DIR,
     MODELING_RULES_DIR,
     XDRC_TEMPLATE_DIR,
+    PARSING_RULES_DIR,
+    CORRELATION_RULES_DIR,
+    XSIAM_DASHBOARDS_DIR,
+    XSIAM_REPORTS_DIR,
+    TRIGGER_DIR,
 ]
 
 CONTENT_ENTITY_UPLOAD_ORDER = [
@@ -604,12 +610,38 @@ class FileType(str, Enum):
 WIZARD_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{WIZARDS_DIR}"
 WIZARD_JSON_REGEX = rf"{WIZARD_DIR_REGEX}\/wizard-([^/]+)\.json"
 
+XSIAM_DASHBOARD_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{XSIAM_DASHBOARDS_DIR}"
+XSIAM_DASHBOARD_JSON_REGEX = rf"{XSIAM_DASHBOARD_DIR_REGEX}\/([^/]+)\.json"
+
+XSIAM_REPORT_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{XSIAM_REPORTS_DIR}"
+XSIAM_REPORT_JSON_REGEX = rf"{XSIAM_REPORT_DIR_REGEX}\/([^/]+)\.json"
+
+TRIGGER_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{TRIGGER_DIR}"
+TRIGGER_JSON_REGEX = rf"{TRIGGER_DIR_REGEX}\/([^/]+)\.json"
+
+XDRC_TEMPLATE_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{XDRC_TEMPLATE_DIR}"
+XDRC_TEMPLATE_PACKAGE_REGEX = rf"{XDRC_TEMPLATE_DIR_REGEX}\/([^\\/]+)"
+XDRC_TEMPLATE_JSON_REGEX = rf"{XDRC_TEMPLATE_PACKAGE_REGEX}\/([^/]+)\.json"
+XDRC_TEMPLATE_YML_REGEX = rf"{XDRC_TEMPLATE_PACKAGE_REGEX}\/([^/]+)\.yml"
+
+CORRELATION_RULES_DIR_REGEX = rf"{PACK_DIR_REGEX}/{CORRELATION_RULES_DIR}"
+CORRELATION_RULES_YML_REGEX = (
+    rf"{CORRELATION_RULES_DIR_REGEX}/(?:correlationrule-)?([^/]+)\.yml"
+)
+
+PARSING_RULES_DIR_REGEX = rf"{PACK_DIR_REGEX}/{PARSING_RULES_DIR}"
+PARSING_RULES_PACKAGE_REGEX = rf"{PARSING_RULES_DIR_REGEX}\/([^\\/]+)"
+PARSING_RULES_YML_REGEX = (
+    rf"{PARSING_RULES_PACKAGE_REGEX}/(?:parsingrule-)?([^/]+)\.yml"
+)
+
+
 # Modeling Rules
 MODELING_RULE_DIR_REGEX = rf"{PACK_DIR_REGEX}\/{MODELING_RULES_DIR}"
 MODELING_RULE_PACKAGE_REGEX = rf"{MODELING_RULE_DIR_REGEX}\/([^\\/]+)"
-MODELING_RULE_YML_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/\2\.yml"
-MODELING_RULE_RULES_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/\2\.xif"
-MODELING_RULE_SCHEMA_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/\2\.json"
+MODELING_RULE_YML_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/([^/]+)\.yml"
+MODELING_RULE_RULES_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/([^/]+)\.xif"
+MODELING_RULE_SCHEMA_REGEX = rf"{MODELING_RULE_PACKAGE_REGEX}\/([^/]+)\.json"
 
 RELATIVE_HREF_URL_REGEX = r'(<.*?href\s*=\s*"((?!(?:https?:\/\/)|#|(?:mailto:)).*?)")'
 RELATIVE_MARKDOWN_URL_REGEX = (
@@ -1186,13 +1218,20 @@ class PB_Status:
     ],
     "report": [PACKS_REPORT_JSON_REGEX],
     "modelingrule": [MODELING_RULE_YML_REGEX],
+    "modelingruleschema": [MODELING_RULE_SCHEMA_REGEX],
     "release-notes": [PACKS_RELEASE_NOTES_REGEX],
     "genericfield": JSON_ALL_GENERIC_FIELDS_REGEXES,
     "generictype": JSON_ALL_GENERIC_TYPES_REGEXES,
     "genericmodule": JSON_ALL_GENERIC_MODULES_REGEXES,
     "genericdefinition": JSON_ALL_GENERIC_DEFINITIONS_REGEXES,
     JOB: JSON_ALL_JOB_REGEXES,
     WIZARD: JSON_ALL_WIZARD_REGEXES,
+    "correlationrule": [CORRELATION_RULES_YML_REGEX],
+    "parsingrule": [PARSING_RULES_YML_REGEX],
+    "xsiamdashboard": [XSIAM_DASHBOARD_JSON_REGEX],
+    "xsiamreport": [XSIAM_REPORT_JSON_REGEX],
+    "trigger": [TRIGGER_JSON_REGEX],
+    "xdrctemplate": [XDRC_TEMPLATE_JSON_REGEX],
 }
 
 EXTERNAL_PR_REGEX = r"^pull/(\d+)$"
@@ -1297,6 +1336,9 @@ class PB_Status:
     FileType.GENERIC_FIELD: "6.5.0",
     FileType.GENERIC_MODULE: "6.5.0",
     FileType.GENERIC_DEFINITION: "6.5.0",
+    FileType.CORRELATION_RULE: "6.10.0",
+    FileType.PARSING_RULE: "6.10.0",
+    FileType.MODELING_RULE: "6.10.0",
 }
 # This constant below should always be two versions before the latest server version
 GENERAL_DEFAULT_FROMVERSION = "6.5.0"

demisto_sdk/commands/common/schemas/correlationrule.yml

@@ -0,0 +1,61 @@
+type: map
+mapping:
+  global_rule_id:
+    type: str
+    required: true
+  name:
+    type: str
+    required: true
+  alert_name:
+    type: str
+    required: true
+  description:
+    type: str
+    required: true
+  alert_description:
+    type: str
+    required: true
+  alert_category:
+    type: str
+    required: true
+  alert_fields:
+    type: map
+    allowempty: true
+  crontab:
+    type: str
+  dataset:
+    type: str
+    required: true
+  drilldown_query_timeframe:
+    type: str
+    required: true
+  execution_mode:
+    type: str
+    required: true
+  mitre_defs:
+    type: map
+    allowempty: true
+  search_window:
+    type: str
+    required: true
+  severity:
+    type: str
+    required: true
+  suppression_enabled:
+    type: bool
+    required: true
+  suppression_duration:
+    type: str
+  suppression_fields:
+    type: str
+  user_defined_category:
+    type: str
+  user_defined_severity:
+    type: str
+  xql_query:
+    type: str
+    required: true
+  investigation_query_link:
+    type: str
+  mapping_strategy:
+    type: str

demisto_sdk/commands/common/schemas/modelingrule.yml

@@ -0,0 +1,19 @@
+type: map
+mapping:
+  id:
+    type: str
+    required: true
+  name:
+    type: str
+    required: true
+  fromversion:
+    type: str
+    required: true
+  toversion:
+    type: str
+  tags:
+    type: str
+  rules:
+    type: str
+  schema:
+    type: str

demisto_sdk/commands/common/schemas/modelingruleschema.yml

@@ -0,0 +1,15 @@
+type: map
+mapping:
+  regex;(.+_raw):
+    type: map
+    mapping:
+      regex;(.+):
+        type: map
+        mapping:
+          type:
+            type: str
+            required: true
+            enum: ["string", "int", "float", "datetime", "boolean"]
+          is_array:
+            type: bool
+            required: true