Merge 8a3e8c8 into 3ecdf20

.changelog/4301.yml

@@ -0,0 +1,4 @@
+changes:
+- description: added support in the new validation. For validators that run on all files and use the graph.
+  type: internal
+pr_number: 4301

TestSuite/pack.py

@@ -76,6 +76,8 @@ class Pack(TestSuiteBase):
     def __init__(self, packs_dir: Path, name: str, repo):
         # Initiate lists:
         self.name = name
+        self.id = name
+        self.node_id = name
         self._repo = repo
         self.repo_path = repo.path
         self.integrations: List[Integration] = list()

demisto_sdk/__main__.py

@@ -28,6 +28,7 @@
 from demisto_sdk.commands.common.constants import (
     DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV,
     ENV_DEMISTO_SDK_MARKETPLACE,
+    ExecutionMode,
     FileType,
     MarketplaceVersions,
 )
@@ -820,8 +821,15 @@ def validate(ctx, config, file_paths: str, **kwargs):
         sys.exit(1)
     try:
         is_external_repo = is_external_repository()
-        # default validate to -g --post-commit
-        if not kwargs.get("validate_all") and not kwargs["use_git"] and not file_path:
+        if kwargs.get("validate_all"):
+            execution_mode = ExecutionMode.ALL_FILES
+        elif kwargs.get("use_git"):
+            execution_mode = ExecutionMode.USE_GIT
+        elif file_path:
+            execution_mode = ExecutionMode.SPECIFIC_FILES
+        else:
+            execution_mode = ExecutionMode.USE_GIT
+            # default validate to -g --post-commit
             kwargs["use_git"] = True
             kwargs["post_commit"] = True
         exit_code = 0
@@ -913,16 +921,14 @@ def validate(ctx, config, file_paths: str, **kwargs):
                 category_to_run=kwargs.get("category_to_run"),
             )
             initializer = Initializer(
-                use_git=kwargs["use_git"],
                 staged=kwargs["staged"],
                 committed_only=kwargs["post_commit"],
                 prev_ver=kwargs["prev_ver"],
                 file_path=file_path,
-                all_files=kwargs.get("validate_all"),
+                execution_mode=execution_mode,
             )
             validator_v2 = ValidateManager(
                 file_path=file_path,
-                validate_all=kwargs.get("validate_all"),
                 initializer=initializer,
                 validation_results=validation_results,
                 config_reader=config_reader,

demisto_sdk/commands/common/constants.py

@@ -1282,6 +1282,12 @@ class GitStatuses(StrEnum):
     DELETED = "D"
 
 
+class ExecutionMode(StrEnum):
+    ALL_FILES = "-a"
+    USE_GIT = "-g"
+    SPECIFIC_FILES = "-i"
+
+
 FILE_TYPES_FOR_TESTING = [".py", ".js", ".yml", ".ps1"]
 
 # python subtypes

demisto_sdk/commands/content_graph/tests/SC_graph_validation_test.py

@@ -4,10 +4,14 @@
 )
 from demisto_sdk.commands.validate.validators.base_validator import BaseValidator
 from demisto_sdk.commands.validate.validators.SC_validators import (
-    SC109_script_name_is_not_unique_validator,
+    SC109_script_name_is_not_unique_validator_all_files,
+    SC109_script_name_is_not_unique_validator_file_list,
 )
-from demisto_sdk.commands.validate.validators.SC_validators.SC109_script_name_is_not_unique_validator import (
-    DuplicatedScriptNameValidator,
+from demisto_sdk.commands.validate.validators.SC_validators.SC109_script_name_is_not_unique_validator_all_files import (
+    DuplicatedScriptNameValidatorAllFiles,
+)
+from demisto_sdk.commands.validate.validators.SC_validators.SC109_script_name_is_not_unique_validator_file_list import (
+    DuplicatedScriptNameValidatorFileList,
 )
 from TestSuite.repo import Repo
 
@@ -19,7 +23,7 @@
 ]
 
 
-def test_DuplicatedScriptNameValidator_is_valid(mocker, graph_repo: Repo):
+def test_DuplicatedScriptNameValidatorFileList_is_valid(mocker, graph_repo: Repo):
     """
     Given
         - A content repo with 8 scripts:
@@ -28,14 +32,61 @@ def test_DuplicatedScriptNameValidator_is_valid(mocker, graph_repo: Repo):
         - 2 scripts (test_alert3, test_incident3) supported by MP V2 with SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
         - 2 scripts (test_alert4, test_incident4) where only one is supported by MP V2 without SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
     When
-        - running DuplicatedScriptNameValidator is_valid function.
+        - running DuplicatedScriptNameValidatorFileList is_valid function.
     Then
         - Validate that only the first pair of scripts appear in the results, and the rest of the scripts is valid.
     """
     mocker.patch.object(
-        SC109_script_name_is_not_unique_validator, "CONTENT_PATH", new=graph_repo.path
+        SC109_script_name_is_not_unique_validator_file_list,
+        "CONTENT_PATH",
+        new=graph_repo.path,
+    )
+    pack = graph_repo.create_pack()
+
+    pack.create_script("test_incident_1").set_data(marketplaces=MP_XSOAR_AND_V2)
+    pack.create_script("test_alert_1").set_data(marketplaces=MP_XSOAR_AND_V2)
+
+    pack.create_script("test_incident_2").set_data(marketplaces=MP_XSOAR)
+    pack.create_script("test_alert_2").set_data(marketplaces=MP_XSOAR)
+
+    pack.create_script("test_incident_3").set_data(
+        skipprepare=[SKIP_PREPARE_SCRIPT_NAME], marketplaces=MP_V2
+    )
+    pack.create_script("test_alert_3").set_data(
+        skipprepare=[SKIP_PREPARE_SCRIPT_NAME], marketplaces=MP_V2
+    )
+
+    pack.create_script("test_incident_4").set_data(marketplaces=MP_XSOAR_AND_V2)
+    pack.create_script("test_alert_4").set_data(marketplaces=MP_XSOAR)
+
+    BaseValidator.graph_interface = graph_repo.create_graph()
+
+    results = DuplicatedScriptNameValidatorFileList().is_valid(
+        [script.object for script in pack.scripts]
     )
 
+    assert len(results) == 1
+    assert "test_alert_1.yml" == results[0].content_object.path.name
+
+
+def test_DuplicatedScriptNameValidatorAllFiles_is_valid(mocker, graph_repo: Repo):
+    """
+    Given
+        - A content repo with 8 scripts:
+        - 2 scripts (test_alert1, test_incident1) supported by MP V2 without SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
+        - 2 scripts (test_alert2, test_incident2) not supported by MP V2 without SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
+        - 2 scripts (test_alert3, test_incident3) supported by MP V2 with SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
+        - 2 scripts (test_alert4, test_incident4) where only one is supported by MP V2 without SKIP_PREPARE_SCRIPT_NAME = "script-name-incident-to-alert".
+    When
+        - running DuplicatedScriptNameValidatorAllFiles is_valid function.
+    Then
+        - Validate that only the first pair of scripts appear in the results, and the rest of the scripts is valid.
+    """
+    mocker.patch.object(
+        SC109_script_name_is_not_unique_validator_all_files,
+        "CONTENT_PATH",
+        new=graph_repo.path,
+    )
     pack = graph_repo.create_pack()
 
     pack.create_script("test_incident_1").set_data(marketplaces=MP_XSOAR_AND_V2)
@@ -56,7 +107,7 @@ def test_DuplicatedScriptNameValidator_is_valid(mocker, graph_repo: Repo):
 
     BaseValidator.graph_interface = graph_repo.create_graph()
 
-    results = DuplicatedScriptNameValidator().is_valid(
+    results = DuplicatedScriptNameValidatorAllFiles().is_valid(
         [script.object for script in pack.scripts]
     )
 

demisto_sdk/commands/validate/config_reader.py

@@ -3,6 +3,7 @@
 
 import toml
 
+from demisto_sdk.commands.common.constants import ExecutionMode
 from demisto_sdk.commands.common.logger import logger
 
 USE_GIT = "use_git"
@@ -42,18 +43,22 @@ def __init__(self, config_file_path=None, category_to_run=None):
             exit(1)
 
     def gather_validations_to_run(
-        self, use_git: bool, ignore_support_level: Optional[bool] = False
+        self, execution_mode: bool, ignore_support_level: Optional[bool] = False
     ) -> ConfiguredValidations:
         """Extract the relevant information from the relevant category in the config file.
 
         Args:
-            use_git (bool): The use_git flag.
+            execution_mode (executionMode): The execution mode.
 
         Returns:
             Tuple[List, List, List, dict]: the select, warning, and ignorable errors sections from the given category,
             and the support_level dict with errors to ignore.
         """
-        flag = self.category_to_run or (USE_GIT if use_git else PATH_BASED_VALIDATIONS)
+        flag = self.category_to_run or (
+            USE_GIT
+            if execution_mode == ExecutionMode.USE_GIT
+            else PATH_BASED_VALIDATIONS
+        )
         section = self.config_file_content.get(flag, {})
         return ConfiguredValidations(
             section.get("select", []),

demisto_sdk/commands/validate/default_config.toml

@@ -45,6 +45,7 @@ select = [
     "RM104", "RM105", "RM113", "RM114",
     "CL100",
     "GF100", "GF101",
+    "GR104",
     "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106",
     "IM100", "IM101", "IM108", "IM109",
     "RP101"
@@ -66,6 +67,7 @@ select = [
     "RM104", "RM105", "RM113", "RM114",
     "CL100",
     "GF100", "GF101",
+    "GR104",
     "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106",
     "IM101", "IM108",
     "RP101"

demisto_sdk/commands/validate/initializer.py

@@ -20,6 +20,7 @@
     PLAYBOOKS_DIR,
     RELEASE_NOTES_DIR,
     SCRIPTS_DIR,
+    ExecutionMode,
     GitStatuses,
     PathLevel,
 )
@@ -50,19 +51,17 @@ class Initializer:
 
     def __init__(
         self,
-        use_git=False,
         staged=None,
         committed_only=None,
         prev_ver=None,
         file_path=None,
-        all_files=False,
+        execution_mode=None,
     ):
         self.staged = staged
-        self.use_git = use_git
         self.file_path = file_path
-        self.all_files = all_files
         self.committed_only = committed_only
         self.prev_ver = prev_ver
+        self.execution_mode = execution_mode
 
     def validate_git_installed(self):
         """Initialize git util."""
@@ -71,7 +70,7 @@ def validate_git_installed(self):
             self.branch_name = self.git_util.get_current_git_branch_or_hash()
         except (InvalidGitRepositoryError, TypeError):
             # if we are using git - fail the validation by raising the exception.
-            if self.use_git:
+            if self.execution_mode == ExecutionMode.USE_GIT:
                 raise
             # if we are not using git - simply move on.
             else:
@@ -311,35 +310,35 @@ def gather_objects_to_run_on(
         content_objects_to_run: Set[BaseContent] = set()
         invalid_content_items: Set[Path] = set()
         non_content_items: Set[Path] = set()
-        if self.use_git:
+        if self.execution_mode == ExecutionMode.USE_GIT:
             (
                 content_objects_to_run,
                 invalid_content_items,
                 non_content_items,
             ) = self.get_files_using_git()
-        elif self.file_path:
+        elif self.execution_mode == ExecutionMode.SPECIFIC_FILES:
             (
                 content_objects_to_run,
                 invalid_content_items,
                 non_content_items,
             ) = self.paths_to_basecontent_set(
                 set(self.load_files(self.file_path.split(",")))
             )
-        elif self.all_files:
+        elif self.execution_mode == ExecutionMode.ALL_FILES:
             logger.info("Running validation on all files.")
             content_dto = ContentDTO.from_path()
             if not isinstance(content_dto, ContentDTO):
                 raise Exception("no content found")
             content_objects_to_run = set(content_dto.packs)
         else:
-            self.use_git = True
+            self.execution_mode = ExecutionMode.USE_GIT
             self.committed_only = True
             (
                 content_objects_to_run,
                 invalid_content_items,
                 non_content_items,
             ) = self.get_files_using_git()
-        if not self.use_git:
+        if self.execution_mode != ExecutionMode.USE_GIT:
             content_objects_to_run_with_packs: Set[
                 BaseContent
             ] = self.get_items_from_packs(content_objects_to_run)

demisto_sdk/commands/validate/sdk_validation_config.toml

@@ -36,6 +36,7 @@ select = [
     "RM101", "RN103", "RM104", "RM105", "RM106", "RM109", "RM113", "RM114",
     "CL100",
     "GF100", "GF101", "GF102",
+    "GR104",
     "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106", "IF116",
     "IM100", "IM101", "IM108", "IM109", "IM106", "IM111",
     "RP101", "BC106", "BC107",
@@ -62,6 +63,7 @@ select = [
     "RM104", "RM105", "RM113", "RM114",
     "CL100",
     "GF100", "GF101",
+    "GR104",
     "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106", "IF116",
     "IM101", "IM108", "IM111",
     "RP101",

demisto_sdk/commands/validate/tests/GR_validators_test.py

@@ -0,0 +1,68 @@
+from demisto_sdk.commands.validate.validators.base_validator import BaseValidator
+from demisto_sdk.commands.validate.validators.GR_validators.GR104_is_pack_display_name_already_exists_all_files import (
+    IsPackDisplayNameAlreadyExistsValidatorAllFiles,
+)
+from demisto_sdk.commands.validate.validators.GR_validators.GR104_is_pack_display_name_already_exists_list_files import (
+    IsPackDisplayNameAlreadyExistsValidatorListFiles,
+)
+from TestSuite.repo import Repo
+
+
+def test_IsPackDisplayNameAlreadyExistsValidatorListFiles_is_valid(graph_repo: Repo):
+    """
+    Given
+        - 3 packs, and 2 of them are with the same name
+    When
+        - running IsPackDisplayNameAlreadyExistsValidatorListFiles is_valid function.
+    Then
+        - Validate that we got the error messages for the duplicate name.
+    """
+
+    graph_repo.create_pack(name="pack1")
+
+    graph_repo.create_pack(name="pack2")
+    graph_repo.packs[1].pack_metadata.update(
+        {
+            "name": "pack1",
+        }
+    )
+
+    graph_repo.create_pack(name="pack3")
+
+    BaseValidator.graph_interface = graph_repo.create_graph()
+
+    results = IsPackDisplayNameAlreadyExistsValidatorListFiles().is_valid(
+        [pack for pack in graph_repo.packs]
+    )
+
+    assert len(results) == 2
+
+
+def test_IsPackDisplayNameAlreadyExistsValidatorAllFiles_is_valid(graph_repo: Repo):
+    """
+    Given
+        - 3 packs, and 2 of them are with the same name
+    When
+        - running IsPackDisplayNameAlreadyExistsValidatorAllFiles is_valid function.
+    Then
+        - Validate that we got the error messages for the duplicate name.
+    """
+
+    graph_repo.create_pack(name="pack1")
+
+    graph_repo.create_pack(name="pack2")
+    graph_repo.packs[1].pack_metadata.update(
+        {
+            "name": "pack1",
+        }
+    )
+
+    graph_repo.create_pack(name="pack3")
+
+    BaseValidator.graph_interface = graph_repo.create_graph()
+
+    results = IsPackDisplayNameAlreadyExistsValidatorAllFiles().is_valid(
+        [pack for pack in graph_repo.packs]
+    )
+
+    assert len(results) == 2