Check Marketplaces Match Server Type (#3875)

* check marketplaces match server type

* fix set update

* add UT

* run pre-commit

* CR fixes

* update logs for testing

* remove test logs

CHANGELOG.md

@@ -9,6 +9,7 @@
 * Fixed an issue where **lint -g** crashed when comparing changes against branches which are not `master`.
 * Fixed an issue in **prepare-content** command where the `incident to alert` process was not triggered when the playbook is set to only XSIAM.
 * Fixed an issue where **validate -g** command crashed when comparing changes against branches when in detached head state.
+* Add support for tests to run on specified marketplaces.
 
 ## 1.25.1
 * Added the `clean` flag to **setup-env** to delete temp files that were created by `lint` from the repo.

demisto_sdk/commands/test_content/TestContentClasses.py

@@ -29,6 +29,7 @@
     DEFAULT_CONTENT_ITEM_FROM_VERSION,
     DEFAULT_CONTENT_ITEM_TO_VERSION,
     FILTER_CONF,
+    MarketplaceVersions,
     PB_Status,
 )
 from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
@@ -52,17 +53,27 @@
 ENV_RESULTS_PATH = "./artifacts/env_results.json"
 FAILED_MATCH_INSTANCE_MSG = (
     "{} Failed to run.\n There are {} instances of {}, please select one of them by using "
-    "the instance_name argument in conf.json. The options are:\n{}"
+    "the instance_names argument in conf.json. The options are:\n{}"
 )
 ENTRY_TYPE_ERROR = 4
 DEFAULT_INTERVAL = 4
 MAX_RETRIES = 3
 RETRIES_THRESHOLD = ceil(MAX_RETRIES / 2)
 
 SLACK_MEM_CHANNEL_ID = "CM55V7J8K"
+XSOAR_SERVER_TYPE = "XSOAR"
 XSIAM_SERVER_TYPE = "XSIAM"
+XPANSE_SERVER_TYPE = "XPANSE"
 XSOAR_SAAS_SERVER_TYPE = "XSOAR SAAS"
 
+MARKETPLACE_VERSIONS_TO_SERVER_TYPE = {
+    MarketplaceVersions.XSOAR: {XSOAR_SERVER_TYPE, XSOAR_SAAS_SERVER_TYPE},
+    MarketplaceVersions.MarketplaceV2: {XSIAM_SERVER_TYPE},
+    MarketplaceVersions.XPANSE: {XPANSE_SERVER_TYPE},
+    MarketplaceVersions.XSOAR_SAAS: {XSOAR_SAAS_SERVER_TYPE},
+    MarketplaceVersions.XSOAR_ON_PREM: {XSOAR_SERVER_TYPE},
+}
+
 __all__ = [
     "BuildContext",
     "Conf",
@@ -148,6 +159,9 @@ def __init__(self, test_configuration: dict, default_test_timeout: int):
         self.test_instance_names: List[str] = self._parse_instance_names_conf(
             test_configuration
         )
+        self.marketplaces: List[MarketplaceVersions] = self._parse_marketplaces_conf(
+            test_configuration
+        )
         self.instance_configuration: dict = test_configuration.get(
             "instance_configuration", {}
         )
@@ -172,6 +186,25 @@ def _parse_instance_names_conf(test_configuration):
             instance_names_conf = [instance_names_conf]
         return instance_names_conf
 
+    @staticmethod
+    def _parse_marketplaces_conf(
+        test_configuration,
+    ) -> List[MarketplaceVersions]:
+        marketplaces_conf = test_configuration.get("marketplaces", [])
+        if not isinstance(marketplaces_conf, list):
+            marketplaces_conf = [marketplaces_conf]
+        marketplaces_conf = [
+            MarketplaceVersions(marketplace) for marketplace in marketplaces_conf
+        ]
+
+        if MarketplaceVersions.XSOAR in marketplaces_conf:
+            marketplaces_conf.append(MarketplaceVersions.XSOAR_SAAS)
+
+        if MarketplaceVersions.XSOAR_ON_PREM in marketplaces_conf:
+            marketplaces_conf.append(MarketplaceVersions.XSOAR)
+
+        return marketplaces_conf
+
     def __str__(self):
         return str(self.raw_dict)
 
@@ -293,6 +326,10 @@ def populate_test_suite(self):
             "playbook.test_instance_names",
             ",".join(self.configuration.test_instance_names),
         )
+        self.test_suite.add_property(
+            "playbook.marketplaces",
+            ",".join(self.configuration.marketplaces),
+        )
         self.test_suite.add_property(
             "playbook.integrations", ",".join(map(str, self.integrations))
         )
@@ -439,12 +476,57 @@ def test_has_skipped_integration():
 
             return False
 
+        def marketplaces_match_server_type() -> bool:
+            """
+            Checks if the test has a marketplace value, and if so- if it matches the server machine we are on.
+            A test playbook might have several entries, each with a different marketplace. This might cause the test playbook to
+            be in the filtered tests list, even when the provided entry is not be the one that runs with the current sever
+            machine marketplace. This function checks that the entry provided is the exact one that needs to run.
+            Entries might differ in any field, the most common one is instance_names.
+            """
+            test_server_types: Set[str] = set()
+            for marketplace in self.configuration.marketplaces or []:
+                test_server_types.update(
+                    MARKETPLACE_VERSIONS_TO_SERVER_TYPE[marketplace]
+                )
+
+            if not test_server_types:
+                return True  # test doesn't have a marketplace value so it runs on all machines
+
+            instance_names_log_message = (
+                f" for instance names: {', '.join(self.configuration.test_instance_names)}"
+                if self.configuration.test_instance_names
+                else ""
+            )
+
+            if self.build_context.server_type in test_server_types:
+                self.log_debug(
+                    f"Running {self} with current server marketplace{instance_names_log_message}"
+                )
+                return True  # test has a marketplace value that matched the build server marketplace
+
+            log_message = (
+                f"Skipping {self} because it's marketplace values are: "
+                f"{', '.join(self.configuration.marketplaces)}{instance_names_log_message}, "
+                f"which is not compatible with the current server marketplace value"
+            )
+            self.close_test_suite([Skipped(log_message)])
+            if self.configuration.playbook_id in self.build_context.filtered_tests:
+                self.log_warning(log_message)
+            else:
+                self.log_debug(log_message)
+            skipped_tests_collected[
+                self.configuration.playbook_id
+            ] = f"test marketplaces are: {', '.join(self.configuration.marketplaces)}{instance_names_log_message}"
+            return False  # test has a marketplace value that doesn't matched the build server marketplace
+
         return (
             in_filtered_tests()
             and not nightly_test_in_non_nightly_build()
             and not skipped_test()
             and not version_mismatch()
             and not test_has_skipped_integration()
+            and marketplaces_match_server_type()
         )
 
     def run_test_module_on_integrations(self, client: DefaultApi) -> bool:

demisto_sdk/commands/test_content/tests/build_context_test.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
 from demisto_sdk.commands.test_content.ParallelLoggingManager import (
     ParallelLoggingManager,
@@ -17,6 +19,7 @@ def generate_test_configuration(
     pid_threshold: int = None,
     is_mockable: bool = None,
     runnable_on_docker_only: bool = None,
+    marketplaces: Union[list, str] = None,
 ) -> dict:
     playbook_config = {
         "playbookID": playbook_id,
@@ -41,6 +44,8 @@ def generate_test_configuration(
         playbook_config["runnable_on_docker_only"] = runnable_on_docker_only
     if is_mockable is not None:
         playbook_config["is_mockable"] = is_mockable
+    if marketplaces:
+        playbook_config["marketplaces"] = marketplaces
     return playbook_config
 
 
@@ -228,10 +233,15 @@ def get_mocked_build_context(
     return BuildContext(kwargs, logging_manager)
 
 
-def create_xsiam_build(mocker, tmp_file):
+def create_xsiam_build(
+    mocker,
+    tmp_file,
+    content_conf_json: dict = None,
+    filtered_tests_content: list = None,
+):
     logging_manager = ParallelLoggingManager(tmp_file / "log_file.log")
     conf_path = tmp_file / "conf_path"
-    conf_path.write_text(json.dumps(generate_content_conf_json()))
+    conf_path.write_text(json.dumps(content_conf_json or generate_content_conf_json()))
 
     secret_conf_path = tmp_file / "secret_conf_path"
     secret_conf_path.write_text(json.dumps(generate_secret_conf_json()))
@@ -252,7 +262,7 @@ def create_xsiam_build(mocker, tmp_file):
     )
 
     filtered_tests_path = tmp_file / "filtered_tests_path"
-    filtered_tests_path.write_text("[]")
+    filtered_tests_path.write_text("\n".join(filtered_tests_content or []) or "[]")
     mocker.patch(
         "demisto_sdk.commands.test_content.TestContentClasses.FILTER_CONF",
         str(filtered_tests_path),
@@ -468,6 +478,63 @@ def test_playbook_with_version_mismatch_is_skipped(mocker, tmp_path):
     )
 
 
+def test_playbook_with_marketplaces(mocker, tmp_path):
+    """
+    Given:
+        - A build context for a server type that matches and that does not match the playbook marketplaces
+    When:
+        - Initializing the BuildContext instance
+    Then:
+        - Ensure that the playbook with marketplaces mismatch is skipped
+        - Ensure that the playbook with the marketplaces match is not skipped
+    """
+    filtered_tests = [
+        "xsiam_playbook_with_marketplaces_mismatch",
+        "xsoar_playbook_with_marketplaces_mismatch",
+    ]
+    tests = [
+        generate_test_configuration(
+            playbook_id="xsiam_playbook_with_marketplaces_mismatch",
+            marketplaces="marketplacev2",
+        ),
+        generate_test_configuration(
+            playbook_id="xsoar_playbook_with_marketplaces_mismatch",
+            marketplaces="xsoar",
+        ),
+    ]
+    content_conf_json = generate_content_conf_json(tests=tests)
+
+    xsoar_build_context = get_mocked_build_context(
+        mocker,
+        tmp_path,
+        content_conf_json=content_conf_json,
+        filtered_tests_content=filtered_tests,
+    )
+    assert (
+        "xsiam_playbook_with_marketplaces_mismatch"
+        in xsoar_build_context.tests_data_keeper.skipped_tests
+    )
+    assert (
+        "xsoar_playbook_with_marketplaces_mismatch"
+        not in xsoar_build_context.tests_data_keeper.skipped_tests
+    )
+
+    xsiam_build_context = create_xsiam_build(
+        mocker,
+        tmp_path,
+        content_conf_json=content_conf_json,
+        filtered_tests_content=filtered_tests,
+    )
+    assert (
+        "xsiam_playbook_with_marketplaces_mismatch"
+        not in xsiam_build_context.tests_data_keeper.skipped_tests
+    )
+    assert (
+        "xsoar_playbook_with_marketplaces_mismatch"
+        in xsiam_build_context.tests_data_keeper.skipped_tests
+    )
+
+
 def test_unmockable_playbook_configuration(mocker, tmp_path):
     """
     Given: