Merge branch 'master' into ay-im106-new-val

.changelog/4114.yml

@@ -0,0 +1,6 @@
+changes:
+- description: A new version of validate is available. For more information, refer to the readme. (a docs web page with further information about each validation will be added soon). Note that the old validate flow is beginning to go under a deprecation process and will eventually be removed.
+  type: feature
+- description: The new validate flow is enabled by default while the old one is disabled. If you want to change this behavior, use the **--run-old-validate** and **--skip-new-validate** flags or the **RUN_OLD_VALIDATE** and **SKIP_NEW_VALIDATE** environment variables.
+  type: breaking
+pr_number: 4114

.github/workflows/on-push.yml

@@ -418,7 +418,7 @@ jobs:
         with:
           artifacts-path-dir: test-old-validate-command
           artifact-name: test-old-validate-command-artifacts
-          validate-arguments: "-a --graph --skip-pack-dependencies --prev-ver origin/master"
+          validate-arguments: "-a --graph --skip-pack-dependencies --skip-new-validate --run-old-validate --prev-ver origin/master"
 
   validate-files-new-validate:
     runs-on: ubuntu-latest
@@ -435,7 +435,7 @@ jobs:
         with:
           artifacts-path-dir: test-new-validate-command
           artifact-name: test-new-validate-command-artifacts
-          validate-arguments: "-a --prev-ver origin/master --run-new-validate --skip-old-validate --config-path sdk_validation_config.toml"
+          validate-arguments: "-a --prev-ver origin/master --config-path sdk_validation_config.toml"
 
   test-lint:
     runs-on: ubuntu-latest

demisto_sdk/__main__.py

@@ -777,18 +777,18 @@ def zip_packs(ctx, **kwargs) -> int:
     help="Wether to skip validations based on their support level or not.",
 )
 @click.option(
-    "--skip-old-validate",
+    "--run-old-validate",
     is_flag=True,
     show_default=True,
     default=False,
-    help="Wether to skip the old validate flow.",
+    help="Wether to run the old validate flow or not. Alteratively, you can configure the RUN_OLD_VALIDATE env variable.",
 )
 @click.option(
-    "--run-new-validate",
+    "--skip-new-validate",
     is_flag=True,
     show_default=True,
     default=False,
-    help="Wether to run the new validate flow.",
+    help="Wether to skip the new validate flow or not. Alteratively, you can configure the SKIP_NEW_VALIDATE env variable.",
 )
 @click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
 @pass_config
@@ -824,8 +824,54 @@ def validate(ctx, config, file_paths: str, **kwargs):
             kwargs["use_git"] = True
             kwargs["post_commit"] = True
         exit_code = 0
-        if not kwargs["skip_old_validate"]:
-            if kwargs["run_new_validate"]:
+        run_new_validate = not kwargs["skip_new_validate"] or (
+            (env_flag := os.getenv("SKIP_NEW_VALIDATE"))
+            and str(env_flag).lower() == "true"
+        )
+        run_old_validate = kwargs["run_old_validate"] or (
+            (env_flag := os.getenv("RUN_OLD_VALIDATE"))
+            and str(env_flag).lower() == "true"
+        )
+        if not run_new_validate:
+            for new_validate_flag in [
+                "fix",
+                "ignore_support_level",
+                "config_path",
+                "category_to_run",
+            ]:
+                if new_validate_flag in kwargs:
+                    logger.warning(
+                        f"The following flag {new_validate_flag.replace('_', '-')} is related only to the new validate and is being called while not running the new validate flow, therefore the flag will be ignored."
+                    )
+        if run_old_validate:
+            for old_validate_flag in [
+                "no_backward_comp",
+                "no_conf_json",
+                "id_set",
+                "graph",
+                "skip_pack_release_notes",
+                "print_ignored_errors",
+                "print_ignored_files",
+                "no_docker_checks",
+                "silence_init_prints",
+                "skip_pack_dependencies",
+                "id_set_path",
+                "create_id_set",
+                "skip_schema_check",
+                "debug_git",
+                "include_untracked",
+                "quiet_bc_validation",
+                "allow_skipped",
+                "run_specific_validations",
+                "no_multiprocessing",
+            ]:
+                if old_validate_flag in kwargs:
+                    logger.warning(
+                        f"The following flag {old_validate_flag.replace('_', '-')} is related only to the old validate and is being called while not running the old validate flow, therefore the flag will be ignored."
+                    )
+
+        if run_old_validate:
+            if not kwargs["skip_new_validate"]:
                 kwargs["graph"] = False
             validator = OldValidateManager(
                 is_backward_check=not kwargs["no_backward_comp"],
@@ -857,7 +903,7 @@ def validate(ctx, config, file_paths: str, **kwargs):
                 specific_validations=kwargs.get("run_specific_validations"),
             )
             exit_code += validator.run_validation()
-        if kwargs["run_new_validate"]:
+        if run_new_validate:
             validation_results = ResultWriter(
                 json_file_path=kwargs.get("json_file"),
             )

demisto_sdk/commands/common/schemas/integration.yml

@@ -142,6 +142,16 @@ mapping:
         type: bool
       isfetch:xsoar_on_prem:
         type: bool
+      isfetchevents:xsoar:
+        type: bool
+      isfetchevents:marketplacev2:
+        type: bool
+      isfetchevents:xpanse:
+        type: bool
+      isfetchevents:xsoar_saas:
+        type: bool
+      isfetchevents:xsoar_on_prem:
+        type: bool
   system:
     type: bool
   hidden:

demisto_sdk/commands/common/tests/pack_unique_files_test.py

@@ -216,7 +216,17 @@ def test_validate_partner_contribute_pack_metadata_no_mail_and_url(
         pack.pack_metadata.write_json(pack_metadata_no_email_and_url)
         with ChangeCWD(repo.path):
             runner = CliRunner(mix_stderr=False)
-            runner.invoke(main, [VALIDATE_CMD, "-i", pack.path], catch_exceptions=False)
+            runner.invoke(
+                main,
+                [
+                    VALIDATE_CMD,
+                    "-i",
+                    pack.path,
+                    "--run-old-validate",
+                    "--skip-new-validate",
+                ],
+                catch_exceptions=False,
+            )
         assert str_in_call_args_list(
             logger_error.call_args_list,
             "Contributed packs must include email or url",
@@ -274,7 +284,17 @@ def test_validate_partner_pack_metadata_url(
         pack.pack_metadata.write_json(pack_metadata_changed_url)
         with ChangeCWD(repo.path):
             runner = CliRunner(mix_stderr=False)
-            runner.invoke(main, [VALIDATE_CMD, "-i", pack.path], catch_exceptions=False)
+            runner.invoke(
+                main,
+                [
+                    VALIDATE_CMD,
+                    "-i",
+                    pack.path,
+                    "--run-old-validate",
+                    "--skip-new-validate",
+                ],
+                catch_exceptions=False,
+            )
 
         error_text = (
             "The metadata URL leads to a GitHub repo instead of a support page."
@@ -328,7 +348,17 @@ def test_validate_partner_contribute_pack_metadata_price_change(
         pack.pack_metadata.write_json(pack_metadata_price_changed)
         with ChangeCWD(repo.path):
             runner = CliRunner(mix_stderr=False)
-            runner.invoke(main, [VALIDATE_CMD, "-i", pack.path], catch_exceptions=False)
+            runner.invoke(
+                main,
+                [
+                    VALIDATE_CMD,
+                    "-i",
+                    pack.path,
+                    "--run-old-validate",
+                    "--skip-new-validate",
+                ],
+                catch_exceptions=False,
+            )
         assert str_in_call_args_list(
             logger_error.call_args_list,
             "The pack price was changed from 2 to 3 - revert the change",

demisto_sdk/commands/common/tools.py

@@ -2034,6 +2034,26 @@ def is_external_repository() -> bool:
         return True
 
 
+def is_external_repo() -> bool:
+    """
+    Returns True if script executed from an external repository (use this instead of is_external_repository)
+
+    """
+    try:
+        remote = GitUtil().repo.remote()
+        return (
+            not remote
+            or (not (parsed_url := giturlparse.parse(remote.url)))
+            or (
+                f"{parsed_url.owner}/{parsed_url.name}"
+                not in ("cortex-xdr/content", "demisto/content")
+            )
+        )
+    except Exception as e:
+        logger.debug(f"failed to get repo information, {str(e)}")
+        return True
+
+
 def get_content_id_set() -> dict:
     """Getting the ID Set from official content's bucket"""
     return requests.get(OFFICIAL_CONTENT_ID_SET_PATH).json()