Add run_isolated and pass_docker_extra_args to docker hook

demisto_sdk/__main__.py

@@ -213,9 +213,6 @@ def main(ctx, config, version, release_notes, **kwargs):
     handle_deprecated_args(ctx.args)
 
     config.configuration = Configuration()
-    import dotenv
-
-    dotenv.load_dotenv(CONTENT_PATH / ".env", override=True)  # type: ignore # load .env file from the cwd
 
     if platform.system() == "Windows":
         logger.warning(

demisto_sdk/commands/common/docker_helper.py

@@ -12,7 +12,7 @@
 import requests
 import urllib3
 from docker.types import Mount
-from packaging.version import Version
+from packaging.version import InvalidVersion, Version
 from requests import JSONDecodeError
 from requests.exceptions import RequestException
 
@@ -159,6 +159,16 @@ def __init__(self):
     def __del__(self):
         del self.tmp_dir_name
 
+    @staticmethod
+    @functools.lru_cache
+    def version() -> Version:
+        version = init_global_docker_client().version()["Version"]
+        try:
+            return Version(version)
+        except InvalidVersion:
+            # we don't care about the build number
+            return Version(version.split("-")[0])
+
     def installation_files(self, container_type: str) -> FILES_SRC_TARGET:
         files = self._files_to_push_on_installation.copy()
         files.append((self.installation_scripts[container_type], "/install.sh"))

demisto_sdk/commands/content_graph/objects/content_item.py

@@ -121,9 +121,6 @@ def in_pack(self) -> Optional["Pack"]:
         Returns:
             Pack: Pack model.
         """
-        # This function converts the pack attribute, which is a parser object to the pack model
-        # This happens since we cant mark the pack type as `Pack` because it is a forward reference.
-        # When upgrading to pydantic v2, remove this method and change pack type to `Pack` directly.
         pack = self.pack
         if not pack or isinstance(pack, fields.FieldInfo):
             pack = None

demisto_sdk/commands/pre_commit/hooks/docker.py

@@ -4,16 +4,23 @@
 import subprocess
 import time
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from functools import lru_cache
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Set, Tuple
 
 from docker.errors import DockerException
+from packaging.version import Version
 
-from demisto_sdk.commands.common.constants import TYPE_PWSH, TYPE_PYTHON
+from demisto_sdk.commands.common.constants import (
+    TYPE_PWSH,
+    TYPE_PYTHON,
+)
 from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH, PYTHONPATH
+from demisto_sdk.commands.common.cpu_count import cpu_count
 from demisto_sdk.commands.common.docker_helper import (
+    DockerBase,
     docker_login,
     get_docker,
     init_global_docker_client,
@@ -153,15 +160,15 @@ def get_environment_flag(env: dict) -> str:
 def _split_by_objects(
     files_with_objects: List[Tuple[Path, IntegrationScript]],
     config_arg: Optional[Tuple],
-    split_by_obj: bool = False,
+    run_isolated: bool = False,
 ) -> Dict[Optional[IntegrationScript], Set[Tuple[Path, IntegrationScript]]]:
     """
     Will group files into groups that share the same configuration file.
     If there is no config file, they get set to the NO_CONFIG_VALUE group
     Args:
         files: the files to split
         config_arg: a tuple, argument_name, file_name
-        split_by_obj: a boolean. If true it will split all the objects into separate hooks.
+        run_isolated: a boolean. If true it will split all the objects into separate hooks.
 
     Returns:
         a dict where the keys are the names of the folder of the config and the value is a set of files for that config
@@ -171,8 +178,9 @@ def _split_by_objects(
     ] = defaultdict(set)
 
     for file, obj in files_with_objects:
-        if split_by_obj or (config_arg and (obj.path.parent / config_arg[1]).exists()):
+        if run_isolated or (config_arg and (obj.path.parent / config_arg[1]).exists()):
             object_to_files[obj].add((file, obj))
+
         else:
             object_to_files[NO_SPLIT].add((file, obj))
 
@@ -184,6 +192,38 @@ class DockerHook(Hook):
     This class will make common manipulations on commands that need to run in docker
     """
 
+    def clean_args_from_hook(self, hooks: List[Dict]):
+        """This clean unsupported args from the generated hooks
+
+        Args:
+            hooks (List[Dict]): The hooks generated
+        """
+        for hook in hooks:
+            hook.pop("docker_image", None)
+            hook.pop("config_file_arg", None)
+            hook.pop("copy_files", None)
+            hook.pop("run_isolated", None)
+            hook.pop("pass_docker_extra_args", None)
+
+    def process_image(
+        self,
+        image: str,
+        files_with_objects: List[Tuple[Path, IntegrationScript]],
+        config_arg: Optional[Tuple],
+        run_isolated: bool,
+    ) -> List[Dict]:
+        object_to_files = _split_by_objects(
+            files_with_objects,
+            config_arg,
+            run_isolated,
+        )
+        image_is_powershell = any(obj.is_powershell for _, obj in files_with_objects)
+
+        dev_image = devtest_image(image, image_is_powershell, self.context.dry_run)
+        hooks = self.generate_hooks(dev_image, image, object_to_files, config_arg)
+        logger.debug(f"Generated {len(hooks)} hooks for image {image}")
+        return hooks
+
     def prepare_hook(
         self,
     ):
@@ -222,39 +262,39 @@ def prepare_hook(
                 for file in copy_files:
                     source: Path = CONTENT_PATH / file
                     target = obj.path.parent / Path(file).name
-                    if source != target and source.exists():
+                    if source != target and source.exists() and not target.exists():
                         shutil.copy(
                             CONTENT_PATH / file, obj.path.parent / Path(file).name
                         )
-        split_by_obj = self._get_property("split_by_object", False)
+        run_isolated = self._get_property("run_isolated", False)
         config_arg = self._get_config_file_arg()
         start_time = time.time()
         logger.debug(f"{len(tag_to_files_objs)} images were collected from files")
         logger.debug(f'collected images: {" ".join(tag_to_files_objs.keys())}')
-        for image, files_with_objects in sorted(
-            tag_to_files_objs.items(), key=lambda item: item[0]
-        ):
-            object_to_files = _split_by_objects(
-                files_with_objects, config_arg, split_by_obj
-            )
-            image_is_powershell = any(
-                obj.is_powershell for _, obj in files_with_objects
-            )
-
-            dev_image = devtest_image(image, image_is_powershell, self.context.dry_run)
-            hooks = self.get_new_hooks(
-                dev_image,
-                image,
-                object_to_files,
-                config_arg,
-            )
+        with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
+            results = []
+            for image, files_objs in sorted(
+                tag_to_files_objs.items(), key=lambda item: item[0]
+            ):
+                results.append(
+                    executor.submit(
+                        self.process_image,
+                        image,
+                        files_objs,
+                        config_arg,
+                        run_isolated,
+                    )
+                )
+        for result in results:
+            hooks = result.result()
             self.hooks.extend(hooks)
+
         end_time = time.time()
         logger.debug(
             f"DockerHook - prepared images in {round(end_time - start_time, 2)} seconds"
         )
 
-    def get_new_hooks(
+    def generate_hooks(
         self,
         dev_image,
         image,
@@ -270,6 +310,7 @@ def get_new_hooks(
             image: name of the base image (for naming)
             object_to_files_with_objects: A dict where the key is the object (or None) and value is the set of files to run together.
             config_arg: The config arg to set where relevant. This will be appended to the end of "args"
+
         Returns:
             All the hooks to be appended for this image
         """
@@ -278,24 +319,35 @@ def get_new_hooks(
         new_hook["name"] = f"{new_hook.get('name')}-{image}"
         new_hook["language"] = "docker_image"
         env = new_hook.pop("env", {})
+        docker_version = DockerBase.version()
+        quiet = True
+        # quiet mode is silently pull the image, and it is supported only above 19.03
+        if docker_version < Version("19.03"):
+            quiet = False
+        docker_extra_args = self._get_property("pass_docker_extra_args", "")
         new_hook[
             "entry"
-        ] = f'--entrypoint {new_hook.get("entry")} {get_environment_flag(env)} --quiet {dev_image}'
+        ] = f'--entrypoint {new_hook.get("entry")} {docker_extra_args} {get_environment_flag(env)} {"--quiet" if quiet else ""} -u 4000:4000 {dev_image}'
         ret_hooks = []
         for (
             integration_script,
             files_with_objects,
         ) in object_to_files_with_objects.items():
+            change_working_directory = False
             files = {file for file, _ in files_with_objects}
             hook = deepcopy(new_hook)
             if integration_script is not None:
+                change_working_directory = (
+                    True  # isolate container, so run in the same directory
+                )
                 if config_arg:
                     args = deepcopy(self._get_property("args", []))
                     args.extend(
                         [
                             config_arg[0],
                             str(
-                                (
+                                Path("/src")
+                                / (
                                     integration_script.path.parent / config_arg[1]
                                 ).relative_to(CONTENT_PATH)
                             ),
@@ -308,17 +360,22 @@ def get_new_hooks(
                 hook[
                     "name"
                 ] = f"{hook['name']}-{integration_script.object_id}"  # for uniqueness
+                # change the working directory to the integration script, as it runs in an isolated container
+                hook[
+                    "entry"
+                ] = f"-w {Path('/src') / integration_script.path.parent.relative_to(CONTENT_PATH)} {hook['entry']}"
+
             if self._set_files_on_hook(
-                hook, files, should_filter=False
+                hook,
+                files,
+                should_filter=False,
+                use_args=change_working_directory,
+                base_path=Path("/src"),
             ):  # no need to filter again, we have only filtered files
                 # disable multiprocessing on hook
                 hook["require_serial"] = True
                 ret_hooks.append(hook)
-        for hook in ret_hooks:
-            hook.pop("docker_image", None)
-            hook.pop("config_file_arg", None)
-            hook.pop("copy_files", None)
-            hook.pop("split_by_object", None)
+        self.clean_args_from_hook(ret_hooks)
         return ret_hooks
 
     def _get_config_file_arg(self) -> Optional[Tuple]:

demisto_sdk/commands/pre_commit/hooks/hook.py

@@ -6,6 +6,7 @@
 
 from packaging.version import Version
 
+from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.pre_commit.hooks.utils import get_property
 from demisto_sdk.commands.pre_commit.pre_commit_context import PreCommitContext
@@ -44,21 +45,33 @@ def exclude_irrelevant_files(self):
         self._exclude_hooks_by_support_level()
 
     def _set_files_on_hook(
-        self, hook: dict, files: Iterable[Path], should_filter: bool = True
+        self,
+        hook: dict,
+        files: Iterable[Path],
+        should_filter: bool = True,
+        use_args: bool = False,
+        base_path: Path = CONTENT_PATH,
     ) -> int:
         """
 
         Args:
             hook: mutates the hook with files returned from filter_files_matching_hook_config
             files: the list of files to set on the hook
-
+            use_args: if True, the files will be added to the args of the hook, and pass_filenames will be set to False
         Returns: the number of files that ultimately are set on the hook. Use this to decide if to run the hook at all
 
         """
         files_to_run_on_hook = set(files)
         if should_filter:
             files_to_run_on_hook = self.filter_files_matching_hook_config(files)
         hook["files"] = join_files(files_to_run_on_hook)
+        if use_args:
+            if "args" not in hook:
+                hook["args"] = []
+            hook["args"].extend(
+                (str(base_path / file) for file in files_to_run_on_hook)
+            )
+            hook["pass_filenames"] = False
         return len(files_to_run_on_hook)
 
     def filter_files_matching_hook_config(

demisto_sdk/commands/pre_commit/pre_commit_command.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Set, Tuple
 
+import more_itertools
 from packaging.version import Version
 
 from demisto_sdk.commands.common.constants import (
@@ -45,6 +46,7 @@
 SKIPPED_HOOKS = {"format", "validate", "secrets"}
 
 INTEGRATION_SCRIPT_REGEX = re.compile(r"^Packs/.*/(?:Integrations|Scripts)/.*.yml$")
+INTEGRATIONS_BATCH = 300
 
 
 class PreCommitRunner:
@@ -345,11 +347,14 @@ def group_by_language(
             infra_files.append(file)
 
     language_to_files: Dict[str, Set] = defaultdict(set)
-    with multiprocessing.Pool() as pool:
-        integrations_scripts = pool.map(
-            BaseContent.from_path, integrations_scripts_mapping.keys()
-        )
-
+    integrations_scripts = []
+    for integration_script_paths in more_itertools.chunked_even(
+        integrations_scripts_mapping.keys(), INTEGRATIONS_BATCH
+    ):
+        with multiprocessing.Pool(processes=cpu_count()) as pool:
+            integrations_scripts.extend(
+                pool.map(BaseContent.from_path, integration_script_paths)
+            )
     exclude_integration_script = set()
     for integration_script in integrations_scripts:
         if not integration_script or not isinstance(

demisto_sdk/commands/pre_commit/pre_commit_context.py

@@ -52,10 +52,12 @@ def __post_init__(self):
         """
         shutil.rmtree(PRECOMMIT_FOLDER, ignore_errors=True)
         PRECOMMIT_FOLDER.mkdir(parents=True)
+        (PRECOMMIT_FOLDER / "coverage").mkdir()
+        (PRECOMMIT_FOLDER / "pytest-junit").mkdir()
         PRECOMMIT_CONFIG.mkdir()
         PRECOMMIT_DOCKER_CONFIGS.mkdir()
 
-        self.precommit_template = get_file_or_remote(PRECOMMIT_TEMPLATE_PATH)
+        self.precommit_template: dict = get_file_or_remote(PRECOMMIT_TEMPLATE_PATH)  # type: ignore[assignment]
         remote_config_file = get_remote_file(str(PRECOMMIT_TEMPLATE_PATH))
         if remote_config_file and remote_config_file != self.precommit_template:
             logger.info(
@@ -108,7 +110,7 @@ def python_version_to_files(self) -> Dict[str, Set[Path]]:
     def support_level_to_files(self) -> Dict[str, Set[Path]]:
         support_level_to_files = defaultdict(set)
         for path, obj in self.files_to_run_with_objects:
-            if obj:
+            if obj is not None:
                 support_level_to_files[obj.support_level].add(path)
         return support_level_to_files