Merge remote-tracking branch 'origin/price-tag-fix' into price-tag-fix

CHANGELOG.md

@@ -3,7 +3,11 @@
 * Fixed an issue in getting a README/CHANGELOG files from git and loading it.
 * Removed release notes validation for new content.
 * Fixed secretes validations for files with the same name in a different directory.
-
+* demisto-sdk lint - parralel working with specify the number of workers.
+* demisto-sdk lint - logging levels output, 3 levels.
+* demisto-sdk lint - json report, structured error reports in json format.
+* demisto-sdk lint - xml junit report for unit-tests.
+* demisto-sdk lint - New packages used in order to excellarate execution time.
 
 #### 0.5.0
 [PyPI History][1]

demisto_sdk/commands/common/configuration.py

@@ -1,5 +1,5 @@
 import logging
-import os
+from pathlib import Path
 
 
 class Configuration:
@@ -15,6 +15,7 @@ class Configuration:
     def __init__(self, log_verbose=False, logging_level=logging.INFO):
         logging.basicConfig(level=logging_level)
         self.log_verbose = log_verbose
-        self.sdk_env_dir = os.path.dirname(os.path.dirname(os.path.join(__file__)))
-        self.env_dir = os.getcwd()
-        self.envs_dirs_base = os.path.join(self.sdk_env_dir, 'lint', 'dev_envs', 'default_python')
+        # refers to "demisto_sdk/commands" dir
+        self.sdk_env_dir = str(Path(__file__).parent.parent)
+        self.env_dir = str(Path().cwd())
+        self.envs_dirs_base = str(Path(self.sdk_env_dir) / 'lint' / 'resources' / 'pipfile_python')

demisto_sdk/commands/common/hook_validations/pack_unique_files.py

@@ -145,7 +145,8 @@ def _is_pack_meta_file_structure_valid(self):
             if not isinstance(dependencies_field, dict):
                 self._add_error('The dependencies field in the pack must be a dictionary.')
                 return False
-            # price_field = metadata[PACK_METADATA_PRICE]
+            # TODO: add it back after #23546 is ready.
+            # price_field = metadata.get(PACK_METADATA_PRICE)
             # try:
             #     int(price_field)
             # except Exception:

demisto_sdk/commands/common/tests/pack_metadata_validator_test.py

@@ -20,8 +20,13 @@ def test_metadata_validator_valid(self, mocker):
         validator = PackUniqueFilesValidator('fake')
         assert validator.validate_pack_meta_file()
 
+    # TODO: add the validation for price after #23546 is ready.
     @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'),
                                           # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'),
+                                          os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'),
+                                          os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'),
+                                          os.path.join(FILES_PATH, 'pack_metadata_empty_category.json'),
+                                          os.path.join(FILES_PATH, 'pack_metadata_invalid_keywords.json'),
                                           os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'),
                                           os.path.join(FILES_PATH, 'pack_metadata_list.json')])
     def test_metadata_validator_invalid(self, mocker, metadata):

demisto_sdk/commands/init/initiator.py

@@ -232,7 +232,7 @@ def create_metadata(fill_manually: bool) -> Dict:
         tags = input("\nTags of the pack, comma separated values: ")
         tags_list = [t.strip() for t in tags.split(',')]
         metadata['tags'] = tags_list
-
+        # TODO: add it back after #23546 is ready.
         # price = input("\nThe price of the pack: ")
         # metadata['price'] = price
 

demisto_sdk/commands/init/tests/initiator_test.py

@@ -64,6 +64,7 @@ def test_get_object_id(monkeypatch, initiator):
     assert initiator.id == 'SomeIntegrationID'
 
 
+# TODO: add the validation for price after #23546 is ready.
 def test_create_metadata(monkeypatch, initiator):
     # test create_metadata without user filling manually
     pack_metadata = initiator.create_metadata(False)

demisto_sdk/commands/lint/lint_manager.py

@@ -124,7 +124,7 @@ def _gather_facts() -> Dict[str, Any]:
             print_warning("Can't communicate with Docker daemon - check your docker Engine is ON - Skiping lint, "
                           "test which require docker!")
             logger.info(f"demisto-sdk-Can't communicate with Docker daemon")
-        logger.info(f"Docker daemon test passed")
+        logger.debug(f"Docker daemon test passed")
 
         return facts
 
@@ -152,7 +152,7 @@ def _get_packages(self, content_repo: git.Repo, input: str, git: bool, all_packs
             pkgs = LintManager._filter_changed_packages(content_repo=content_repo,
                                                         pkgs=pkgs)
             for pkg in pkgs:
-                print_v(f"Package added after comparing to git {Colors.Fg.cyan}{pkg}{Colors.reset}",
+                print_v(f"Found changed package {Colors.Fg.cyan}{pkg}{Colors.reset}",
                         log_verbose=self._verbose)
         print(f"Execute lint and test on {Colors.Fg.cyan}{len(pkgs)}/{total_found}{Colors.reset} packages")
 
@@ -191,9 +191,8 @@ def _filter_changed_packages(content_repo: git.Repo, pkgs: List[Path]) -> List[P
         """
         print(f"Comparing to {Colors.Fg.cyan}{content_repo.remote()}/master{Colors.reset} using branch {Colors.Fg.cyan}"
               f"{content_repo.active_branch}{Colors.reset}")
-        # untracked_files = {content_repo.working_dir / Path(item).parent for item in content_repo.untracked_files}
         staged_files = {content_repo.working_dir / Path(item.b_path).parent for item in
-                        content_repo.index.diff(None, paths=pkgs)}
+                        content_repo.active_branch.commit.tree.diff(None, paths=pkgs)}
         last_common_commit = content_repo.merge_base(content_repo.active_branch.commit,
                                                      content_repo.remote().refs.master)
         changed_from_master = {content_repo.working_dir / Path(item.b_path).parent for item in

demisto_sdk/commands/lint/linter.py

@@ -6,8 +6,8 @@
 import os
 from typing import List, Optional, Tuple
 
-import docker
 # 3-rd party packages
+import docker
 import docker.errors
 import docker.models.containers
 import requests.exceptions
@@ -214,7 +214,9 @@ def _gather_facts(self, modules: dict) -> bool:
         # Facts for Powershell pack
         elif self._pkg_lint_status["pack_type"] == TYPE_PWSH:
             # Get lint files
-            lint_files = set(self._pack_abs_dir.glob(["*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1", "demistomock.ps1'"], flags=NEGATE))
+            lint_files = set(
+                self._pack_abs_dir.glob(["*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1", "demistomock.ps1'"],
+                                        flags=NEGATE))
         if 'commonserver' in self._pack_abs_dir.name.lower():
             if self._pkg_lint_status["pack_type"] == TYPE_PWSH:
                 self._facts["lint_files"] = [Path(self._pack_abs_dir / 'CommonServerPowerShell.ps1')]
@@ -353,8 +355,8 @@ def _run_vulture(self, py_num: float, lint_files: List[Path]) -> Tuple[int, str]
             lint_files(List[Path]): file to perform lint
 
         Returns:
-           int:  0 on successful else 1, errors
-           str: Bandit errors
+           int: 0 on successful else 1, errors
+           str: Vulture errors
         """
         log_prompt = f"{self._pack_name} - Vulture"
         logger.info(f"{log_prompt} - Start")
@@ -403,8 +405,7 @@ def _run_lint_on_docker_image(self, no_pylint: list, no_test: bool, no_pwsh_anal
             image_id = ""
             errors = ""
             for trial in range(2):
-                image_id, errors = self._docker_image_create(docker_base_image=image,
-                                                             no_test=no_test)
+                image_id, errors = self._docker_image_create(docker_base_image=image)
                 if not errors:
                     break
 
@@ -453,6 +454,14 @@ def _run_lint_on_docker_image(self, no_pylint: list, no_test: bool, no_pwsh_anal
                 pass
 
     def _docker_login(self) -> bool:
+        """ Login to docker-hub using enviorment varaibles:
+                1. DOCKERHUB_USER - User for docker hub.
+                2. DOCKERHUB_PASSWORD - Password for docker-hub.
+            Used in Circle-CI for pushing into repo devtestdemisto
+
+        Returns:
+            bool: True if logged in successfully.
+        """
         docker_user = os.getenv('DOCKERHUB_USER')
         docker_pass = os.getenv('DOCKERHUB_PASSWORD')
         try:
@@ -463,7 +472,7 @@ def _docker_login(self) -> bool:
         except docker.errors.APIError:
             return False
 
-    def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
+    def _docker_image_create(self, docker_base_image: str) -> str:
         """ Create docker image:
             1. Installing 'build base' if required in alpine images version - https://wiki.alpinelinux.org/wiki/GCC
             2. Installing pypi packs - if only pylint required - only pylint installed otherwise all pytest and pylint
@@ -472,8 +481,7 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
                 demisto_sdk/commands/lint/templates/dockerfile.jinja2
 
         Args:
-            docker_base_image(str): docker image to use as base for installing dev deps.
-            no_test(bool): wheter to run tests or not - will install required packages if True.
+            docker_base_image(str): docker image to use as base for installing dev deps..
 
         Returns:
             string. image name to use
@@ -493,7 +501,6 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
         try:
             dockerfile = template.render(image=docker_base_image[0],
                                          pypi_packs=requirements + self._facts["additional_requirements"],
-                                         no_test=(self._facts["test"] and no_test),
                                          pack_type=self._pkg_lint_status["pack_type"])
         except exceptions.TemplateError as e:
             logger.debug(f"{log_prompt} - Error when build image - {e.message()}")
@@ -562,7 +569,7 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
         return test_image_id, errors
 
     def _docker_run_pylint(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Pylint in container based to created test image
+        """ Run Pylint in created test image
 
         Args:
             test_image(str): test image id/name
@@ -635,7 +642,7 @@ def _docker_run_pylint(self, test_image: str, keep_container: bool) -> Tuple[int
         return exit_code, output
 
     def _docker_run_pytest(self, test_image: str, keep_container: bool, test_xml: str) -> Tuple[int, str]:
-        """ Run Pytest in container based to created test image
+        """ Run Pytest in created test image
 
         Args:
             test_image(str): Test image id/name
@@ -718,7 +725,7 @@ def _docker_run_pytest(self, test_image: str, keep_container: bool, test_xml: st
         return exit_code, test_json
 
     def _docker_run_pwsh_analyze(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Powershell code analyze in container based to created test image
+        """ Run Powershell code analyze in created test image
 
         Args:
             test_image(str): test image id/name
@@ -781,7 +788,7 @@ def _docker_run_pwsh_analyze(self, test_image: str, keep_container: bool) -> Tup
         return exit_code, output
 
     def _docker_run_pwsh_test(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Powershell tests in container based to created test image
+        """ Run Powershell tests in created test image
 
         Args:
             test_image(str): test image id/name

demisto_sdk/commands/lint/tests/test_linter/docker_runner_test.py

@@ -34,8 +34,7 @@ def test_build_image_no_errors(self, linter_obj: Linter, mocker):
         docker_build_response.short_id = exp_test_image_id
         linter_obj._docker_client.containers.create().commit().short_id = exp_test_image_id
 
-        act_test_image_id, act_errors = linter_obj._docker_image_create(docker_base_image=[exp_test_image_id, 3.7],
-                                                                        no_test=False)
+        act_test_image_id, act_errors = linter_obj._docker_image_create(docker_base_image=[exp_test_image_id, 3.7])
 
         assert act_test_image_id == exp_test_image_id
         assert act_errors == exp_errors

demisto_sdk/tests/test_files/pack_metadata_empty_category.json

@@ -0,0 +1,28 @@
+{
+    "name": "AWS Feed",
+    "description": "Indicators feed from AWS",
+    "support": "Cortex XSOAR",
+    "serverMinVersion": "5.5.0",
+    "currentVersion": "1.0.0",
+    "author": "Cortex XSOAR",
+    "url": "https://www.paloaltonetworks.com/cortex",
+    "email": "",
+    "categories": [""],
+    "tags": [],
+    "created": "2020-03-09T16:04:45Z",
+    "beta": false,
+    "deprecated": false,
+    "certification": "certified",
+    "useCases": [],
+    "keywords": ["AWS", "Feed"],
+    "price": 0,
+    "dependencies": {
+        "Base": {
+            "mandatory": true,
+            "minVersion": "1.0.0",
+            "name": "Base",
+            "certification": "certified",
+            "author": "Cortex XSOAR"
+          }
+    }
+}

demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json

@@ -0,0 +1,29 @@
+{
+    "name": "AWS Feed",
+    "description": "Indicators feed from AWS",
+    "support": "Cortex XSOAR",
+    "serverMinVersion": "5.5.0",
+    "currentVersion": "1.0.0",
+    "author": "Cortex XSOAR",
+    "url": "https://www.paloaltonetworks.com/cortex",
+    "email": "",
+    "categories": [
+        "Data Enrichment & Threat Intelligence"
+    ],
+    "tags": [""],
+    "created": "2020-03-09T16:04:45Z",
+    "beta": false,
+    "deprecated": false,
+    "certification": "certified",
+    "useCases": [],
+    "keywords": ["AWS", "Feed", "test"],
+    "price": 0,
+    "dependencies": {
+        "Base": {
+            "mandatory": true,
+            "minVersion": "1.0.0",
+            "name": "Base",
+            "certification": "certified"
+          }
+    }
+}

demisto_sdk/tests/test_files/pack_metadata_invalid_keywords.json

@@ -0,0 +1,30 @@
+{
+    "name": "AWS Feed",
+    "description": "Indicators feed from AWS",
+    "support": "Cortex XSOAR",
+    "serverMinVersion": "5.5.0",
+    "currentVersion": "1.0.0",
+    "author": "Cortex XSOAR",
+    "url": "https://www.paloaltonetworks.com/cortex",
+    "email": "",
+    "categories": [
+        "Data Enrichment & Threat Intelligence"
+    ],
+    "tags": [],
+    "created": "2020-03-09T16:04:45Z",
+    "beta": false,
+    "deprecated": false,
+    "certification": "certified",
+    "useCases": [],
+    "keywords": [""],
+    "price": 0,
+    "dependencies": {
+        "Base": {
+            "mandatory": true,
+            "minVersion": "1.0.0",
+            "name": "Base",
+            "certification": "certified",
+            "author": "Cortex XSOAR"
+          }
+    }
+}

demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json

@@ -0,0 +1,30 @@
+{
+    "name": "AWS Feed",
+    "description": "Indicators feed from AWS",
+    "support": "Cortex XSOAR",
+    "serverMinVersion": "5.5.0",
+    "currentVersion": "1.0.0",
+    "author": "Cortex XSOAR",
+    "url": "https://www.paloaltonetworks.com/cortex",
+    "email": "",
+    "categories": [
+        "Data Enrichment & Threat Intelligence"
+    ],
+    "tags": [],
+    "created": "2020-03-09T16:04:45Z",
+    "beta": false,
+    "deprecated": false,
+    "certification": "certified",
+    "useCases": [],
+    "keywords": ["AWS", "Feed"],
+    "price": 0,
+    "dependencies": [{
+        "Base": {
+            "mandatory": true,
+            "minVersion": "1.0.0",
+            "name": "Base",
+            "certification": "certified",
+            "author": "Cortex XSOAR"
+          }
+    }]
+}