Bug - git index, pipfile configuration, changelog lint command

* Git index bug fix.
* Pipfile path in the Configuration.
* Changelog for the lint.

CHANGELOG.md

@@ -3,7 +3,11 @@
 * Fixed an issue in getting a README/CHANGELOG files from git and loading it.
 * Removed release notes validation for new content.
 * Fixed secretes validations for files with the same name in a different directory.
-
+* demisto-sdk lint - parralel working with specify the number of workers.
+* demisto-sdk lint - logging levels output, 3 levels.
+* demisto-sdk lint - json report, structured error reports in json format.
+* demisto-sdk lint - xml junit report for unit-tests.
+* demisto-sdk lint - New packages used in order to excellarate execution time.
 
 #### 0.5.0
 [PyPI History][1]

demisto_sdk/commands/common/configuration.py

@@ -1,5 +1,5 @@
 import logging
-import os
+from pathlib import Path
 
 
 class Configuration:
@@ -15,6 +15,7 @@ class Configuration:
     def __init__(self, log_verbose=False, logging_level=logging.INFO):
         logging.basicConfig(level=logging_level)
         self.log_verbose = log_verbose
-        self.sdk_env_dir = os.path.dirname(os.path.dirname(os.path.join(__file__)))
-        self.env_dir = os.getcwd()
-        self.envs_dirs_base = os.path.join(self.sdk_env_dir, 'lint', 'dev_envs', 'default_python')
+        # refers to "demisto_sdk/commands" dir
+        self.sdk_env_dir = str(Path(__file__).parent.parent)
+        self.env_dir = str(Path().cwd())
+        self.envs_dirs_base = str(Path(self.sdk_env_dir) / 'lint' / 'resources' / 'pipfile_python')

demisto_sdk/commands/lint/lint_manager.py

@@ -124,7 +124,7 @@ def _gather_facts() -> Dict[str, Any]:
             print_warning("Can't communicate with Docker daemon - check your docker Engine is ON - Skiping lint, "
                           "test which require docker!")
             logger.info(f"demisto-sdk-Can't communicate with Docker daemon")
-        logger.info(f"Docker daemon test passed")
+        logger.debug(f"Docker daemon test passed")
 
         return facts
 
@@ -152,7 +152,7 @@ def _get_packages(self, content_repo: git.Repo, input: str, git: bool, all_packs
             pkgs = LintManager._filter_changed_packages(content_repo=content_repo,
                                                         pkgs=pkgs)
             for pkg in pkgs:
-                print_v(f"Package added after comparing to git {Colors.Fg.cyan}{pkg}{Colors.reset}",
+                print_v(f"Found changed package {Colors.Fg.cyan}{pkg}{Colors.reset}",
                         log_verbose=self._verbose)
         print(f"Execute lint and test on {Colors.Fg.cyan}{len(pkgs)}/{total_found}{Colors.reset} packages")
 
@@ -191,9 +191,8 @@ def _filter_changed_packages(content_repo: git.Repo, pkgs: List[Path]) -> List[P
         """
         print(f"Comparing to {Colors.Fg.cyan}{content_repo.remote()}/master{Colors.reset} using branch {Colors.Fg.cyan}"
               f"{content_repo.active_branch}{Colors.reset}")
-        # untracked_files = {content_repo.working_dir / Path(item).parent for item in content_repo.untracked_files}
         staged_files = {content_repo.working_dir / Path(item.b_path).parent for item in
-                        content_repo.index.diff(None, paths=pkgs)}
+                        content_repo.active_branch.commit.tree.diff(None, paths=pkgs)}
         last_common_commit = content_repo.merge_base(content_repo.active_branch.commit,
                                                      content_repo.remote().refs.master)
         changed_from_master = {content_repo.working_dir / Path(item.b_path).parent for item in

demisto_sdk/commands/lint/linter.py

@@ -6,8 +6,8 @@
 import os
 from typing import List, Optional, Tuple
 
-import docker
 # 3-rd party packages
+import docker
 import docker.errors
 import docker.models.containers
 import requests.exceptions
@@ -214,7 +214,9 @@ def _gather_facts(self, modules: dict) -> bool:
         # Facts for Powershell pack
         elif self._pkg_lint_status["pack_type"] == TYPE_PWSH:
             # Get lint files
-            lint_files = set(self._pack_abs_dir.glob(["*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1", "demistomock.ps1'"], flags=NEGATE))
+            lint_files = set(
+                self._pack_abs_dir.glob(["*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1", "demistomock.ps1'"],
+                                        flags=NEGATE))
         if 'commonserver' in self._pack_abs_dir.name.lower():
             if self._pkg_lint_status["pack_type"] == TYPE_PWSH:
                 self._facts["lint_files"] = [Path(self._pack_abs_dir / 'CommonServerPowerShell.ps1')]
@@ -353,8 +355,8 @@ def _run_vulture(self, py_num: float, lint_files: List[Path]) -> Tuple[int, str]
             lint_files(List[Path]): file to perform lint
 
         Returns:
-           int:  0 on successful else 1, errors
-           str: Bandit errors
+           int: 0 on successful else 1, errors
+           str: Vulture errors
         """
         log_prompt = f"{self._pack_name} - Vulture"
         logger.info(f"{log_prompt} - Start")
@@ -403,8 +405,7 @@ def _run_lint_on_docker_image(self, no_pylint: list, no_test: bool, no_pwsh_anal
             image_id = ""
             errors = ""
             for trial in range(2):
-                image_id, errors = self._docker_image_create(docker_base_image=image,
-                                                             no_test=no_test)
+                image_id, errors = self._docker_image_create(docker_base_image=image)
                 if not errors:
                     break
 
@@ -453,6 +454,14 @@ def _run_lint_on_docker_image(self, no_pylint: list, no_test: bool, no_pwsh_anal
                 pass
 
     def _docker_login(self) -> bool:
+        """ Login to docker-hub using enviorment varaibles:
+                1. DOCKERHUB_USER - User for docker hub.
+                2. DOCKERHUB_PASSWORD - Password for docker-hub.
+            Used in Circle-CI for pushing into repo devtestdemisto
+
+        Returns:
+            bool: True if logged in successfully.
+        """
         docker_user = os.getenv('DOCKERHUB_USER')
         docker_pass = os.getenv('DOCKERHUB_PASSWORD')
         try:
@@ -463,7 +472,7 @@ def _docker_login(self) -> bool:
         except docker.errors.APIError:
             return False
 
-    def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
+    def _docker_image_create(self, docker_base_image: str) -> str:
         """ Create docker image:
             1. Installing 'build base' if required in alpine images version - https://wiki.alpinelinux.org/wiki/GCC
             2. Installing pypi packs - if only pylint required - only pylint installed otherwise all pytest and pylint
@@ -472,8 +481,7 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
                 demisto_sdk/commands/lint/templates/dockerfile.jinja2
 
         Args:
-            docker_base_image(str): docker image to use as base for installing dev deps.
-            no_test(bool): wheter to run tests or not - will install required packages if True.
+            docker_base_image(str): docker image to use as base for installing dev deps..
 
         Returns:
             string. image name to use
@@ -493,7 +501,6 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
         try:
             dockerfile = template.render(image=docker_base_image[0],
                                          pypi_packs=requirements + self._facts["additional_requirements"],
-                                         no_test=(self._facts["test"] and no_test),
                                          pack_type=self._pkg_lint_status["pack_type"])
         except exceptions.TemplateError as e:
             logger.debug(f"{log_prompt} - Error when build image - {e.message()}")
@@ -562,7 +569,7 @@ def _docker_image_create(self, docker_base_image: str, no_test: bool) -> str:
         return test_image_id, errors
 
     def _docker_run_pylint(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Pylint in container based to created test image
+        """ Run Pylint in created test image
 
         Args:
             test_image(str): test image id/name
@@ -635,7 +642,7 @@ def _docker_run_pylint(self, test_image: str, keep_container: bool) -> Tuple[int
         return exit_code, output
 
     def _docker_run_pytest(self, test_image: str, keep_container: bool, test_xml: str) -> Tuple[int, str]:
-        """ Run Pytest in container based to created test image
+        """ Run Pytest in created test image
 
         Args:
             test_image(str): Test image id/name
@@ -718,7 +725,7 @@ def _docker_run_pytest(self, test_image: str, keep_container: bool, test_xml: st
         return exit_code, test_json
 
     def _docker_run_pwsh_analyze(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Powershell code analyze in container based to created test image
+        """ Run Powershell code analyze in created test image
 
         Args:
             test_image(str): test image id/name
@@ -781,7 +788,7 @@ def _docker_run_pwsh_analyze(self, test_image: str, keep_container: bool) -> Tup
         return exit_code, output
 
     def _docker_run_pwsh_test(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
-        """ Run Powershell tests in container based to created test image
+        """ Run Powershell tests in created test image
 
         Args:
             test_image(str): test image id/name

demisto_sdk/commands/lint/tests/test_linter/docker_runner_test.py

@@ -34,8 +34,7 @@ def test_build_image_no_errors(self, linter_obj: Linter, mocker):
         docker_build_response.short_id = exp_test_image_id
         linter_obj._docker_client.containers.create().commit().short_id = exp_test_image_id
 
-        act_test_image_id, act_errors = linter_obj._docker_image_create(docker_base_image=[exp_test_image_id, 3.7],
-                                                                        no_test=False)
+        act_test_image_id, act_errors = linter_obj._docker_image_create(docker_base_image=[exp_test_image_id, 3.7])
 
         assert act_test_image_id == exp_test_image_id
         assert act_errors == exp_errors