Merge branch 'master' into ds-verify-pip-installation

.changelog/4203.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Fixed an issue where validate spammed logs when running.
+  type: fix
+pr_number: 4203

.changelog/4205.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added support to delete packs in **graph update**
+  type: fix
+pr_number: 4205

CHANGELOG.md

@@ -1,4 +1,18 @@
 # Changelog
+## 1.29.0
+* The new validate flow is enabled by default while the old one is disabled. If you want to change this behavior, use the **--run-old-validate** and **--skip-new-validate** flags or the **RUN_OLD_VALIDATE** and **SKIP_NEW_VALIDATE** environment variables. [#4114](https://github.com/demisto/demisto-sdk/pull/4114)
+* A new version of validate is available. For more information, refer to the readme. (a docs web page with further information about each validation will be added soon). Note that the old validate flow is beginning to go under a deprecation process and will eventually be removed. [#4114](https://github.com/demisto/demisto-sdk/pull/4114)
+* Added the `--template-path` argument to the **pre-commit** command. [#4184](https://github.com/demisto/demisto-sdk/pull/4184)
+* Added support for a new argument **parallel* for each hook to determine whether a hook run in parallel or not. [#4130](https://github.com/demisto/demisto-sdk/pull/4130)
+* Added handling for API Modules in **pre-commit**. [#4175](https://github.com/demisto/demisto-sdk/pull/4175)
+* Fixed an issue in **pre-commit** when using `--all-files` it crashed with memory error. [#4196](https://github.com/demisto/demisto-sdk/pull/4196)
+* Fixed an issue in **graph update** where all content packs were being parsed. [#4186](https://github.com/demisto/demisto-sdk/pull/4186)
+* Fixed an issue in **graph update** to create the graph from scratch if changed packs can not be determined. [#4186](https://github.com/demisto/demisto-sdk/pull/4186)
+* Fixed an issue in **graph update** where the graph failed to update in external repositories. [#4190](https://github.com/demisto/demisto-sdk/pull/4190)
+* Fixed an issue where **pre-commit** command showed wrong hooks failing. [#4130](https://github.com/demisto/demisto-sdk/pull/4130)
+* Fixes an issue where *Fetch Events* and *Fetch Assets* checkboxes appeared in Xsoar Machines. [#4160](https://github.com/demisto/demisto-sdk/pull/4160)
+* Added an implicit validation for the graph's *relationship* object by converting it to utilize Pydantic. [#4142](https://github.com/demisto/demisto-sdk/pull/4142)
+
 ## 1.28.0
 * Added the `validate-xsoar-config` pre-commit hook, replacing `XC` validations. [#4150](https://github.com/demisto/demisto-sdk/pull/4150)
 * Add `--prev-version` argument to **pre-commit** command. [#4177](https://github.com/demisto/demisto-sdk/pull/4177)

demisto_sdk/__main__.py

@@ -839,11 +839,11 @@ def validate(ctx, config, file_paths: str, **kwargs):
                 "config_path",
                 "category_to_run",
             ]:
-                if new_validate_flag in kwargs:
+                if kwargs.get(new_validate_flag):
                     logger.warning(
                         f"The following flag {new_validate_flag.replace('_', '-')} is related only to the new validate and is being called while not running the new validate flow, therefore the flag will be ignored."
                     )
-        if run_old_validate:
+        if not run_old_validate:
             for old_validate_flag in [
                 "no_backward_comp",
                 "no_conf_json",
@@ -865,7 +865,7 @@ def validate(ctx, config, file_paths: str, **kwargs):
                 "run_specific_validations",
                 "no_multiprocessing",
             ]:
-                if old_validate_flag in kwargs:
+                if kwargs.get(old_validate_flag):
                     logger.warning(
                         f"The following flag {old_validate_flag.replace('_', '-')} is related only to the old validate and is being called while not running the old validate flow, therefore the flag will be ignored."
                     )

demisto_sdk/commands/common/constants.py

@@ -419,6 +419,8 @@ class FileType(str, Enum):
     FileType.GENERIC_FIELD,
 ]
 
+DEFAULT_IMAGE = "demisto_sdk/tests/test_files/default_image.png"
+
 DEFAULT_IMAGE_PREFIX = "data:image/png;base64,"
 DEFAULT_IMAGE_BASE64 = (
     "iVBORw0KGgoAAAANSUhEUgAAAFAAAABQCAMAAAC5zwKfAAACYVBMVEVHcEwAT4UAT4UAT4YAf/8A//8AT4UAf78AT4U"
@@ -2052,6 +2054,8 @@ class ParameterType(Enum):
 FORMATTING_SCRIPT = "indicator-format"
 
 DOCKERFILES_INFO_REPO = "demisto/dockerfiles-info"
+DOCKERFILES_REPO = "demisto/dockerfiles"
+CONTENT_REPO = "demisto/content"
 
 TEST_COVERAGE_DEFAULT_URL = f"https://storage.googleapis.com/{DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV}/code-coverage-reports/coverage-min.json"
 

demisto_sdk/commands/common/docker_images_metadata.py

@@ -12,7 +12,6 @@
 from demisto_sdk.commands.common.docker.docker_image import DockerImage
 from demisto_sdk.commands.common.files.errors import FileReadError
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.singleton import PydanticSingleton
 from demisto_sdk.commands.common.tools import NoInternetConnectionException
@@ -54,8 +53,8 @@ def __from_github(
         try:
             dockerfiles_metadata = JsonFile.read_from_github_api(
                 file_name,
+                repo=DOCKERFILES_INFO_REPO,
                 tag=tag,
-                git_content_config=GitContentConfig(repo_name=DOCKERFILES_INFO_REPO),
                 verify_ssl=False,
                 encoding="utf-8-sig",
             )

demisto_sdk/commands/common/files/file.py

@@ -1,4 +1,5 @@
 import inspect
+import os
 import shutil
 import urllib.parse
 from abc import ABC, abstractmethod
@@ -13,6 +14,7 @@
 from requests.exceptions import ConnectionError, RequestException, Timeout
 
 from demisto_sdk.commands.common.constants import (
+    CONTENT_REPO,
     DEMISTO_GIT_PRIMARY_BRANCH,
     DEMISTO_GIT_UPSTREAM,
     urljoin,
@@ -26,7 +28,10 @@
     MemoryFileReadError,
     UnknownFileError,
 )
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
+from demisto_sdk.commands.common.git_content_config import (
+    GitContentConfig,
+    GitCredentials,
+)
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.handlers.xsoar_handler import XSOAR_Handler
 from demisto_sdk.commands.common.logger import logger
@@ -317,7 +322,7 @@ def __read_git_file(self, tag: str, from_remote: bool = True) -> Any:
     def read_from_github_api(
         cls,
         path: str,
-        git_content_config: Optional[GitContentConfig] = None,
+        repo: str = CONTENT_REPO,
         encoding: Optional[str] = None,
         tag: str = DEMISTO_GIT_PRIMARY_BRANCH,
         handler: Optional[XSOAR_Handler] = None,
@@ -328,8 +333,8 @@ def read_from_github_api(
         Reads a file from Github api.
 
         Args:
-            path: the path to the file in github
-            git_content_config: git content config object
+            path: the path to the file in Github from the repo's root
+            repo: the repository name, e.g.: demisto/content
             encoding: any custom encoding if needed
             tag: the branch/sha to take the file from within Github
             handler: whether a custom handler is required, if not takes the default.
@@ -339,17 +344,16 @@ def read_from_github_api(
         Returns:
             Any: the file content in the desired format
         """
-        if not git_content_config:
-            git_content_config = GitContentConfig()
-
-        git_path_url = urljoin(git_content_config.base_api, tag, path)
-        github_token = git_content_config.CREDENTIALS.github_token
+        if not path.startswith("/"):
+            path = f"/{path}"
+        url = f"https://raw.githubusercontent.com/{repo}/{tag}{path}"
+        github_token = os.getenv(GitCredentials.ENV_GITHUB_TOKEN_NAME, "")
 
         timeout = 10
 
         try:
             return cls.read_from_http_request(
-                git_path_url,
+                url,
                 headers=frozenset(
                     {
                         "Authorization": f"Bearer {github_token}"
@@ -366,17 +370,17 @@ def read_from_github_api(
             )
         except FileReadError as e:
             logger.warning(
-                f"Received error {e} when trying to retrieve {git_path_url} content from Github, retrying"
+                f"Received error {e} when trying to retrieve {url} content from Github, retrying"
             )
             try:
                 return cls.read_from_http_request(
-                    git_path_url,
+                    url,
                     params=frozenset({"token": github_token}.items()),
                     timeout=timeout,
                 )
             except FileReadError:
                 logger.error(
-                    f"Could not retrieve the content of {git_path_url} file from Github"
+                    f"Could not retrieve the content of {url} file from Github"
                 )
                 raise
 

demisto_sdk/commands/content_graph/parsers/related_files.py

@@ -1,3 +1,4 @@
+import base64
 from abc import ABC
 from enum import Enum
 from pathlib import Path
@@ -198,14 +199,30 @@ def get_file_size(self):
     def get_file_dimensions(self):
         raise NotImplementedError
 
+    def load_image(self):
+        raise NotImplementedError
+
 
 class PNGFiles(ImageFiles):
     def get_file_size(self):
         return self.file_path.stat()
 
+    def load_image(self) -> Union[str, bytearray, memoryview]:
+        encoded_image = ""
+        with open(self.file_path, "rb") as image:
+            image_data = image.read()
+            encoded_image = base64.b64encode(image_data)  # type: ignore
+            if isinstance(encoded_image, bytes):
+                encoded_image = encoded_image.decode("utf-8")
+        return encoded_image
+
 
 class SVGFiles(ImageFiles):
-    pass
+    def load_image(self) -> bytes:
+        encoded_image = b""
+        with open(self.file_path, "rb") as image_file:
+            encoded_image = image_file.read()  # type: ignore
+        return encoded_image
 
 
 class DarkSVGRelatedFile(RelatedFile):

demisto_sdk/commands/content_graph/parsers/repository.py

@@ -60,8 +60,8 @@ def parse(
     def parse_pack(pack_path: Path) -> Optional[PackParser]:
         try:
             return PackParser(pack_path)
-        except NotAContentItemException:
-            logger.error(f"Pack {pack_path.name} is not a valid pack. Skipping")
+        except (NotAContentItemException, FileNotFoundError):
+            logger.warning(f"Pack {pack_path.name} is not a valid pack. Skipping")
             return None
 
     @staticmethod

demisto_sdk/commands/validate/sdk_validation_config.toml

@@ -36,7 +36,7 @@ select = [
     "CL100",
     "GF100", "GF101", "GF102",
     "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106",
-    "IM100", "IM108", "IM109",
+    "IM100", "IM108", "IM109", "IM106",
     "RP101", "BC106", "BC107"
 ]
 warning = []

demisto_sdk/commands/validate/tests/IM_validators_test.py

@@ -1,12 +1,20 @@
 import pytest
+from pytest_mock import MockerFixture
 
+from demisto_sdk.commands.common.constants import DEFAULT_IMAGE
+from demisto_sdk.commands.content_graph.parsers.related_files import (
+    ImageRelatedFile,
+)
 from demisto_sdk.commands.validate.tests.test_tools import (
     create_integration_object,
     create_pack_object,
 )
 from demisto_sdk.commands.validate.validators.IM_validators.IM100_image_exists_validation import (
     ImageExistsValidator,
 )
+from demisto_sdk.commands.validate.validators.IM_validators.IM106_default_image_validator import (
+    DefaultImageValidator,
+)
 from demisto_sdk.commands.validate.validators.IM_validators.IM108_author_image_is_empty import (
     AuthorImageIsEmptyValidator,
 )
@@ -112,3 +120,40 @@ def test_AuthorImageIsEmptyValidator_is_valid(
             for result, expected_msg in zip(results, expected_msgs)
         ]
     )
+
+
+def test_DefaultImageValidator_is_valid(mocker: MockerFixture):
+    """
+    Given:
+        - First integration with a default image.
+        - Second integration with a sample image
+
+    When:
+        - Calling the DefaultImageValidator is_valid function.
+
+    Then:
+        - Make sure the right amount of integration image validation failed, and that the right error message is returned.
+        - Case 1: Should fail.
+        - Case 2: Shouldn't fail.
+    """
+    from pathlib import Path
+
+    default_image = ImageRelatedFile(main_file_path=Path(DEFAULT_IMAGE))
+    sample_image = ImageRelatedFile(
+        main_file_path=Path("TestSuite/assets/default_integration/sample_image.png")
+    )
+
+    content_items = [create_integration_object(), create_integration_object()]
+    mocker.patch.object(
+        content_items[0].image, "load_image", return_value=default_image.load_image()
+    )
+    mocker.patch.object(
+        content_items[1].image, "load_image", return_value=sample_image.load_image()
+    )
+    results = DefaultImageValidator().is_valid(content_items)
+    assert len(results) == 1
+    assert results[
+        0
+    ].message == "The integration is using the default image at {0}, please change to the integration image.".format(
+        DEFAULT_IMAGE
+    )

demisto_sdk/commands/validate/validators/DO_validators/DO105_docker_image_is_not_deprecated.py

@@ -2,8 +2,8 @@
 
 from typing import ClassVar, Dict, Iterable, List, Union
 
+from demisto_sdk.commands.common.constants import DOCKERFILES_REPO
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.content_graph.objects.integration import Integration
 from demisto_sdk.commands.content_graph.objects.script import Script
 from demisto_sdk.commands.validate.validators.base_validator import (
@@ -30,7 +30,7 @@ def is_valid(self, content_items: Iterable[ContentTypes]) -> List[ValidationResu
         if not self.deprecated_dockers_to_reasons:
             deprecated_dockers = JsonFile.read_from_github_api(
                 path="/docker/deprecated_images.json",
-                git_content_config=GitContentConfig(repo_name="demisto/dockerfiles"),
+                repo=DOCKERFILES_REPO,
                 verify_ssl=False,
             )
             DockerImageIsNotDeprecatedValidator.deprecated_dockers_to_reasons = {