Add python version for content graph

CHANGELOG.md

@@ -3,6 +3,7 @@
 * Added a period at the end of lines produced by the **generate-docs** command that state the tested version of the product.
 * Update `RN112` validation's docs reference link.
 * Fixed an issue in calculating content graph hash when creating or updating it.
+* The content graph will now include the **python_version** field that each script/integration uses.
 * Calling **graph create** or **graph update** now run the commands with default arguments, instead of showing the command help.
 * Removed the use of chunks when calculating content relationships.
 

demisto_sdk/commands/common/constants.py

@@ -1845,5 +1845,6 @@ class ParameterType(Enum):
 FORMATTING_SCRIPT = "indicator-format"
 
 ENV_SDK_WORKING_OFFLINE = "DEMISTO_SDK_OFFLINE_ENV"
+DOCKERFILES_INFO_REPO = "demisto/dockerfiles-info"
 
 TEST_COVERAGE_DEFAULT_URL = "https://storage.googleapis.com/marketplace-dist-dev/code-coverage-reports/coverage-min.json"

demisto_sdk/commands/common/docker_helper.py

@@ -18,11 +18,13 @@
 from demisto_sdk.commands.common.constants import (
     DEFAULT_PYTHON2_VERSION,
     DEFAULT_PYTHON_VERSION,
+    DOCKERFILES_INFO_REPO,
     TYPE_PWSH,
     TYPE_PYTHON,
     TYPE_PYTHON2,
     TYPE_PYTHON3,
 )
+from demisto_sdk.commands.common.docker_images_metadata import DockerImagesMetadata
 from demisto_sdk.commands.common.logger import logger
 
 DOCKER_CLIENT = None
@@ -36,7 +38,9 @@
     )
 )
 
-PYTHON_IMAGE_REGEX = re.compile(r"[\d\w]+/python3?:(?P<python_version>[23]\.\d+)")
+DEMISTO_PYTHON_BASE_IMAGE_REGEX = re.compile(
+    r"[\d\w]+/python3?:(?P<python_version>[23]\.\d+(\.\d+)?)"
+)
 
 TEST_REQUIREMENTS_DIR = Path(__file__).parent.parent / "lint" / "resources"
 
@@ -286,8 +290,12 @@ def pull_or_create_test_image(
             The test image name and errors to create it if any
         """
         errors = ""
-        if not python_version and container_type != TYPE_PWSH:
-            python_version = get_python_version(base_image).major
+        if (
+            not python_version
+            and container_type != TYPE_PWSH
+            and (version := get_python_version(base_image))
+        ):
+            python_version = version.major
         python3_requirements = get_pip_requirements_from_file(
             TEST_REQUIREMENTS_DIR / "python3_requirements" / "dev-requirements.txt"
         )
@@ -400,6 +408,14 @@ def get_docker():
     return MountableDocker() if CAN_MOUNT_FILES else DockerBase()
 
 
+def _get_python_version_from_tag_by_regex(image: str) -> Optional[Version]:
+
+    if match := DEMISTO_PYTHON_BASE_IMAGE_REGEX.match(image):
+        return Version(match.group("python_version"))
+
+    return None
+
+
 def _get_docker_hub_token(repo: str) -> str:
     auth = None
 
@@ -467,25 +483,49 @@ def _get_python_version_from_env(env: List[str]) -> Version:
 
 
 @functools.lru_cache
-def get_python_version(image: Optional[str]) -> Version:
-    log_prompt = f"Get python version from image {image}"
-    logger.debug(f"{log_prompt} - Start")
-    if not image or "pwsh" in image or "powershell" in image:
+def get_python_version(image: Optional[str]):
+    """
+    Get the python version of a docker image if exist.
+
+    Args:
+        image (str): the docker image
+
+    Returns:
+        Version: Python version X.Y (3.7, 3.6, ..)
+    """
+    logger.debug(f"Get python version from image {image=}")
+
+    if not image:
         # When no docker_image is specified, we use the default python version which is Python 2.7.18
         logger.debug(
             f"No docker image specified or a powershell image, using default python version: {DEFAULT_PYTHON2_VERSION}"
         )
         return Version(DEFAULT_PYTHON2_VERSION)
-    if match := PYTHON_IMAGE_REGEX.match(image):
-        return Version(match.group("python_version"))
+
+    if "pwsh" in image or "powershell" in image:
+        logger.debug(
+            f"The {image=} is a powershell image, does not have python version"
+        )
+        return None
+
+    if python_version := DockerImagesMetadata.get_instance().python_version(image):
+        return python_version
+    logger.debug(
+        f"Could not get python version for {image=} from {DOCKERFILES_INFO_REPO} repo"
+    )
+
+    if python_version := _get_python_version_from_tag_by_regex(image):
+        return python_version
+    logger.debug(f"Could not get python version for {image=} from regex")
+
     try:
-        return _get_python_version_from_image_client(image)
+        logger.debug(f"get python version for {image=} from dockerhub api")
+        return _get_python_version_from_dockerhub_api(image)
     except Exception:
         logger.debug(
-            "Could not get the python version from client. Trying with API",
-            exc_info=True,
+            f"Getting python version from {image=} by pulling its image and query its env"
         )
-        return _get_python_version_from_dockerhub_api(image)
+        return _get_python_version_from_image_client(image)
 
 
 def _get_python_version_from_image_client(image: str) -> Version:
@@ -499,15 +539,24 @@ def _get_python_version_from_image_client(image: str) -> Version:
     """
     try:
         image_model = DockerBase.pull_image(image)
-        env = image_model.attrs["Config"]["Env"]
-        logger.debug(f"Got {env=} from {image=}")
-        return _get_python_version_from_env(env)
+        image_env = image_model.attrs["Config"]["Env"]
+        logger.debug(f"Got {image_env=} from {image=}")
+        return _get_python_version_from_env(image_env)
     except Exception:
         logger.exception(f"Failed detecting Python version for {image=}")
         raise
 
 
-def _get_python_version_from_dockerhub_api(image: str):
+def _get_python_version_from_dockerhub_api(image: str) -> Version:
+    """
+    Get python version for a docker image from the dockerhub api
+
+    Args:
+        image (str): the docker image.
+
+    Returns:
+        Version: Python version X.Y (3.7, 3.6, ..)
+    """
     if ":" not in image:
         repo = image
         tag = "latest"

demisto_sdk/commands/common/docker_images_metadata.py

@@ -0,0 +1,101 @@
+import re
+from enum import Enum
+from typing import Dict, Optional
+
+from packaging.version import Version
+from pydantic import BaseModel
+
+from demisto_sdk.commands.common.constants import (
+    DOCKERFILES_INFO_REPO,
+)
+from demisto_sdk.commands.common.git_content_config import GitContentConfig
+from demisto_sdk.commands.common.logger import logger
+from demisto_sdk.commands.common.singleton import PydanticSingleton
+from demisto_sdk.commands.common.tools import get_remote_file_from_api
+
+DOCKER_IMAGES_METADATA_NAME = "docker_images_metadata.json"
+
+# regex to extract docker-images that are specific to content / dockerfiles
+DOCKER_IMAGE_REGEX = r"^demisto/([^\s:]+):(\d+(\.\d+)*)$"
+
+
+class DockerImageTagMetadata(BaseModel):
+    python_version: Optional[str]
+
+
+class DockerImagesMetadata(PydanticSingleton, BaseModel):
+    docker_images: Dict[str, Dict[str, DockerImageTagMetadata]]
+
+    class MetadataValues(str, Enum):
+        PYTHON_VERSION = "python_version"
+
+    @classmethod
+    def get_instance_from(cls, *args, **kwargs):
+        return cls.__from_github(*args, **kwargs)
+
+    @classmethod
+    def __from_github(
+        cls, file_name: str = DOCKER_IMAGES_METADATA_NAME, tag: str = "master"
+    ):
+        """
+        Get the docker_images_metadata.json from the dockerfiles-info repo and load it to a pydnatic object.
+
+        Args:
+            file_name (str): the file path for the docker_images_metadata.json
+            tag (str): branch/commit to get a specific docker_images_metadata.json
+
+        """
+        logger.debug(
+            f"Trying to load the {DOCKER_IMAGES_METADATA_NAME} from {DOCKERFILES_INFO_REPO}"
+        )
+        dockerfiles_metadata = get_remote_file_from_api(
+            file_name,
+            tag=tag,
+            git_content_config=GitContentConfig(repo_name=DOCKERFILES_INFO_REPO),
+            encoding="utf-8-sig",
+        )
+        if not dockerfiles_metadata:
+            logger.error(
+                f"Could not retrieve the {DOCKER_IMAGES_METADATA_NAME} from {DOCKERFILES_INFO_REPO} repo"
+            )
+            dockerfiles_metadata = {"docker_images": {}}
+
+        return cls.parse_obj(dockerfiles_metadata)
+
+    def __get_metadata_value(
+        self, docker_image: str, docker_metadata_key: str
+    ) -> Optional[str]:
+        """
+        Get the content of the requested key in the metadata
+
+        Args:
+            docker_image (str): the docker image from the script/integration yml
+            docker_metadata_key (str): the key in the DockerImageTagMetadata class
+        """
+        try:
+            # if we were not able to load the file
+            if not self.docker_images:
+                return None
+            match = re.match(DOCKER_IMAGE_REGEX, docker_image)
+            docker_name, tag = match.group(1), match.group(2)  # type: ignore[union-attr]
+            docker_image_metadata = (self.docker_images.get(docker_name) or {}).get(tag)
+            return getattr(docker_image_metadata, docker_metadata_key)
+        except (AttributeError, ValueError, TypeError) as err:
+            logger.debug(
+                f"Could not get {docker_metadata_key} for {docker_image=} because {err=} occurred"
+            )
+            return None
+
+    def python_version(self, docker_image: str) -> Optional[Version]:
+        """
+        Get the python version of a docker image.
+        """
+        if python_version := self.__get_metadata_value(
+            docker_image, self.MetadataValues.PYTHON_VERSION
+        ):
+            logger.debug(
+                f"successfully got {python_version=} for {docker_image=} from {DOCKER_IMAGES_METADATA_NAME}"
+            )
+            return Version(python_version)
+
+        return None

demisto_sdk/commands/common/git_content_config.py

@@ -13,6 +13,7 @@
 # dirs
 import requests
 
+from demisto_sdk.commands.common.constants import DOCKERFILES_INFO_REPO
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
 
@@ -67,6 +68,7 @@ class GitContentConfig:
         (GITHUB_USER_CONTENT, OFFICIAL_CONTENT_REPO_NAME),
         (GITHUB, OFFICIAL_CONTENT_REPO_NAME),
         (CODE_PAN_RUN, OFFICIAL_CONTENT_PROJECT_ID),
+        (GITHUB_USER_CONTENT, DOCKERFILES_INFO_REPO),
     }
 
     CREDENTIALS = GitCredentials()

demisto_sdk/commands/common/native_image.py

@@ -4,11 +4,8 @@
 
 from demisto_sdk.commands.common.constants import NATIVE_IMAGE_FILE_NAME
 from demisto_sdk.commands.common.logger import logger
-from demisto_sdk.commands.common.singleton import Singleton
-from demisto_sdk.commands.common.tools import (
-    extract_docker_image_from_text,
-    get_dict_from_file,
-)
+from demisto_sdk.commands.common.singleton import PydanticSingleton
+from demisto_sdk.commands.common.tools import extract_docker_image_from_text
 
 
 class NativeImage(BaseModel):
@@ -26,19 +23,23 @@ def _extract_native_image_version_for_server(native_image: str) -> str:
     return native_image.replace("native:", "")
 
 
-class NativeImageConfig(Singleton, BaseModel):
+class NativeImageConfig(PydanticSingleton, BaseModel):
     native_images: Dict[str, NativeImage]
     ignored_content_items: List[IgnoredContentItem]
     flags_versions_mapping: Dict[str, str] = {}
     docker_images_to_native_images_mapping: Dict[str, List] = {}
 
-    def __init__(
-        self, native_image_config_file_path: str = f"Tests/{NATIVE_IMAGE_FILE_NAME}"
+    @classmethod
+    def get_instance_from(cls, *args, **kwargs):
+        return cls.from_path(*args, **kwargs)
+
+    @classmethod
+    def from_path(
+        cls, native_image_config_file_path: str = f"Tests/{NATIVE_IMAGE_FILE_NAME}"
     ):
-        super().__init__(**self.load(native_image_config_file_path))
-        self.docker_images_to_native_images_mapping = (
-            self.__docker_images_to_native_images_support()
-        )
+        native_image_config = cls.parse_file(native_image_config_file_path)
+        native_image_config.__docker_images_to_native_images_support()
+        return native_image_config
 
     def __docker_images_to_native_images_support(self):
         """
@@ -53,29 +54,18 @@ def __docker_images_to_native_images_support(self):
            chromium docker image is supported in both 8.1.0, 8.2.0 native images
            while tesseract is only supported in 8.1.0
         """
-        docker_images_to_native_images_mapping: Dict = {}
-
         for native_image_name, native_image_obj in self.native_images.items():
             for supported_docker_image in native_image_obj.supported_docker_images:
-                if supported_docker_image not in docker_images_to_native_images_mapping:
-                    docker_images_to_native_images_mapping[supported_docker_image] = []
-                docker_images_to_native_images_mapping[supported_docker_image].append(
-                    native_image_name
-                )
-
-        return docker_images_to_native_images_mapping
-
-    @staticmethod
-    def load(
-        native_image_config_file_path: str = f"Tests/{NATIVE_IMAGE_FILE_NAME}",
-    ) -> Dict:
-        """
-        Load the native image configuration file
-        """
-        native_image_config_content, _ = get_dict_from_file(
-            native_image_config_file_path
-        )
-        return native_image_config_content
+                if (
+                    supported_docker_image
+                    not in self.docker_images_to_native_images_mapping
+                ):
+                    self.docker_images_to_native_images_mapping[
+                        supported_docker_image
+                    ] = []
+                self.docker_images_to_native_images_mapping[
+                    supported_docker_image
+                ].append(native_image_name)
 
     def get_native_image_reference(self, native_image) -> Optional[str]:
         """
@@ -182,12 +172,3 @@ def get_supported_native_image_versions(
                 )
             return native_images
         return []
-
-
-def file_to_native_image_config(
-    native_image_config_file_path: str = f"Tests/{NATIVE_IMAGE_FILE_NAME}",
-) -> NativeImageConfig:
-    """
-    Converts the native image file to NativeImageConfig object.
-    """
-    return NativeImageConfig(native_image_config_file_path)

demisto_sdk/commands/common/singleton.py

@@ -1,7 +1,20 @@
-class Singleton:
+from abc import abstractmethod
+
+
+class PydanticSingleton:
     _instance = None
 
-    def __new__(cls, *args, **kwargs):
-        if not isinstance(cls._instance, cls):
-            cls._instance = super().__new__(cls)
+    @classmethod
+    def get_instance(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = cls.get_instance_from(*args, **kwargs)
         return cls._instance
+
+    @classmethod
+    @abstractmethod
+    def get_instance_from(cls, *args, **kwargs):
+        """
+        Pydantic objects should be initialized with class methods and not from the class constructor.
+        Each Singleton that is based on pydantic should implement this abstract method.
+        """
+        pass