Skip to content

Commit

Permalink
Docker image improvements (#3943)
Browse files Browse the repository at this point in the history
* [graph objects] - add supported native images property

* update native image obj

* enhance docker-image class

* add encoding to structured file

* fix cache issues

* revert docker_images_metadata

* fix tests and revert linter and native-image

* revert file

* comments

* fix test_DockerImageExistValidator_is_valid test

* revert config
  • Loading branch information
GuyAfik committed Jan 14, 2024
1 parent 4cee61a commit 173da66
Show file tree
Hide file tree
Showing 13 changed files with 288 additions and 114 deletions.
154 changes: 154 additions & 0 deletions demisto_sdk/commands/common/docker/docker_image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import re
from datetime import datetime
from typing import Optional

from packaging.version import Version

from demisto_sdk.commands.common.constants import (
NATIVE_IMAGE_DOCKER_NAME,
)
from demisto_sdk.commands.common.docker.dockerhub_client import DockerHubClient
from demisto_sdk.commands.common.logger import logger


class DockerImage:

DOCKER_IMAGE_REGX = (
r"^([^/]+)/(.*?)(?::(.*))?$" # regex to extract parts of a docker-image
)
DEMISTO_PYTHON_BASE_IMAGE_REGEX = re.compile(
r"[\d\w]+/python3?:(?P<python_version>[23]\.\d+(\.\d+)?)" # regex to extract python version for image name
)

def __init__(
self,
dockerhub_client: DockerHubClient,
repository: str = "",
image_name: str = "",
tag: str = "",
):
self._dockerhub_client = dockerhub_client
self.repository = repository # the repository e.g.: demisto
self.image_name = image_name # the image name e.g.: python3, pan-os-python
self.tag = tag # the tag

@classmethod
def parse(
cls,
docker_image: str,
dockerhub_client: Optional[DockerHubClient] = None,
raise_if_not_valid: bool = False,
) -> "DockerImage":
"""
Parses a docker-image into repository, image name and its tag
Args:
docker_image: the docker image to parse
dockerhub_client: client to interact with dockerhub client
raise_if_not_valid: raise ValueError if the docker-image structure is not valid
"""
_dockerhub_client = dockerhub_client or DockerHubClient()
pattern = re.compile(cls.DOCKER_IMAGE_REGX)
if matches := pattern.match(docker_image):
docker_image_object = cls(
_dockerhub_client,
repository=matches.group(1),
image_name=matches.group(2),
tag=matches.group(3),
)
else:
docker_image_object = cls(_dockerhub_client)

if raise_if_not_valid and not docker_image_object.is_valid:
raise ValueError(
f"Docker image {docker_image} is not valid, should be in the form of repository/image-name:tag"
)

return docker_image_object

def __str__(self):
return f"{self.repository}/{self.image_name}:{self.tag}"

@property
def name(self):
"""
Returns the repository + image name. .e.g: demisto/python3, demisto/pan-os-python
"""
return f"{self.repository}/{self.image_name}"

@property
def is_valid(self) -> bool:
"""
Validates that the structure of the docker-image is valid.
Returns:
bool: True if the structure is valid, False if not.
"""
if not self.repository or not self.image_name or not self.tag:
logger.warning(
f"Docker image {self} is not valid, should be in the form of repository/image-name:tag"
)
return False
return True

@property
def is_tag_latest(self) -> bool:
return self.tag == "latest"

@property
def is_demisto_repository(self) -> bool:
return self.repository == "demisto"

@property
def is_native_image(self) -> bool:
return self.name == NATIVE_IMAGE_DOCKER_NAME

@property
def creation_date(self) -> datetime:
return self._dockerhub_client.get_docker_image_tag_creation_date(
self.name, tag=self.tag
)

@property
def python_version(self) -> Optional[Version]:
if self.is_valid:
if "pwsh" == self.image_name or "powershell" == self.image_name:
logger.debug(
f"The {self} is a powershell image, does not have python version"
)
return None

if match := self.DEMISTO_PYTHON_BASE_IMAGE_REGEX.match(str(self)):
return Version(match.group("python_version"))

logger.debug(f"Could not get python version for image {self} from regex")
image_env = self._dockerhub_client.get_image_env(self.name, tag=self.tag)

if python_version := next(
(
var.split("=")[1]
for var in image_env
if var.startswith("PYTHON_VERSION=")
),
None,
):
return Version(python_version)

logger.error(f"Could not find python-version of docker-image {self}")
return None

logger.debug(
f"docker-image {self} is not valid, could not get its python-version"
)
return None

@property
def is_image_exist(self) -> bool:
"""
Returns True if the docker-image exist in dockerhub
"""
return self._dockerhub_client.is_docker_image_exist(self.name, tag=self.tag)

@property
def latest_tag(self) -> Version:
return self._dockerhub_client.get_latest_docker_image_tag(self.name)
40 changes: 28 additions & 12 deletions demisto_sdk/commands/common/docker/dockerhub_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@ def __str__(self):
return f"Error - {self.message} - Exception - {self.exception}"


@lru_cache
class DockerHubClient:

DEFAULT_REGISTRY = "https://registry-1.docker.io/v2"
DOCKER_HUB_API_BASE_URL = "https://hub.docker.com/v2"
TOKEN_URL = "https://auth.docker.io/token"

def __init__(
self,
Expand Down Expand Up @@ -87,12 +89,14 @@ def get_token(
if _expiration_time.replace(tzinfo=None) < now:
return token_metadata.get("token")

params = {
"service": "registry.docker.io",
"scope": f"repository:{repo}:{scope}",
}

response = self._session.get(
"https://auth.docker.io/token",
params={
"service": "registry.docker.io",
"scope": f"repository:{repo}:{scope}",
},
self.TOKEN_URL,
params=params,
auth=self.auth,
)
try:
Expand All @@ -105,11 +109,8 @@ def get_token(
logger.debug("Trying to get dockerhub token without username:password")
try:
response = self._session.get(
"https://auth.docker.io/token",
params={
"service": "registry.docker.io",
"scope": f"repository:{repo}:{scope}",
},
self.TOKEN_URL,
params=params,
)
response.raise_for_status()
except RequestException as error:
Expand Down Expand Up @@ -382,7 +383,9 @@ def is_docker_image_exist(self, docker_image: str, tag: str) -> bool:
return True
except DockerHubRequestException as error:
if error.exception.response.status_code == requests.codes.not_found:
logger.debug(f"docker-image {docker_image}:{tag} does not exist")
logger.debug(
f"docker-image {docker_image}:{tag} does not exist in dockerhub"
)
return False
logger.debug(
f"Error when trying to fetch {docker_image}:{tag} metadata: {error}"
Expand Down Expand Up @@ -428,6 +431,19 @@ def get_latest_docker_image_tag(self, docker_image: str) -> Version:

return max(version_tags)

def get_latest_docker_image(self, docker_image: str) -> str:
"""
Returns the latest docker-image including the tag.
Args:
docker_image: The docker-image name, e.g: demisto/pan-os-python
Returns:
str: the full docker-image included the tag, for example demisto/pan-os-python:2.0.0
"""
return f"{docker_image}/{self.get_latest_docker_image_tag(docker_image)}"

def get_repository_images(
self, repo: str = DEFAULT_REPOSITORY
) -> List[Dict[str, Any]]:
Expand All @@ -441,7 +457,7 @@ def get_repository_images(
return self.do_docker_hub_get_request(f"/repositories/{repo}")
except RequestException as error:
raise DockerHubRequestException(
f"Failed to retreive images of repository {repo}", exception=error
f"Failed to retrieve images of repository {repo}", exception=error
)

def get_repository_images_names(self, repo: str = DEFAULT_REPOSITORY) -> List[str]:
Expand Down
71 changes: 71 additions & 0 deletions demisto_sdk/commands/common/docker/tests/docker_image_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import pytest

from demisto_sdk.commands.common.docker.docker_image import DockerImage


@pytest.mark.parametrize(
"docker_image, expected_repo, expected_image_name, expected_tag",
[
(
"demisto/pan-os-python:1.0.0.83880",
"demisto",
"pan-os-python",
"1.0.0.83880",
),
("demisto/py3-tools:0.0.1.25751", "demisto", "py3-tools", "0.0.1.25751"),
("demisto/boto3py3:1.0.0.52713", "demisto", "boto3py3", "1.0.0.52713"),
(
"demisto/googleapi-python3:1.0.0.12698",
"demisto",
"googleapi-python3",
"1.0.0.12698",
),
("demisto/ml:1.0.0.84027", "demisto", "ml", "1.0.0.84027"),
("demisto/fetch-data:1.0.0.22177", "demisto", "fetch-data", "1.0.0.22177"),
("demisto/python3:3.10.13.78960", "demisto", "python3", "3.10.13.78960"),
("custom-repo/test-image:7.8.9", "custom-repo", "test-image", "7.8.9"),
("some-repo/test-image-2:1.1.1.1", "some-repo", "test-image-2", "1.1.1.1"),
],
)
def test_docker_image_parse_valid_docker_images(
docker_image: str, expected_repo: str, expected_image_name: str, expected_tag: str
):
"""
Given:
- valid docker images
When:
- trying to parse docker-images parts (repository, image-name and tag)
Then:
- make sure images are parsed correctly
"""
docker_image_object = DockerImage.parse(docker_image)
assert docker_image_object.repository == expected_repo
assert docker_image_object.image_name == expected_image_name
assert docker_image_object.tag == expected_tag


@pytest.mark.parametrize(
"docker_image",
[
"demisto/pan-os-python",
"demisto",
"1.0.0.52713",
"test-image:7.8.9",
"some-repo",
],
)
def test_docker_image_parse_invalid_docker_images(docker_image: str):
"""
Given:
- invalid docker images
When:
- trying to parse docker-images parts (repository, image-name and tag)
Then:
- make sure ValueError is raised
"""
with pytest.raises(ValueError):
DockerImage.parse(docker_image, raise_if_not_valid=True)
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@

@pytest.fixture()
def dockerhub_client() -> DockerHubClient:
return DockerHubClient(username="test", password="test")
dockerhub_client = DockerHubClient(username="test", password="test")
dockerhub_client.do_registry_get_request.cache_clear()
dockerhub_client.do_docker_hub_get_request.cache_clear()
return dockerhub_client


def test_get_token_with_new_token(requests_mock, dockerhub_client: DockerHubClient):
Expand Down
1 change: 0 additions & 1 deletion demisto_sdk/commands/content_graph/objects/base_script.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ def prepare_for_upload(
data = super().prepare_for_upload(current_marketplace, **kwargs)

if supported_native_images := self.get_supported_native_images(
marketplace=current_marketplace,
ignore_native_image=kwargs.get("ignore_native_image") or False,
):
logger.debug(
Expand Down
1 change: 0 additions & 1 deletion demisto_sdk/commands/content_graph/objects/integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ def prepare_for_upload(
data = super().prepare_for_upload(current_marketplace, **kwargs)

if supported_native_images := self.get_supported_native_images(
marketplace=current_marketplace,
ignore_native_image=kwargs.get("ignore_native_image") or False,
):
logger.debug(
Expand Down
Loading

0 comments on commit 173da66

Please sign in to comment.