[files] - Refactor read_from_github method (#4192)

* Refactor read_from_github method * fix ut * black * fix tests * change to startswith
demisto · Apr 7, 2024 · 16ed5d7 · 16ed5d7
1 parent 2055959
commit 16ed5d7
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 17 deletions.
diff --git a/demisto_sdk/commands/common/constants.py b/demisto_sdk/commands/common/constants.py
@@ -2054,6 +2054,8 @@ class ParameterType(Enum):
 FORMATTING_SCRIPT = "indicator-format"
 
 DOCKERFILES_INFO_REPO = "demisto/dockerfiles-info"
+DOCKERFILES_REPO = "demisto/dockerfiles"
+CONTENT_REPO = "demisto/content"
 
 TEST_COVERAGE_DEFAULT_URL = f"https://storage.googleapis.com/{DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV}/code-coverage-reports/coverage-min.json"
 

diff --git a/demisto_sdk/commands/common/docker_images_metadata.py b/demisto_sdk/commands/common/docker_images_metadata.py
@@ -12,7 +12,6 @@
 from demisto_sdk.commands.common.docker.docker_image import DockerImage
 from demisto_sdk.commands.common.files.errors import FileReadError
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.singleton import PydanticSingleton
 from demisto_sdk.commands.common.tools import NoInternetConnectionException
@@ -54,8 +53,8 @@ def __from_github(
         try:
             dockerfiles_metadata = JsonFile.read_from_github_api(
                 file_name,
+                repo=DOCKERFILES_INFO_REPO,
                 tag=tag,
-                git_content_config=GitContentConfig(repo_name=DOCKERFILES_INFO_REPO),
                 verify_ssl=False,
                 encoding="utf-8-sig",
             )

diff --git a/demisto_sdk/commands/common/files/file.py b/demisto_sdk/commands/common/files/file.py
@@ -1,4 +1,5 @@
 import inspect
+import os
 import shutil
 import urllib.parse
 from abc import ABC, abstractmethod
@@ -13,6 +14,7 @@
 from requests.exceptions import ConnectionError, RequestException, Timeout
 
 from demisto_sdk.commands.common.constants import (
+    CONTENT_REPO,
     DEMISTO_GIT_PRIMARY_BRANCH,
     DEMISTO_GIT_UPSTREAM,
     urljoin,
@@ -26,7 +28,10 @@
     MemoryFileReadError,
     UnknownFileError,
 )
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
+from demisto_sdk.commands.common.git_content_config import (
+    GitContentConfig,
+    GitCredentials,
+)
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.handlers.xsoar_handler import XSOAR_Handler
 from demisto_sdk.commands.common.logger import logger
@@ -317,7 +322,7 @@ def __read_git_file(self, tag: str, from_remote: bool = True) -> Any:
     def read_from_github_api(
         cls,
         path: str,
-        git_content_config: Optional[GitContentConfig] = None,
+        repo: str = CONTENT_REPO,
         encoding: Optional[str] = None,
         tag: str = DEMISTO_GIT_PRIMARY_BRANCH,
         handler: Optional[XSOAR_Handler] = None,
@@ -328,8 +333,8 @@ def read_from_github_api(
         Reads a file from Github api.
 
         Args:
-            path: the path to the file in github
-            git_content_config: git content config object
+            path: the path to the file in Github from the repo's root
+            repo: the repository name, e.g.: demisto/content
             encoding: any custom encoding if needed
             tag: the branch/sha to take the file from within Github
             handler: whether a custom handler is required, if not takes the default.
@@ -339,17 +344,16 @@ def read_from_github_api(
         Returns:
             Any: the file content in the desired format
         """
-        if not git_content_config:
-            git_content_config = GitContentConfig()
-
-        git_path_url = urljoin(git_content_config.base_api, tag, path)
-        github_token = git_content_config.CREDENTIALS.github_token
+        if not path.startswith("/"):
+            path = f"/{path}"
+        url = f"https://raw.githubusercontent.com/{repo}/{tag}{path}"
+        github_token = os.getenv(GitCredentials.ENV_GITHUB_TOKEN_NAME, "")
 
         timeout = 10
 
         try:
             return cls.read_from_http_request(
-                git_path_url,
+                url,
                 headers=frozenset(
                     {
                         "Authorization": f"Bearer {github_token}"
@@ -366,17 +370,17 @@ def read_from_github_api(
             )
         except FileReadError as e:
             logger.warning(
-                f"Received error {e} when trying to retrieve {git_path_url} content from Github, retrying"
+                f"Received error {e} when trying to retrieve {url} content from Github, retrying"
             )
             try:
                 return cls.read_from_http_request(
-                    git_path_url,
+                    url,
                     params=frozenset({"token": github_token}.items()),
                     timeout=timeout,
                 )
             except FileReadError:
                 logger.error(
-                    f"Could not retrieve the content of {git_path_url} file from Github"
+                    f"Could not retrieve the content of {url} file from Github"
                 )
                 raise
 

diff --git a/...to_sdk/commands/validate/validators/DO_validators/DO105_docker_image_is_not_deprecated.py b/...to_sdk/commands/validate/validators/DO_validators/DO105_docker_image_is_not_deprecated.py
@@ -2,8 +2,8 @@
 
 from typing import ClassVar, Dict, Iterable, List, Union
 
+from demisto_sdk.commands.common.constants import DOCKERFILES_REPO
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.content_graph.objects.integration import Integration
 from demisto_sdk.commands.content_graph.objects.script import Script
 from demisto_sdk.commands.validate.validators.base_validator import (
@@ -30,7 +30,7 @@ def is_valid(self, content_items: Iterable[ContentTypes]) -> List[ValidationResu
         if not self.deprecated_dockers_to_reasons:
             deprecated_dockers = JsonFile.read_from_github_api(
                 path="/docker/deprecated_images.json",
-                git_content_config=GitContentConfig(repo_name="demisto/dockerfiles"),
+                repo=DOCKERFILES_REPO,
                 verify_ssl=False,
             )
             DockerImageIsNotDeprecatedValidator.deprecated_dockers_to_reasons = {