Merge e0017f2 into 6404079

demisto_sdk/commands/common/constants.py

@@ -11,7 +11,10 @@
 
 
 PROJECT_DATA_DIR = Path.home() / ".demisto-sdk"
+CACHE_DIR = PROJECT_DATA_DIR / "cache"
 LOGS_DIR = PROJECT_DATA_DIR / "logs"
+NEO4J_DIR = PROJECT_DATA_DIR / "neo4j"
+
 LOG_FILE_NAME = "demisto_sdk_debug.log"
 
 # --- Environment Variables ---
@@ -43,6 +46,12 @@
 DEMISTO_SDK_LOG_FILE_SIZE = "DEMISTO_SDK_LOG_FILE_SIZE"
 DEMISTO_SDK_LOG_FILE_COUNT = "DEMISTO_SDK_LOG_FILE_COUNT"
 DEMISTO_SDK_LOG_NO_COLORS = "DEMISTO_SDK_LOG_NO_COLORS"
+
+# Neo4j
+DEMISTO_SDK_NEO4J_DATABASE_HTTP = "DEMISTO_SDK_NEO4J_DATABASE_HTTP"
+DEMISTO_SDK_NEO4J_DATABASE_URL = "DEMISTO_SDK_NEO4J_DATABASE_URL"
+DEMISTO_SDK_NEO4J_USERNAME = "DEMISTO_SDK_NEO4J_USERNAME"
+DEMISTO_SDK_NEO4J_PASSWORD = "DEMISTO_SDK_NEO4J_PASSWORD"
 # --- Environment Variables ---
 
 

demisto_sdk/commands/common/singleton.py

@@ -1,4 +1,15 @@
 from abc import abstractmethod
+from typing import Any, Dict
+
+
+class SingletonMeta(type):
+    _instances: Dict[type, Any] = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+
+        return cls._instances[cls]
 
 
 class PydanticSingleton:

demisto_sdk/commands/content_graph/common.py

@@ -6,7 +6,13 @@
 
 from neo4j import graph
 
-from demisto_sdk.commands.common.constants import PACKS_FOLDER
+from demisto_sdk.commands.common.constants import (
+    DEMISTO_SDK_NEO4J_DATABASE_HTTP,
+    DEMISTO_SDK_NEO4J_DATABASE_URL,
+    DEMISTO_SDK_NEO4J_PASSWORD,
+    DEMISTO_SDK_NEO4J_USERNAME,
+    PACKS_FOLDER,
+)
 from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.common.tools import (
     get_dict_from_file,
@@ -17,15 +23,11 @@
 NEO4J_ADMIN_DOCKER = ""
 
 NEO4J_DATABASE_HTTP = os.getenv(
-    "DEMISTO_SDK_NEO4J_DATABASE_HTTP", "http://127.0.0.1:7474"
-)
-NEO4J_DATABASE_URL = os.getenv(
-    "DEMISTO_SDK_NEO4J_DATABASE_URL", "neo4j://127.0.0.1:7687"
+    DEMISTO_SDK_NEO4J_DATABASE_HTTP, "http://127.0.0.1:7474"
 )
-NEO4J_USERNAME = os.getenv("DEMISTO_SDK_NEO4J_USERNAME", "neo4j")
-NEO4J_PASSWORD = os.getenv("DEMISTO_SDK_NEO4J_PASSWORD", "contentgraph")
-
-NEO4J_FOLDER = "neo4j-data"
+NEO4J_DATABASE_URL = os.getenv(DEMISTO_SDK_NEO4J_DATABASE_URL, "neo4j://127.0.0.1:7687")
+NEO4J_USERNAME = os.getenv(DEMISTO_SDK_NEO4J_USERNAME, "neo4j")
+NEO4J_PASSWORD = os.getenv(DEMISTO_SDK_NEO4J_PASSWORD, "contentgraph")
 
 PACK_METADATA_FILENAME = "pack_metadata.json"
 PACK_CONTRIBUTORS_FILENAME = "CONTRIBUTORS.json"

demisto_sdk/commands/content_graph/interface/neo4j/import_utils.py

@@ -4,12 +4,13 @@
 from zipfile import ZipFile
 
 from demisto_sdk.commands.common.logger import logger
+from demisto_sdk.commands.common.singleton import SingletonMeta
 from demisto_sdk.commands.content_graph.neo4j_service import get_neo4j_import_path
 
 GRAPHML_FILE_SUFFIX = ".graphml"
 
 
-class Neo4jImportHandler:
+class Neo4jImportHandler(metaclass=SingletonMeta):
     def __init__(self) -> None:
         """This class handles the import of data to neo4j.
         import_path is the path to the directory where the data is located.

demisto_sdk/commands/content_graph/interface/neo4j/neo4j_graph.py

@@ -108,12 +108,10 @@ class NoModelException(Exception):
 
 
 class Neo4jContentGraphInterface(ContentGraphInterface):
-    # this is used to save cache of packs and integrations which queried
-    _import_handler = Neo4jImportHandler()
-
     def __init__(
         self,
     ) -> None:
+        self._import_handler = Neo4jImportHandler()
         self._id_to_obj: Dict[str, BaseNode] = {}
 
         if not self.is_alive():

demisto_sdk/commands/content_graph/neo4j_service.py

@@ -6,16 +6,14 @@
 import docker
 import requests
 
-from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
+from demisto_sdk.commands.common.constants import NEO4J_DIR
 from demisto_sdk.commands.common.docker_helper import init_global_docker_client
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.content_graph.common import (
     NEO4J_DATABASE_HTTP,
-    NEO4J_FOLDER,
     NEO4J_PASSWORD,
 )
 
-REPO_PATH = CONTENT_PATH.absolute()
 NEO4J_VERSION = "5.13.0"
 
 NEO4J_SERVICE_IMAGE = f"neo4j:{NEO4J_VERSION}"
@@ -70,7 +68,7 @@ def _download_apoc():
         return
     download_url = apocs[0].get("downloadUrl")
     sha1 = apocs[0].get("sha1")
-    plugins_folder = REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER
+    plugins_folder = NEO4J_DIR / NEO4J_PLUGINS_FOLDER
     plugins_folder.mkdir(parents=True, exist_ok=True)
 
     if _is_apoc_available(plugins_folder, sha1):
@@ -88,19 +86,19 @@ def _docker_start():
     logger.debug("Starting neo4j service")
     docker_client = init_global_docker_client()
     _stop_neo4j_service_docker(docker_client)
-    (REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER).mkdir(parents=True, exist_ok=True)
-    (REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER).mkdir(parents=True, exist_ok=True)
-    (REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER).mkdir(parents=True, exist_ok=True)
+    (NEO4J_DIR / NEO4J_DATA_FOLDER).mkdir(parents=True, exist_ok=True)
+    (NEO4J_DIR / NEO4J_IMPORT_FOLDER).mkdir(parents=True, exist_ok=True)
+    (NEO4J_DIR / NEO4J_PLUGINS_FOLDER).mkdir(parents=True, exist_ok=True)
     # suppress logs in docker init to avoid spamming
 
     docker_client.containers.run(
         image=NEO4J_SERVICE_IMAGE,
         name="neo4j-content",
         ports={"7474/tcp": 7474, "7687/tcp": 7687, "7473/tcp": 7473},
         volumes=[
-            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER}:/{NEO4J_DATA_FOLDER}",
-            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER}:{LOCAL_NEO4J_PATH / NEO4J_IMPORT_FOLDER}",
-            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER}:/{NEO4J_PLUGINS_FOLDER}",
+            f"{NEO4J_DIR / NEO4J_DATA_FOLDER}:/{NEO4J_DATA_FOLDER}",
+            f"{NEO4J_DIR / NEO4J_IMPORT_FOLDER}:{LOCAL_NEO4J_PATH / NEO4J_IMPORT_FOLDER}",
+            f"{NEO4J_DIR / NEO4J_PLUGINS_FOLDER}:/{NEO4J_PLUGINS_FOLDER}",
         ],
         detach=True,
         environment={
@@ -137,7 +135,7 @@ def start():
         logger.debug("Neo4j is running locally. Start manually")
         return
 
-    Path.mkdir(REPO_PATH / NEO4J_FOLDER, exist_ok=True, parents=True)
+    NEO4J_DIR.mkdir(exist_ok=True, parents=True)
     # we download apoc only if we are running on docker
     # if the user is running locally he needs to setup apoc manually
     _download_apoc()
@@ -147,7 +145,7 @@ def start():
         logger.debug(
             f"Could not start neo4j container, delete data folder and trying again. {e}"
         )
-        shutil.rmtree(REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER, ignore_errors=True)
+        shutil.rmtree(NEO4J_DIR / NEO4J_DATA_FOLDER, ignore_errors=True)
         _docker_start()
 
 
@@ -161,10 +159,8 @@ def stop(force: bool = False, clean: bool = False):
     docker_client = init_global_docker_client()
     _stop_neo4j_service_docker(docker_client)
     if clean:
-        shutil.rmtree(REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER, ignore_errors=True)
-        shutil.rmtree(
-            REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER, ignore_errors=True
-        )
+        shutil.rmtree(NEO4J_DIR / NEO4J_DATA_FOLDER, ignore_errors=True)
+        shutil.rmtree(NEO4J_DIR / NEO4J_PLUGINS_FOLDER, ignore_errors=True)
 
 
 def is_alive():
@@ -181,4 +177,4 @@ def is_running_on_docker():
 def get_neo4j_import_path() -> Path:
     if not is_running_on_docker():
         return LOCAL_NEO4J_PATH / NEO4J_IMPORT_FOLDER
-    return REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER
+    return NEO4J_DIR / NEO4J_IMPORT_FOLDER

demisto_sdk/commands/content_graph/tests/format_with_graph_test.py

@@ -28,18 +28,21 @@
 from demisto_sdk.commands.content_graph.tests.update_content_graph_test import (
     _get_pack_by_id,
 )
+from TestSuite.repo import Repo
 from TestSuite.test_tools import ChangeCWD, str_in_call_args_list
 
 FORMAT_CMD = "format"
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker, repo):
+def setup_method(mocker, tmp_path_factory, repo: Repo):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = Path(repo.path)
-    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
     mocker.patch(
         "demisto_sdk.commands.common.docker_images_metadata.get_remote_file_from_api",

demisto_sdk/commands/content_graph/tests/generate_docs_script_test.py

@@ -23,6 +23,7 @@
 )
 from demisto_sdk.commands.generate_docs.generate_script_doc import generate_script_doc
 from demisto_sdk.commands.generate_docs.tests.generate_docs_test import handle_example
+from TestSuite.repo import Repo
 
 INPUT_SCRIPT = "SampleScript"
 USES_SCRIPT = "UsesScript"
@@ -32,12 +33,14 @@
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker, repo):
+def setup_method(mocker, tmp_path_factory, repo: Repo):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = Path(repo.path)
-    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
 
 

demisto_sdk/commands/content_graph/tests/graph_validator_test.py

@@ -40,12 +40,14 @@
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker):
+def setup_method(mocker, tmp_path_factory):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = GIT_PATH
-    mocker.patch.object(neo4j_service, "REPO_PATH", GIT_PATH)
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", GIT_PATH)
     mocker.patch.object(ContentGraphInterface, "export_graph", return_value=None)
     mocker.patch(

demisto_sdk/commands/content_graph/tests/pack_metadata_graph_test.py

@@ -24,13 +24,15 @@
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker, repo: Repo):
+def setup_method(mocker, tmp_path_factory, repo: Repo):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = Path(repo.path)
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
-    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
 
 
 @pytest.fixture

demisto_sdk/commands/content_graph/tests/prepare_content_graph_test.py

@@ -32,13 +32,15 @@
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker, repo: Repo):
+def setup_method(mocker, tmp_path_factory, repo: Repo):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = Path(repo.path)
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
-    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
     mocker.patch(
         "demisto_sdk.commands.common.docker_images_metadata.get_remote_file_from_api",
         return_value={

demisto_sdk/commands/content_graph/tests/update_content_graph_test.py

@@ -40,12 +40,14 @@
 
 
 @pytest.fixture(autouse=True)
-def setup_method(mocker):
+def setup_method(mocker, tmp_path_factory):
     """Auto-used fixture for setup before every test run"""
     import demisto_sdk.commands.content_graph.objects.base_content as bc
 
     bc.CONTENT_PATH = GIT_PATH
-    mocker.patch.object(neo4j_service, "REPO_PATH", GIT_PATH)
+    mocker.patch.object(
+        neo4j_service, "NEO4J_DIR", new=tmp_path_factory.mktemp("neo4j")
+    )
     mocker.patch.object(ContentGraphInterface, "repo_path", GIT_PATH)
     mocker.patch(
         "demisto_sdk.commands.common.docker_images_metadata.get_remote_file_from_api",

demisto_sdk/commands/pre_commit/pre_commit_context.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Tuple
 
+from demisto_sdk.commands.common.constants import CACHE_DIR
 from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.tools import (
@@ -23,9 +24,9 @@
 
 PRECOMMIT_TEMPLATE_NAME = ".pre-commit-config_template.yaml"
 PRECOMMIT_TEMPLATE_PATH = CONTENT_PATH / PRECOMMIT_TEMPLATE_NAME
-PRECOMMIT_FOLDER = CONTENT_PATH / ".pre-commit"
+PRECOMMIT_FOLDER = CACHE_DIR / "pre-commit"
 PRECOMMIT_CONFIG = PRECOMMIT_FOLDER / "config"
-PRECOMMIT_CONFIG_MAIN_PATH = PRECOMMIT_CONFIG / ".pre-commit-config-main.yaml"
+PRECOMMIT_CONFIG_MAIN_PATH = PRECOMMIT_CONFIG / "pre-commit-config-main.yaml"
 PRECOMMIT_DOCKER_CONFIGS = PRECOMMIT_CONFIG / "docker"
 
 
@@ -50,7 +51,7 @@ def __post_init__(self):
         We initialize the hooks and all_files for later use.
         """
         shutil.rmtree(PRECOMMIT_FOLDER, ignore_errors=True)
-        PRECOMMIT_FOLDER.mkdir()
+        PRECOMMIT_FOLDER.mkdir(parents=True)
         PRECOMMIT_CONFIG.mkdir()
         PRECOMMIT_DOCKER_CONFIGS.mkdir()