Merge remote-tracking branch 'origin/master' into YR/-BC-104-New-form…

…at-validation/CIAC-10001

.changelog/4205.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added support to delete packs in **graph update**
+  type: fix
+pr_number: 4205

.changelog/4208.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Fix metadata (i.e. description) for XSOAR6 marketplace.
+  type: fix
+pr_number: 4208

.changelog/4214.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added the **dump-api** command to dump the demisto-sdk API to a JSON file.
+  type: internal
+pr_number: 4214

.changelog/4215.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Fixed an issue where the graph was not fully cleaned before import in **graph update**
+  type: fix
+pr_number: 4215

.changelog/4219.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Fixed an issue where tests were not collected in VSCode after **setup-env**
+  type: fix
+pr_number: 4219

.github/workflows/on-push.yml

@@ -207,9 +207,11 @@ jobs:
           coverage xml
       - name: Coveralls-Action
         uses: coverallsapp/github-action@v2
+        continue-on-error: true
       - name: Coveralls-Comment
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        continue-on-error: true
         run: |
           if [ -n "$COVERALLS_REPO_TOKEN" ]; then
             pip install coveralls
@@ -311,7 +313,6 @@ jobs:
 
       - name: run pre-commit on input files
         timeout-minutes: 60
-        continue-on-error: true # TODO - fail if there is an error here
         run: |
           source $(poetry env info --path)/bin/activate
           cd content
@@ -553,3 +554,23 @@ jobs:
       - name: Output to GitHub Action Summary
         if: steps.changed-files.outputs.any_changed == 'true'
         run: cat validation_docs.md >> $GITHUB_STEP_SUMMARY
+
+  verify-pip-installation:
+    name: Verify SDK Pip Installation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: 'pip'
+
+      - name: pip intsall current project
+        run: pip install .
+
+      - name: Verify Installation
+        run: demisto-sdk --version

demisto_sdk/__main__.py

@@ -44,6 +44,7 @@
     logging_setup,
 )
 from demisto_sdk.commands.common.tools import (
+    convert_path_to_str,
     find_type,
     get_last_remote_release_version,
     get_release_note_entries,
@@ -3809,5 +3810,49 @@ def xsoar_linter(
 main.add_command(typer.main.get_command(xsoar_linter_app), "xsoar-lint")
 
 
+# ====================== export ====================== #
+
+export_app = typer.Typer(name="dump-api")
+
+
+@export_app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def dump_api(
+    ctx: typer.Context,
+    output_path: Path = typer.Option(
+        CONTENT_PATH,
+        "-o",
+        "--output",
+        help="The output directory or JSON file to save the demisto-sdk api.",
+    ),
+):
+    """
+    This commands dumps the `demisto-sdk` API to a file.
+    It is used to view the help of all commands in one file.
+
+    Args:
+        ctx (typer.Context):
+        output_path (Path, optional): The output directory or JSON file to save the demisto-sdk api.
+    """
+    output_json: dict = {}
+    for command_name, command in main.commands.items():
+        if isinstance(command, click.Group):
+            output_json[command_name] = {}
+            for sub_command_name, sub_command in command.commands.items():
+                output_json[command_name][sub_command_name] = sub_command.to_info_dict(
+                    ctx
+                )
+        else:
+            output_json[command_name] = command.to_info_dict(ctx)
+    convert_path_to_str(output_json)
+    if output_path.is_dir():
+        output_path = output_path / "demisto-sdk-api.json"
+    output_path.write_text(json.dumps(output_json, indent=4))
+
+
+main.add_command(typer.main.get_command(export_app), "dump-api")
+
+
 if __name__ == "__main__":
     main()

demisto_sdk/commands/common/constants.py

@@ -2054,6 +2054,8 @@ class ParameterType(Enum):
 FORMATTING_SCRIPT = "indicator-format"
 
 DOCKERFILES_INFO_REPO = "demisto/dockerfiles-info"
+DOCKERFILES_REPO = "demisto/dockerfiles"
+CONTENT_REPO = "demisto/content"
 
 TEST_COVERAGE_DEFAULT_URL = f"https://storage.googleapis.com/{DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV}/code-coverage-reports/coverage-min.json"
 

demisto_sdk/commands/common/content/tests/objects/pack_objects/pack_metadata/pack_metadata_test.py

@@ -9,6 +9,7 @@
 from pytest import MonkeyPatch
 
 from demisto_sdk.commands.common.constants import (
+    DEFAULT_CONTENT_ITEM_TO_VERSION,
     PACKS_DIR,
     XSOAR_AUTHOR,
     XSOAR_SUPPORT,
@@ -24,6 +25,9 @@
     PackContentItems,
 )
 from demisto_sdk.commands.content_graph.objects.pack_metadata import PackMetadata
+from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
+    mock_integration,
+)
 from TestSuite.test_tools import ChangeCWD, str_in_call_args_list
 
 TEST_DATA = src_root() / "tests" / "test_files"
@@ -448,3 +452,81 @@ def test__enhance_pack_properties__internal_and_external(
         content_items=PackContentItems(),  # type: ignore
     )
     assert my_instance.version_info == expected
+
+
+@pytest.mark.parametrize(
+    "marketplace_version, current_fromversion, new_fromversion, current_toversion, new_toversion, expected_toversion",
+    [
+        (MarketplaceVersions.XSOAR, "5.5.0", "8.0.0", "7.9.9", "8.2.0", "7.9.9"),
+        (MarketplaceVersions.XSOAR, "5.5.0", "6.5.0", "7.2.0", "7.9.9", "7.9.9"),
+        (
+            MarketplaceVersions.XSOAR,
+            "5.5.0",
+            "6.5.0",
+            "7.9.9",
+            DEFAULT_CONTENT_ITEM_TO_VERSION,
+            "",
+        ),
+        (MarketplaceVersions.XSOAR_SAAS, "5.5.0", "8.0.0", "6.2.0", "8.5.0", "8.5.0"),
+    ],
+)
+def test_replace_item_if_has_higher_toversion(
+    marketplace_version,
+    current_fromversion,
+    new_fromversion,
+    current_toversion,
+    new_toversion,
+    expected_toversion,
+):
+    """Tests the _replace_item_if_has_higher_toversion
+    updates to the highest version supported by the MarketplaceVersions.XSOAR
+    ARGS:
+        marketplace_version: MarketplaceVersions the flow is running on.
+        current_fromversion: current fromversion of content item in the pack metadata
+        new_fromversion: the fromversion of content item in the pack metadata
+        current_toversion: current toversion of content item in the pack metadata
+        new_toversion: a new toversion of content item
+        expected_toversion
+    Given:
+        - a Pack Metadata and an integration uploading to MarketplaceVersions.XSOAR
+    When:
+        - Calling the _replace_item_if_has_higher_toversion method.
+    Then:
+        - Verify that the content_item_metadata toversion is set correctly.
+        Scenario 1: On MarketplaceVersions.XSOAR should not update the metadata to a version higher than 7.9.9
+        Scenario 2: On MarketplaceVersions.XSOAR should update to higher version while still lower than the max 7.9.9
+        Scenario 3: On all marketplaces will update the metdata of content item toversion to empty if new toversion is DEFAULT_CONTENT_ITEM_TO_VERSION
+        Scenario 4: On MarketplaceVersions.XSOAR_SAAS should update metadata to the highest version.
+    """
+    content_item_metadata = {
+        "fromversion": current_fromversion,
+        "toversion": current_toversion,
+    }
+    marketplace = marketplace_version
+    my_instance = PackMetadata(
+        name="test",
+        display_name="",
+        description="",
+        created="",
+        legacy=False,
+        support="",
+        url="",
+        email="",
+        eulaLink="",
+        price=0,
+        hidden=False,
+        commit="",
+        downloads=0,
+        keywords=[],
+        searchRank=0,
+        excludedDependencies=[],
+        videos=[],
+        modules=[],
+    )  # type: ignore
+    integration = mock_integration()
+    integration.toversion = new_toversion
+    integration.fromversion = new_fromversion
+    my_instance._replace_item_if_has_higher_toversion(
+        integration, content_item_metadata, integration.summary(), marketplace
+    )
+    assert content_item_metadata["toversion"] == expected_toversion

demisto_sdk/commands/common/docker_images_metadata.py

@@ -12,7 +12,6 @@
 from demisto_sdk.commands.common.docker.docker_image import DockerImage
 from demisto_sdk.commands.common.files.errors import FileReadError
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.singleton import PydanticSingleton
 from demisto_sdk.commands.common.tools import NoInternetConnectionException
@@ -54,8 +53,8 @@ def __from_github(
         try:
             dockerfiles_metadata = JsonFile.read_from_github_api(
                 file_name,
+                repo=DOCKERFILES_INFO_REPO,
                 tag=tag,
-                git_content_config=GitContentConfig(repo_name=DOCKERFILES_INFO_REPO),
                 verify_ssl=False,
                 encoding="utf-8-sig",
             )

demisto_sdk/commands/common/files/file.py

@@ -1,4 +1,5 @@
 import inspect
+import os
 import shutil
 import urllib.parse
 from abc import ABC, abstractmethod
@@ -13,6 +14,7 @@
 from requests.exceptions import ConnectionError, RequestException, Timeout
 
 from demisto_sdk.commands.common.constants import (
+    CONTENT_REPO,
     DEMISTO_GIT_PRIMARY_BRANCH,
     DEMISTO_GIT_UPSTREAM,
     urljoin,
@@ -26,7 +28,10 @@
     MemoryFileReadError,
     UnknownFileError,
 )
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
+from demisto_sdk.commands.common.git_content_config import (
+    GitContentConfig,
+    GitCredentials,
+)
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.handlers.xsoar_handler import XSOAR_Handler
 from demisto_sdk.commands.common.logger import logger
@@ -317,7 +322,7 @@ def __read_git_file(self, tag: str, from_remote: bool = True) -> Any:
     def read_from_github_api(
         cls,
         path: str,
-        git_content_config: Optional[GitContentConfig] = None,
+        repo: str = CONTENT_REPO,
         encoding: Optional[str] = None,
         tag: str = DEMISTO_GIT_PRIMARY_BRANCH,
         handler: Optional[XSOAR_Handler] = None,
@@ -328,8 +333,8 @@ def read_from_github_api(
         Reads a file from Github api.
 
         Args:
-            path: the path to the file in github
-            git_content_config: git content config object
+            path: the path to the file in Github from the repo's root
+            repo: the repository name, e.g.: demisto/content
             encoding: any custom encoding if needed
             tag: the branch/sha to take the file from within Github
             handler: whether a custom handler is required, if not takes the default.
@@ -339,17 +344,16 @@ def read_from_github_api(
         Returns:
             Any: the file content in the desired format
         """
-        if not git_content_config:
-            git_content_config = GitContentConfig()
-
-        git_path_url = urljoin(git_content_config.base_api, tag, path)
-        github_token = git_content_config.CREDENTIALS.github_token
+        if not path.startswith("/"):
+            path = f"/{path}"
+        url = f"https://raw.githubusercontent.com/{repo}/{tag}{path}"
+        github_token = os.getenv(GitCredentials.ENV_GITHUB_TOKEN_NAME, "")
 
         timeout = 10
 
         try:
             return cls.read_from_http_request(
-                git_path_url,
+                url,
                 headers=frozenset(
                     {
                         "Authorization": f"Bearer {github_token}"
@@ -366,17 +370,17 @@ def read_from_github_api(
             )
         except FileReadError as e:
             logger.warning(
-                f"Received error {e} when trying to retrieve {git_path_url} content from Github, retrying"
+                f"Received error {e} when trying to retrieve {url} content from Github, retrying"
             )
             try:
                 return cls.read_from_http_request(
-                    git_path_url,
+                    url,
                     params=frozenset({"token": github_token}.items()),
                     timeout=timeout,
                 )
             except FileReadError:
                 logger.error(
-                    f"Could not retrieve the content of {git_path_url} file from Github"
+                    f"Could not retrieve the content of {url} file from Github"
                 )
                 raise
 

demisto_sdk/commands/common/tools.py

@@ -4490,3 +4490,23 @@ def get_relative_path(file_path: Union[str, Path], relative_to: Path) -> Path:
     if file_path.is_absolute():
         file_path = file_path.relative_to(relative_to)
     return file_path
+
+
+def convert_path_to_str(data: Union[dict, list]):
+    """This converts recursively all Path objects to strings in the given data.
+
+    Args:
+        data (Union[dict, list]): The data to convert.
+    """
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if isinstance(value, (dict, list)):
+                convert_path_to_str(value)
+            elif isinstance(value, Path):
+                data[key] = str(value)
+    elif isinstance(data, list):
+        for index, item in enumerate(data):
+            if isinstance(item, (dict, list)):
+                convert_path_to_str(item)
+            elif isinstance(item, Path):
+                data[index] = str(item)

demisto_sdk/commands/content_graph/interface/neo4j/neo4j_graph.py

@@ -55,6 +55,7 @@
 from demisto_sdk.commands.content_graph.interface.neo4j.queries.relationships import (
     _match_relationships,
     create_relationships,
+    delete_all_graph_relationships,
     get_sources_by_path,
     get_targets_by_path,
 )
@@ -663,6 +664,7 @@ def export_graph(
 
     def clean_graph(self):
         with self.driver.session() as session:
+            session.execute_write(delete_all_graph_relationships)
             session.execute_write(delete_all_graph_nodes)
         self._id_to_obj = {}
 

demisto_sdk/commands/content_graph/interface/neo4j/queries/nodes.py

@@ -255,9 +255,12 @@ def get_list_properties(tx: Transaction) -> List[str]:
 
 
 def delete_all_graph_nodes(tx: Transaction) -> None:
-    query = """// Deletes all graph nodes and relationships
-MATCH (n)
-DETACH DELETE n"""
+    query = """// Deletes all graph nodes
+        MATCH (n)
+        CALL { WITH n
+        DETACH DELETE n
+        };
+    """
     run_query(tx, query)