Merge branch 'master' of https://github.com/demisto/demisto-sdk into …

…support-python-311

.changelog/4225.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Removed the `update-additional-dependencies` script and pre-commit hook.
+  type: internal
+pr_number: 4225

.github/workflows/on-push.yml

@@ -210,9 +210,11 @@ jobs:
           coverage xml
       - name: Coveralls-Action
         uses: coverallsapp/github-action@v2
+        continue-on-error: true
       - name: Coveralls-Comment
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        continue-on-error: true
         run: |
           if [ -n "$COVERALLS_REPO_TOKEN" ]; then
             pip install coveralls
@@ -316,12 +318,19 @@ jobs:
 
       - name: run pre-commit on input files
         timeout-minutes: 60
-        continue-on-error: true # TODO - fail if there is an error here
         run: |
           source $(poetry env info --path)/bin/activate
           cd content
           demisto-sdk pre-commit --validate --show-diff-on-failure --verbose -i Packs/HelloWorld -i Packs/QRadar/Integrations/QRadar_v3 --mode=nightly
 
+      - name: run pre-commit with default pre-commit config shipped with the SDK
+        timeout-minutes: 60
+        run: |
+          source $(poetry env info --path)/bin/activate
+          cd content
+          rm -rf .pre-commit-config_template.yaml
+          demisto-sdk pre-commit --validate --show-diff-on-failure --verbose --mode=ci
+
       - name: Upload artifacts
         if: always()
         uses: ./.github/actions/upload_artifacts
@@ -559,3 +568,23 @@ jobs:
       - name: Output to GitHub Action Summary
         if: steps.changed-files.outputs.any_changed == 'true'
         run: cat validation_docs.md >> $GITHUB_STEP_SUMMARY
+
+  verify-pip-installation:
+    name: Verify SDK Pip Installation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: 'pip'
+
+      - name: pip intsall current project
+        run: pip install .
+
+      - name: Verify Installation
+        run: demisto-sdk --version

.pre-commit-hooks.yaml

@@ -49,14 +49,6 @@
   types: [python]
   exclude: tests/.*|demisto_sdk/commands/init/templates/.*
 
-- id: update-additional-dependencies
-  name: update-additional-dependencies
-  entry: update-additional-dependencies
-  language: python
-  files: poetry.lock
-  require_serial: true
-  pass_filenames: false
-
 - id: coverage-analyze
   name: coverage-analyze
   entry: demisto-sdk coverage-analyze

CHANGELOG.md

@@ -1,4 +1,14 @@
 # Changelog
+## 1.29.1
+* Fix metadata (i.e. description) for XSOAR6 marketplace. [#4208](https://github.com/demisto/demisto-sdk/pull/4208)
+* Fixed an issue where `xsoar_on_prem` tag, was not removed as part of the parsing process. [#4218](https://github.com/demisto/demisto-sdk/pull/4218)
+* Added support to delete packs in **graph update** [#4205](https://github.com/demisto/demisto-sdk/pull/4205)
+* Added a default `.pre-commit-config_template.yaml` file that will be used by **pre-commit** command in case such does not exist in the target repository. [#4194](https://github.com/demisto/demisto-sdk/pull/4194)
+* Fixed an issue where the graph was not fully cleaned before import in **graph update** [#4215](https://github.com/demisto/demisto-sdk/pull/4215)
+* Fixed an issue where tests were not collected in VSCode after **setup-env** [#4219](https://github.com/demisto/demisto-sdk/pull/4219)
+* Fixed an issue where validate spammed logs when running. [#4203](https://github.com/demisto/demisto-sdk/pull/4203)
+* Added the **dump-api** command to dump the demisto-sdk API to a JSON file. [#4214](https://github.com/demisto/demisto-sdk/pull/4214)
+
 ## 1.29.0
 * The new validate flow is enabled by default while the old one is disabled. If you want to change this behavior, use the **--run-old-validate** and **--skip-new-validate** flags or the **RUN_OLD_VALIDATE** and **SKIP_NEW_VALIDATE** environment variables. [#4114](https://github.com/demisto/demisto-sdk/pull/4114)
 * A new version of validate is available. For more information, refer to the readme. (a docs web page with further information about each validation will be added soon). Note that the old validate flow is beginning to go under a deprecation process and will eventually be removed. [#4114](https://github.com/demisto/demisto-sdk/pull/4114)

demisto_sdk/__main__.py

@@ -44,6 +44,7 @@
     logging_setup,
 )
 from demisto_sdk.commands.common.tools import (
+    convert_path_to_str,
     find_type,
     get_last_remote_release_version,
     get_release_note_entries,
@@ -3809,5 +3810,49 @@ def xsoar_linter(
 main.add_command(typer.main.get_command(xsoar_linter_app), "xsoar-lint")
 
 
+# ====================== export ====================== #
+
+export_app = typer.Typer(name="dump-api")
+
+
+@export_app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def dump_api(
+    ctx: typer.Context,
+    output_path: Path = typer.Option(
+        CONTENT_PATH,
+        "-o",
+        "--output",
+        help="The output directory or JSON file to save the demisto-sdk api.",
+    ),
+):
+    """
+    This commands dumps the `demisto-sdk` API to a file.
+    It is used to view the help of all commands in one file.
+
+    Args:
+        ctx (typer.Context):
+        output_path (Path, optional): The output directory or JSON file to save the demisto-sdk api.
+    """
+    output_json: dict = {}
+    for command_name, command in main.commands.items():
+        if isinstance(command, click.Group):
+            output_json[command_name] = {}
+            for sub_command_name, sub_command in command.commands.items():
+                output_json[command_name][sub_command_name] = sub_command.to_info_dict(
+                    ctx
+                )
+        else:
+            output_json[command_name] = command.to_info_dict(ctx)
+    convert_path_to_str(output_json)
+    if output_path.is_dir():
+        output_path = output_path / "demisto-sdk-api.json"
+    output_path.write_text(json.dumps(output_json, indent=4))
+
+
+main.add_command(typer.main.get_command(export_app), "dump-api")
+
+
 if __name__ == "__main__":
     main()

demisto_sdk/commands/common/constants.py

@@ -2056,6 +2056,8 @@ class ParameterType(Enum):
 FORMATTING_SCRIPT = "indicator-format"
 
 DOCKERFILES_INFO_REPO = "demisto/dockerfiles-info"
+DOCKERFILES_REPO = "demisto/dockerfiles"
+CONTENT_REPO = "demisto/content"
 
 TEST_COVERAGE_DEFAULT_URL = f"https://storage.googleapis.com/{DEMISTO_SDK_MARKETPLACE_XSOAR_DIST_DEV}/code-coverage-reports/coverage-min.json"
 

demisto_sdk/commands/common/content/tests/objects/pack_objects/pack_metadata/pack_metadata_test.py

@@ -9,6 +9,7 @@
 from pytest import MonkeyPatch
 
 from demisto_sdk.commands.common.constants import (
+    DEFAULT_CONTENT_ITEM_TO_VERSION,
     PACKS_DIR,
     XSOAR_AUTHOR,
     XSOAR_SUPPORT,
@@ -24,6 +25,9 @@
     PackContentItems,
 )
 from demisto_sdk.commands.content_graph.objects.pack_metadata import PackMetadata
+from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
+    mock_integration,
+)
 from TestSuite.test_tools import ChangeCWD, str_in_call_args_list
 
 TEST_DATA = src_root() / "tests" / "test_files"
@@ -448,3 +452,81 @@ def test__enhance_pack_properties__internal_and_external(
         content_items=PackContentItems(),  # type: ignore
     )
     assert my_instance.version_info == expected
+
+
+@pytest.mark.parametrize(
+    "marketplace_version, current_fromversion, new_fromversion, current_toversion, new_toversion, expected_toversion",
+    [
+        (MarketplaceVersions.XSOAR, "5.5.0", "8.0.0", "7.9.9", "8.2.0", "7.9.9"),
+        (MarketplaceVersions.XSOAR, "5.5.0", "6.5.0", "7.2.0", "7.9.9", "7.9.9"),
+        (
+            MarketplaceVersions.XSOAR,
+            "5.5.0",
+            "6.5.0",
+            "7.9.9",
+            DEFAULT_CONTENT_ITEM_TO_VERSION,
+            "",
+        ),
+        (MarketplaceVersions.XSOAR_SAAS, "5.5.0", "8.0.0", "6.2.0", "8.5.0", "8.5.0"),
+    ],
+)
+def test_replace_item_if_has_higher_toversion(
+    marketplace_version,
+    current_fromversion,
+    new_fromversion,
+    current_toversion,
+    new_toversion,
+    expected_toversion,
+):
+    """Tests the _replace_item_if_has_higher_toversion
+    updates to the highest version supported by the MarketplaceVersions.XSOAR
+    ARGS:
+        marketplace_version: MarketplaceVersions the flow is running on.
+        current_fromversion: current fromversion of content item in the pack metadata
+        new_fromversion: the fromversion of content item in the pack metadata
+        current_toversion: current toversion of content item in the pack metadata
+        new_toversion: a new toversion of content item
+        expected_toversion
+    Given:
+        - a Pack Metadata and an integration uploading to MarketplaceVersions.XSOAR
+    When:
+        - Calling the _replace_item_if_has_higher_toversion method.
+    Then:
+        - Verify that the content_item_metadata toversion is set correctly.
+        Scenario 1: On MarketplaceVersions.XSOAR should not update the metadata to a version higher than 7.9.9
+        Scenario 2: On MarketplaceVersions.XSOAR should update to higher version while still lower than the max 7.9.9
+        Scenario 3: On all marketplaces will update the metdata of content item toversion to empty if new toversion is DEFAULT_CONTENT_ITEM_TO_VERSION
+        Scenario 4: On MarketplaceVersions.XSOAR_SAAS should update metadata to the highest version.
+    """
+    content_item_metadata = {
+        "fromversion": current_fromversion,
+        "toversion": current_toversion,
+    }
+    marketplace = marketplace_version
+    my_instance = PackMetadata(
+        name="test",
+        display_name="",
+        description="",
+        created="",
+        legacy=False,
+        support="",
+        url="",
+        email="",
+        eulaLink="",
+        price=0,
+        hidden=False,
+        commit="",
+        downloads=0,
+        keywords=[],
+        searchRank=0,
+        excludedDependencies=[],
+        videos=[],
+        modules=[],
+    )  # type: ignore
+    integration = mock_integration()
+    integration.toversion = new_toversion
+    integration.fromversion = new_fromversion
+    my_instance._replace_item_if_has_higher_toversion(
+        integration, content_item_metadata, integration.summary(), marketplace
+    )
+    assert content_item_metadata["toversion"] == expected_toversion

demisto_sdk/commands/common/docker_images_metadata.py

@@ -11,7 +11,6 @@
 from demisto_sdk.commands.common.docker.docker_image import DockerImage
 from demisto_sdk.commands.common.files.errors import FileReadError
 from demisto_sdk.commands.common.files.json_file import JsonFile
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.singleton import PydanticSingleton
 from demisto_sdk.commands.common.StrEnum import StrEnum
@@ -54,8 +53,8 @@ def __from_github(
         try:
             dockerfiles_metadata = JsonFile.read_from_github_api(
                 file_name,
+                repo=DOCKERFILES_INFO_REPO,
                 tag=tag,
-                git_content_config=GitContentConfig(repo_name=DOCKERFILES_INFO_REPO),
                 verify_ssl=False,
                 encoding="utf-8-sig",
             )

demisto_sdk/commands/common/files/file.py

@@ -1,4 +1,5 @@
 import inspect
+import os
 import shutil
 import urllib.parse
 from abc import ABC, abstractmethod
@@ -13,6 +14,7 @@
 from requests.exceptions import ConnectionError, RequestException, Timeout
 
 from demisto_sdk.commands.common.constants import (
+    CONTENT_REPO,
     DEMISTO_GIT_PRIMARY_BRANCH,
     DEMISTO_GIT_UPSTREAM,
     urljoin,
@@ -26,7 +28,10 @@
     MemoryFileReadError,
     UnknownFileError,
 )
-from demisto_sdk.commands.common.git_content_config import GitContentConfig
+from demisto_sdk.commands.common.git_content_config import (
+    GitContentConfig,
+    GitCredentials,
+)
 from demisto_sdk.commands.common.git_util import GitUtil
 from demisto_sdk.commands.common.handlers.xsoar_handler import XSOAR_Handler
 from demisto_sdk.commands.common.logger import logger
@@ -317,7 +322,7 @@ def __read_git_file(self, tag: str, from_remote: bool = True) -> Any:
     def read_from_github_api(
         cls,
         path: str,
-        git_content_config: Optional[GitContentConfig] = None,
+        repo: str = CONTENT_REPO,
         encoding: Optional[str] = None,
         tag: str = DEMISTO_GIT_PRIMARY_BRANCH,
         handler: Optional[XSOAR_Handler] = None,
@@ -328,8 +333,8 @@ def read_from_github_api(
         Reads a file from Github api.
 
         Args:
-            path: the path to the file in github
-            git_content_config: git content config object
+            path: the path to the file in Github from the repo's root
+            repo: the repository name, e.g.: demisto/content
             encoding: any custom encoding if needed
             tag: the branch/sha to take the file from within Github
             handler: whether a custom handler is required, if not takes the default.
@@ -339,17 +344,16 @@ def read_from_github_api(
         Returns:
             Any: the file content in the desired format
         """
-        if not git_content_config:
-            git_content_config = GitContentConfig()
-
-        git_path_url = urljoin(git_content_config.base_api, tag, path)
-        github_token = git_content_config.CREDENTIALS.github_token
+        if not path.startswith("/"):
+            path = f"/{path}"
+        url = f"https://raw.githubusercontent.com/{repo}/{tag}{path}"
+        github_token = os.getenv(GitCredentials.ENV_GITHUB_TOKEN_NAME, "")
 
         timeout = 10
 
         try:
             return cls.read_from_http_request(
-                git_path_url,
+                url,
                 headers=frozenset(
                     {
                         "Authorization": f"Bearer {github_token}"
@@ -366,17 +370,17 @@ def read_from_github_api(
             )
         except FileReadError as e:
             logger.warning(
-                f"Received error {e} when trying to retrieve {git_path_url} content from Github, retrying"
+                f"Received error {e} when trying to retrieve {url} content from Github, retrying"
             )
             try:
                 return cls.read_from_http_request(
-                    git_path_url,
+                    url,
                     params=frozenset({"token": github_token}.items()),
                     timeout=timeout,
                 )
             except FileReadError:
                 logger.error(
-                    f"Could not retrieve the content of {git_path_url} file from Github"
+                    f"Could not retrieve the content of {url} file from Github"
                 )
                 raise
 

demisto_sdk/commands/common/tools.py

@@ -4490,3 +4490,23 @@ def get_relative_path(file_path: Union[str, Path], relative_to: Path) -> Path:
     if file_path.is_absolute():
         file_path = file_path.relative_to(relative_to)
     return file_path
+
+
+def convert_path_to_str(data: Union[dict, list]):
+    """This converts recursively all Path objects to strings in the given data.
+
+    Args:
+        data (Union[dict, list]): The data to convert.
+    """
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if isinstance(value, (dict, list)):
+                convert_path_to_str(value)
+            elif isinstance(value, Path):
+                data[key] = str(value)
+    elif isinstance(data, list):
+        for index, item in enumerate(data):
+            if isinstance(item, (dict, list)):
+                convert_path_to_str(item)
+            elif isinstance(item, Path):
+                data[index] = str(item)