Merge 950a1c3 into fb36525

.changelog/4051.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added the `validate-conf-json` pre-commit hook, checking for structure and linked content.
+  type: internal
+pr_number: 4051

TestSuite/conf_json.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from demisto_sdk.commands.common.handlers import JSON_Handler
 
@@ -16,22 +16,22 @@ def __init__(self, dir_path: Path, name: str, prefix: str):
 
     def write_json(
         self,
-        tests: Optional[List[str]] = None,
-        skipped_tests: Optional[List[str]] = None,
-        skipped_integrations: Optional[List[str]] = None,
+        tests: Optional[List[dict]] = None,
+        skipped_tests: Optional[Dict[str, str]] = None,
+        skipped_integrations: Optional[Dict[str, str]] = None,
         unmockable_integrations: Optional[List[str]] = None,
         docker_thresholds: Optional[dict] = None,
     ):
         if tests is None:
             tests = []
         if skipped_tests is None:
-            skipped_tests = None
+            skipped_tests = {}
         if skipped_integrations is None:
-            skipped_integrations = []
+            skipped_integrations = {}
         if unmockable_integrations is None:
             unmockable_integrations = []
         if docker_thresholds is None:
-            docker_thresholds = {}
+            docker_thresholds = {"_comment": "", "images": []}
         self._file_path.write_text(
             json.dumps(
                 {
@@ -40,6 +40,15 @@ def write_json(
                     "skipped_integrations": skipped_integrations,
                     "unmockable_integrations": unmockable_integrations,
                     "docker_thresholds": docker_thresholds,
+                    # the next fields are not modified in tests (hence lack of args), but are structurally required.
+                    "available_tests_fields": [],
+                    "testTimeout": 100,
+                    "testInterval": 100,
+                    "nightly_packs": [],
+                    "parallel_integrations": [],
+                    "private_tests": [],
+                    "test_marketplacev2": [],
+                    "reputation_tests": [],
                 }
             ),
             None,

demisto_sdk/commands/content_graph/objects/conf_json.py

@@ -0,0 +1,133 @@
+"""
+NOTE: This is not a standard Content item. There's no model for it.
+I's only used (at least at the time of writing these lines) in the validate_conf_json.py script
+"""
+import itertools
+from collections import defaultdict
+from pathlib import Path
+from typing import DefaultDict, Dict, List, Optional, Set, Union
+
+from more_itertools import always_iterable
+from packaging.version import Version
+from pydantic import BaseModel, Extra, Field, validator
+
+from demisto_sdk.commands.common.constants import MarketplaceVersions
+from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
+from demisto_sdk.commands.common.tools import get_json
+from demisto_sdk.commands.content_graph.common import ContentType
+
+
+class StrictBaseModel(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+
+class DictWithSingleSimpleString(StrictBaseModel):
+    simple: str
+
+
+class ExternalPlaybookConfig(StrictBaseModel):
+    playbookID: str
+    input_parameters: Dict[str, DictWithSingleSimpleString]
+
+
+class InstanceConfiguration(StrictBaseModel):
+    classifier_id: str
+    incoming_mapper_id: str
+
+
+class Test(StrictBaseModel):
+    playbookID: str
+    integrations: Optional[Union[str, List[str]]] = None
+    instance_names: Optional[Union[str, List[str]]] = None
+    timeout: Optional[int] = None
+    is_mockable: Optional[bool] = None
+    memory_threshold: Optional[int] = None
+    pid_threshold: Optional[int] = None
+    has_api: Optional[bool] = None
+    fromversion: Optional[str] = None
+    toversion: Optional[str] = None
+    nightly: Optional[bool] = None
+    context_print_dt: Optional[str] = None
+    scripts: Optional[Union[str, List[str]]]
+    runnable_on_docker_only: Optional[bool] = None
+    external_playbook_config: Optional[ExternalPlaybookConfig] = None
+    instance_configuration: Optional[InstanceConfiguration] = None
+    marketplaces: Optional[MarketplaceVersions] = None
+
+    @validator("fromversion", "toversion")
+    def validate_version(cls, v):
+        Version(v)
+
+
+class ImageConfig(StrictBaseModel):
+    memory_threshold: Optional[int] = None
+    pid_threshold: Optional[int] = None
+
+
+class DockerThresholds(StrictBaseModel):
+    field_comment: str = Field(..., alias="_comment")
+    images: Dict[str, ImageConfig]
+
+
+class ConfJSON(StrictBaseModel):
+    available_tests_fields: Dict[str, str]
+    testTimeout: int
+    testInterval: int
+    tests: List[Test]
+    skipped_tests: Dict[str, str]
+    skipped_integrations: Dict[str, str]
+    nightly_packs: List[str]
+    unmockable_integrations: Dict[str, str]
+    parallel_integrations: List[str]
+    private_tests: List[str]
+    docker_thresholds: DockerThresholds
+    test_marketplacev2: List[str]
+    reputation_tests: List[str]
+
+    @staticmethod
+    def from_path(path: Path = CONF_PATH) -> "ConfJSON":
+        return ConfJSON(**get_json(path))  # type:ignore[assignment]
+
+    @property
+    def linked_content_items(self) -> Dict[ContentType, Set[str]]:
+        result: DefaultDict[ContentType, Set[str]] = defaultdict(set)
+
+        for content_type, id_sources in (
+            (
+                ContentType.INTEGRATION,
+                (
+                    itertools.chain.from_iterable(
+                        always_iterable(test.integrations) for test in self.tests
+                    ),
+                    self.unmockable_integrations.keys(),
+                    self.parallel_integrations,
+                    (
+                        v
+                        for v in self.skipped_integrations.keys()
+                        if not v.startswith("_comment")
+                    ),
+                ),
+            ),
+            (
+                ContentType.TEST_PLAYBOOK,
+                (
+                    (test.playbookID for test in self.tests),
+                    # self.skipped_tests.keys(), # not collecting skipped tests as this section preserves for skip reasons.
+                    self.private_tests,
+                    self.reputation_tests,
+                    self.test_marketplacev2,
+                ),
+            ),
+            (
+                ContentType.SCRIPT,
+                (test.scripts for test in self.tests),
+            ),
+            (
+                ContentType.PACK,
+                (self.nightly_packs,),
+            ),
+        ):
+            for id_source in id_sources:
+                result[content_type].update(filter(None, (always_iterable(id_source))))
+        return dict(result)

demisto_sdk/commands/content_graph/tests/validate_conf_json_test.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+import pytest
+
+from demisto_sdk.scripts.validate_conf_json import ConfJsonValidator
+from TestSuite.repo import Repo
+
+
+def test_sanity(graph_repo: Repo):
+    """
+    Given   an empty repo with an empty conf.json
+    When    calling validate
+    Then    make sure it passes
+    """
+    graph_repo.create_pack()
+    graph = graph_repo.create_graph()
+    validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
+    assert validator.validate()
+
+
+@pytest.mark.parametrize("list_of_integrations", (True, False))
+def test_integration_playbook_positive(
+    graph_repo: Repo, list_of_integrations: bool
+) -> None:
+    """
+    Given   a repo with one integration and one playbook
+    When    calling validate
+    Then    make sure it passes
+    """
+    pack = graph_repo.create_pack()
+    playbook = pack.create_test_playbook("SamplePlaybookTest")
+    integration = pack.create_integration()
+
+    graph_repo.conf.write_json(
+        tests=[
+            {
+                "playbookID": playbook.name,
+                "integrations": [integration.name]
+                if list_of_integrations
+                else integration.name,
+            }
+        ]
+    )
+    graph = graph_repo.create_graph()
+    validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
+    assert validator._validate_content_exists()
+    assert validator.validate()
+
+
+def test_integration_mistyped(graph_repo: Repo) -> None:
+    """
+    Given   a repo with one integration, whose name is mistyped in conf.json
+    When    calling validate
+    Then    make sure it fails
+    """
+    pack = graph_repo.create_pack()
+    pack.create_test_playbook("SamplePlaybookTest")
+    pack.create_integration(name="foo")
+    graph_repo.conf.write_json(
+        tests=[
+            {
+                "playbookID": "SamplePlaybookTest",
+                "integrations": "FOO",
+            }
+        ]
+    )
+    graph = graph_repo.create_graph()
+    validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
+    assert not validator._validate_content_exists()
+    assert not validator.validate()
+
+
+def test_invalid_skipped_integration(graph_repo: Repo) -> None:
+    """
+    Given   a repo with one skipped integration configured, but doesn't exist in the repo
+    When    calling validate
+    Then    make sure it fails
+    """
+    pack = graph_repo.create_pack()
+    pack.create_test_playbook("SamplePlaybookTest")
+    graph_repo.conf.write_json(skipped_integrations={"hello": "world"})
+    graph = graph_repo.create_graph()
+    validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
+    assert not validator.validate()

demisto_sdk/scripts/validate_conf_json.py

@@ -0,0 +1,94 @@
+import os
+from pathlib import Path
+from typing import List, Optional, cast
+
+from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
+from demisto_sdk.commands.common.logger import logger, logging_setup
+from demisto_sdk.commands.common.tools import string_to_bool
+from demisto_sdk.commands.content_graph.commands.update import update_content_graph
+from demisto_sdk.commands.content_graph.interface import ContentGraphInterface
+from demisto_sdk.commands.content_graph.objects.base_content import UnknownContent
+from demisto_sdk.commands.content_graph.objects.conf_json import ConfJSON
+from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
+
+
+class ConfJsonValidator:
+    def __init__(
+        self,
+        conf_json_path: Path = CONF_PATH,
+        graph: Optional[ContentGraphInterface] = None,  # Pass None to generate
+    ) -> None:
+        self._conf_path = conf_json_path
+        self.conf = ConfJSON.from_path(conf_json_path)
+
+        logger.info("Creating content graph - this may take a few minutes")
+        if graph is None:
+            update_content_graph(graph := ContentGraphInterface())
+        self.graph_ids_by_type = {
+            content_type: cast(
+                List[ContentItem],
+                [
+                    item
+                    for item in graph.search(
+                        content_type=content_type, object_id=conf_ids
+                    )
+                    if not isinstance(
+                        item, UnknownContent
+                    )  # UnknownContent items are artificially generated in relationships, are not part of the repo
+                ],
+            )
+            for content_type, conf_ids in self.conf.linked_content_items.items()
+        }
+
+    def _validate_content_exists(self) -> bool:
+        is_valid = True
+
+        for content_type, linked_ids in self.conf.linked_content_items.items():
+            if linked_ids_missing_in_graph := linked_ids.difference(
+                {
+                    item.object_id
+                    for item in self.graph_ids_by_type.get(content_type, ())
+                }
+            ):
+                message = f"{len(linked_ids_missing_in_graph)} {content_type.value}s are not found in the graph: {','.join(sorted(linked_ids_missing_in_graph))}"
+                logger.error(message)
+                if string_to_bool(os.getenv("GITHUB_ACTIONS", False)):
+                    print(  # noqa: T201
+                        f"::error file={self._conf_path},line=1,endLine=1,title=Conf.JSON Error::{message}"
+                    )
+
+                is_valid = False
+        return is_valid
+
+    # def _validate_content_not_deprecated(self) -> bool:
+    #
+    #     This is commented out as we decided to not treat deprecated content as errorneous.
+    #
+    #     is_valid = True
+    #     for content_type, linked_ids in self.conf.linked_content_items.items():
+    #         graph_deprecated_ids = {
+    #             item.object_id
+    #             for item in self.graph_ids_by_type.get(content_type, ())
+    #             if item.deprecated
+    #         }
+    #         if linked_deprecated_ids := linked_ids.intersection(graph_deprecated_ids):
+    #             logger.error(
+    #                 f"{len(linked_deprecated_ids)} {content_type.value}s are deprecated: {','.join(sorted(linked_deprecated_ids))}"
+    #             )
+    #             is_valid = False
+    #     return is_valid
+
+    def validate(self) -> bool:
+        return self._validate_content_exists()
+
+
+def main():
+    logging_setup()
+    if not ConfJsonValidator().validate():
+        logger.error("conf.json is not valid")
+        exit(1)
+    logger.info("[green]conf.json is valid[/green]")
+
+
+if __name__ == "__main__":
+    main()

demisto_sdk/tests/test_files/Packs/CortexXDR/Playbooks/Valid_Deprecated_Playbook.yml

@@ -1,6 +1,6 @@
-id: Cortex XDR Incident Handling
+id: Cortex_XDR_Incident_Handling
 version: -1
-name: Cortex XDR Incident Handling
+name: Cortex_XDR_Incident_Handling
 description: Deprecated. Use "Cortex XDR Incident Handling v2" playbook instead.
   This playbook is triggered by fetchinga Palo Alto Networks Cortex XDR
   incident. \nThe playbook syncs and updates new XDR alerts that construct the incident.

pyproject.toml

@@ -22,6 +22,7 @@ update-additional-dependencies = "demisto_sdk.scripts.update_additional_dependen
 sdk-changelog = "demisto_sdk.scripts.changelog.changelog:main"
 merge-coverage-report = "demisto_sdk.scripts.merge_coverage_report:main"
 merge-pytest-reports = "demisto_sdk.scripts.merge_pytest_reports:main"
+validate-conf-json = "demisto_sdk.scripts.validate_conf_json:main"
 init-validation = "demisto_sdk.scripts.init_validation_script:main"
 
 [tool.poetry.dependencies]