Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

validate conf.json #4051

Merged
merged 31 commits into from
Feb 28, 2024
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changelog/4051.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
changes:
- description: Added the `validate-conf-json` pre-commit hook, checking for structure and linked content.
type: internal
pr_number: 4051
23 changes: 16 additions & 7 deletions TestSuite/conf_json.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from pathlib import Path
from typing import List, Optional
from typing import Dict, List, Optional

from demisto_sdk.commands.common.handlers import JSON_Handler

Expand All @@ -16,22 +16,22 @@ def __init__(self, dir_path: Path, name: str, prefix: str):

def write_json(
self,
tests: Optional[List[str]] = None,
skipped_tests: Optional[List[str]] = None,
skipped_integrations: Optional[List[str]] = None,
tests: Optional[List[dict]] = None,
skipped_tests: Optional[Dict[str, str]] = None,
skipped_integrations: Optional[Dict[str, str]] = None,
unmockable_integrations: Optional[List[str]] = None,
docker_thresholds: Optional[dict] = None,
):
if tests is None:
tests = []
if skipped_tests is None:
skipped_tests = None
skipped_tests = {}
if skipped_integrations is None:
skipped_integrations = []
skipped_integrations = {}
if unmockable_integrations is None:
unmockable_integrations = []
if docker_thresholds is None:
docker_thresholds = {}
docker_thresholds = {"_comment": "", "images": []}
self._file_path.write_text(
json.dumps(
{
Expand All @@ -40,6 +40,15 @@ def write_json(
"skipped_integrations": skipped_integrations,
"unmockable_integrations": unmockable_integrations,
"docker_thresholds": docker_thresholds,
# the next fields are not modified in tests (hence lack of args), but are structurally required.
"available_tests_fields": [],
"testTimeout": 100,
"testInterval": 100,
"nightly_packs": [],
"parallel_integrations": [],
"private_tests": [],
"test_marketplacev2": [],
"reputation_tests": [],
}
),
None,
Expand Down
133 changes: 133 additions & 0 deletions demisto_sdk/commands/content_graph/objects/conf_json.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
"""
NOTE: This is not a standard Content item. There's no model for it.
I's only used (at least at the time of writing these lines) in the validate_conf_json.py script
"""
import itertools
from collections import defaultdict
from pathlib import Path
from typing import DefaultDict, Dict, List, Optional, Set, Union

from more_itertools import always_iterable
from packaging.version import Version
from pydantic import BaseModel, Extra, Field, validator

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
from demisto_sdk.commands.common.tools import get_json
from demisto_sdk.commands.content_graph.common import ContentType


class StrictBaseModel(BaseModel):
class Config:
extra = Extra.forbid


class DictWithSingleSimpleString(StrictBaseModel):
simple: str


class ExternalPlaybookConfig(StrictBaseModel):
playbookID: str
input_parameters: Dict[str, DictWithSingleSimpleString]


class InstanceConfiguration(StrictBaseModel):
classifier_id: str
incoming_mapper_id: str


class Test(StrictBaseModel):
playbookID: str
integrations: Optional[Union[str, List[str]]] = None
instance_names: Optional[Union[str, List[str]]] = None
timeout: Optional[int] = None
is_mockable: Optional[bool] = None
memory_threshold: Optional[int] = None
pid_threshold: Optional[int] = None
has_api: Optional[bool] = None
fromversion: Optional[str] = None
toversion: Optional[str] = None
nightly: Optional[bool] = None
context_print_dt: Optional[str] = None
scripts: Optional[Union[str, List[str]]]
runnable_on_docker_only: Optional[bool] = None
external_playbook_config: Optional[ExternalPlaybookConfig] = None
instance_configuration: Optional[InstanceConfiguration] = None
marketplaces: Optional[MarketplaceVersions] = None

@validator("fromversion", "toversion")
def validate_version(cls, v):
Version(v)


class ImageConfig(StrictBaseModel):
memory_threshold: Optional[int] = None
pid_threshold: Optional[int] = None


class DockerThresholds(StrictBaseModel):
field_comment: str = Field(..., alias="_comment")
images: Dict[str, ImageConfig]


class ConfJSON(StrictBaseModel):
available_tests_fields: Dict[str, str]
testTimeout: int
testInterval: int
tests: List[Test]
skipped_tests: Dict[str, str]
skipped_integrations: Dict[str, str]
nightly_packs: List[str]
unmockable_integrations: Dict[str, str]
parallel_integrations: List[str]
private_tests: List[str]
docker_thresholds: DockerThresholds
test_marketplacev2: List[str]
reputation_tests: List[str]

@staticmethod
def from_path(path: Path = CONF_PATH) -> "ConfJSON":
return ConfJSON(**get_json(path)) # type:ignore[assignment]

@property
def linked_content_items(self) -> Dict[ContentType, Set[str]]:
result: DefaultDict[ContentType, Set[str]] = defaultdict(set)

for content_type, id_sources in (
(
ContentType.INTEGRATION,
(
itertools.chain.from_iterable(
always_iterable(test.integrations) for test in self.tests
),
self.unmockable_integrations.keys(),
self.parallel_integrations,
(
v
for v in self.skipped_integrations.keys()
if not v.startswith("_comment")
),
),
),
(
ContentType.TEST_PLAYBOOK,
(
(test.playbookID for test in self.tests),
# self.skipped_tests.keys(), # not collecting skipped tests as this section preserves for skip reasons.
self.private_tests,
self.reputation_tests,
self.test_marketplacev2,
),
),
(
ContentType.SCRIPT,
(test.scripts for test in self.tests),
),
(
ContentType.PACK,
(self.nightly_packs,),
),
):
for id_source in id_sources:
result[content_type].update(filter(None, (always_iterable(id_source))))
return dict(result)
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
from pathlib import Path

import pytest

from demisto_sdk.scripts.validate_conf_json import ConfJsonValidator
from TestSuite.repo import Repo


def test_sanity(graph_repo: Repo):
"""
Given an empty repo with an empty conf.json
When calling validate
Then make sure it passes
"""
graph_repo.create_pack()
graph = graph_repo.create_graph()
validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
assert validator.validate()


@pytest.mark.parametrize("list_of_integrations", (True, False))
def test_integration_playbook_positive(
graph_repo: Repo, list_of_integrations: bool
) -> None:
"""
Given a repo with one integration and one playbook
When calling validate
Then make sure it passes
"""
pack = graph_repo.create_pack()
playbook = pack.create_test_playbook("SamplePlaybookTest")
integration = pack.create_integration()

graph_repo.conf.write_json(
tests=[
{
"playbookID": playbook.name,
"integrations": [integration.name]
if list_of_integrations
else integration.name,
}
]
)
graph = graph_repo.create_graph()
validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
assert validator._validate_content_exists()
assert validator.validate()


def test_integration_mistyped(graph_repo: Repo) -> None:
"""
Given a repo with one integration, whose name is mistyped in conf.json
When calling validate
Then make sure it fails
"""
pack = graph_repo.create_pack()
pack.create_test_playbook("SamplePlaybookTest")
pack.create_integration(name="foo")
graph_repo.conf.write_json(
tests=[
{
"playbookID": "SamplePlaybookTest",
"integrations": "FOO",
}
]
)
graph = graph_repo.create_graph()
validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
assert not validator._validate_content_exists()
assert not validator.validate()


def test_invalid_skipped_integration(graph_repo: Repo) -> None:
"""
Given a repo with one skipped integration configured, but doesn't exist in the repo
When calling validate
Then make sure it fails
"""
pack = graph_repo.create_pack()
pack.create_test_playbook("SamplePlaybookTest")
graph_repo.conf.write_json(skipped_integrations={"hello": "world"})
graph = graph_repo.create_graph()
validator = ConfJsonValidator(Path(graph_repo.conf.path), graph)
assert not validator.validate()
88 changes: 88 additions & 0 deletions demisto_sdk/scripts/validate_conf_json.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
from pathlib import Path
from typing import List, Optional, cast

from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
from demisto_sdk.commands.common.logger import logger, logging_setup
from demisto_sdk.commands.content_graph.commands.update import update_content_graph
from demisto_sdk.commands.content_graph.interface import ContentGraphInterface
from demisto_sdk.commands.content_graph.objects.base_content import UnknownContent
from demisto_sdk.commands.content_graph.objects.conf_json import ConfJSON
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class ConfJsonValidator:
def __init__(
self,
conf_json_path: Path = CONF_PATH,
graph: Optional[ContentGraphInterface] = None, # Pass None to generate
) -> None:
self.conf = ConfJSON.from_path(conf_json_path)

logger.info("Creating content graph - this may take a few minutes")
if graph is None:
update_content_graph(graph := ContentGraphInterface())
self.graph_ids_by_type = {
content_type: cast(
List[ContentItem],
[
item
for item in graph.search(
content_type=content_type, object_id=conf_ids
)
if not isinstance(
item, UnknownContent
) # UnknownContent items are artificially generated in relationships, are not part of the repo
],
)
for content_type, conf_ids in self.conf.linked_content_items.items()
}
logger.info(f"{self.graph_ids_by_type.keys()=}")

def _validate_content_exists(self) -> bool:
is_valid = True

for content_type, linked_ids in self.conf.linked_content_items.items():
if linked_ids_missing_in_graph := linked_ids.difference(
{
item.object_id
for item in self.graph_ids_by_type.get(content_type, ())
}
):
logger.error(
f"{len(linked_ids_missing_in_graph)} {content_type.value}s are not found in the graph: {','.join(sorted(linked_ids_missing_in_graph))}"
)
is_valid = False
return is_valid

# def _validate_content_not_deprecated(self) -> bool:
#
# This is commented out as we decided to not treat deprecated content as errorneous.
#
# is_valid = True
# for content_type, linked_ids in self.conf.linked_content_items.items():
# graph_deprecated_ids = {
# item.object_id
# for item in self.graph_ids_by_type.get(content_type, ())
# if item.deprecated
# }
# if linked_deprecated_ids := linked_ids.intersection(graph_deprecated_ids):
# logger.error(
# f"{len(linked_deprecated_ids)} {content_type.value}s are deprecated: {','.join(sorted(linked_deprecated_ids))}"
# )
# is_valid = False
# return is_valid
dorschw marked this conversation as resolved.
Show resolved Hide resolved

def validate(self) -> bool:
return self._validate_content_exists()


def main():
logging_setup()
if not ConfJsonValidator().validate():
logger.error("conf.json is not valid")
exit(1)
logger.info("[green]conf.json is valid[/green]")


if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
id: Cortex XDR Incident Handling
id: Cortex_XDR_Incident_Handling
version: -1
name: Cortex XDR Incident Handling
name: Cortex_XDR_Incident_Handling
description: Deprecated. Use "Cortex XDR Incident Handling v2" playbook instead.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is intended? because the example in the repo is with spaces.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spaces are allowed, but in the unit tests we need the underscores

This playbook is triggered by fetchinga Palo Alto Networks Cortex XDR
incident. \nThe playbook syncs and updates new XDR alerts that construct the incident.
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ update-additional-dependencies = "demisto_sdk.scripts.update_additional_dependen
sdk-changelog = "demisto_sdk.scripts.changelog.changelog:main"
merge-coverage-report = "demisto_sdk.scripts.merge_coverage_report:main"
merge-pytest-reports = "demisto_sdk.scripts.merge_pytest_reports:main"
validate-conf-json = "demisto_sdk.scripts.validate_conf_json:main"
init-validation = "demisto_sdk.scripts.init_validation_script:main"

[tool.poetry.dependencies]
Expand Down