Skip to content

Commit

Permalink
Adding the defaultDataSource field to the pack metadata (#4139)
Browse files Browse the repository at this point in the history 
* Add validation

* Change to reflect prior logic

* Merge remote-tracking branch 'origin/datasource-feature' into datasource-feature

# Conflicts:
#	.pre-commit-config.yaml

* fix conflicts

* Fix UT

* rename defaultDataSourceName

* add defaultDataSourceName to metadata

* fix validation

* precommit

* add is valid default datasource name validator

* add integration doc

* add validations to lists

* fix PA131

* add fix for PA131

* raise Exception in fix PA131

* fix PA132

* more logs

* add data source id

* run pre-commit

* support contribution display name

* handle contrib display name

* enhance get_valid_data_source_integrations

* add is_data_source to integration

* fix contrib name

* post-demo - keep default

* logs

* change defaultDataSource to dict

* change field to str

* remove Field

* exclude Field

* exclude Field

* add UT

* CR

* fix UTs

* fix field alias

* remove exclude field

* remove Field

* exclude name

* CR error

* change to error error

* Field alias

* rename fields

* revert _

* deal with XSOAR

* add marketplace

* fix generate docs

* fix generate docs

---------

Authored-by: BEAdi <72088126+BEAdi@users.noreply.github.com>
  • Loading branch information
@BEAdi
BEAdi committed May 28, 2024
1 parent 97cb746 commit 9ea32e5
Show file tree
Hide file tree
Showing 14 changed files with 904 additions and 26 deletions.
4 changes: 4 additions & 0 deletions .changelog/4139.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
changes:
- description: Added the `defaultDataSource` field to the pack metadata.
type: feature
pr_number: 4139
288 changes: 288 additions & 0 deletions ...dk/commands/common/content/tests/objects/pack_objects/pack_metadata/pack_metadata_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,13 @@
from demisto_sdk.commands.common.content.objects.pack_objects import PackMetaData
from demisto_sdk.commands.common.content.objects.pack_objects.pack import Pack
from demisto_sdk.commands.common.content.objects_factory import path_to_pack_object
from demisto_sdk.commands.common.docker.docker_image import DockerImage
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import src_root
from demisto_sdk.commands.content_graph.common import (
ContentType,
)
from demisto_sdk.commands.content_graph.objects.integration import Command, Integration
from demisto_sdk.commands.content_graph.objects.pack_content_items import (
PackContentItems,
)
Expand Down Expand Up @@ -530,3 +535,286 @@ def test_replace_item_if_has_higher_toversion(
integration, content_item_metadata, integration.summary(), marketplace
)
assert content_item_metadata["toversion"] == expected_toversion


def mock_integration_for_data_source(
name,
display_name,
is_fetch=False,
is_fetch_events=False,
is_remote_sync_in=False,
is_fetch_samples=False,
is_feed=False,
deprecated=False,
marketplaces=MarketplaceVersions.MarketplaceV2,
path=Path("Packs"),
):
if not isinstance(marketplaces, list):
marketplaces = [marketplaces]
return Integration(
id=name,
content_type=ContentType.INTEGRATION,
node_id=f"{ContentType.INTEGRATION}:{name}",
path=path,
fromversion="5.0.0",
toversion="99.99.99",
display_name=display_name,
name=name,
marketplaces=marketplaces,
deprecated=deprecated,
type="python3",
docker_image=DockerImage("demisto/python3:3.10.11.54799"),
category="blabla",
commands=[Command(name="test-command", description="")],
is_fetch=is_fetch,
is_fetch_events=is_fetch_events,
is_remote_sync_in=is_remote_sync_in,
is_fetch_samples=is_fetch_samples,
is_feed=is_feed,
)


@pytest.mark.parametrize(
"support_level, include_name, expected_output",
(
(
None,
False,
[
"Partner Contribution Same Name",
"partner_contribution_different_name",
"regular_integration_different_name",
"Samples Fetch",
"Some Mirroring",
],
),
(
"partner",
False,
[
"Partner Contribution Same Name",
"partner_contribution_different_name",
"regular_integration_different_name",
"Samples Fetch",
"Some Mirroring",
],
),
(
None,
True,
[
{
"id": "Partner Contribution Same Name",
"name": "Partner Contribution Same Name (Partner Contribution)",
},
{
"id": "partner_contribution_different_name",
"name": "Partner Contribution Different Name (Partner Contribution)",
},
{
"id": "regular_integration_different_name",
"name": "Regular Integration Different Name",
},
{"id": "Samples Fetch", "name": "Samples Fetch"},
{"id": "Some Mirroring", "name": "Some Mirroring"},
],
),
(
"partner",
True,
[
{
"id": "Partner Contribution Same Name",
"name": "Partner Contribution Same Name",
},
{
"id": "partner_contribution_different_name",
"name": "Partner Contribution Different Name",
},
{
"id": "regular_integration_different_name",
"name": "Regular Integration Different Name",
},
{"id": "Samples Fetch", "name": "Samples Fetch"},
{"id": "Some Mirroring", "name": "Some Mirroring"},
],
),
),
)
def test_get_valid_data_source_integrations(
support_level, include_name, expected_output
):
"""
Given:
- Support level and whether to include the name
When:
- Getting valid data source integrations for a pack
Then:
- The correct data source integration return, with the expected name
"""
integrations = [
mock_integration_for_data_source(
"Partner Contribution Same Name",
"Partner Contribution Same Name (Partner Contribution)",
is_fetch=True,
),
mock_integration_for_data_source(
"partner_contribution_different_name",
"Partner Contribution Different Name (Partner Contribution)",
is_fetch_events=True,
),
mock_integration_for_data_source(
"regular_integration_different_name",
"Regular Integration Different Name",
is_fetch=True,
),
mock_integration_for_data_source(
"Not Fetching Integration", "Not Fetching Integration", is_fetch=False
),
mock_integration_for_data_source(
"Deprecated Integration",
"Deprecated Integration",
is_fetch=True,
deprecated=True,
),
mock_integration_for_data_source(
"Not XSIAM Integration",
"Not XSIAM Integration",
is_fetch=True,
marketplaces=MarketplaceVersions.XSOAR_ON_PREM,
),
mock_integration_for_data_source(
"Some Feed", "Some Feed", is_fetch=True, is_feed=True
),
mock_integration_for_data_source(
"Samples Fetch", "Samples Fetch", is_fetch_samples=True
),
mock_integration_for_data_source(
"Some Mirroring", "Some Mirroring", is_remote_sync_in=True
),
]

content_items = PackContentItems()
content_items.integration.extend(integrations)

result = PackMetadata.get_valid_data_source_integrations(
content_items, support_level, include_name
)
assert result == expected_output


@pytest.mark.parametrize(
"support, given_default_data_source_id, integrations, expected_default_data_source",
(
(
"partner",
"partner_support",
[
mock_integration_for_data_source(
"partner_support",
"Partner Support (Partner Contribution)",
is_fetch=True,
),
mock_integration_for_data_source(
"other_partner_support",
"Other Partner Support (Partner Contribution)",
is_fetch=True,
),
],
{"id": "partner_support", "name": "Partner Support"},
),
(
"xsoar",
"xsoar_support",
[
mock_integration_for_data_source(
"xsoar_support", "XSOAR Support", is_fetch=True
),
mock_integration_for_data_source(
"Other XSOAR Support", "Other XSOAR Support", is_fetch=True
),
],
{"id": "xsoar_support", "name": "XSOAR Support"},
),
(
"xsoar",
None,
[
mock_integration_for_data_source(
"One Fetching Integration",
"One Fetching Integration",
is_fetch=True,
),
mock_integration_for_data_source(
"Not Fetching Integration",
"Not Fetching Integration",
is_fetch=False,
),
mock_integration_for_data_source(
"Deprecated Integration",
"Deprecated Integration",
is_fetch=True,
deprecated=True,
),
mock_integration_for_data_source(
"Feed Integration", "Feed Integration", is_fetch=True, is_feed=True
),
],
{"id": "One Fetching Integration", "name": "One Fetching Integration"},
),
),
)
def test_set_default_data_source(
support, given_default_data_source_id, integrations, expected_default_data_source
):
"""
Given:
- Support level, default data source name and id, pack integrations names and ids
When:
- Setting a default data source to a pack
Then:
- The correct data source integration is set
"""
content_items, my_instance = mock_pack_metadata_for_data_source(
support=support,
default_data_source=given_default_data_source_id,
integrations=integrations,
)

my_instance._set_default_data_source(content_items)
assert my_instance.default_data_source_id == expected_default_data_source.get("id")
assert my_instance.default_data_source_name == expected_default_data_source.get(
"name"
)


def mock_pack_metadata_for_data_source(support, default_data_source, integrations):
my_instance = PackMetadata(
name="test",
display_name="",
description="",
created="",
legacy=False,
support=support,
url="",
email="",
eulaLink="",
price=0,
hidden=False,
commit="",
downloads=0,
keywords=[],
searchRank=0,
excludedDependencies=[],
videos=[],
modules=[],
default_data_source_id=default_data_source,
) # type: ignore

content_items = PackContentItems()
content_items.integration.extend(integrations)
return content_items, my_instance
16 changes: 16 additions & 0 deletions demisto_sdk/commands/content_graph/objects/integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,10 @@ class Integration(IntegrationScript, content_type=ContentType.INTEGRATION): # t
is_fetch_events: bool = Field(False, alias="isfetchevents")
is_fetch_assets: bool = Field(False, alias="isfetchassets")
is_fetch_events_and_assets: bool = False
is_fetch_samples: bool = False
is_feed: bool = False
is_beta: bool = False
is_remote_sync_in: bool = False
is_mappable: bool = False
long_running: bool = False
category: str
Expand Down Expand Up @@ -218,3 +220,17 @@ def light_svg(self) -> LightSVGRelatedFile:
@cached_property
def image(self) -> ImageRelatedFile:
return ImageRelatedFile(self.path, git_sha=self.git_sha)

def is_data_source(self):
return (
MarketplaceVersions.MarketplaceV2 in self.marketplaces
and not self.deprecated
and not self.is_feed
and (
self.is_fetch
or self.is_fetch_events
or self.is_remote_sync_in
or self.is_fetch_events_and_assets
or self.is_fetch_samples
)
)
Loading

0 comments on commit 9ea32e5

Please sign in to comment.