From 4e131a37f8f4ab19c8947ce1613d47c4790b39a8 Mon Sep 17 00:00:00 2001
From: yedidyacohenpalo <162107504+yedidyacohenpalo@users.noreply.github.com>
Date: Thu, 6 Jun 2024 08:57:48 +0300
Subject: [PATCH] validation rm101 in the new format (#4322)
* validation rm101 in the new format
* expected error msg fixed
* fixed tests and find_regex_on_data moved to tools file
* format
* conflict solved
* test doc string
* doc string improved
---
demisto_sdk/commands/common/constants.py | 4 ++
demisto_sdk/commands/common/tools.py | 18 +++++
.../validate/sdk_validation_config.toml | 3 +-
.../validate/tests/RM_validators_test.py | 69 +++++++++++++++++++
.../RM101_is_image_path_valid.py | 62 +++++++++++++++++
5 files changed, 154 insertions(+), 2 deletions(-)
create mode 100644 demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py
diff --git a/demisto_sdk/commands/common/constants.py b/demisto_sdk/commands/common/constants.py
index f84e377ab6..177bf50b8e 100644
--- a/demisto_sdk/commands/common/constants.py
+++ b/demisto_sdk/commands/common/constants.py
@@ -2186,3 +2186,7 @@ class IncidentState(StrEnum):
MarketplaceVersions.XSOAR.value,
MarketplaceVersions.MarketplaceV2.value,
]
+
+INVALID_IMAGE_PATH_REGEX = (
+ r"(\!\[.*?\]|src\=)(\(|\")(https://github.com/demisto/content/blob/.*?)(\)|\")"
+)
diff --git a/demisto_sdk/commands/common/tools.py b/demisto_sdk/commands/common/tools.py
index 8551335868..35d7fee73e 100644
--- a/demisto_sdk/commands/common/tools.py
+++ b/demisto_sdk/commands/common/tools.py
@@ -4522,3 +4522,21 @@ def convert_path_to_str(data: Union[dict, list]):
convert_path_to_str(item)
elif isinstance(item, Path):
data[index] = str(item)
+
+
+def find_regex_on_data(data: str, regex: str):
+ """
+ Finds all matches of a given regex pattern in the provided data.
+
+ Args:
+ data (str): The string data to search within.
+ regex (str): The regex pattern to use for finding matches.
+
+ Returns:
+ List[str]: A list of all matches found in the data. If no matches are found, returns an empty list.
+ """
+ return re.findall(
+ regex,
+ data,
+ re.IGNORECASE,
+ )
diff --git a/demisto_sdk/commands/validate/sdk_validation_config.toml b/demisto_sdk/commands/validate/sdk_validation_config.toml
index 39e9591503..37f075147b 100644
--- a/demisto_sdk/commands/validate/sdk_validation_config.toml
+++ b/demisto_sdk/commands/validate/sdk_validation_config.toml
@@ -33,8 +33,7 @@ select = [
"DO100", "DO101", "DO102", "DO103", "DO104", "DO105", "DO106",
"DS100", "DS107",
"SC100", "SC105", "SC106", "SC109",
- "RM104", "RM105", "RM106", "RM109", "RM113", "RM114",
- "RN103",
+ "RM101", "RN103", "RM104", "RM105", "RM106", "RM109", "RM113", "RM114",
"CL100",
"GF100", "GF101", "GF102",
"IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106", "IF116",
diff --git a/demisto_sdk/commands/validate/tests/RM_validators_test.py b/demisto_sdk/commands/validate/tests/RM_validators_test.py
index 7bbb44d2d4..37787d9bad 100644
--- a/demisto_sdk/commands/validate/tests/RM_validators_test.py
+++ b/demisto_sdk/commands/validate/tests/RM_validators_test.py
@@ -9,6 +9,9 @@
create_playbook_object,
create_script_object,
)
+from demisto_sdk.commands.validate.validators.RM_validators.RM101_is_image_path_valid import (
+ IsImagePathValidValidator,
+)
from demisto_sdk.commands.validate.validators.RM_validators.RM104_empty_readme import (
EmptyReadmeValidator,
)
@@ -153,6 +156,72 @@ def test_empty_readme_validator(
)
+@pytest.mark.parametrize(
+ "content_items, expected_number_of_failures",
+ [
+ ([create_integration_object()], 0),
+ (
+ [
+ create_integration_object(
+ readme_content='
'
+ )
+ ],
+ 1,
+ ),
+ (
+ [
+ create_script_object(
+ readme_content='
'
+ )
+ ],
+ 1,
+ ),
+ (
+ [
+ create_pack_object(
+ readme_text='
'
+ )
+ ],
+ 1,
+ ),
+ (
+ [
+ create_playbook_object(
+ readme_content='
'
+ )
+ ],
+ 1,
+ ),
+ ],
+)
+def test_is_image_path_validator(content_items, expected_number_of_failures):
+ """
+ Given:
+ - A list of content items with their respective readme contents.
+ When:
+ - The IsImagePathValidValidator is run on the provided content items.
+ - A content item with no images (expected failures: 0).
+ - A content item with a non-raw image URL in the readme (expected failures: 1).
+ - A script object with a non-raw image URL in the readme (expected failures: 1).
+ - A pack object with a non-raw image URL in the readme (expected failures: 1).
+ - A playbook object with a non-raw image URL in the readme (expected failures: 1).
+
+ Then:
+ - Validate that the number of detected invalid image paths matches the expected number of failures.
+ - Ensure that each failure message correctly identifies the non-raw GitHub image URL and suggests the proper raw URL format.
+ """
+ results = IsImagePathValidValidator().is_valid(content_items)
+ assert len(results) == expected_number_of_failures
+ assert all(
+ [
+ result.message.endswith(
+ "detected the following images URLs which are not raw links: https://github.com/demisto/content/blob/path/to/image.jpg suggested URL https://github.com/demisto/content/raw/path/to/image.jpg"
+ )
+ for result in results
+ ]
+ )
+
+
@pytest.mark.parametrize(
"content_items, is_file_exist, expected_number_of_failures, expected_msgs",
[
diff --git a/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py b/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py
new file mode 100644
index 0000000000..886290b6e6
--- /dev/null
+++ b/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py
@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Union
+
+from demisto_sdk.commands.common.constants import INVALID_IMAGE_PATH_REGEX, GitStatuses
+from demisto_sdk.commands.common.tools import find_regex_on_data
+from demisto_sdk.commands.content_graph.objects.integration import Integration
+from demisto_sdk.commands.content_graph.objects.pack import Pack
+from demisto_sdk.commands.content_graph.objects.playbook import Playbook
+from demisto_sdk.commands.content_graph.objects.script import Script
+from demisto_sdk.commands.content_graph.parsers.related_files import RelatedFileType
+from demisto_sdk.commands.validate.validators.base_validator import (
+ BaseValidator,
+ ValidationResult,
+)
+
+ContentTypes = Union[Integration, Script, Playbook, Pack]
+
+
+class IsImagePathValidValidator(BaseValidator[ContentTypes]):
+ error_code = "RM101"
+ description = "Validate images absolute paths, and prints the suggested path if it's not valid."
+ rationale = "In official marketplace content, ensures that the images can be used in the upload flow properly."
+ error_message = "In {filename} detected the following images URLs which are not raw links: {params}"
+ related_field = "readme"
+ is_auto_fixable = False
+ expected_git_statuses = [GitStatuses.ADDED, GitStatuses.MODIFIED]
+ related_file_type = [RelatedFileType.README]
+
+ def is_valid(self, content_items: Iterable[ContentTypes]) -> List[ValidationResult]:
+ return [
+ ValidationResult(
+ validator=self,
+ message=self.error_message.format(filename=params[0], params=params[1]),
+ content_object=content_item,
+ )
+ for content_item in content_items
+ if (params := self.is_image_path_valid(content_item))
+ ]
+
+ def is_image_path_valid(self, content_item):
+ if invalid_paths := find_regex_on_data(
+ content_item.readme.file_content, INVALID_IMAGE_PATH_REGEX
+ ):
+ handled_errors = []
+ for path in invalid_paths:
+ path = path[2]
+ alternative_path = path.replace("blob", "raw")
+ handled_errors.append((path, alternative_path))
+ return self.format_error_message(handled_errors, content_item)
+ return ""
+
+ def format_error_message(self, handled_errors, content_item):
+ urls = "\n".join(
+ f"{error[0]} suggested URL {error[1]}" for error in handled_errors
+ )
+ filename = "/".join(
+ content_item.readme.file_path.parts[
+ content_item.readme.file_path.parts.index("Packs") :
+ ]
+ )
+ return (filename, urls)