From 4e131a37f8f4ab19c8947ce1613d47c4790b39a8 Mon Sep 17 00:00:00 2001 From: yedidyacohenpalo <162107504+yedidyacohenpalo@users.noreply.github.com> Date: Thu, 6 Jun 2024 08:57:48 +0300 Subject: [PATCH] validation rm101 in the new format (#4322) * validation rm101 in the new format * expected error msg fixed * fixed tests and find_regex_on_data moved to tools file * format * conflict solved * test doc string * doc string improved --- demisto_sdk/commands/common/constants.py | 4 ++ demisto_sdk/commands/common/tools.py | 18 +++++ .../validate/sdk_validation_config.toml | 3 +- .../validate/tests/RM_validators_test.py | 69 +++++++++++++++++++ .../RM101_is_image_path_valid.py | 62 +++++++++++++++++ 5 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py diff --git a/demisto_sdk/commands/common/constants.py b/demisto_sdk/commands/common/constants.py index f84e377ab6..177bf50b8e 100644 --- a/demisto_sdk/commands/common/constants.py +++ b/demisto_sdk/commands/common/constants.py @@ -2186,3 +2186,7 @@ class IncidentState(StrEnum): MarketplaceVersions.XSOAR.value, MarketplaceVersions.MarketplaceV2.value, ] + +INVALID_IMAGE_PATH_REGEX = ( + r"(\!\[.*?\]|src\=)(\(|\")(https://github.com/demisto/content/blob/.*?)(\)|\")" +) diff --git a/demisto_sdk/commands/common/tools.py b/demisto_sdk/commands/common/tools.py index 8551335868..35d7fee73e 100644 --- a/demisto_sdk/commands/common/tools.py +++ b/demisto_sdk/commands/common/tools.py @@ -4522,3 +4522,21 @@ def convert_path_to_str(data: Union[dict, list]): convert_path_to_str(item) elif isinstance(item, Path): data[index] = str(item) + + +def find_regex_on_data(data: str, regex: str): + """ + Finds all matches of a given regex pattern in the provided data. + + Args: + data (str): The string data to search within. + regex (str): The regex pattern to use for finding matches. + + Returns: + List[str]: A list of all matches found in the data. If no matches are found, returns an empty list. + """ + return re.findall( + regex, + data, + re.IGNORECASE, + ) diff --git a/demisto_sdk/commands/validate/sdk_validation_config.toml b/demisto_sdk/commands/validate/sdk_validation_config.toml index 39e9591503..37f075147b 100644 --- a/demisto_sdk/commands/validate/sdk_validation_config.toml +++ b/demisto_sdk/commands/validate/sdk_validation_config.toml @@ -33,8 +33,7 @@ select = [ "DO100", "DO101", "DO102", "DO103", "DO104", "DO105", "DO106", "DS100", "DS107", "SC100", "SC105", "SC106", "SC109", - "RM104", "RM105", "RM106", "RM109", "RM113", "RM114", - "RN103", + "RM101", "RN103", "RM104", "RM105", "RM106", "RM109", "RM113", "RM114", "CL100", "GF100", "GF101", "GF102", "IF100", "IF101", "IF102", "IF103", "IF104", "IF105", "IF106", "IF116", diff --git a/demisto_sdk/commands/validate/tests/RM_validators_test.py b/demisto_sdk/commands/validate/tests/RM_validators_test.py index 7bbb44d2d4..37787d9bad 100644 --- a/demisto_sdk/commands/validate/tests/RM_validators_test.py +++ b/demisto_sdk/commands/validate/tests/RM_validators_test.py @@ -9,6 +9,9 @@ create_playbook_object, create_script_object, ) +from demisto_sdk.commands.validate.validators.RM_validators.RM101_is_image_path_valid import ( + IsImagePathValidValidator, +) from demisto_sdk.commands.validate.validators.RM_validators.RM104_empty_readme import ( EmptyReadmeValidator, ) @@ -153,6 +156,72 @@ def test_empty_readme_validator( ) +@pytest.mark.parametrize( + "content_items, expected_number_of_failures", + [ + ([create_integration_object()], 0), + ( + [ + create_integration_object( + readme_content='Alt text' + ) + ], + 1, + ), + ( + [ + create_script_object( + readme_content='Alt text' + ) + ], + 1, + ), + ( + [ + create_pack_object( + readme_text='Alt text' + ) + ], + 1, + ), + ( + [ + create_playbook_object( + readme_content='Alt text' + ) + ], + 1, + ), + ], +) +def test_is_image_path_validator(content_items, expected_number_of_failures): + """ + Given: + - A list of content items with their respective readme contents. + When: + - The IsImagePathValidValidator is run on the provided content items. + - A content item with no images (expected failures: 0). + - A content item with a non-raw image URL in the readme (expected failures: 1). + - A script object with a non-raw image URL in the readme (expected failures: 1). + - A pack object with a non-raw image URL in the readme (expected failures: 1). + - A playbook object with a non-raw image URL in the readme (expected failures: 1). + + Then: + - Validate that the number of detected invalid image paths matches the expected number of failures. + - Ensure that each failure message correctly identifies the non-raw GitHub image URL and suggests the proper raw URL format. + """ + results = IsImagePathValidValidator().is_valid(content_items) + assert len(results) == expected_number_of_failures + assert all( + [ + result.message.endswith( + "detected the following images URLs which are not raw links: https://github.com/demisto/content/blob/path/to/image.jpg suggested URL https://github.com/demisto/content/raw/path/to/image.jpg" + ) + for result in results + ] + ) + + @pytest.mark.parametrize( "content_items, is_file_exist, expected_number_of_failures, expected_msgs", [ diff --git a/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py b/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py new file mode 100644 index 0000000000..886290b6e6 --- /dev/null +++ b/demisto_sdk/commands/validate/validators/RM_validators/RM101_is_image_path_valid.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Iterable, List, Union + +from demisto_sdk.commands.common.constants import INVALID_IMAGE_PATH_REGEX, GitStatuses +from demisto_sdk.commands.common.tools import find_regex_on_data +from demisto_sdk.commands.content_graph.objects.integration import Integration +from demisto_sdk.commands.content_graph.objects.pack import Pack +from demisto_sdk.commands.content_graph.objects.playbook import Playbook +from demisto_sdk.commands.content_graph.objects.script import Script +from demisto_sdk.commands.content_graph.parsers.related_files import RelatedFileType +from demisto_sdk.commands.validate.validators.base_validator import ( + BaseValidator, + ValidationResult, +) + +ContentTypes = Union[Integration, Script, Playbook, Pack] + + +class IsImagePathValidValidator(BaseValidator[ContentTypes]): + error_code = "RM101" + description = "Validate images absolute paths, and prints the suggested path if it's not valid." + rationale = "In official marketplace content, ensures that the images can be used in the upload flow properly." + error_message = "In {filename} detected the following images URLs which are not raw links: {params}" + related_field = "readme" + is_auto_fixable = False + expected_git_statuses = [GitStatuses.ADDED, GitStatuses.MODIFIED] + related_file_type = [RelatedFileType.README] + + def is_valid(self, content_items: Iterable[ContentTypes]) -> List[ValidationResult]: + return [ + ValidationResult( + validator=self, + message=self.error_message.format(filename=params[0], params=params[1]), + content_object=content_item, + ) + for content_item in content_items + if (params := self.is_image_path_valid(content_item)) + ] + + def is_image_path_valid(self, content_item): + if invalid_paths := find_regex_on_data( + content_item.readme.file_content, INVALID_IMAGE_PATH_REGEX + ): + handled_errors = [] + for path in invalid_paths: + path = path[2] + alternative_path = path.replace("blob", "raw") + handled_errors.append((path, alternative_path)) + return self.format_error_message(handled_errors, content_item) + return "" + + def format_error_message(self, handled_errors, content_item): + urls = "\n".join( + f"{error[0]} suggested URL {error[1]}" for error in handled_errors + ) + filename = "/".join( + content_item.readme.file_path.parts[ + content_item.readme.file_path.parts.index("Packs") : + ] + ) + return (filename, urls)