From 0f2631a3757ab35a09c97ef9ffb29d52b8ebb45f Mon Sep 17 00:00:00 2001 From: sberman Date: Sun, 12 Apr 2020 12:33:20 +0300 Subject: [PATCH 01/23] Removed validation for price in pack metadata --- .../common/hook_validations/pack_unique_files.py | 14 +++++++------- .../common/tests/pack_metadata_validator_test.py | 2 +- demisto_sdk/commands/init/initiator.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index fbd9b8a3a1..5369f2527c 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -11,7 +11,7 @@ PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, - PACK_METADATA_PRICE, + # PACK_METADATA_PRICE, PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, @@ -151,12 +151,12 @@ def _is_pack_meta_file_structure_valid(self): if not isinstance(dependencies_field, dict): self._add_error('The dependencies field in the pack must be a dictionary.') return False - price_field = metadata[PACK_METADATA_PRICE] - try: - int(price_field) - except Exception: - self._add_error('The price field in the pack must be a number.') - return False + # price_field = metadata[PACK_METADATA_PRICE] + # try: + # int(price_field) + # except Exception: + # self._add_error('The price field in the pack must be a number.') + # return False for list_field in (PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_CATEGORIES, PACK_METADATA_USE_CASES): field = metadata[list_field] diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index bdf5103598..b803206197 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -21,7 +21,7 @@ def test_metadata_validator_valid(self, mocker): assert validator.validate_pack_meta_file() @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), - os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), + # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), os.path.join(FILES_PATH, 'pack_metadata_list.json')]) def test_metadata_validator_invalid(self, mocker, metadata): diff --git a/demisto_sdk/commands/init/initiator.py b/demisto_sdk/commands/init/initiator.py index a97994104a..8ea9bc26bf 100644 --- a/demisto_sdk/commands/init/initiator.py +++ b/demisto_sdk/commands/init/initiator.py @@ -199,7 +199,7 @@ def create_metadata(fill_manually): 'certification': 'certified', 'useCases': [], 'keywords': [], - 'price': '0', + # 'price': '0', 'dependencies': {}, } @@ -225,8 +225,8 @@ def create_metadata(fill_manually): tags_list = [t.strip() for t in tags.split(',')] metadata['tags'] = tags_list - price = input("\nThe price of the pack: ") - metadata['price'] = price + # price = input("\nThe price of the pack: ") + # metadata['price'] = price return metadata From cfdbcef94525e4cfd0ce4bbed6d8490d9120d1c7 Mon Sep 17 00:00:00 2001 From: sberman Date: Sun, 12 Apr 2020 12:45:06 +0300 Subject: [PATCH 02/23] Fixing isort --- .../hook_validations/pack_unique_files.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 5369f2527c..321ec1b16a 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,18 +6,12 @@ import os import re -from demisto_sdk.commands.common.constants import (API_MODULES_PACK, - PACK_METADATA_CATEGORIES, - PACK_METADATA_DEPENDENCIES, - PACK_METADATA_FIELDS, - PACK_METADATA_KEYWORDS, - # PACK_METADATA_PRICE, - PACK_METADATA_TAGS, - PACK_METADATA_USE_CASES, - PACKS_PACK_IGNORE_FILE_NAME, - PACKS_PACK_META_FILE_NAME, - PACKS_README_FILE_NAME, - PACKS_WHITELIST_FILE_NAME) +from demisto_sdk.commands.common.constants import ( # PACK_METADATA_PRICE, + API_MODULES_PACK, PACK_METADATA_CATEGORIES, PACK_METADATA_DEPENDENCIES, + PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, + PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, + PACKS_PACK_META_FILE_NAME, PACKS_README_FILE_NAME, + PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path From 7ba18f4cfb24863afab260f76aaf6de3295d8ba2 Mon Sep 17 00:00:00 2001 From: sberman Date: Sun, 12 Apr 2020 12:53:13 +0300 Subject: [PATCH 03/23] Fixed test --- demisto_sdk/commands/init/tests/initiator_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/demisto_sdk/commands/init/tests/initiator_test.py b/demisto_sdk/commands/init/tests/initiator_test.py index 0b14f31f4f..170418b6af 100644 --- a/demisto_sdk/commands/init/tests/initiator_test.py +++ b/demisto_sdk/commands/init/tests/initiator_test.py @@ -17,7 +17,7 @@ PACK_URL = 'https://www.github.com/pack' PACK_EMAIL = 'author@mail.com' PACK_TAGS = 'Tag1,Tag2' -PACK_PRICE = '0' +# PACK_PRICE = '0' @pytest.fixture @@ -82,7 +82,7 @@ def test_create_metadata(monkeypatch, initiator): 'certification': 'certified', 'useCases': [], 'keywords': [], - 'price': '0', + # 'price': '0', 'dependencies': {}, } @@ -92,7 +92,8 @@ def test_create_metadata(monkeypatch, initiator): generate_multiple_inputs( deque([ PACK_NAME, PACK_DESC, '1', PACK_SERVER_MIN_VERSION, PACK_AUTHOR, - PACK_URL, PACK_EMAIL, '1', PACK_TAGS, PACK_PRICE + PACK_URL, PACK_EMAIL, '1', PACK_TAGS + # PACK_PRICE ]) ) ) @@ -109,7 +110,7 @@ def test_create_metadata(monkeypatch, initiator): 'email': PACK_EMAIL, 'keywords': [], 'name': PACK_NAME, - 'price': PACK_PRICE, + # 'price': PACK_PRICE, 'serverMinVersion': PACK_SERVER_MIN_VERSION, 'support': PACK_SUPPORT_OPTIONS[0], 'tags': ['Tag1', 'Tag2'], From 47f8d3a1a96c3089924d1144e9440afa29e978c8 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 11:05:03 +0300 Subject: [PATCH 04/23] added TODO comments --- .../commands/common/hook_validations/pack_unique_files.py | 1 + .../commands/common/tests/pack_metadata_validator_test.py | 1 + demisto_sdk/commands/init/initiator.py | 2 +- demisto_sdk/commands/init/tests/initiator_test.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 321ec1b16a..6bafc6a313 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -145,6 +145,7 @@ def _is_pack_meta_file_structure_valid(self): if not isinstance(dependencies_field, dict): self._add_error('The dependencies field in the pack must be a dictionary.') return False + # TODO: add it back after #23546 is ready. # price_field = metadata[PACK_METADATA_PRICE] # try: # int(price_field) diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index b803206197..41d84cb56b 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -20,6 +20,7 @@ def test_metadata_validator_valid(self, mocker): validator = PackUniqueFilesValidator('fake') assert validator.validate_pack_meta_file() + # TODO: add the validation for price after #23546 is ready. @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), diff --git a/demisto_sdk/commands/init/initiator.py b/demisto_sdk/commands/init/initiator.py index ade64ce531..cf73db4041 100644 --- a/demisto_sdk/commands/init/initiator.py +++ b/demisto_sdk/commands/init/initiator.py @@ -232,7 +232,7 @@ def create_metadata(fill_manually: bool) -> Dict: tags = input("\nTags of the pack, comma separated values: ") tags_list = [t.strip() for t in tags.split(',')] metadata['tags'] = tags_list - + # TODO: add it back after #23546 is ready. # price = input("\nThe price of the pack: ") # metadata['price'] = price diff --git a/demisto_sdk/commands/init/tests/initiator_test.py b/demisto_sdk/commands/init/tests/initiator_test.py index 170418b6af..69f4ed414e 100644 --- a/demisto_sdk/commands/init/tests/initiator_test.py +++ b/demisto_sdk/commands/init/tests/initiator_test.py @@ -61,6 +61,7 @@ def test_get_object_id(monkeypatch, initiator): assert initiator.id == 'SomeIntegrationID' +# TODO: add the validation for price after #23546 is ready. def test_create_metadata(monkeypatch, initiator): # test create_metadata without user filling manually pack_metadata = initiator.create_metadata(False) From d96dedc8fb95fd34844bdbf96e6d5ae9e0e3077b Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 15:23:47 +0300 Subject: [PATCH 05/23] added another test --- .../tests/pack_metadata_validator_test.py | 1 + .../pack_metadata_invalid_dependencies.json | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index 41d84cb56b..e2ceb25d17 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -23,6 +23,7 @@ def test_metadata_validator_valid(self, mocker): # TODO: add the validation for price after #23546 is ready. @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), + os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), os.path.join(FILES_PATH, 'pack_metadata_list.json')]) def test_metadata_validator_invalid(self, mocker, metadata): diff --git a/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json new file mode 100644 index 0000000000..04dee0995e --- /dev/null +++ b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json @@ -0,0 +1,29 @@ +{ + "name": "AWS Feed", + "description": "Indicators feed from AWS", + "support": "Cortex XSOAR", + "serverMinVersion": "5.5.0", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Data Enrichment & Threat Intelligence" + ], + "tags": [""], + "created": "2020-03-09T16:04:45Z", + "beta": false, + "deprecated": false, + "certification": "certified", + "useCases": [], + "keywords": ["AWS", "Feed"], + "price": 0, + "dependencies": { + "Base": { + "mandatory": true, + "minVersion": "1.0.0", + "name": "Base", + "certification": "certified", + } + } +} From 878b522936863c367f8d1dd1af3095480fef4f37 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 15:29:22 +0300 Subject: [PATCH 06/23] Fixed json --- .../tests/test_files/pack_metadata_invalid_dependencies.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json index 04dee0995e..92fa79f74d 100644 --- a/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json +++ b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json @@ -23,7 +23,7 @@ "mandatory": true, "minVersion": "1.0.0", "name": "Base", - "certification": "certified", + "certification": "certified" } } } From 36293ce249e0270313d526340ac196182573d18c Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 15:55:20 +0300 Subject: [PATCH 07/23] - --- .../tests/test_files/pack_metadata_invalid_dependencies.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json index 92fa79f74d..a49031aebe 100644 --- a/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json +++ b/demisto_sdk/tests/test_files/pack_metadata_invalid_dependencies.json @@ -16,7 +16,7 @@ "deprecated": false, "certification": "certified", "useCases": [], - "keywords": ["AWS", "Feed"], + "keywords": ["AWS", "Feed", "test"], "price": 0, "dependencies": { "Base": { From 164e58841a3c9836bc7be0b122df79093336e73a Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 16:10:15 +0300 Subject: [PATCH 08/23] one more test --- .../tests/pack_metadata_validator_test.py | 1 + .../pack_metadata_list_dependencies.json | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index e2ceb25d17..786aa88a80 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -24,6 +24,7 @@ def test_metadata_validator_valid(self, mocker): @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), + os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), os.path.join(FILES_PATH, 'pack_metadata_list.json')]) def test_metadata_validator_invalid(self, mocker, metadata): diff --git a/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json new file mode 100644 index 0000000000..e295bd03de --- /dev/null +++ b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json @@ -0,0 +1,30 @@ +{ + "name": "AWS Feed", + "description": "Indicators feed from AWS", + "support": "Cortex XSOAR", + "serverMinVersion": "5.5.0", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Data Enrichment & Threat Intelligence" + ], + "tags": [], + "created": "2020-03-09T16:04:45Z", + "beta": false, + "deprecated": false, + "certification": "certified", + "useCases": [], + "keywords": ["AWS", "Feed"], + "price": 0, + "dependencies": { + "Base": [{ + "mandatory": true, + "minVersion": "1.0.0", + "name": "Base", + "certification": "certified", + "author": "Cortex XSOAR" + } + }] +} From 70434875bb79b51cb108b4879e7a6d237657624a Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 16:15:08 +0300 Subject: [PATCH 09/23] Fixed json --- .../tests/test_files/pack_metadata_list_dependencies.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json index e295bd03de..c6bf8a4ca6 100644 --- a/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json +++ b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json @@ -25,6 +25,6 @@ "name": "Base", "certification": "certified", "author": "Cortex XSOAR" - } - }] + }] + } } From 04d047284d1712961437a1751bbacef6a67f2591 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 16:23:30 +0300 Subject: [PATCH 10/23] - --- .../tests/test_files/pack_metadata_list_dependencies.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json index c6bf8a4ca6..8880bcf135 100644 --- a/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json +++ b/demisto_sdk/tests/test_files/pack_metadata_list_dependencies.json @@ -18,13 +18,13 @@ "useCases": [], "keywords": ["AWS", "Feed"], "price": 0, - "dependencies": { - "Base": [{ + "dependencies": [{ + "Base": { "mandatory": true, "minVersion": "1.0.0", "name": "Base", "certification": "certified", "author": "Cortex XSOAR" - }] - } + } + }] } From b9e9dae9b5e69d4b3a8d299d6e252b4bc8a15004 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 16:31:09 +0300 Subject: [PATCH 11/23] And another test --- .../tests/pack_metadata_validator_test.py | 1 + .../pack_metadata_empty_category.json | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 demisto_sdk/tests/test_files/pack_metadata_empty_category.json diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index 786aa88a80..422fa3322c 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -25,6 +25,7 @@ def test_metadata_validator_valid(self, mocker): # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'), + os.path.join(FILES_PATH, 'pack_metadata_empty_category.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), os.path.join(FILES_PATH, 'pack_metadata_list.json')]) def test_metadata_validator_invalid(self, mocker, metadata): diff --git a/demisto_sdk/tests/test_files/pack_metadata_empty_category.json b/demisto_sdk/tests/test_files/pack_metadata_empty_category.json new file mode 100644 index 0000000000..5f4b27b978 --- /dev/null +++ b/demisto_sdk/tests/test_files/pack_metadata_empty_category.json @@ -0,0 +1,28 @@ +{ + "name": "AWS Feed", + "description": "Indicators feed from AWS", + "support": "Cortex XSOAR", + "serverMinVersion": "5.5.0", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [""], + "tags": [], + "created": "2020-03-09T16:04:45Z", + "beta": false, + "deprecated": false, + "certification": "certified", + "useCases": [], + "keywords": ["AWS", "Feed"], + "price": 0, + "dependencies": { + "Base": { + "mandatory": true, + "minVersion": "1.0.0", + "name": "Base", + "certification": "certified", + "author": "Cortex XSOAR" + } + } +} From f1dd8f5c71027e232ea574b632df21ee8acc3aa2 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 16:59:39 +0300 Subject: [PATCH 12/23] Added back the price, but it will not validate on it if not given --- .../hook_validations/pack_unique_files.py | 26 ++++++++++--------- .../tests/pack_metadata_validator_test.py | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 6bafc6a313..02b4e587c2 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,12 +6,11 @@ import os import re -from demisto_sdk.commands.common.constants import ( # PACK_METADATA_PRICE, - API_MODULES_PACK, PACK_METADATA_CATEGORIES, PACK_METADATA_DEPENDENCIES, - PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, - PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, - PACKS_PACK_META_FILE_NAME, PACKS_README_FILE_NAME, - PACKS_WHITELIST_FILE_NAME) +from demisto_sdk.commands.common.constants import (PACK_METADATA_PRICE, API_MODULES_PACK, PACK_METADATA_CATEGORIES, + PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, + PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, + PACKS_PACK_IGNORE_FILE_NAME, PACKS_PACK_META_FILE_NAME, + PACKS_README_FILE_NAME, PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path @@ -146,12 +145,15 @@ def _is_pack_meta_file_structure_valid(self): self._add_error('The dependencies field in the pack must be a dictionary.') return False # TODO: add it back after #23546 is ready. - # price_field = metadata[PACK_METADATA_PRICE] - # try: - # int(price_field) - # except Exception: - # self._add_error('The price field in the pack must be a number.') - # return False + price_field = metadata.get(PACK_METADATA_PRICE) + if price_field: + try: + int(price_field) + except Exception: + self._add_error('The price field in the pack must be a number.') + return False + else: + pass for list_field in (PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_CATEGORIES, PACK_METADATA_USE_CASES): field = metadata[list_field] diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index 422fa3322c..c6e6d4a893 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -22,7 +22,7 @@ def test_metadata_validator_valid(self, mocker): # TODO: add the validation for price after #23546 is ready. @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), - # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), + os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_empty_category.json'), From 1fc49e8131fc5b1cf3cc2ed39c3c102ef8b70ebd Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:01:59 +0300 Subject: [PATCH 13/23] Removed comment --- .../commands/common/hook_validations/pack_unique_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 02b4e587c2..db846a577a 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -144,7 +144,6 @@ def _is_pack_meta_file_structure_valid(self): if not isinstance(dependencies_field, dict): self._add_error('The dependencies field in the pack must be a dictionary.') return False - # TODO: add it back after #23546 is ready. price_field = metadata.get(PACK_METADATA_PRICE) if price_field: try: From 01a98444d00878b235bcd3a40b263873c22ca2a9 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:04:56 +0300 Subject: [PATCH 14/23] fixing isort --- .../hook_validations/pack_unique_files.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index db846a577a..c887c1b02d 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,11 +6,18 @@ import os import re -from demisto_sdk.commands.common.constants import (PACK_METADATA_PRICE, API_MODULES_PACK, PACK_METADATA_CATEGORIES, - PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, - PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, - PACKS_PACK_IGNORE_FILE_NAME, PACKS_PACK_META_FILE_NAME, - PACKS_README_FILE_NAME, PACKS_WHITELIST_FILE_NAME) +from demisto_sdk.commands.common.constants import (API_MODULES_PACK, + PACK_METADATA_CATEGORIES, + PACK_METADATA_DEPENDENCIES, + PACK_METADATA_FIELDS, + PACK_METADATA_KEYWORDS, + PACK_METADATA_PRICE, + PACK_METADATA_TAGS, + PACK_METADATA_USE_CASES, + PACKS_PACK_IGNORE_FILE_NAME, + PACKS_PACK_META_FILE_NAME, + PACKS_README_FILE_NAME, + PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path From 0ad368202347c56702b1d332cd6943870ddaa1c2 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:25:17 +0300 Subject: [PATCH 15/23] removed again validation --- demisto_sdk/__main__.py | 3 ++ .../hook_validations/pack_unique_files.py | 32 +++++++------------ .../tests/pack_metadata_validator_test.py | 2 +- .../commands/validate/file_validator.py | 4 ++- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/demisto_sdk/__main__.py b/demisto_sdk/__main__.py index 0efde5718c..9db4b78b60 100644 --- a/demisto_sdk/__main__.py +++ b/demisto_sdk/__main__.py @@ -204,6 +204,9 @@ def unify(**kwargs): @click.option( '-p', '--path', help='Path of file to validate specifically, outside of a git directory.' ) +@click.option( + '-i', '--input', help='Path of file to validate specifically, outside of a git directory.' +) @pass_config def validate(config, **kwargs): sys.path.append(config.configuration.env_dir) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index c887c1b02d..0558cedbfe 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,18 +6,11 @@ import os import re -from demisto_sdk.commands.common.constants import (API_MODULES_PACK, - PACK_METADATA_CATEGORIES, - PACK_METADATA_DEPENDENCIES, - PACK_METADATA_FIELDS, - PACK_METADATA_KEYWORDS, - PACK_METADATA_PRICE, - PACK_METADATA_TAGS, - PACK_METADATA_USE_CASES, - PACKS_PACK_IGNORE_FILE_NAME, - PACKS_PACK_META_FILE_NAME, - PACKS_README_FILE_NAME, - PACKS_WHITELIST_FILE_NAME) +from demisto_sdk.commands.common.constants import (PACK_METADATA_PRICE, API_MODULES_PACK, PACK_METADATA_CATEGORIES, + PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, + PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, + PACKS_PACK_IGNORE_FILE_NAME, PACKS_PACK_META_FILE_NAME, + PACKS_README_FILE_NAME, PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path @@ -151,15 +144,12 @@ def _is_pack_meta_file_structure_valid(self): if not isinstance(dependencies_field, dict): self._add_error('The dependencies field in the pack must be a dictionary.') return False - price_field = metadata.get(PACK_METADATA_PRICE) - if price_field: - try: - int(price_field) - except Exception: - self._add_error('The price field in the pack must be a number.') - return False - else: - pass + # price_field = metadata.get(PACK_METADATA_PRICE) + # try: + # int(price_field) + # except Exception: + # self._add_error('The price field in the pack must be a number.') + # return False for list_field in (PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_CATEGORIES, PACK_METADATA_USE_CASES): field = metadata[list_field] diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index c6e6d4a893..422fa3322c 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -22,7 +22,7 @@ def test_metadata_validator_valid(self, mocker): # TODO: add the validation for price after #23546 is ready. @pytest.mark.parametrize('metadata', [os.path.join(FILES_PATH, 'pack_metadata_missing_fields.json'), - os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), + # os.path.join(FILES_PATH, 'pack_metadata_invalid_price.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_empty_category.json'), diff --git a/demisto_sdk/commands/validate/file_validator.py b/demisto_sdk/commands/validate/file_validator.py index cf405396f9..20f8e1fe64 100644 --- a/demisto_sdk/commands/validate/file_validator.py +++ b/demisto_sdk/commands/validate/file_validator.py @@ -58,7 +58,7 @@ get_yml_paths_in_dir, is_file_path_in_pack, print_color, print_error, - print_warning, run_command) + print_warning, run_command, get_files_in_dir) from demisto_sdk.commands.unify.unifier import Unifier @@ -590,6 +590,8 @@ def is_valid_structure(self): Returns: (bool). Whether the structure is valid or not. """ + if self.file_path: + files = get_files_in_dir(self.file_path, ("json", "yml", "md")) if self.validate_conf_json: if not self.conf_json_validator.is_valid_conf_json(): self._is_valid = False From cce93c9d44558dd95d2edaa157f741a2db7c5724 Mon Sep 17 00:00:00 2001 From: ikeren Date: Mon, 13 Apr 2020 17:26:30 +0300 Subject: [PATCH 16/23] remove unnecessary function --- demisto_sdk/commands/init/initiator.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/demisto_sdk/commands/init/initiator.py b/demisto_sdk/commands/init/initiator.py index ade64ce531..f0edf47043 100644 --- a/demisto_sdk/commands/init/initiator.py +++ b/demisto_sdk/commands/init/initiator.py @@ -269,19 +269,6 @@ def get_valid_user_input(options_list: List[str], option_message: str) -> str: return options_list[user_choice - 1] - @staticmethod - def get_yml_data_as_dict(file_path: str) -> Dict: - """Converts YML file data to Dict. - - Args: - file_path (str): The path to the .yml file - - Returns: - Dict. Data from YML. - """ - with open(file_path) as f: - return yaml.load(f, Loader=yamlordereddictloader.SafeLoader) - def integration_init(self) -> bool: """Creates a new integration according to a template. @@ -365,7 +352,8 @@ def yml_reformatting(self, current_suffix: str, integration: bool = False): current_suffix (str): The yml file name (HelloWorld or HelloWorldScript) integration (bool): Indicates if integration yml is being reformatted. """ - yml_dict = self.get_yml_data_as_dict(file_path=os.path.join(self.full_output_path, f"{current_suffix}.yml")) + with open(os.path.join(self.full_output_path, f"{current_suffix}.yml")) as f: + yml_dict = yaml.load(f, Loader=yamlordereddictloader.SafeLoader) yml_dict["commonfields"]["id"] = self.id yml_dict['name'] = self.id if integration: From 99ec5a135467016a567e71d4f7a14c47a75b2d5d Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:27:00 +0300 Subject: [PATCH 17/23] - --- demisto_sdk/__main__.py | 3 --- demisto_sdk/commands/validate/file_validator.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/demisto_sdk/__main__.py b/demisto_sdk/__main__.py index 9db4b78b60..0efde5718c 100644 --- a/demisto_sdk/__main__.py +++ b/demisto_sdk/__main__.py @@ -204,9 +204,6 @@ def unify(**kwargs): @click.option( '-p', '--path', help='Path of file to validate specifically, outside of a git directory.' ) -@click.option( - '-i', '--input', help='Path of file to validate specifically, outside of a git directory.' -) @pass_config def validate(config, **kwargs): sys.path.append(config.configuration.env_dir) diff --git a/demisto_sdk/commands/validate/file_validator.py b/demisto_sdk/commands/validate/file_validator.py index 20f8e1fe64..dfa7ba6810 100644 --- a/demisto_sdk/commands/validate/file_validator.py +++ b/demisto_sdk/commands/validate/file_validator.py @@ -590,8 +590,6 @@ def is_valid_structure(self): Returns: (bool). Whether the structure is valid or not. """ - if self.file_path: - files = get_files_in_dir(self.file_path, ("json", "yml", "md")) if self.validate_conf_json: if not self.conf_json_validator.is_valid_conf_json(): self._is_valid = False From 9b59677a00f1682003c1bef3ccfbb4a9333bc218 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:28:33 +0300 Subject: [PATCH 18/23] isort --- .../common/hook_validations/pack_unique_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 0558cedbfe..4aafe1356d 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,11 +6,11 @@ import os import re -from demisto_sdk.commands.common.constants import (PACK_METADATA_PRICE, API_MODULES_PACK, PACK_METADATA_CATEGORIES, - PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, - PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, - PACKS_PACK_IGNORE_FILE_NAME, PACKS_PACK_META_FILE_NAME, - PACKS_README_FILE_NAME, PACKS_WHITELIST_FILE_NAME) +from demisto_sdk.commands.common.constants import ( # PACK_METADATA_PRICE,; API_MODULES_PACK, PACK_METADATA_CATEGORIES, + PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, + PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, + PACKS_PACK_META_FILE_NAME, PACKS_README_FILE_NAME, + PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path From 9bb15e0bc8d0410521e459e257da0ebe7192e86b Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:29:36 +0300 Subject: [PATCH 19/23] isort --- .../commands/common/hook_validations/pack_unique_files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 4aafe1356d..0f7936ce2e 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -6,9 +6,10 @@ import os import re -from demisto_sdk.commands.common.constants import ( # PACK_METADATA_PRICE,; API_MODULES_PACK, PACK_METADATA_CATEGORIES, - PACK_METADATA_DEPENDENCIES, PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, - PACK_METADATA_TAGS, PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, +from demisto_sdk.commands.common.constants import ( # PACK_METADATA_PRICE, + API_MODULES_PACK, PACK_METADATA_CATEGORIES, PACK_METADATA_DEPENDENCIES, + PACK_METADATA_FIELDS, PACK_METADATA_KEYWORDS, PACK_METADATA_TAGS, + PACK_METADATA_USE_CASES, PACKS_PACK_IGNORE_FILE_NAME, PACKS_PACK_META_FILE_NAME, PACKS_README_FILE_NAME, PACKS_WHITELIST_FILE_NAME) from demisto_sdk.commands.common.tools import pack_name_to_path From a339730ce153729733007ec8a1237aae72bb3f1b Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:34:19 +0300 Subject: [PATCH 20/23] revert change --- demisto_sdk/commands/validate/file_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demisto_sdk/commands/validate/file_validator.py b/demisto_sdk/commands/validate/file_validator.py index dfa7ba6810..cf405396f9 100644 --- a/demisto_sdk/commands/validate/file_validator.py +++ b/demisto_sdk/commands/validate/file_validator.py @@ -58,7 +58,7 @@ get_yml_paths_in_dir, is_file_path_in_pack, print_color, print_error, - print_warning, run_command, get_files_in_dir) + print_warning, run_command) from demisto_sdk.commands.unify.unifier import Unifier From ed50d612e3e326300cad8e62946f68c7c2cdda57 Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:37:58 +0300 Subject: [PATCH 21/23] added todo --- .../commands/common/hook_validations/pack_unique_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py index 0f7936ce2e..7120a10798 100644 --- a/demisto_sdk/commands/common/hook_validations/pack_unique_files.py +++ b/demisto_sdk/commands/common/hook_validations/pack_unique_files.py @@ -145,6 +145,7 @@ def _is_pack_meta_file_structure_valid(self): if not isinstance(dependencies_field, dict): self._add_error('The dependencies field in the pack must be a dictionary.') return False + # TODO: add it back after #23546 is ready. # price_field = metadata.get(PACK_METADATA_PRICE) # try: # int(price_field) From 7c78fc866f54983fc7dd52288bd5a5e3d05d8d0b Mon Sep 17 00:00:00 2001 From: sberman Date: Mon, 13 Apr 2020 17:56:44 +0300 Subject: [PATCH 22/23] more tests --- .../tests/pack_metadata_validator_test.py | 1 + .../pack_metadata_invalid_keywords.json | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 demisto_sdk/tests/test_files/pack_metadata_invalid_keywords.json diff --git a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py index 422fa3322c..2c64eb9515 100644 --- a/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py +++ b/demisto_sdk/commands/common/tests/pack_metadata_validator_test.py @@ -26,6 +26,7 @@ def test_metadata_validator_valid(self, mocker): os.path.join(FILES_PATH, 'pack_metadata_invalid_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_list_dependencies.json'), os.path.join(FILES_PATH, 'pack_metadata_empty_category.json'), + os.path.join(FILES_PATH, 'pack_metadata_invalid_keywords.json'), os.path.join(FILES_PATH, 'pack_metadata_invalid_tags.json'), os.path.join(FILES_PATH, 'pack_metadata_list.json')]) def test_metadata_validator_invalid(self, mocker, metadata): diff --git a/demisto_sdk/tests/test_files/pack_metadata_invalid_keywords.json b/demisto_sdk/tests/test_files/pack_metadata_invalid_keywords.json new file mode 100644 index 0000000000..6c653ffe65 --- /dev/null +++ b/demisto_sdk/tests/test_files/pack_metadata_invalid_keywords.json @@ -0,0 +1,30 @@ +{ + "name": "AWS Feed", + "description": "Indicators feed from AWS", + "support": "Cortex XSOAR", + "serverMinVersion": "5.5.0", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Data Enrichment & Threat Intelligence" + ], + "tags": [], + "created": "2020-03-09T16:04:45Z", + "beta": false, + "deprecated": false, + "certification": "certified", + "useCases": [], + "keywords": [""], + "price": 0, + "dependencies": { + "Base": { + "mandatory": true, + "minVersion": "1.0.0", + "name": "Base", + "certification": "certified", + "author": "Cortex XSOAR" + } + } +} From 476d2a0bad56f93fb2f32778292d339cff0339b0 Mon Sep 17 00:00:00 2001 From: ikeren Date: Mon, 13 Apr 2020 18:10:26 +0300 Subject: [PATCH 23/23] add test for initiator.yml_reformatting --- .../commands/init/tests/initiator_test.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/demisto_sdk/commands/init/tests/initiator_test.py b/demisto_sdk/commands/init/tests/initiator_test.py index 170418b6af..d07d53f585 100644 --- a/demisto_sdk/commands/init/tests/initiator_test.py +++ b/demisto_sdk/commands/init/tests/initiator_test.py @@ -1,9 +1,12 @@ import os -from collections import deque +from collections import OrderedDict, deque from datetime import datetime +from pathlib import Path from typing import Callable import pytest +import yaml +import yamlordereddictloader from demisto_sdk.commands.common.constants import (INTEGRATION_CATEGORIES, PACK_INITIAL_VERSION, PACK_SUPPORT_OPTIONS) @@ -144,3 +147,37 @@ def test_create_new_directory(mocker, monkeypatch, initiator): # fail to create pack cause of existing dir without overriding it monkeypatch.setattr('builtins.input', lambda _: 'N') assert initiator.create_new_directory() is False + + +def test_yml_reformatting(tmp_path, initiator): + integration_id = 'HelloWorld' + initiator.id = integration_id + d = tmp_path / integration_id + d.mkdir() + full_output_path = Path(d) + initiator.full_output_path = full_output_path + + p = d / f'{integration_id}.yml' + with p.open(mode='w') as f: + yaml.dump( + { + 'commonfields': { + 'id': '' + }, + 'name': '', + 'display': '' + }, + f + ) + dir_name = 'HelloWorldTest' + initiator.dir_name = dir_name + initiator.yml_reformatting(current_suffix=initiator.HELLO_WORLD_INTEGRATION, integration=True) + with open(full_output_path / f'{dir_name}.yml', 'r') as f: + yml_dict = yaml.load(f, Loader=yamlordereddictloader.SafeLoader) + assert yml_dict == OrderedDict({ + 'commonfields': OrderedDict({ + 'id': 'HelloWorld' + }), + 'display': 'HelloWorld', + 'name': 'HelloWorld' + })