From fdf1a8ec1c4c8198e54d1eaa289a8e6196051fdf Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Mon, 5 Feb 2024 17:44:49 +0200 Subject: [PATCH 01/15] regex and url formatter fix --- Packs/CommonScripts/ReleaseNotes/1_13_34.md | 6 ++++++ Packs/CommonScripts/Scripts/FormatURL/FormatURL.py | 8 ++++++-- .../CommonScripts/Scripts/FormatURL/FormatURL_test.py | 10 ++++++---- Packs/CommonScripts/pack_metadata.json | 4 ++-- Packs/CommonTypes/IndicatorTypes/reputation-url.json | 2 +- Packs/CommonTypes/ReleaseNotes/3_4_1.md | 4 ++++ Packs/CommonTypes/pack_metadata.json | 2 +- 7 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_34.md create mode 100644 Packs/CommonTypes/ReleaseNotes/3_4_1.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_34.md b/Packs/CommonScripts/ReleaseNotes/1_13_34.md new file mode 100644 index 000000000000..0b0ea3178bd8 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_34.md @@ -0,0 +1,6 @@ + +#### Scripts + +##### FormatURL + +- %%UPDATE_RN%% diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py index 782b9a74ce4d..60c8e584b253 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py @@ -120,8 +120,12 @@ def __init__(self, original_url: str): if not self.done and self.fragment: self.fragment_check() - if self.quoted: - self.output = urllib.parse.unquote(self.output) + while '%' in self.output: + unquoted = urllib.parse.unquote(self.output) + if unquoted != self.output: + self.output = unquoted + else: + break def __str__(self): return f"{self.output}" diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py index 082413d2b3f0..4086574254f4 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py @@ -78,7 +78,9 @@ 'https://google.com:443/search?q=a*test&gs=ps'), ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' 'hxxps%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', - 'https://google.com:443/search?q=a*test&gs=ps') + 'https://google.com:443/search?q=a*test&gs=ps'), + ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3DUnsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', + 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966&email_address=paulameixner@curo.com') ] PROOF_POINT_REDIRECTS = [ @@ -190,9 +192,9 @@ ] FORMAT_HEX = [ - ('ftps://foo.bar/baz%20%21%22%23%24%25%26', 'ftps://foo.bar/baz%20%21%22%23%24%25%26'), - ('foo.bar/baz%20%21%22%23%24%25%26', 'foo.bar/baz%20%21%22%23%24%25%26'), - ('https://foo.com/?key=foo%26bar', 'https://foo.com/?key=foo%26bar'), # disable-secrets-detection + ('ftps://foo.bar/baz%26bar', 'ftps://foo.bar/baz&bar'), + ('foo.bar/baz%26bar', 'foo.bar/baz&bar'), + ('https://foo.com/?key=foo%26bar', 'https://foo.com/?key=foo&bar'), # disable-secrets-detection ('https%3A//foo.com/?key=foo%26bar', 'https://foo.com/?key=foo&bar'), # disable-secrets-detection ] diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 8285660f2c4e..d24285c5c977 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.32", + "currentVersion": "1.13.34", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", @@ -58,4 +58,4 @@ "marketplacev2", "xpanse" ] -} +} \ No newline at end of file diff --git a/Packs/CommonTypes/IndicatorTypes/reputation-url.json b/Packs/CommonTypes/IndicatorTypes/reputation-url.json index 11b7ff6f2db6..86e85167601f 100644 --- a/Packs/CommonTypes/IndicatorTypes/reputation-url.json +++ b/Packs/CommonTypes/IndicatorTypes/reputation-url.json @@ -7,7 +7,7 @@ "commitMessage": "", "shouldPublish": false, "shouldCommit": false, - "regex": "(?i)\\b(?:(?P(?P(?:https?|hxxps?|s?ftps?|meows?)\\[?[:-]]?(?://|\\\\|3A__)|mailto:)?(?P(?P[\\p{L}][-_\\p{L}\\d\\[\\]]+@)?(?P(?:[-\\d\\p{L}]+\\[?\\.\\]?)+(?P[\\p{L}][-\\p{L}\\d\\[\\]]+)[.]?)|(?P(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?P\\[?(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:\\])?))(?P:[\\d]+)?(?P[/*#?]+(?:[?\\d\\p{L}+&@'#%=~_\\/$!:.;*–—―\\-|\\[\\]]*|[(][\\-\\d\\p{L}+&@#\\/%=~_$?!:,;*–—―|\\[\\]]*[)])*))|(?P(?:(?:https?|hxxps?|s?ftps?|meows?)\\[?[:-]]?(?://|\\\\|3A__)|mailto:)(?:(?:[\\p{L}][-_\\p{L}\\d\\[\\]]+@)?(?:(?:[-\\d\\p{L}]+\\[?\\.\\]?)+(?:[\\p{L}][-\\p{L}\\d\\[\\]]+)[.]?)|(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:\\[?(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:\\])?))))", + "regex": "(?i)\\b(?:(?P(?P(?:https?|hxxps?|s?ftps?|meows?)(?:%3A|\\[?[:-]]?)(?://|\\\\|3A__)|mailto:)?(?P(?P[\\p{L}][-_\\p{L}\\d\\[\\]]+@)?(?P(?:[-\\d\\p{L}]+\\[?\\.\\]?)+(?P[\\p{L}][-\\p{L}\\d\\[\\]]+)[.]?)|(?P(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?P\\[?(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:\\])?))(?P:[\\d]+)?(?P[/*#?]+(?:[?\\d\\p{L}+&@'#%=~_\\/$!:.;*–—―\\-|\\[\\]]*|[(][\\-\\d\\p{L}+&@#\\/%=~_$?!:,;*–—―|\\[\\]]*[)])*))|(?P(?:(?:https?|hxxps?|s?ftps?|meows?)\\[?[:-]]?(?://|\\\\|3A__)|mailto:)(?:(?:[\\p{L}][-_\\p{L}\\d\\[\\]]+@)?(?:(?:[-\\d\\p{L}]+\\[?\\.\\]?)+(?:[\\p{L}][-\\p{L}\\d\\[\\]]+)[.]?)|(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:\\[?(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:\\])?))))", "details": "URL", "prevDetails": "URL", "reputationScriptName": "", diff --git a/Packs/CommonTypes/ReleaseNotes/3_4_1.md b/Packs/CommonTypes/ReleaseNotes/3_4_1.md new file mode 100644 index 000000000000..8d1def3d639f --- /dev/null +++ b/Packs/CommonTypes/ReleaseNotes/3_4_1.md @@ -0,0 +1,4 @@ + +#### Indicator Types + +- **URL** diff --git a/Packs/CommonTypes/pack_metadata.json b/Packs/CommonTypes/pack_metadata.json index 73dd885f9f0d..8e6be00b3c37 100644 --- a/Packs/CommonTypes/pack_metadata.json +++ b/Packs/CommonTypes/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Types", "description": "This Content Pack will get you up and running in no-time and provide you with the most commonly used incident & indicator fields and types.", "support": "xsoar", - "currentVersion": "3.4.0", + "currentVersion": "3.4.1", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From bd23f6e484a22c3005d43c0d77c08c97b643db82 Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Tue, 6 Feb 2024 15:19:56 +0200 Subject: [PATCH 02/15] RN --- Packs/CommonScripts/ReleaseNotes/1_13_35.md | 6 ++++++ Packs/CommonScripts/pack_metadata.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_35.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_35.md b/Packs/CommonScripts/ReleaseNotes/1_13_35.md new file mode 100644 index 000000000000..5ecc61e48f56 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_35.md @@ -0,0 +1,6 @@ + +#### Scripts + +##### FormatURL + +- Fixed an issue in which double quoted URLs did not parse correctly. diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index d24285c5c977..3fd963a61fdf 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.34", + "currentVersion": "1.13.35", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From ab8e10c61e7fdf9eceb0d2520892422ec6f2589d Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Tue, 6 Feb 2024 16:34:18 +0200 Subject: [PATCH 03/15] docker bump --- Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml index 5a9faeee848f..9fb18ffd7c7d 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml @@ -18,7 +18,7 @@ tags: timeout: '0' type: python subtype: python3 -dockerimage: demisto/python3:3.10.13.80593 +dockerimage: demisto/python3:3.10.13.86272 fromversion: 5.5.0 tests: - FormatURL-Test From e959e42658b0c69033b4b3f44838ba5a5c391684 Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Tue, 6 Feb 2024 18:29:04 +0200 Subject: [PATCH 04/15] hyphen --- Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py index 4086574254f4..6f6e82e8efcc 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py @@ -181,8 +181,8 @@ ('https://testö.com/test.html', 'https://testö.com/test.html'), ('www.testö.com/test.aspx', 'www.testö.com/test.aspx'), ('https://www.teöst.com/', 'https://www.teöst.com/'), - ('https://www.test.se/Auth/?&rUrl=https://test.com/wp–images/amclimore@test.com', # disable-secrets-detection - 'https://www.test.se/Auth/?&rUrl=https://test.com/wp–images/amclimore@test.com'), # disable-secrets-detection + ('https://www.test.se/Auth/?&rUrl=https://test.com/wp-images/amclimore@test.com', # disable-secrets-detection + 'https://www.test.se/Auth/?&rUrl=https://test.com/wp-images/amclimore@test.com'), # disable-secrets-detection ('test.com/#/?q=(1,2)', "test.com/#/?q=(1,2)"), # disable-secrets-detection ] From 05bf2f3e8b0b21bdb056b8d15f551bcf7ee72464 Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Wed, 7 Feb 2024 11:08:43 +0200 Subject: [PATCH 05/15] Flake8 --- Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py index 6f6e82e8efcc..d7e9ebdb5591 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py @@ -79,7 +79,9 @@ ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' 'hxxps%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', 'https://google.com:443/search?q=a*test&gs=ps'), - ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3DUnsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', + ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3DUnsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966' + '%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7C' + 'Unknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966&email_address=paulameixner@curo.com') ] From 5570a49b32abd742227f9ba9b1884ddf12bb3bcb Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Wed, 7 Feb 2024 11:27:12 +0200 Subject: [PATCH 06/15] flake8 --- .../Scripts/FormatURL/FormatURL_test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py index d7e9ebdb5591..016f6ca86546 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py @@ -79,10 +79,14 @@ ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' 'hxxps%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', 'https://google.com:443/search?q=a*test&gs=ps'), - ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3DUnsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966' - '%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7C' - 'Unknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', - 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966&email_address=paulameixner@curo.com') + ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3D' + 'Unsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966' + '%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C' + '93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7C' + 'Unknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C' + '3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', + 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966' + '&email_address=paulameixner@curo.com') ] PROOF_POINT_REDIRECTS = [ From bd341c7dfa1c04a68992c6edd978e8a0aac919fb Mon Sep 17 00:00:00 2001 From: Content Bot Date: Wed, 7 Feb 2024 11:35:32 +0000 Subject: [PATCH 07/15] Bump pack from version CommonScripts to 1.13.36. --- Packs/CommonScripts/ReleaseNotes/1_13_36.md | 6 ++++++ Packs/CommonScripts/pack_metadata.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_36.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_36.md b/Packs/CommonScripts/ReleaseNotes/1_13_36.md new file mode 100644 index 000000000000..5ecc61e48f56 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_36.md @@ -0,0 +1,6 @@ + +#### Scripts + +##### FormatURL + +- Fixed an issue in which double quoted URLs did not parse correctly. diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 3fd963a61fdf..e0c3c812bdc2 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.35", + "currentVersion": "1.13.36", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From eb44871d1e691aed2e1e397fd4dfd435f73086aa Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Wed, 7 Feb 2024 13:54:51 +0200 Subject: [PATCH 08/15] RN --- .../playbook-URLextraction-Test.yml | 124 ++++++++++++------ Packs/CommonScripts/ReleaseNotes/1_13_36.md | 7 + Packs/CommonScripts/pack_metadata.json | 2 +- 3 files changed, 91 insertions(+), 42 deletions(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_36.md diff --git a/Packs/Base/TestPlaybooks/playbook-URLextraction-Test.yml b/Packs/Base/TestPlaybooks/playbook-URLextraction-Test.yml index 747c0814b556..85993c98b97c 100644 --- a/Packs/Base/TestPlaybooks/playbook-URLextraction-Test.yml +++ b/Packs/Base/TestPlaybooks/playbook-URLextraction-Test.yml @@ -5,10 +5,10 @@ starttaskid: "0" tasks: "0": id: "0" - taskid: 0bf25d8b-9488-4cfb-8dcf-dae5f0a1cded + taskid: f9d57fd7-0c7b-40a3-8f8e-1e4eafee8866 type: start task: - id: 0bf25d8b-9488-4cfb-8dcf-dae5f0a1cded + id: f9d57fd7-0c7b-40a3-8f8e-1e4eafee8866 version: -1 name: "" iscommand: false @@ -35,10 +35,10 @@ tasks: isautoswitchedtoquietmode: false "2": id: "2" - taskid: 8e8a7766-da6d-432c-8780-17dd405baf43 + taskid: 7114d8d9-fbbc-418b-8907-84471a45623d type: regular task: - id: 8e8a7766-da6d-432c-8780-17dd405baf43 + id: 7114d8d9-fbbc-418b-8907-84471a45623d version: -1 name: Set valid URLs description: Sets a value into the context with the given context key @@ -53,7 +53,7 @@ tasks: key: simple: valid_urls value: - simple: '"www.ru.wikipedia.org/wiki/Елизавета_I", "www.golang.org/pkg/regexp/syntax/", "http://www.mock.com?e=P6wGLG", "https://Test.com/this-that" "http://_23_11.redacted.com./#redactedredactedredacted", "http://www.mock.com?gbdfs","http://test.com#fragment3","http://test.com#fragment3/","(http://www.foo.bar/taz?())", "http://test.com#fragment3","http://test.com#fragment3/","http://test.com#fragment3#fragment3", "(http://www.foo.bar/taz?())","http://öevil.com/","http://öevil.com:5000/","http://öevil.com/anypath", "www.evilö.com/evil.aspx","https://www.evöl.com/","https://www.evöl.com/anypath", "hxxps://www.xn--e1v2i3l4.com","www.evil.com:443/path/to/resource.html", "https://www.evil.com:443/path/to/resource.html","1.2.3.4/path", "google.com/path","2001:db8:3333:4444:5555:6666:7777:8888/path/path", "ftp://foo.bar/resource","ftp://foo.bar/"' + simple: '"www.ru.wikipedia.org/wiki/Елизавета_I", "www.golang.org/pkg/regexp/syntax/", "http://www.mock.com?e=P6wGLG", "https://Test.com/this-that" "http://_23_11.redacted.com./#redactedredactedredacted", "http://www.mock.com?gbdfs","http://test.com#fragment3","http://test.com#fragment3/","(http://www.foo.bar/taz?())", "http://test.com#fragment3","http://test.com#fragment3/","http://test.com#fragment3#fragment3", "(http://www.foo.bar/taz?())","http://öevil.com/","http://öevil.com:5000/","http://öevil.com/anypath", "www.evilö.com/evil.aspx","https://www.evöl.com/","https://www.evöl.com/anypath", "hxxps://www.xn--e1v2i3l4.com","www.evil.com:443/path/to/resource.html", "https://www.evil.com:443/path/to/resource.html","1.2.3.4/path", "google.com/path","2001:db8:3333:4444:5555:6666:7777:8888/path/path", "ftp://foo.bar/resource","ftp://foo.bar/","http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Ftest.net%2Fsubscribe%3Fserver_action%3DUnsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966%26email_address%3Dtest%2540test.com&data=05%7C02%7Ctest%40test.com%7C93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C"' separatecontext: false continueonerrortype: "" view: |- @@ -72,10 +72,10 @@ tasks: isautoswitchedtoquietmode: false "3": id: "3" - taskid: afb63060-5959-4813-8cb6-24d8f2c6a2bc + taskid: 8168af80-26d1-4f07-83aa-b888a0ec7dc7 type: regular task: - id: afb63060-5959-4813-8cb6-24d8f2c6a2bc + id: 8168af80-26d1-4f07-83aa-b888a0ec7dc7 version: -1 name: Print valid URLs description: Prints text to war room (Markdown supported) @@ -93,6 +93,7 @@ tasks: - "10" - "14" - "15" + - "21" scriptarguments: value: simple: ${valid_urls} @@ -115,10 +116,10 @@ tasks: isautoswitchedtoquietmode: false "4": id: "4" - taskid: e3155f24-44bd-44ee-80e0-797ea9d74a45 + taskid: 4adf0fcd-9213-4254-88c8-c7c7fe8dcd13 type: condition task: - id: e3155f24-44bd-44ee-80e0-797ea9d74a45 + id: 4adf0fcd-9213-4254-88c8-c7c7fe8dcd13 version: -1 name: Check URL case type: condition @@ -156,10 +157,10 @@ tasks: isautoswitchedtoquietmode: false "5": id: "5" - taskid: c5185b66-195c-44d3-88f0-0c539eef628a + taskid: 0b373a18-c460-4e09-8363-13c6e828c252 type: regular task: - id: c5185b66-195c-44d3-88f0-0c539eef628a + id: 0b373a18-c460-4e09-8363-13c6e828c252 version: -1 name: Set invalid URLs description: Sets a value into the context with the given context key @@ -193,10 +194,10 @@ tasks: isautoswitchedtoquietmode: false "6": id: "6" - taskid: 37d4c822-62a7-488d-86ae-2e202be860a6 + taskid: 2ade05f6-7cb4-4b1f-8e61-e73af116d2b2 type: regular task: - id: 37d4c822-62a7-488d-86ae-2e202be860a6 + id: 2ade05f6-7cb4-4b1f-8e61-e73af116d2b2 version: -1 name: Print invalid URLs description: Prints text to war room (Markdown supported) @@ -232,10 +233,10 @@ tasks: isautoswitchedtoquietmode: false "7": id: "7" - taskid: d5057c4c-e39f-4455-8782-4da74a01117a + taskid: 0efb6f82-5a7f-4641-8492-9b2a3719e744 type: condition task: - id: d5057c4c-e39f-4455-8782-4da74a01117a + id: 0efb6f82-5a7f-4641-8492-9b2a3719e744 version: -1 name: Check non extraction of invalid URLs - Numbers type: condition @@ -273,10 +274,10 @@ tasks: isautoswitchedtoquietmode: false "8": id: "8" - taskid: ff568217-5862-4da2-8be8-331c4a6435a8 + taskid: e286a452-0331-424e-8de3-28a41be1180c type: regular task: - id: ff568217-5862-4da2-8be8-331c4a6435a8 + id: e286a452-0331-424e-8de3-28a41be1180c version: -1 name: DeleteContext description: Delete field from context @@ -305,10 +306,10 @@ tasks: isautoswitchedtoquietmode: false "9": id: "9" - taskid: c4ff4dee-0de4-4714-8585-31b68cfdbf7e + taskid: b1e15b32-162a-4f1c-8cac-d21150f89508 type: condition task: - id: c4ff4dee-0de4-4714-8585-31b68cfdbf7e + id: b1e15b32-162a-4f1c-8cac-d21150f89508 version: -1 name: Check URL with port type: condition @@ -346,10 +347,10 @@ tasks: isautoswitchedtoquietmode: false "10": id: "10" - taskid: 683209f8-f9a8-4a78-83d8-d528f28418cf + taskid: 2316f4c7-b110-4ffc-82ca-da6fd7411ff4 type: condition task: - id: 683209f8-f9a8-4a78-83d8-d528f28418cf + id: 2316f4c7-b110-4ffc-82ca-da6fd7411ff4 version: -1 name: URL with port and path type: condition @@ -387,10 +388,10 @@ tasks: isautoswitchedtoquietmode: false "11": id: "11" - taskid: 93548cb0-040b-4810-8f14-e494ec92c841 + taskid: 2d152797-a7f4-48b5-8a3d-7bf7f1eb7d23 type: condition task: - id: 93548cb0-040b-4810-8f14-e494ec92c841 + id: 2d152797-a7f4-48b5-8a3d-7bf7f1eb7d23 version: -1 name: Check URL with non ASCII type: condition @@ -428,10 +429,10 @@ tasks: isautoswitchedtoquietmode: false "12": id: "12" - taskid: 73a75fcc-4c8c-4b24-81a6-479622f1e4cc + taskid: 84381f2c-df33-4b42-888b-f0662b7325b4 type: condition task: - id: 73a75fcc-4c8c-4b24-81a6-479622f1e4cc + id: 84381f2c-df33-4b42-888b-f0662b7325b4 version: -1 name: Check URL with path type: condition @@ -469,10 +470,10 @@ tasks: isautoswitchedtoquietmode: false "13": id: "13" - taskid: d0802bb0-96b9-4b10-89ed-ae85cde45102 + taskid: 839f3d8d-3f12-4d6c-8e6e-10db3e9c0850 type: condition task: - id: d0802bb0-96b9-4b10-89ed-ae85cde45102 + id: 839f3d8d-3f12-4d6c-8e6e-10db3e9c0850 version: -1 name: IP as a URL type: condition @@ -510,10 +511,10 @@ tasks: isautoswitchedtoquietmode: false "14": id: "14" - taskid: 517a280d-15e1-4b3a-84f3-8c026883092a + taskid: 36088adb-3917-4600-8ac0-9a18dca320f9 type: condition task: - id: 517a280d-15e1-4b3a-84f3-8c026883092a + id: 36088adb-3917-4600-8ac0-9a18dca320f9 version: -1 name: Check URL Query type: condition @@ -551,10 +552,10 @@ tasks: isautoswitchedtoquietmode: false "15": id: "15" - taskid: f48805f8-3ecc-4276-82c1-e86244ed1c3a + taskid: d427c006-446a-460e-8447-6622364245d7 type: condition task: - id: f48805f8-3ecc-4276-82c1-e86244ed1c3a + id: d427c006-446a-460e-8447-6622364245d7 version: -1 name: Check URL fragment type: condition @@ -592,10 +593,10 @@ tasks: isautoswitchedtoquietmode: false "17": id: "17" - taskid: e4ecb1b3-d026-4ecd-88b6-76f5a95f6e4f + taskid: b3a1d6db-f146-44c7-8d2a-729790163b09 type: condition task: - id: e4ecb1b3-d026-4ecd-88b6-76f5a95f6e4f + id: b3a1d6db-f146-44c7-8d2a-729790163b09 version: -1 name: Check non extraction of invalid URLs - invalid path type: condition @@ -633,10 +634,10 @@ tasks: isautoswitchedtoquietmode: false "18": id: "18" - taskid: 268a4639-c147-45c6-83e9-4bf749692ae8 + taskid: 75acccb9-ab74-4b7e-84b3-10ad422e2bbf type: condition task: - id: 268a4639-c147-45c6-83e9-4bf749692ae8 + id: 75acccb9-ab74-4b7e-84b3-10ad422e2bbf version: -1 name: Check non extraction of invalid URLs - space in sub domain type: condition @@ -674,10 +675,10 @@ tasks: isautoswitchedtoquietmode: false "19": id: "19" - taskid: ebaf78fa-016f-4245-8f6d-6e67084aa7ce + taskid: 1113c93b-c181-418c-8264-9a4e148e6822 type: condition task: - id: ebaf78fa-016f-4245-8f6d-6e67084aa7ce + id: 1113c93b-c181-418c-8264-9a4e148e6822 version: -1 name: Check non extraction of invalid URLs - invalid subdomain type: condition @@ -715,10 +716,10 @@ tasks: isautoswitchedtoquietmode: false "20": id: "20" - taskid: 476e05e6-a4d3-4089-855a-45a47cd625af + taskid: 96328373-35f4-4a43-8fd6-317bc503c475 type: regular task: - id: 476e05e6-a4d3-4089-855a-45a47cd625af + id: 96328373-35f4-4a43-8fd6-317bc503c475 version: -1 name: DeleteContext description: |- @@ -753,13 +754,54 @@ tasks: quietmode: 0 isoversize: false isautoswitchedtoquietmode: false + "21": + id: "21" + taskid: 8635fec1-9a51-4a29-8368-9c318f56f0d0 + type: condition + task: + id: 8635fec1-9a51-4a29-8368-9c318f56f0d0 + version: -1 + name: Double quoted + type: condition + iscommand: false + brand: "" + nexttasks: + "yes": + - "5" + separatecontext: false + conditions: + - label: "yes" + condition: + - - operator: containsGeneral + left: + value: + simple: ${URL.Data} + iscontext: true + right: + value: + simple: http://test.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966&email_address=test@test.com + continueonerrortype: "" + view: |- + { + "position": { + "x": 3490, + "y": 720 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false view: |- { "linkLabelsPosition": {}, "paper": { "dimensions": { "height": 1465, - "width": 3390, + "width": 3820, "x": 50, "y": 50 } @@ -768,4 +810,4 @@ view: |- inputs: [] outputs: [] fromversion: 6.5.0 -description: Test playbook for URL extraction flow +description: Test playbook for URL extraction flow. diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_36.md b/Packs/CommonScripts/ReleaseNotes/1_13_36.md new file mode 100644 index 000000000000..f18608b3cc42 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_36.md @@ -0,0 +1,7 @@ + +#### Scripts + +##### FormatURL + +- Better handling of double quoted URLs. +- Updated the Docker image to: *demisto/python3:3.10.13.86272*. diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 3fd963a61fdf..e0c3c812bdc2 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.35", + "currentVersion": "1.13.36", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 9627793c0cf2bf86094f6401b0c693a58610c48e Mon Sep 17 00:00:00 2001 From: Dror Avrahami Date: Wed, 7 Feb 2024 14:02:35 +0200 Subject: [PATCH 09/15] Update 3_4_1.md --- Packs/CommonTypes/ReleaseNotes/3_4_1.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Packs/CommonTypes/ReleaseNotes/3_4_1.md b/Packs/CommonTypes/ReleaseNotes/3_4_1.md index 8d1def3d639f..00b58dc1666c 100644 --- a/Packs/CommonTypes/ReleaseNotes/3_4_1.md +++ b/Packs/CommonTypes/ReleaseNotes/3_4_1.md @@ -2,3 +2,4 @@ #### Indicator Types - **URL** +Fixed an issue with the regex not capturing quoted slashes. From 79b6975c5ba2ec2efed505f77e3ca8d3dbbbeef0 Mon Sep 17 00:00:00 2001 From: Content Bot Date: Thu, 8 Feb 2024 13:03:14 +0000 Subject: [PATCH 10/15] Bump pack from version CommonScripts to 1.13.37. --- Packs/CommonScripts/ReleaseNotes/1_13_37.md | 7 +++++++ Packs/CommonScripts/pack_metadata.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_37.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_37.md b/Packs/CommonScripts/ReleaseNotes/1_13_37.md new file mode 100644 index 000000000000..f18608b3cc42 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_37.md @@ -0,0 +1,7 @@ + +#### Scripts + +##### FormatURL + +- Better handling of double quoted URLs. +- Updated the Docker image to: *demisto/python3:3.10.13.86272*. diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index e0c3c812bdc2..65e7759e7214 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.36", + "currentVersion": "1.13.37", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 0eeec75e6d43a833c312c6c0a8d67740f83fcd3d Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Sun, 11 Feb 2024 11:20:54 +0200 Subject: [PATCH 11/15] docker --- Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml index 9fb18ffd7c7d..3a902c8ebf19 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml @@ -18,7 +18,7 @@ tags: timeout: '0' type: python subtype: python3 -dockerimage: demisto/python3:3.10.13.86272 +dockerimage: demisto/python3:3.10.13.87159 fromversion: 5.5.0 tests: - FormatURL-Test From 55d029f1da76019b45188533d02c237f506d9279 Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Sun, 11 Feb 2024 11:23:16 +0200 Subject: [PATCH 12/15] RN --- Packs/CommonScripts/ReleaseNotes/1_13_37.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_37.md b/Packs/CommonScripts/ReleaseNotes/1_13_37.md index f18608b3cc42..d39c1f0fc4c8 100644 --- a/Packs/CommonScripts/ReleaseNotes/1_13_37.md +++ b/Packs/CommonScripts/ReleaseNotes/1_13_37.md @@ -2,6 +2,7 @@ #### Scripts ##### FormatURL +- Updated the Docker image to: *demisto/python3:3.10.13.87159*. - Better handling of double quoted URLs. -- Updated the Docker image to: *demisto/python3:3.10.13.86272*. + From 91a508b4e709c5c1385af84d64e871ff2259484b Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Sun, 11 Feb 2024 11:32:49 +0200 Subject: [PATCH 13/15] RN --- Packs/CommonScripts/ReleaseNotes/1_13_37.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_37.md b/Packs/CommonScripts/ReleaseNotes/1_13_37.md index d39c1f0fc4c8..8cd4bc8d062f 100644 --- a/Packs/CommonScripts/ReleaseNotes/1_13_37.md +++ b/Packs/CommonScripts/ReleaseNotes/1_13_37.md @@ -3,6 +3,5 @@ ##### FormatURL - Updated the Docker image to: *demisto/python3:3.10.13.87159*. - -- Better handling of double quoted URLs. +- Improved implementation when unquoting double quoted URLs. From d86939e6f04e74603fa3ccbd10e259c534524a69 Mon Sep 17 00:00:00 2001 From: Ni-Knight Date: Sun, 11 Feb 2024 11:49:00 +0200 Subject: [PATCH 14/15] small fix --- .../Scripts/FormatURL/FormatURL_test.py | 259 +++++++++--------- 1 file changed, 136 insertions(+), 123 deletions(-) diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py index 016f6ca86546..d34ec8b7eb90 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL_test.py @@ -2,51 +2,51 @@ import demistomock as demisto from FormatURL import * -TEST_URL_HTTP = 'http://www.test.com' -TEST_URL_HTTPS = 'https://www.test.com' -TEST_URL_INNER_HXXP = 'http://www.testhxxp.com' +TEST_URL_HTTP = 'http://www.test.com' # disable-secrets-detection +TEST_URL_HTTPS = 'https://www.test.com' # disable-secrets-detection +TEST_URL_INNER_HXXP = 'http://www.testhxxp.com' # disable-secrets-detection NOT_FORMAT_TO_FORMAT = [ # Start of http:/ replacements. - ('http:/www.test.com', TEST_URL_HTTP), - ('https:/www.test.com', TEST_URL_HTTPS), - ('http:\\\\www.test.com', TEST_URL_HTTP), - ('https:\\\\www.test.com', TEST_URL_HTTPS), - ('http:\\www.test.com', TEST_URL_HTTP), - ('https:\\www.test.com', TEST_URL_HTTPS), - ('http:www.test.com', TEST_URL_HTTP), - ('https:www.test.com', TEST_URL_HTTPS), + ('http:/www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('https:/www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('http:\\\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('https:\\\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('http:\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('https:\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('http:www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('https:www.test.com', TEST_URL_HTTPS), # disable-secrets-detection # End of http/s replacements. # Start of hxxp/s replacements. - ('hxxp:/www.test.com', TEST_URL_HTTP), - ('hxxps:/www.test.com', TEST_URL_HTTPS), - ('hXXp:/www.test.com', TEST_URL_HTTP), - ('hXXps:/www.test.com', TEST_URL_HTTPS), - ('hxxp:/www.testhxxp.com', 'http://www.testhxxp.com'), - ('hXxp:/www.testhxxp.com', 'http://www.testhxxp.com'), - - - ('hxxp:\\www.test.com', TEST_URL_HTTP), - ('hxxps:\\www.test.com', TEST_URL_HTTPS), - ('hXXp:\\www.test.com', TEST_URL_HTTP), - ('hXXps:\\www.test.com', TEST_URL_HTTPS), - ('hxxps:/www.testhxxp.com', 'https://www.testhxxp.com'), - - ('hxxp:\\\\www.test.com', TEST_URL_HTTP), - ('hxxps:\\\\www.test.com', TEST_URL_HTTPS), - ('hXXp:\\\\www.test.com', TEST_URL_HTTP), - ('hXXps:\\\\www.test.com', TEST_URL_HTTPS), + ('hxxp:/www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hxxps:/www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('hXXp:/www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hXXps:/www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('hxxp:/www.testhxxp.com', 'http://www.testhxxp.com'), # disable-secrets-detection + ('hXxp:/www.testhxxp.com', 'http://www.testhxxp.com'), # disable-secrets-detection + + + ('hxxp:\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hxxps:\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('hXXp:\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hXXps:\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('hxxps:/www.testhxxp.com', 'https://www.testhxxp.com'), # disable-secrets-detection + + ('hxxp:\\\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hxxps:\\\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('hXXp:\\\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('hXXps:\\\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection # End of hxxp/s replacements. # start of meow/s replacements. - ('meow:/www.test.com', TEST_URL_HTTP), - ('meows:/www.test.com', TEST_URL_HTTPS), - ('meow:\\\\www.test.com', TEST_URL_HTTP), - ('meows:\\\\www.test.com', TEST_URL_HTTPS), - ('meow:\\www.test.com', TEST_URL_HTTP), - ('meow:\\www.meow.com', 'http://www.meow.com'), - ('meows:\\www.test.com', TEST_URL_HTTPS), - ('meows:\\www.meow.com', 'https://www.meow.com'), + ('meow:/www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('meows:/www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('meow:\\\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('meows:\\\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('meow:\\www.test.com', TEST_URL_HTTP), # disable-secrets-detection + ('meow:\\www.meow.com', 'http://www.meow.com'), # disable-secrets-detection + ('meows:\\www.test.com', TEST_URL_HTTPS), # disable-secrets-detection + ('meows:\\www.meow.com', 'https://www.meow.com'), # disable-secrets-detection # end of meow/s replacements. # Start of Sanity test, no replacement should be done. @@ -56,92 +56,103 @@ ] BRACKETS_URL_TO_FORMAT = [ - ('{[https://test1.test-api.com/test1/test2/s.testing]}', 'https://test1.test-api.com/test1/test2/s.testing'), - ('"https://test1.test-api.com"', 'https://test1.test-api.com'), - ('[[https://test1.test-api.com]]', 'https://test1.test-api.com'), - ('[https://www.test.com]', 'https://www.test.com'), - ('https://www.test.com]', 'https://www.test.com'), - ('[https://www.test.com', 'https://www.test.com'), - ('[[https://www.test.com', 'https://www.test.com'), - ('\'https://www.test.com/test\'', 'https://www.test.com/test'), - ('\'https://www.test.com/?a=\'b\'\'', 'https://www.test.com/?a=\'b\''), + ('{[https://test1.test-api.com/test1/test2/s.testing]}', # disable-secrets-detection + 'https://test1.test-api.com/test1/test2/s.testing'), # disable-secrets-detection + ('"https://test1.test-api.com"', 'https://test1.test-api.com'), # disable-secrets-detection + ('[[https://test1.test-api.com]]', 'https://test1.test-api.com'), # disable-secrets-detection + ('[https://www.test.com]', 'https://www.test.com'), # disable-secrets-detection + ('https://www.test.com]', 'https://www.test.com'), # disable-secrets-detection + ('[https://www.test.com', 'https://www.test.com'), # disable-secrets-detection + ('[[https://www.test.com', 'https://www.test.com'), # disable-secrets-detection + ('\'https://www.test.com/test\'', 'https://www.test.com/test'), # disable-secrets-detection + ('\'https://www.test.com/?a=\'b\'\'', 'https://www.test.com/?a=\'b\''), # disable-secrets-detection ] ATP_REDIRECTS = [ - ('https://na01.safelinks.protection.outlook.com/?url=https%3A%2F%2Foffice.memoriesflower.com' - '%2FPermission%2Foffice.php&data=01%7C01%7Cdavid.levin%40mheducation.com' - '%7C0ac9a3770fe64fbb21fb08d50764c401%7Cf919b1efc0c347358fca0928ec39d8d5%7C0&sdata=PEoDOerQnha' - '%2FACafNx8JAep8O9MdllcKCsHET2Ye%2B4%3D&reserved=0', - 'https://office.memoriesflower.com/Permission/office.php'), - ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' - 'https%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', - 'https://google.com:443/search?q=a*test&gs=ps'), - ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' - 'hxxps%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', - 'https://google.com:443/search?q=a*test&gs=ps'), - ('http://nam12.safelinks.protection.outlook.com/?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3D' - 'Unsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966' - '%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C' - '93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867%7C1%7C0%7C638393716982915257%7C' - 'Unknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C' - '3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', - 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&sublist=*&msgid=1703700099.20966' - '&email_address=paulameixner@curo.com') + ('https://na01.safelinks.protection.outlook.com/?url=https%3A%2F%2Foffice.memoriesflower.com' # disable-secrets-detection + '%2FPermission%2Foffice.php&data=01%7C01%7Cdavid.levin%40mheducation.com' # disable-secrets-detection + '%7C0ac9a3770fe64fbb21fb08d50764c401%7Cf919b1efc0c347358fca0928ec39d8d5%7C0&sdata=PEoDOerQnha' # disable-secrets-detection + '%2FACafNx8JAep8O9MdllcKCsHET2Ye%2B4%3D&reserved=0', # disable-secrets-detection + 'https://office.memoriesflower.com/Permission/office.php'), # disable-secrets-detection + ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' # disable-secrets-detection + 'https%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', # disable-secrets-detection + 'https://google.com:443/search?q=a*test&gs=ps'), # disable-secrets-detection + ('https://na01.safelinks.protection.outlook.com/?url=https%3A//urldefense.com/v3/__' # disable-secrets-detection + 'hxxps%3A//google.com%3A443/search%3Fq%3Da%2Atest%26gs%3Dps__%3BKw%21-612Flbf0JvQ3kNJkRi5Jg&', # disable-secrets-detection + 'https://google.com:443/search?q=a*test&gs=ps'), # disable-secrets-detection + ('http://nam12.safelinks.protection.outlook.com/' # disable-secrets-detection + '?url=http%3A%2F%2Fi.ms00.net%2Fsubscribe%3Fserver_action%3D' # disable-secrets-detection + 'Unsubscribe%26list%3Dvalintry2%26sublist%3D*%26msgid%3D1703700099.20966' # disable-secrets-detection + '%26email_address%3Dpaulameixner%2540curo.com&data=05%7C02%7Cpaulameixner%40curo.com%7C' # disable-secrets-detection + '93f0eea20f1c47350eb508dc07b40542%7C2dc14abb79414377a7d259f436e42867' # disable-secrets-detection + '%7C1%7C0%7C638393716982915257%7C' # disable-secrets-detection + 'Unknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C' # disable-secrets-detection + '3000%7C%7C%7C&sdata=%2FwfuIapNXRbZBgLVK651uTH%2FwXrSZFqwdvhvWK6Azwk%3D&reserved=0', # disable-secrets-detection + 'http://i.ms00.net/subscribe?server_action=Unsubscribe&list=valintry2&' # disable-secrets-detection + 'sublist=*&msgid=1703700099.20966' # disable-secrets-detection + '&email_address=paulameixner@curo.com') # disable-secrets-detection ] PROOF_POINT_REDIRECTS = [ - ('https://urldefense.proofpoint.com/v2/url?u=https-3A__example.com_something.html', - 'https://example.com/something.html'), - ('https://urldefense.proofpoint.com/v2/url?u=http-3A__links.mkt3337.com_ctt-3Fkn-3D3-26ms-3DMzQ3OTg3MDQS1-26r' - '-3DMzkxNzk3NDkwMDA0S0-26b-3D0-26j-3DMTMwMjA1ODYzNQS2-26mt-3D1-26rt-3D0&d=DwMFaQ&c' - '=Vxt5e0Osvvt2gflwSlsJ5DmPGcPvTRKLJyp031rXjhg&r=MujLDFBJstxoxZI_GKbsW7wxGM7nnIK__qZvVy6j9Wc&m' - '=QJGhloAyfD0UZ6n8r6y9dF-khNKqvRAIWDRU_K65xPI&s=ew-rOtBFjiX1Hgv71XQJ5BEgl9TPaoWRm_Xp9Nuo8bk&e=', - 'http://links.mkt3337.com/ctt?kn=3&ms=MzQ3OTg3MDQS1&r=MzkxNzk3NDkwMDA0S0&b=0&j=MTMwMjA1ODYzNQS2&mt=1&rt=0'), - ('https://urldefense.proofpoint.com/v1/url?u=http://www.bouncycastle.org/&k=oIvRg1%2BdGAgOoM1BIlLLqw%3D%3D%0A' - '&r=IKM5u8%2B%2F%2Fi8EBhWOS%2BqGbTqCC%2BrMqWI%2FVfEAEsQO%2F0Y%3D%0A&m' - '=Ww6iaHO73mDQpPQwOwfLfN8WMapqHyvtu8jM8SjqmVQ%3D%0A&s' - '=d3583cfa53dade97025bc6274c6c8951dc29fe0f38830cf8e5a447723b9f1c9a', - 'http://www.bouncycastle.org/'), - ('https://urldefense.com/v3/__https://google.com:443/search?q=a*test&gs=ps__;Kw!-612Flbf0JvQ3kNJkRi5Jg' - '!Ue6tQudNKaShHg93trcdjqDP8se2ySE65jyCIe2K1D_uNjZ1Lnf6YLQERujngZv9UWf66ujQIQ$', - 'https://google.com:443/search?q=a*test&gs=ps'), - ('https://urldefense.us/v3/__https://google.com:443/search?q=a*test&gs=ps__;Kw!-612Flbf0JvQ3kNJkRi5Jg' - '!Ue6tQudNKaShHg93trcdjqDP8se2ySE65jyCIe2K1D_uNjZ1Lnf6YLQERujngZv9UWf66ujQIQ$', - 'https://google.com:443/search?q=a*test&gs=ps') + ('https://urldefense.proofpoint.com/v2/url?u=https-3A__example.com_something.html', # disable-secrets-detection + 'https://example.com/something.html'), # disable-secrets-detection + ('https://urldefense.proofpoint.com/v2/url?' # disable-secrets-detection + 'u=http-3A__links.mkt3337.com_ctt-3Fkn-3D3-26ms-3DMzQ3OTg3MDQS1-26r' # disable-secrets-detection + '-3DMzkxNzk3NDkwMDA0S0-26b-3D0-26j-3DMTMwMjA1ODYzNQS2-26mt-3D1-26rt-3D0&d=DwMFaQ&c' # disable-secrets-detection + '=Vxt5e0Osvvt2gflwSlsJ5DmPGcPvTRKLJyp031rXjhg&r=MujLDFBJstxoxZI_GKbsW7wxGM7nnIK__qZvVy6j9Wc&m' # disable-secrets-detection + '=QJGhloAyfD0UZ6n8r6y9dF-khNKqvRAIWDRU_K65xPI&s=ew-rOtBFjiX1Hgv71XQJ5BEgl9TPaoWRm_Xp9Nuo8bk&e=', # disable-secrets-detection + 'http://links.mkt3337.com/ctt?kn=3&ms=MzQ3OTg3MDQS1&r=MzkxNzk3NDkwMDA0S0&b=0&j=' # disable-secrets-detection + 'MTMwMjA1ODYzNQS2&mt=1&rt=0'), # disable-secrets-detection + ('https://urldefense.proofpoint.com/v1/url?u=http://www.bouncycastle.org/' # disable-secrets-detection + '&k=oIvRg1%2BdGAgOoM1BIlLLqw%3D%3D%0A' # disable-secrets-detection + '&r=IKM5u8%2B%2F%2Fi8EBhWOS%2BqGbTqCC%2BrMqWI%2FVfEAEsQO%2F0Y%3D%0A&m' # disable-secrets-detection + '=Ww6iaHO73mDQpPQwOwfLfN8WMapqHyvtu8jM8SjqmVQ%3D%0A&s' # disable-secrets-detection + '=d3583cfa53dade97025bc6274c6c8951dc29fe0f38830cf8e5a447723b9f1c9a', # disable-secrets-detection + 'http://www.bouncycastle.org/'), # disable-secrets-detection + ('https://urldefense.com/v3/__https://google.com:443/' # disable-secrets-detection + 'search?q=a*test&gs=ps__;Kw!-612Flbf0JvQ3kNJkRi5Jg' # disable-secrets-detection + '!Ue6tQudNKaShHg93trcdjqDP8se2ySE65jyCIe2K1D_uNjZ1Lnf6YLQERujngZv9UWf66ujQIQ$', # disable-secrets-detection + 'https://google.com:443/search?q=a*test&gs=ps'), # disable-secrets-detection + ('https://urldefense.us/v3/__https://google.com:443/' # disable-secrets-detection + 'search?q=a*test&gs=ps__;Kw!-612Flbf0JvQ3kNJkRi5Jg' # disable-secrets-detection + '!Ue6tQudNKaShHg93trcdjqDP8se2ySE65jyCIe2K1D_uNjZ1Lnf6YLQERujngZv9UWf66ujQIQ$', # disable-secrets-detection + 'https://google.com:443/search?q=a*test&gs=ps') # disable-secrets-detection ] FIREEYE_REDIRECT = [ - ('https://protect2.fireeye.com/v1/url?k=00bf92e9-5f24adeb-00beb0cd-0cc47aa88f82-a1f32e4f84d91cbe&q=1' - '&e=221919da-9d68-429a-a70e-9d8d836ca107&u=https%3A%2F%2Fwww.facebook.com%2FNamshiOfficial', - 'https://www.facebook.com/NamshiOfficial'), + ('https://protect2.fireeye.com/v1/url?' # disable-secrets-detection + 'k=00bf92e9-5f24adeb-00beb0cd-0cc47aa88f82-a1f32e4f84d91cbe&q=1' # disable-secrets-detection + '&e=221919da-9d68-429a-a70e-9d8d836ca107&u=https%3A%2F%2Fwww.facebook.com%2FNamshiOfficial', # disable-secrets-detection + 'https://www.facebook.com/NamshiOfficial'), # disable-secrets-detection ] TRENDMICRO_REDIRECT = [ - ('https://imsva91-ctp.trendmicro.com:443/wis/clicktime/v1/query?' + ('https://imsva91-ctp.trendmicro.com:443/wis/clicktime/v1/query?' # disable-secrets-detection 'url==3Dhttp%3a%2f%2fclick.sanantonioshoemakers.com' # disable-secrets-detection - '%2f%3fqs%3dba654fa7d9346fec1b=3fa6c55906d045be350d0ee6e3edc4ff33ef33eacb79b79602f5aaf719ee16c3d24e8489293=4d3&' - 'umid=3DB8AB568B-E738-A205-9C9E-ECD7B0A0383F&auth==3D00e18db2b3f9ca3ba6337946518e0b003516e16e-' - '5a8d41640e706acd29c760ae7a8cd40=f664d6489', + '%2f%3fqs%3dba654fa7d9346fec1b=3fa6c55906d045be350d0ee6e3ed' # disable-secrets-detection + 'c4ff33ef33eacb79b79602f5aaf719ee16c3d24e8489293=4d3&' # disable-secrets-detection + 'umid=3DB8AB568B-E738-A205-9C9E-ECD7B0A0383F&auth==3D00e18db2b3f9ca3ba6337946518e0b003516e16e-' # disable-secrets-detection + '5a8d41640e706acd29c760ae7a8cd40=f664d6489', # disable-secrets-detection 'http://click.sanantonioshoemakers.com/?qs=ba654fa7d9346fec1b=' # disable-secrets-detection - '3fa6c55906d045be350d0ee6e3edc4ff33ef33eacb' - '79b79602f5aaf719ee16c3d24e8489293=4d3'), + '3fa6c55906d045be350d0ee6e3edc4ff33ef33eacb' # disable-secrets-detection + '79b79602f5aaf719ee16c3d24e8489293=4d3'), # disable-secrets-detection ] FORMAT_USERINFO = [ - ('https://user@domain.com', 'https://user@domain.com') + ('https://user@domain.com', 'https://user@domain.com') # disable-secrets-detection ] FORMAT_PORT = [ - ('www.test.com:443/path/to/file.html', 'www.test.com:443/path/to/file.html'), + ('www.test.com:443/path/to/file.html', 'www.test.com:443/path/to/file.html'), # disable-secrets-detection ] FORMAT_IPv4 = [ - ('https://1.2.3.4/path/to/file.html', 'https://1.2.3.4/path/to/file.html'), - ('1.2.3.4/path', '1.2.3.4/path'), - ('1.2.3.4/path/to/file.html', '1.2.3.4/path/to/file.html'), - ('http://142.42.1.1:8080/', 'http://142.42.1.1:8080/'), - ('http://142.42.1.1:8080', 'http://142.42.1.1:8080'), - ('http://223.255.255.254', 'http://223.255.255.254'), + ('https://1.2.3.4/path/to/file.html', 'https://1.2.3.4/path/to/file.html'), # disable-secrets-detection + ('1.2.3.4/path', '1.2.3.4/path'), # disable-secrets-detection + ('1.2.3.4/path/to/file.html', '1.2.3.4/path/to/file.html'), # disable-secrets-detection + ('http://142.42.1.1:8080/', 'http://142.42.1.1:8080/'), # disable-secrets-detection + ('http://142.42.1.1:8080', 'http://142.42.1.1:8080'), # disable-secrets-detection + ('http://223.255.255.254', 'http://223.255.255.254'), # disable-secrets-detection ] FORMAT_IPv6 = [ @@ -161,18 +172,20 @@ ] FORMAT_QUERY = [ - ('www.test.test.com/test.html?paramaters=testagain', 'www.test.test.com/test.html?paramaters=testagain'), - ('https://www.test.test.com/test.html?paramaters=testagain', - 'https://www.test.test.com/test.html?paramaters=testagain'), + ('www.test.test.com/test.html?paramaters=testagain', # disable-secrets-detection + 'www.test.test.com/test.html?paramaters=testagain'), # disable-secrets-detection + ('https://www.test.test.com/test.html?paramaters=testagain', # disable-secrets-detection + 'https://www.test.test.com/test.html?paramaters=testagain'), # disable-secrets-detection ('https://test.test.com/v2/test?test&test=[test]test', # disable-secrets-detection 'https://test.test.com/v2/test?test&test=[test]test') # disable-secrets-detection ] FORMAT_FRAGMENT = [ - ('https://test.com#fragment3', 'https://test.com#fragment3'), - ('http://_23_11.redacted.com./#redactedredactedredacted', 'http://_23_11.redacted.com./#redactedredactedredacted'), - ('https://test.com?a=b#fragment3', 'https://test.com?a=b#fragment3'), - ('https://test.com/?a=b#fragment3', 'https://test.com/?a=b#fragment3'), + ('https://test.com#fragment3', 'https://test.com#fragment3'), # disable-secrets-detection + ('http://_23_11.redacted.com./#redactedredactedredacted', # disable-secrets-detection + 'http://_23_11.redacted.com./#redactedredactedredacted'), # disable-secrets-detection + ('https://test.com?a=b#fragment3', 'https://test.com?a=b#fragment3'), # disable-secrets-detection + ('https://test.com/?a=b#fragment3', 'https://test.com/?a=b#fragment3'), # disable-secrets-detection ] FORMAT_REFANG = [ @@ -182,24 +195,24 @@ ] FORMAT_NON_ASCII = [ - ('http://☺.damowmow.com/', 'http://☺.damowmow.com/'), - ('http://ötest.com/', 'http://ötest.com/'), - ('https://testö.com/test.html', 'https://testö.com/test.html'), - ('www.testö.com/test.aspx', 'www.testö.com/test.aspx'), - ('https://www.teöst.com/', 'https://www.teöst.com/'), + ('http://☺.damowmow.com/', 'http://☺.damowmow.com/'), # disable-secrets-detection + ('http://ötest.com/', 'http://ötest.com/'), # disable-secrets-detection + ('https://testö.com/test.html', 'https://testö.com/test.html'), # disable-secrets-detection + ('www.testö.com/test.aspx', 'www.testö.com/test.aspx'), # disable-secrets-detection + ('https://www.teöst.com/', 'https://www.teöst.com/'), # disable-secrets-detection ('https://www.test.se/Auth/?&rUrl=https://test.com/wp-images/amclimore@test.com', # disable-secrets-detection 'https://www.test.se/Auth/?&rUrl=https://test.com/wp-images/amclimore@test.com'), # disable-secrets-detection ('test.com/#/?q=(1,2)', "test.com/#/?q=(1,2)"), # disable-secrets-detection ] FORMAT_PUNYCODE = [ - ('http://xn--t1e2s3t4.com/testagain.aspx', 'http://xn--t1e2s3t4.com/testagain.aspx'), - ('https://www.xn--t1e2s3t4.com', 'https://www.xn--t1e2s3t4.com'), + ('http://xn--t1e2s3t4.com/testagain.aspx', 'http://xn--t1e2s3t4.com/testagain.aspx'), # disable-secrets-detection + ('https://www.xn--t1e2s3t4.com', 'https://www.xn--t1e2s3t4.com'), # disable-secrets-detection ] FORMAT_HEX = [ - ('ftps://foo.bar/baz%26bar', 'ftps://foo.bar/baz&bar'), - ('foo.bar/baz%26bar', 'foo.bar/baz&bar'), + ('ftps://foo.bar/baz%26bar', 'ftps://foo.bar/baz&bar'), # disable-secrets-detection + ('foo.bar/baz%26bar', 'foo.bar/baz&bar'), # disable-secrets-detection ('https://foo.com/?key=foo%26bar', 'https://foo.com/?key=foo&bar'), # disable-secrets-detection ('https%3A//foo.com/?key=foo%26bar', 'https://foo.com/?key=foo&bar'), # disable-secrets-detection ] @@ -325,13 +338,13 @@ def test_wrappers(self, url_: str, expected: str): assert URLFormatter(url_).__str__() == expected @pytest.mark.parametrize('url_, expected', [ - ('[https://urldefense.com/v3/__https://google.com:443/search?66ujQIQ$]', - 'https://google.com:443/search?66ujQIQ$'), - ('(https://urldefense.us/v3/__https://google.com:443/searchERujngZv9UWf66ujQIQ$)', - 'https://google.com:443/searchERujngZv9UWf66ujQIQ$'), - ('[https://testURL.com)', 'https://testURL.com'), - ('[https://testURL.com', 'https://testURL.com'), - ('[(https://testURL.com)]', 'https://testURL.com') + ('[https://urldefense.com/v3/__https://google.com:443/search?66ujQIQ$]', # disable-secrets-detection + 'https://google.com:443/search?66ujQIQ$'), # disable-secrets-detection + ('(https://urldefense.us/v3/__https://google.com:443/searchERujngZv9UWf66ujQIQ$)', # disable-secrets-detection + 'https://google.com:443/searchERujngZv9UWf66ujQIQ$'), # disable-secrets-detection + ('[https://testURL.com)', 'https://testURL.com'), # disable-secrets-detection + ('[https://testURL.com', 'https://testURL.com'), # disable-secrets-detection + ('[(https://testURL.com)]', 'https://testURL.com') # disable-secrets-detection ]) def test_remove_special_chars_from_start_and_end_of_url(self, url_, expected): """ From 01672a23f7dee648f8ed59d27d242e587f456e00 Mon Sep 17 00:00:00 2001 From: Content Bot Date: Sun, 11 Feb 2024 11:24:56 +0000 Subject: [PATCH 15/15] Bump pack from version CommonScripts to 1.13.38. --- Packs/CommonScripts/ReleaseNotes/1_13_38.md | 7 +++++++ Packs/CommonScripts/pack_metadata.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_13_38.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_13_38.md b/Packs/CommonScripts/ReleaseNotes/1_13_38.md new file mode 100644 index 000000000000..8cd4bc8d062f --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_13_38.md @@ -0,0 +1,7 @@ + +#### Scripts + +##### FormatURL +- Updated the Docker image to: *demisto/python3:3.10.13.87159*. +- Improved implementation when unquoting double quoted URLs. + diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 65e7759e7214..515986e726be 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.13.37", + "currentVersion": "1.13.38", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",