Fix images special chars when uploading to GCP (#31421)

* added urllib.parse.unquote * added a UT and some explenation * bumped version to all packs that need upload * commit * commit * remove trailing white space * removed unrelated file * added the RN116 to ignore
demisto · Dec 19, 2023 · 1c7bc70 · 1c7bc70
1 parent f56abaa
commit 1c7bc70
Show file tree

Hide file tree

Showing 20 changed files with 69 additions and 10 deletions.
diff --git a/Packs/DevSecOps/.pack-ignore b/Packs/DevSecOps/.pack-ignore
@@ -28,3 +28,5 @@ ignore=BA101
 [file:Packs/DevSecOps/Integrations/LGTM/README.md]
 ignore=RM112
 
+[file:1_1_8.md]
+ignore=RN116
diff --git a/Packs/DevSecOps/ReleaseNotes/1_1_8.md b/Packs/DevSecOps/ReleaseNotes/1_1_8.md
@@ -0,0 +1,2 @@
+##### DevSecOps
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/DevSecOps/pack_metadata.json b/Packs/DevSecOps/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "DevSecOps",
     "description": "DevSecOps CI/CD Orchestration Integration Pack.",
     "support": "community",
-    "currentVersion": "1.1.7",
+    "currentVersion": "1.1.8",
     "author": "Ayman Mahmoud",
     "githubUser": [
         "ayman-m"

diff --git a/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore b/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore
@@ -14,4 +14,7 @@ ignore=IM111
 ignore=IM111
 
 [file:classifier-Group-IB_Threat_Intelligence_&_Attribution_(classifier).json]
-ignore=BA101
+ignore=BA101
+
+[file:1_4_1.md]
+ignore=RN116
diff --git a/Packs/GroupIB_ThreatIntelligenceAttribution/ReleaseNotes/1_4_1.md b/Packs/GroupIB_ThreatIntelligenceAttribution/ReleaseNotes/1_4_1.md
@@ -0,0 +1,2 @@
+##### GroupIB_ThreatIntelligenceAttribution
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/GroupIB_ThreatIntelligenceAttribution/pack_metadata.json b/Packs/GroupIB_ThreatIntelligenceAttribution/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "Group-IB Threat Intelligence",
     "description": "Group-IB Threat Intelligence is a system for analyzing and attributing cyberattacks, threat hunting, and protecting network infrastructure based on data relating to adversary tactics, tools, and activity. Use this pack to fast receive incidents related to you, attribute them to adversaries to do instant response, enrich your security with an enormous IOCs collection, and provide possibilities for manual investigation through Group-IB data via Cortex XSOAR interface.",
     "support": "partner",
-    "currentVersion": "1.4.0",
+    "currentVersion": "1.4.1",
     "author": "Group-IB",
     "url": "https://www.group-ib.com/",
     "email": "integration@group-ib.com",

diff --git a/Packs/MITRECoA/.pack-ignore b/Packs/MITRECoA/.pack-ignore
@@ -157,3 +157,5 @@ ignore=BA124
 [file:EntryWidgetCoAHandled.yml]
 ignore=BA124
 
+[file:1_0_4.md]
+ignore=RN116
diff --git a/Packs/MITRECoA/ReleaseNotes/1_0_4.md b/Packs/MITRECoA/ReleaseNotes/1_0_4.md
@@ -0,0 +1,2 @@
+##### MITRECoA
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/MITRECoA/pack_metadata.json b/Packs/MITRECoA/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "MITRE ATT&CK - Courses of Action",
     "description": "Looking for actionable intelligence? This intelligence-driven Pack provides manual or automated remediation of MITRE ATT&CK techniques.",
     "support": "xsoar",
-    "currentVersion": "1.0.3",
+    "currentVersion": "1.0.4",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

diff --git a/Packs/MalwareInvestigationAndResponse/.pack-ignore b/Packs/MalwareInvestigationAndResponse/.pack-ignore
@@ -4,3 +4,5 @@ ignore=BA124
 [file:ReadProcessFileWrapper.yml]
 ignore=BA124
 
+[file:2_0_9.md]
+ignore=RN116
diff --git a/Packs/MalwareInvestigationAndResponse/ReleaseNotes/2_0_9.md b/Packs/MalwareInvestigationAndResponse/ReleaseNotes/2_0_9.md
@@ -0,0 +1,3 @@
+
+##### MalwareInvestigationAndResponse
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/MalwareInvestigationAndResponse/pack_metadata.json b/Packs/MalwareInvestigationAndResponse/pack_metadata.json
@@ -5,7 +5,7 @@
     "videos": [
         "https://www.youtube.com/watch?v=DtGIefyoTao"
     ],
-    "currentVersion": "2.0.8",
+    "currentVersion": "2.0.9",
     "serverMinVersion": "6.5.0",
     "author": "Cortex XSOAR",
     "hidden": false,

diff --git a/Packs/OTSecurity/.pack-ignore b/Packs/OTSecurity/.pack-ignore
@@ -1,2 +1,5 @@
 [file:classifier-OTSecurity_API_Classifier_v1.json]
-ignore=BA101
+ignore=BA101
+
+[file:1_0_3.md]
+ignore=RN116
diff --git a/Packs/OTSecurity/ReleaseNotes/1_0_3.md b/Packs/OTSecurity/ReleaseNotes/1_0_3.md
@@ -0,0 +1,3 @@
+
+##### OTSecurity
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/OTSecurity/pack_metadata.json b/Packs/OTSecurity/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "OTSecurity",
     "description": "An OT Security Automation Pack",
     "support": "community",
-    "currentVersion": "1.0.2",
+    "currentVersion": "1.0.3",
     "author": "Ayman Mahmoud",
     "url": "",
     "email": "amahmoud@paloaltonetworks.com",

diff --git a/Packs/WhisperGateCVE-2021-32648/.pack-ignore b/Packs/WhisperGateCVE-2021-32648/.pack-ignore
@@ -1,2 +1,5 @@
 [file:playbook-WhisperGate_&_CVE-2021-32648.yml]
-ignore=BA101
+ignore=BA101
+
+[file:1_0_5.md]
+ignore=RN116
diff --git a/Packs/WhisperGateCVE-2021-32648/ReleaseNotes/1_0_5.md b/Packs/WhisperGateCVE-2021-32648/ReleaseNotes/1_0_5.md
@@ -0,0 +1,3 @@
+
+##### WhisperGateCVE-2021-32648
+- Fixed an issue where images with special chars were not displyed properly in the pack README file.
diff --git a/Packs/WhisperGateCVE-2021-32648/pack_metadata.json b/Packs/WhisperGateCVE-2021-32648/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "WhisperGate and HermeticWiper & CVE-2021-32648",
     "description": "On January 14th, 2022, reports began on a malware operation dubbed \"WhisperGate\" targeting multiple organizations in Ukraine. \nOn February 23, 2022, a new wiper malware known as \"HermeticWiper\" was disclosed by several cybersecurity researchers. The new wiper \"HermeticWiper\" was also being used against organizations in Ukraine.",
     "support": "xsoar",
-    "currentVersion": "1.0.4",
+    "currentVersion": "1.0.5",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

diff --git a/Tests/Marketplace/Tests/pack_readme_handler_test.py b/Tests/Marketplace/Tests/pack_readme_handler_test.py
@@ -1,10 +1,12 @@
 import os
+import pytest
 from google.cloud.storage.blob import Blob
 from pathlib import Path
 from Tests.Marketplace.marketplace_constants import GCPConfig, BucketUploadFlow
 from Tests.Marketplace.pack_readme_handler import (
     copy_markdown_images,
     download_markdown_images_from_artifacts,
+    decode_before_upload
 )
 
 
@@ -71,3 +73,21 @@ def test_download_markdown_images_from_artifacts(mocker):
         "pack_3": {},
     }
     assert pack_images_names == expected_res
+
+
+@pytest.mark.parametrize("name, res", [('%26%20%25', '& %'),
+                                       ('ABC', 'ABC'),
+                                       ('A B C', 'A B C'),
+                                       ('$$ $$', '$$ $$'),
+                                       ('%24%24%20%24%24', '$$ $$'),
+                                       ('', '')])
+def test_decode_before_upload(name, res):
+    """
+    Given:
+        - A name of a file or path
+    When:
+        - Uploaing the file path or file name to GCP
+    Then:
+        - Decode the path/name before upload to prevent double encoding.
+    """
+    assert decode_before_upload(name) == res
diff --git a/Tests/Marketplace/pack_readme_handler.py b/Tests/Marketplace/pack_readme_handler.py
@@ -45,13 +45,20 @@ def download_markdown_images_from_artifacts(
                 )
 
             pack_images_names[pack_name][readme_desc_data] = [
-                image_name.get("image_name") for image_name in images_data
+                decode_before_upload(image_name.get("image_name")) for image_name in images_data
             ]
 
     logging.debug(f'{pack_images_names=}')
     return pack_images_names
 
 
+def decode_before_upload(image_name):
+    """
+    To prevent double encoding when uploading the images to GCP.
+    """
+    return urllib.parse.unquote(image_name)
+
+
 def download_markdown_image_from_url_and_upload_to_gcs(
     original_markdown_url: str,
     relative_image_path: str,
@@ -85,6 +92,8 @@ def download_markdown_image_from_url_and_upload_to_gcs(
 
             with open(image_name, "wb") as f:
                 shutil.copyfileobj(r.raw, f)
+
+            relative_image_path = decode_before_upload(relative_image_path)
             # init the blob with the correct path to save the image on gcs
             gcs_storage_path = os.path.join(storage_base_path, relative_image_path)
             logging.debug(f"{gcs_storage_path=}")