CreateHash, DeduplicateValuesbyKey, JsonUnescape [Common Scripts Util…

…ity Consolidations] (#23672)

* Moved CreateHash script to CommonScripts

* Standardized CreateHash code

* Added createHash UT

* Moved DeduplicateValuesbyKey to CommonScripts

* Added DeduplicateValuesbyKey UT

* Updated release notes, Moved JsonUnescape

* Added JsonUnescape UT

* fixed DeduplicateValuesbyKey UT

* Updated CommonScripts RN

* Removed nosec from hashlib functions

* Added to known_words.txt

* Fixed linting issue, Updated RN

* added noseq to ignore hash Bandit error

* Updated RN

* Updated RN

* Updated RN

* Updated RN

* Updated RN

* Updated RN

* Updated RN

* Updated docker images & RN

Co-authored-by: Shmuel Fainberg <sfainberg@paloaltonetworks.com>

Packs/CommonScripts/ReleaseNotes/1_11_3.md

@@ -0,0 +1,12 @@
+
+#### Scripts
+##### New: JsonUnescape
+- Recursively un-escapes JSON data if escaped JSON is found (Available from Cortex XSOAR 6.0.0).
+- Updated the Docker image to: *demisto/python3-deb:3.10.9.43863*.
+##### New: DeduplicateValuesbyKey
+- Given a list of objects and a key found in each of those objects, return a unique list of values associated with that key. Returns error if the objects provided do not contain the key of interest.  (Available from Cortex XSOAR 6.0.0).
+- Updated the Docker image to: *demisto/python3:3.10.9.43882*.
+##### New: CreateHash
+- Creating a hash of a given input, support sha1, sha256, sha512, md5 and blake. Wrapper for https://docs.python.org/3/library/hashlib.html.
+ (Available from Cortex XSOAR 6.0.0).
+ - Updated the Docker image to: *demisto/python3:3.10.9.43882*.

Packs/CommonScripts/Scripts/CreateHash/CreateHash.py

@@ -0,0 +1,52 @@
+
+import hashlib
+from hashlib import blake2b
+
+import demistomock as demisto  # noqa: F401
+from CommonServerPython import *  # noqa: F401
+
+
+def create_hash(text, hashtype):
+    """Create a hash from a given input and return it as a context outputs
+
+    Args:
+        text (str): input to hash
+        hashtype (str): hash type
+
+    Returns:
+        Dict[str,str]: Dictionary representing the command results context
+    """
+    if hashtype == "sha512":
+        h = hashlib.sha512()
+        h.update(text.encode('utf-8'))
+    elif hashtype == "sha256":
+        h = hashlib.sha256()
+        h.update(text.encode('utf-8'))
+    elif hashtype == 'sha1':
+        h = hashlib.sha1()  # nosec
+        h.update(text.encode('utf-8'))
+    elif hashtype == 'md5':
+        h = hashlib.md5()  # nosec
+        h.update(text.encode('utf-8'))
+    else:
+        h = blake2b()
+        h.update(text.encode('utf-8'))
+
+    context = {
+        "CreateHash": str(h.hexdigest())
+    }
+
+    return context
+
+
+def main():  # pragma: no cover
+    args = demisto.args()
+    text = args.get('text')
+    hashtype = args.get('type')
+
+    context = create_hash(text, hashtype)
+    return_results(CommandResults(outputs=context))
+
+
+if __name__ in ('__builtin__', 'builtins', '__main__'):
+    main()

Packs/CreateHash/Scripts/CreateHash/CreateHash.yml → Packs/CommonScripts/Scripts/CreateHash/CreateHash.yml

@@ -21,7 +21,7 @@ commonfields:
 contentitemexportablefields:
   contentitemfields:
     fromServerVersion: ''
-dockerimage: demisto/python3:3.10.6.33415
+dockerimage: demisto/python3:3.10.9.43882
 enabled: true
 name: CreateHash
 outputs:

Packs/CommonScripts/Scripts/CreateHash/CreateHash_test.py

@@ -0,0 +1,18 @@
+import pytest
+
+
+@pytest.mark.parametrize("hash_method", ['sha512', 'sha256', 'sha1', 'md5', ''])
+def test_create_hash(hash_method):
+    """
+    Given:
+        - A string
+    When:
+        - Running the script
+    Then:
+        - Ensure the expected hash is returned
+    """
+    from CreateHash import create_hash
+    context = create_hash('test', hash_method)
+
+    assert isinstance(context, dict)
+    assert isinstance(context.get('CreateHash'), str)

Packs/DeduplicateValuesbyKey/Scripts/DeduplicateValuesbyKey/DeduplicateValuesbyKey.py → Packs/CommonScripts/Scripts/DeduplicateValuesbyKey/DeduplicateValuesbyKey.py

@@ -1,23 +1,24 @@
 import demistomock as demisto  # noqa: F401
 from CommonServerPython import *  # noqa: F401
 
-"""Given a list of objects (dictionaries) and a key, generate a list
-of unique values of that key in the list of objects."""
 
+def generate_unique_values_from_objects(object_list, key, keep_none):
+    """Given a list of objects (dictionaries) and a key,
+    generate a list of unique values of that key in the list of objects
+    and return the unique values list.
 
-def main():
-
-    object_list = demisto.args().get("object_list")
-    key = demisto.args().get("key_of_interest")
-    keep_none = argToBoolean(demisto.args().get("keep_none"))
 
+    Args:
+        object_list (List[Dict]): list of objects (dictionaries)
+        key (Object): key of interest
+        keep_none (bool): whether to keep None values
+    """
     values = set()
 
     for obj in object_list:
         # Initially attempt to retrieve value using built-in object get method.
-        #
         # This accounts for keys that may contain dots that are NOT intended
-        # to be retrieved from subkeys.
+        # to be retrieved from sub-keys.
         if key in obj:
             values.add(obj[key])
         # Otherwise, use demisto.get to access values.
@@ -31,10 +32,21 @@ def main():
     if not values:
         return_error("The objects provided did not contain the key of interest.")
 
+    return list(values)
+
+
+def main():  # pragma: no cover
+    args = demisto.args()
+    object_list = args.get("object_list")
+    key = args.get("key_of_interest")
+    keep_none = argToBoolean(args.get("keep_none"))
+
+    values = generate_unique_values_from_objects(object_list, key, keep_none)
+
     return_results(CommandResults(
         outputs_prefix="unique_values",
         outputs_key_field="unique_values",
-        outputs={"unique_values": list(values)}))
+        outputs={"unique_values": values}))
 
 
 if __name__ in ("__main__", "__builtin__", "builtins"):

Packs/DeduplicateValuesbyKey/Scripts/DeduplicateValuesbyKey/DeduplicateValuesbyKey.yml → Packs/CommonScripts/Scripts/DeduplicateValuesbyKey/DeduplicateValuesbyKey.yml

@@ -3,29 +3,25 @@ args:
   isArray: true
   name: object_list
   required: true
-- description: String representing key from which unique values should be retrieved.
-    Use dot notation to access subkeys (e.g. 'key.subkey')
+- description: String representing key from which unique values should be retrieved. Use dot notation to access subkeys (e.g. 'key.subkey')
   name: key_of_interest
   required: true
 - auto: PREDEFINED
   default: true
   defaultValue: "False"
-  description: Default is False. If set to True, will return None in the unique list
-    if the key is not found
+  description: Default is False. If set to True, will return None in the unique list if the key is not found
   name: keep_none
   predefined:
   - "True"
   - "False"
-comment: 'Given a list of objects and a key found in each of those objects, return
-  a unique list of values associated with that key. Returns error if the objects provided
-  do not contain the key of interest. '
+comment: 'Given a list of objects and a key found in each of those objects, return a unique list of values associated with that key. Returns error if the objects provided do not contain the key of interest. '
 commonfields:
   id: DeduplicateValuesbyKey
   version: -1
 contentitemexportablefields:
   contentitemfields:
     fromServerVersion: ""
-dockerimage: demisto/python3:3.9.8.24399
+dockerimage: demisto/python3:3.10.9.43882
 enabled: true
 name: DeduplicateValuesbyKey
 outputs:

Packs/CommonScripts/Scripts/DeduplicateValuesbyKey/DeduplicateValuesbyKey_test.py

@@ -0,0 +1,71 @@
+
+import pytest
+
+
+def test_generate_unique_values_from_objects():
+    from DeduplicateValuesbyKey import generate_unique_values_from_objects
+    objects = [
+        {
+            "key": "value1",
+            "value": "value1"
+        },
+        {
+            "key": "value1",
+            "value": "value2"
+        },
+        {
+            "key": "value2",
+            "value": "value3"
+        },
+        {
+            "key": "value2",
+            "value": "value4"
+        },
+        {
+            "key": "value3",
+            "value": "value5"
+        },
+        {
+            "key": "value3",
+            "value": "value6"
+        }]
+    values = generate_unique_values_from_objects(objects, "key", False)
+    assert set(values) == set(["value1", "value2", "value3"])
+
+
+def test_generate_unique_values_from_objects_with_none():
+    from DeduplicateValuesbyKey import generate_unique_values_from_objects
+    objects = [
+        {
+            "key": "value1",
+            "value": "value1"
+        },
+        {
+            "key": "value1",
+            "value": "value2"
+        },
+        {
+            "key": "value2",
+            "value": "value3"
+        },
+        {
+            "key": "value2",
+            "value": "value4"
+        },
+        {
+            "key": "value3",
+            "value": "value5"
+        },
+        {
+            "key": "None_value",
+            "value": None
+        }]
+    values = generate_unique_values_from_objects(objects, "key", True)
+    assert set(values) == set(["None_value", "value1", "value2", "value3"])
+
+
+def test_generate_unique_values_from_objects_fail():
+    from DeduplicateValuesbyKey import generate_unique_values_from_objects
+
+    with pytest.raises(SystemExit):
+        generate_unique_values_from_objects([], "key", True)

Packs/JsonUnescape/Scripts/JsonUnescape/JsonUnescape.py → Packs/CommonScripts/Scripts/JsonUnescape/JsonUnescape.py

@@ -7,6 +7,7 @@
 
 
 def hook(obj: Dict) -> Dict:
+    """ Hook to convert string to json if possible """
     new_obj = {}
     for k, v in obj.items():
         try:
@@ -17,13 +18,15 @@ def hook(obj: Dict) -> Dict:
 
 
 def unescape(args: Dict) -> Union[Dict, List]:
+    """ Unescape json string """
     json_str = json.dumps(args.get("value"))
     return json.loads(json_str, object_hook=hook)
 
 
-def main():
+def main():  # noqa: F841
+    args = demisto.args()
     try:
-        return_results(unescape(demisto.args()))
+        return_results(unescape(args))
     except Exception as ex:
         demisto.error(traceback.format_exc())
         return_error(f"Error: {str(ex)}")

Packs/JsonUnescape/Scripts/JsonUnescape/JsonUnescape.yml → Packs/CommonScripts/Scripts/JsonUnescape/JsonUnescape.yml

@@ -8,7 +8,7 @@ commonfields:
 contentitemexportablefields:
   contentitemfields:
     fromServerVersion: ""
-dockerimage: demisto/python3-deb:3.9.7.24076
+dockerimage: demisto/python3-deb:3.10.9.43863
 enabled: true
 name: JsonUnescape
 outputs:

Packs/CommonScripts/Scripts/JsonUnescape/JsonUnescape_test.py

@@ -0,0 +1,26 @@
+def test_hook():
+    """
+    Given
+    - A string value representing a json
+    When
+    - Running the hook function
+    Then
+    - Ensure the string value is converted to json
+    """
+    from JsonUnescape import hook
+    assert hook({"key": "value"}) == {"key": "value"}
+    assert hook({"key": "{'key': 'value'}"}) == {'key': "{'key': 'value'}"}
+
+
+def test_unescape():
+    """
+    Given
+    - A dictionary with a string value
+    When
+    - Running the unescape function
+    Then
+    - Ensure the string value is converted to json
+    """
+    from JsonUnescape import unescape
+    assert unescape({"value": "value"}) == 'value'
+    assert unescape({"value": "{'key': 'value'}"}) == "{'key': 'value'}"

Packs/CommonScripts/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Common Scripts",
     "description": "Frequently used scripts pack.",
     "support": "xsoar",
-    "currentVersion": "1.11.1",
+    "currentVersion": "1.11.3",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/CreateHash/ReleaseNotes/1_0_3.json

@@ -0,0 +1 @@
+{"breakingChanges":true,"breakingChangesNotes":"The script: CreateHash moved to *CommonScripts* pack. Make sure to update the *CommonScripts* pack to the latest version in order to use this script."}

Packs/CreateHash/ReleaseNotes/1_0_3.md

@@ -0,0 +1,2 @@
+##### CreateHash
+- Deprecated. **CreateHash** script moved to *Common Scripts* pack.

Packs/CreateHash/pack_metadata.json

@@ -1,11 +1,12 @@
 {
-    "name": "CreateHash",
-    "description": "Simple wrapper of the https://docs.python.org/3/library/hashlib.html to produce sha512, sha256, sha1, md5 and Blake (simple hash) of a given text input",
+    "name": "CreateHash (Deprecated)",
+    "description": "Deprecated. CreateHash script moved to CommonScripts pack.",
     "support": "community",
-    "currentVersion": "1.0.2",
+    "currentVersion": "1.0.3",
     "author": "Joerg Stephan",
     "url": "",
     "email": "",
+    "hidden": true,
     "created": "2021-12-06T11:19:01Z",
     "categories": [
         "Utilities"
@@ -20,4 +21,4 @@
         "xsoar",
         "marketplacev2"
     ]
-}
+}

Packs/DeduplicateValuesbyKey/ReleaseNotes/1_0_1.json

@@ -0,0 +1 @@
+{"breakingChanges":true,"breakingChangesNotes":"The script: DeduplicateValuesbyKey moved to *CommonScripts* pack. Make sure to update the *CommonScripts* pack to the latest version in order to use this script."}

Packs/DeduplicateValuesbyKey/ReleaseNotes/1_0_1.md

@@ -0,0 +1,2 @@
+##### DeduplicateValuesbyKey
+- Deprecated. **DeduplicateValuesbyKey** script moved to *Common Scripts* pack.