SDO connector cleanup and table of mappings (#1484)

stix_shifter/requirements.txt

@@ -17,4 +17,5 @@ stix2-matcher==3.0.0
 stix2-patterns==1.3.2
 stix2-validator==3.1.3
 xmltodict==0.13.0
-urllib3==1.26.15
+urllib3==1.26.15
+uuid==1.30

stix_shifter/scripts/supported_property_exporter.py

@@ -1,15 +1,21 @@
 import json
+import argparse
 from os import path
 import re
 from datetime import datetime
 
+## Script for generating a table of mappings for each connector based on the operator, from-stix, and to-stix mapping files
+## Add --sdo SDO argument for connectors that use SDO mappings instead of a to-stix SCO mapping file
+## python supported_property_exporter.py --sdo SDO
+
 current_dir = path.abspath(path.dirname(__file__))
 
 CONNECTOR_MODULE_PATH = path.abspath(path.join(current_dir, "../../stix_shifter_modules"))
 ADAPTER_GUIDE_PATH = path.abspath(path.join(current_dir, '../../adapter-guide'))
 
 # Add new connectors to this dictionary as they become available. The key must match the name of the translation module.
-CONNECTORS = {
+# Comment out any connectors you wish to ommit.
+SCO_CONNECTORS = {
     "qradar": "IBM QRadar", 
     "splunk": "Splunk Enterprise Security", 
     "bigfix": "HCL BigFix", 
@@ -20,7 +26,7 @@
     # "security_advisor": "IBM Cloud Security Advisor",
     "guardium": "IBM Guardium Data Protection",
     "aws_cloud_watch_logs": "Amazon CloudWatch Logs",
-    "azure_sentinel": "Microsoft Graph Security",
+    # "azure_sentinel": "Microsoft Graph Security",
     "alertflex": "Alertflex",
     "arcsight": "Micro Focus ArcSight",
     "aws_athena": "Amazon Athena",
@@ -44,6 +50,18 @@
     "okta": "Okta"
 }
 
+SDO_CONNECTORS = {
+    "abuseipdb": "AbuseIPDB",
+    "alienvault_otx": "AlienVault OTX",
+    "dshield": "SANS ISC DShield",
+    "intezer": "Intezer",
+    "recorded_future": "Recorded Future",
+    "reversinglabs": "ReversingLabs",
+    "threat_grid": "Cisco Threat Grid",
+    "threat_q": "ThreatQ",
+    "virus_total": "VirusTotal"
+}
+
 DEFAULT_DIALECT = "default"
 
 DIALECTS = {
@@ -85,6 +103,17 @@
 
 def __main__():
 
+    # process arguments
+    parent_parser = argparse.ArgumentParser(description='mapping_table_generator')
+    parent_parser.add_argument('--sdo',help='Generate tables for connectors that use SDO mapping')
+
+    args = parent_parser.parse_args()
+
+    if args.sdo:
+        CONNECTORS = SDO_CONNECTORS
+    else:
+        CONNECTORS = SCO_CONNECTORS
+
     table_of_contents = "# Currently supported STIX objects and properties\n\n"
     table_of_contents += "Each connector supports a set of STIX objects and properties as defined in the connector's mapping files. There is also a set of common STIX properties that all cyber observable objects must contain. See [STIX™ Version 2.0. Part 4: Cyber Observable Objects](http://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part4-cyber-observable-objects.html) for more information on STIX objects.\n"
     table_of_contents += "## Common cyber observable properties\n\n"
@@ -99,16 +128,7 @@ def __main__():
     table_of_contents_file_path = path.abspath(path.join(ADAPTER_GUIDE_PATH, "supported-mappings.md"))
     table_of_contents_file = open(table_of_contents_file_path, "w")
 
-    for index, (key, module) in enumerate(CONNECTORS.items()):
-        try:
-            filepath = path.abspath(path.join(CONNECTOR_MODULE_PATH, key, "stix_translation/json", "to_stix_map.json"))    
-            to_stix_json_file = open(filepath)
-            loaded_to_stix_json = json.loads(to_stix_json_file.read())
-
-
-        except(Exception):
-            print("Error for {} module".format(key))
-            continue
+    for _, (key, module) in enumerate(CONNECTORS.items()):
 
         data_field_alias_mapping = []
         if key == 'qradar':
@@ -122,27 +142,23 @@ def __main__():
                 print("Error for {} module".format(key))
                 continue
 
-        stix_attribute_collection = _parse_attributes(loaded_to_stix_json, key, {})
-
         output_string = ""
         output_string += "##### Updated on " + UPDATED_AT + "\n"
         output_string += "## " + module + "\n"
         table_of_contents += "- [{}]({})\n".format(module, "../stix_shifter_modules/{}/{}_supported_stix.md".format(key, key))
 
+        # SDOs
+        try:
+           output_string = _generate_sdo_list(output_string, args)
+        except Exception as e:
+            print("Error constructing SDO list for {} module: {}".format(key, e))
+            continue
 
         # OPERATORS
         try:
             filepath = path.abspath(path.join(CONNECTOR_MODULE_PATH, key, "stix_translation/json", "operators.json")) 
             operators_json_file = open(filepath)   
-            loaded_operators_json = json.loads(operators_json_file.read())
-            stix_operator_collection = _parse_operators(loaded_operators_json, {})
-            output_string += "### Supported STIX Operators\n"
-            output_string += "*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*\n\n"
-            output_string += "| STIX Operator | Data Source Operator |\n"
-            output_string += "|--|--|\n"
-            for stix_operator, ds_operator in stix_operator_collection.items():
-                output_string += "| {} | {} |\n".format(stix_operator, ds_operator)
-            output_string += "| <br> | |\n"
+            output_string = _generate_operators_table(operators_json_file, output_string)
             operators_json_file.close()
         except Exception as e:
             print("Error constructing STIX operator mapping table for {} module: {}".format(key, e))
@@ -163,65 +179,96 @@ def __main__():
                     output_string += "### Searchable STIX objects and properties for {} dialect\n".format(dialect.capitalize())
                     filepath = path.abspath(path.join(CONNECTOR_MODULE_PATH, key, "stix_translation/json", "{}from_stix_map.json".format(dialect + "_")))    
                 from_stix_json_file = open(filepath)
-                loaded_from_stix_json = json.loads(from_stix_json_file.read())
-                # sorted_from_stix_objects = json.dumps(loaded_from_stix_json, sort_keys=True)
-                # sorted_attribute_objects = json.loads(sorted_attribute_objects)
-                if key == 'cybereason':
-                    output_string += "*The Cybereason connector can only join specific linked fields with the AND operator as defined in its [configmap](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/stix_shifter_modules/aws_athena/stix_translation/json/operators.json).*\n\n"
-                output_string += "| STIX Object and Property | Mapped Data Source Fields |\n"
-                output_string += "|--|--|\n"
-                for stix_object_key, value in loaded_from_stix_json.items():
-                    property_dictionary = value["fields"]
-                    for s_property, fields_list in property_dictionary.items():
-                        if not isinstance(fields_list, list):
-                            fields_list = [fields_list]
-                        orig_fields_list = []
-                        if data_field_alias_mapping:
-                            # TODO: Get real field name for QRadar
-                            for aliased_field in fields_list:
-                                orig_data_field = _get_data_field(aliased_field, data_field_alias_mapping)
-                                orig_fields_list.append(orig_data_field)
-                        else:
-                            orig_fields_list = fields_list
-
-                        # output_string += "| {} | {} | {} |\n".format(stix_object_key, s_property, fields_list)
-                        # fields_string = ', '.join(map(str, fields_list))
-
-                        output_string += "| **{}**:{} | {} |\n".format(stix_object_key, s_property, ', '.join(map(str, orig_fields_list)))
-                output_string += "| <br> | |\n"
+                output_string = _generate_from_stix_table(from_stix_json_file, key, data_field_alias_mapping, output_string)
                 from_stix_json_file.close()
         except Exception as e:
             print("Error constructing from-STIX mapping table for {} module: {}".format(key, e))
             continue
-
 
         # TO-STIX 
+        if not args.sdo:
+            try:
+                filepath = path.abspath(path.join(CONNECTOR_MODULE_PATH, key, "stix_translation/json", "to_stix_map.json"))    
+                to_stix_json_file = open(filepath) 
+                output_string = _generate_to_stix_table(key, to_stix_json_file, data_field_alias_mapping, output_string)
+                to_stix_json_file.close()
+            except Exception as e:
+                print("Error constructing to-STIX mapping table for {} module: {}".format(key, e))
+                continue
+
         try:
-            # supported_stix_file_path = path.abspath(path.join(ADAPTER_GUIDE_PATH, "connectors", "{}_supported_stix.md".format(key)))
             supported_stix_file_path = path.abspath(path.join(CONNECTOR_MODULE_PATH, key, "{}_supported_stix.md".format(key)))
             supported_stix_file = open(supported_stix_file_path, "w")   
-            sorted_attribute_objects = json.dumps(stix_attribute_collection, sort_keys=True)
-            sorted_attribute_objects = json.loads(sorted_attribute_objects)
-            output_string += "### Supported STIX Objects and Properties for Query Results\n"
-            output_string += "| STIX Object | STIX Property | Data Source Field |\n"
-            output_string += "|--|--|--|\n"
-            for stix_object, property_list in sorted_attribute_objects.items():
-                for index, prop in enumerate(property_list):
-                    stix_property, data_field = prop.split(":")
-                    if data_field_alias_mapping:
-                        data_field = _get_data_field(data_field, data_field_alias_mapping)
-                    output_string += "| {} | {} | {} |\n".format(stix_object, stix_property, data_field)
-                output_string += "| <br> | | |\n"
-            to_stix_json_file.close()
             supported_stix_file.write(output_string)
             supported_stix_file.close()
         except Exception as e:
-            print("Error constructing to-STIX mapping table for {} module: {}".format(key, e))
-            continue
+                print("Error writing mapping tables for {} module: {}".format(key, e))
+                continue
 
     table_of_contents_file.write(table_of_contents)
     table_of_contents_file.close()
 
+def _generate_sdo_list(output_string, args):
+    output_string += "### Results STIX Domain Objects\n"
+    if args.sdo:
+        output_string += "* Identity\n* Sighting\n* Infrastructure\n* Malware\n* Extension\n* Indicator\n"
+    else:
+        output_string += "* Identity\n* Observed Data\n"
+    output_string += "<br>\n"
+    return output_string
+
+def _generate_operators_table(operators_json_file, output_string):
+    loaded_operators_json = json.loads(operators_json_file.read())
+    stix_operator_collection = _parse_operators(loaded_operators_json, {})
+    output_string += "### Supported STIX Operators\n"
+    output_string += "*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*\n\n"
+    output_string += "| STIX Operator | Data Source Operator |\n"
+    output_string += "|--|--|\n"
+    for stix_operator, ds_operator in stix_operator_collection.items():
+        output_string += "| {} | {} |\n".format(stix_operator, ds_operator)
+    output_string += "| <br> | |\n"
+    return output_string
+
+def _generate_from_stix_table(from_stix_json_file, key, data_field_alias_mapping, output_string):
+    loaded_from_stix_json = json.loads(from_stix_json_file.read())
+    if key == 'cybereason':
+        output_string += "*The Cybereason connector can only join specific linked fields with the AND operator as defined in its [configmap](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/stix_shifter_modules/aws_athena/stix_translation/json/operators.json).*\n\n"
+    output_string += "| STIX Object and Property | Mapped Data Source Fields |\n"
+    output_string += "|--|--|\n"
+    for stix_object_key, value in loaded_from_stix_json.items():
+        property_dictionary = value["fields"]
+        for s_property, fields_list in property_dictionary.items():
+            if not isinstance(fields_list, list):
+                fields_list = [fields_list]
+            orig_fields_list = []
+            if data_field_alias_mapping:
+                # TODO: Get real field name for QRadar
+                for aliased_field in fields_list:
+                    orig_data_field = _get_data_field(aliased_field, data_field_alias_mapping)
+                    orig_fields_list.append(orig_data_field)
+            else:
+                orig_fields_list = fields_list
+            output_string += "| **{}**:{} | {} |\n".format(stix_object_key, s_property, ', '.join(map(str, orig_fields_list)))
+    output_string += "| <br> | |\n"
+    return output_string
+
+def _generate_to_stix_table(key, to_stix_json_file, data_field_alias_mapping, output_string):
+    loaded_to_stix_json = json.loads(to_stix_json_file.read())
+    stix_attribute_collection = _parse_attributes(loaded_to_stix_json, key, {})
+    sorted_attribute_objects = json.dumps(stix_attribute_collection, sort_keys=True)
+    sorted_attribute_objects = json.loads(sorted_attribute_objects)
+    output_string += "### Supported STIX Objects and Properties for Query Results\n"
+    output_string += "| STIX Object | STIX Property | Data Source Field |\n"
+    output_string += "|--|--|--|\n"
+    for stix_object, property_list in sorted_attribute_objects.items():
+        for index, prop in enumerate(property_list):
+            stix_property, data_field = prop.split(":")
+            if data_field_alias_mapping:
+                data_field = _get_data_field(data_field, data_field_alias_mapping)
+            output_string += "| {} | {} | {} |\n".format(stix_object, stix_property, data_field)
+        output_string += "| <br> | | |\n"
+    return output_string
+
 
 def _get_data_field(data_field, data_field_alias_mapping):
     for value in data_field_alias_mapping:

stix_shifter_modules/abuseipdb/README.md

@@ -0,0 +1,7 @@
+# AbuseIPDB
+
+Determine whether an IP was reported or not as malicious by AbuseIPDB.
+
+## Supported STIX Mappings
+
+See the [table of mappings](abuseipdb_supported_stix.md) for the STIX objects and operators supported by this connector.

stix_shifter_modules/abuseipdb/abuseipdb_supported_stix.md

@@ -0,0 +1,36 @@
+##### Updated on 05/15/23
+## AbuseIPDB
+### Results STIX Domain Objects
+* Identity
+* Sighting
+* Infrastructure
+* Malware
+* Extension
+* Indicator
+<br>
+### Supported STIX Operators
+*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*
+
+| STIX Operator | Data Source Operator |
+|--|--|
+| AND (Comparision) | AND |
+| OR (Comparision) | OR |
+| > | > |
+| >= | >= |
+| < | < |
+| <= | <= |
+| = | = |
+| != | != |
+| LIKE | = |
+| IN | IN |
+| MATCHES | CONTAINS |
+| ISSUBSET | insubnet |
+| OR (Observation) | OR |
+| AND (Observation) | AND |
+| <br> | |
+### Searchable STIX objects and properties
+| STIX Object and Property | Mapped Data Source Fields |
+|--|--|
+| **ipv4-addr**:value | SourceIpV4, DestinationIpV4 |
+| **ipv6-addr**:value | SourceIpV6, DestinationIpV6 |
+| <br> | |

stix_shifter_modules/alertflex/alertflex_supported_stix.md

@@ -1,5 +1,9 @@
-##### Updated on 02/27/23
+##### Updated on 05/15/23
 ## Alertflex
+### Results STIX Domain Objects
+* Identity
+* Observed Data
+<br>
 ### Supported STIX Operators
 *Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*
 

stix_shifter_modules/alienvault_otx/README.md

@@ -0,0 +1,7 @@
+# AlienVault OTX
+
+Query AlienVault OTX for IPs, domains, URLs, or file hashes.
+
+## Supported STIX Mappings
+
+See the [table of mappings](alienvault_otx_supported_stix.md) for the STIX objects and operators supported by this connector.

stix_shifter_modules/alienvault_otx/alienvault_otx_supported_stix.md

@@ -0,0 +1,42 @@
+##### Updated on 05/15/23
+## AlienVault OTX
+### Results STIX Domain Objects
+* Identity
+* Sighting
+* Infrastructure
+* Malware
+* Extension
+* Indicator
+<br>
+### Supported STIX Operators
+*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*
+
+| STIX Operator | Data Source Operator |
+|--|--|
+| AND (Comparision) | AND |
+| OR (Comparision) | OR |
+| > | > |
+| >= | >= |
+| < | < |
+| <= | <= |
+| = | = |
+| != | != |
+| LIKE | = |
+| IN | IN |
+| MATCHES | CONTAINS |
+| ISSUBSET | insubnet |
+| OR (Observation) | OR |
+| AND (Observation) | AND |
+| <br> | |
+### Searchable STIX objects and properties
+| STIX Object and Property | Mapped Data Source Fields |
+|--|--|
+| **url**:value | Url |
+| **ipv4-addr**:value | SourceIpV4, DestinationIpV4 |
+| **ipv6-addr**:value | SourceIpV6, DestinationIpV6 |
+| **domain-name**:value | Url |
+| **file**:hashes.'SHA-256' | sha256hash |
+| **file**:hashes.MD5 | md5hash |
+| **file**:hashes.'MD5' | md5hash |
+| **file**:hashes.'SHA-1' | sha1hash |
+| <br> | |

stix_shifter_modules/arcsight/arcsight_supported_stix.md

@@ -1,5 +1,9 @@
-##### Updated on 02/27/23
+##### Updated on 05/15/23
 ## Micro Focus ArcSight
+### Results STIX Domain Objects
+* Identity
+* Observed Data
+<br>
 ### Supported STIX Operators
 *Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).*