Directory restructure (#76)

MANIFEST.in

@@ -5,4 +5,4 @@ include *.md
 include LICENSE.txt
 
 # Include JSON files
-include stix_shifter/src/modules/qradar/json/*.json
+include stix_shifter/stix_translation/src/modules/qradar/json/*.json

README.md

@@ -48,11 +48,11 @@ Stix-shifter handles two primary functions:
 
 ### Converting from STIX Patterns to data source queries (query) or from data source results to STIX cyber observables (results)
 
-#### Call the stix_shifter in the format of
+#### Call the stix_translation in the format of
 
 ```
-usage: stix_shifter.py translate [-h]
-                                 {qradar, dummy, splunk}
+usage: stix_translation.py translate [-h]
+                                 {'qradar', 'dummy', 'car', 'cim', 'splunk', 'elastic', 'bigfix', 'csa', 'csa:at', 'csa:nf'}
                                  {results, query} data
 
 positional arguments:
@@ -172,11 +172,11 @@ python main.py translate qradar results \
 
 ## Transmission
 
-#### Call the stix_shifter in the format of
+#### Call the stix_transmission in the format of
 
 ```
-usage: stix_shifter.py transmit [-h]
-                      {async_dummy, synchronous_dummy, qradar, splunk, bigfix}
+usage: stix_transmission.py transmit [-h]
+                      {'async_dummy', 'synchronous_dummy', 'qradar', 'splunk', 'bigfix', 'csa'}
 
 positional arguments:
 {<async_dummy, synchronous_dummy, qradar, splunk, bigfix>}         Transmission module to use
@@ -248,11 +248,11 @@ python main.py transmit qradar '{"host":"<ip address>", "port":"<port>", "cert":
 
 ### Example of converting a STIX pattern to an IBM QRadar AQL query:
 
-[See the QRadar module documentation](stix_shifter/src/modules/qradar/README.md)
+[See the QRadar module documentation](stix_shifter/stix_translation/src/modules/qradar/README.md)
 
 ### Example of converting IBM QRadar events to STIX:
 
-[See the QRadar module documentation](stix_shifter/src/modules/qradar/README.md)
+[See the QRadar module documentation](stix_shifter/stix_translation/src/modules/qradar/README.md)
 
 # Contributing
 

main.py

@@ -1,6 +1,7 @@
 import argparse
 import sys
-from stix_shifter import stix_shifter
+from stix_shifter.stix_translation import stix_translation
+from stix_shifter.stix_transmission import stix_transmission
 import json
 
 
@@ -45,9 +46,9 @@ def __main__():
 
     # positional arguments
     translate_parser.add_argument(
-        'module', choices=stix_shifter.TRANSLATION_MODULES, help='what translation module to use')
+        'module', choices=stix_translation.TRANSLATION_MODULES, help='what translation module to use')
     translate_parser.add_argument('translate_type', choices=[
-        stix_shifter.RESULTS, stix_shifter.QUERY], help='what translation action to perform')
+        stix_translation.RESULTS, stix_translation.QUERY], help='what translation action to perform')
     translate_parser.add_argument(
         'data_source', help='STIX identity object representing a datasource')
     translate_parser.add_argument(
@@ -66,7 +67,7 @@ def __main__():
 
     # positional arguments
     transmit_parser.add_argument(
-        'module', choices=stix_shifter.TRANSMISSION_MODULES,
+        'module', choices=stix_transmission.TRANSMISSION_MODULES,
         help='choose which connection module to use'
     )
     transmit_parser.add_argument(
@@ -82,39 +83,81 @@ def __main__():
 
     # operation subparser
     operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command")
-    operation_subparser.add_parser(stix_shifter.PING, help="Pings the data source")
-    query_operation_parser = operation_subparser.add_parser(stix_shifter.QUERY, help="Executes a query on the data source")
+    operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source")
+    query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source")
     query_operation_parser.add_argument('query_string', help='native datasource query string')
-    results_operation_parser = operation_subparser.add_parser(stix_shifter.RESULTS, help="Fetches the results of the data source query")
+    results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query")
     results_operation_parser.add_argument('search_id', help='uuid of executed query')
     results_operation_parser.add_argument('offset', help='offset of results')
     results_operation_parser.add_argument('length', help='length of results')
-    status_operation_parser = operation_subparser.add_parser(stix_shifter.STATUS, help="Gets the current status of the query")
+    status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query")
     status_operation_parser.add_argument('search_id', help='uuid of executed query')
-    operation_subparser.add_parser(stix_shifter.IS_ASYNC, help='Checks if the query operation is asynchronous')
+    operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous')
 
     args = parent_parser.parse_args()
 
     if args.command is None:
         parent_parser.print_help(sys.stderr)
         sys.exit(1)
 
-    shifter = stix_shifter.StixShifter()
 
     if args.command == TRANSLATE:
         options = json.loads(args.options) if bool(args.options) else {}
         if args.stix_validator:
             options['stix_validator'] = args.stix_validator
         if args.data_mapper:
             options['data_mapper'] = args.data_mapper
-        result = shifter.translate(
+
+        translation = stix_translation.StixTranslation()
+        result = translation.translate(
             args.module, args.translate_type, args.data_source, args.data, options=options)
     elif args.command == TRANSMIT:
-        result = shifter.transmit(args)
+        result = transmit(args) # stix_transmission
 
     print(result)
     exit(0)
 
 
+def transmit(args):
+    """
+    Connects to datasource and executes a query, grabs status update or query results
+    :param args:
+    args: <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
+    <
+        query <query string>,
+        status <search id>,
+        results <search id> <offset> <length>,
+        ping,
+        is_async
+    >
+    """
+    connection_dict = json.loads(args.connection)
+    configuration_dict = json.loads(args.configuration)
+    transmission = stix_transmission.StixTransmission(args.module, connection_dict, configuration_dict)
+
+    operation_command = args.operation_command
+    if operation_command == stix_transmission.QUERY:
+        query = args.query_string
+        result = transmission.query(query)
+    elif operation_command == stix_transmission.STATUS:
+        search_id = args.search_id
+        result = transmission.status(search_id)
+    elif operation_command == stix_transmission.RESULTS:
+        search_id = args.search_id
+        offset = args.offset
+        length = args.length
+        result = transmission.results(search_id, offset, length)
+    elif operation_command == stix_transmission.DELETE:
+        search_id = args.search_id
+        result = transmission.delete(search_id)
+    elif operation_command == stix_transmission.PING:
+        result = transmission.ping()
+    elif operation_command == stix_transmission.IS_ASYNC:
+        result = transmission.is_async()
+    else:
+        raise NotImplementedError
+    return result
+
+
 if __name__ == "__main__":
-    __main__()
+    __main__()

setup.py

@@ -181,7 +181,8 @@
     # },
     entry_points={
         'console_scripts': [
-            'stix-transmission=stix_transmission.stix_transmission:main',
+            'stix-transmission=stix_shifter.stix_transmission.stix_transmission:main',
+            'stix-translation=stix_shifter.stix_translation.stix_translation:main',
         ],
     },
 

stix_shifter/src/json_to_stix/json_to_stix.py → stix_shifter/stix_translation/src/json_to_stix/json_to_stix.py

@@ -1,7 +1,7 @@
 import json
 from . import json_to_stix_translator
 from ..modules.base.base_result_translator import BaseResultTranslator
-from stix_shifter.src import transformers
+from stix_shifter.stix_translation.src import transformers
 
 # Concrete BaseResultTranslator
 

stix_shifter/src/modules/bigfix/bigfix_data_mapping.py → stix_shifter/stix_translation/src/modules/bigfix/bigfix_data_mapping.py

@@ -1,6 +1,6 @@
 from os import path
 import json
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 
 def _fetch_mapping():

stix_shifter/src/modules/bigfix/bigfix_query_constructor.py → stix_shifter/stix_translation/src/modules/bigfix/bigfix_query_constructor.py

@@ -1,7 +1,7 @@
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.patterns.errors import SearchFeatureNotSupportedError
+from stix_shifter.stix_translation.src.patterns.errors import SearchFeatureNotSupportedError
 
 
 class RelevanceQueryStringPatternTranslator:

stix_shifter/src/modules/bigfix/stix_to_bigfix.py → stix_shifter/stix_translation/src/modules/bigfix/stix_to_bigfix.py

@@ -18,7 +18,7 @@ def transform_query(self, data, options, mapping=None):
         Transforms STIX query into Relevance query format. Based on a mapping file
         :param data: STIX query string to transform into aql query format
         :type data: str
-        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into aql format. This defaults to the from_stix_map.json in the stix_shifter/src/modules/qradar/json/ directory
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into aql format. This defaults to the from_stix_map.json in the stix_shifter/stix_translation/src/modules/qradar/json/ directory
         :type mapping: str (filepath)
         :return: aql query string
         :rtype: str

stix_shifter/src/modules/car/car_data_mapping.py → stix_shifter/stix_translation/src/modules/car/car_data_mapping.py

@@ -1,4 +1,4 @@
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 # TODO: should this really be a class? Could be a module, made it a class
 # in case we have configuration arguments at some point

stix_shifter/src/modules/cim/cim_data_mapping.py → stix_shifter/stix_translation/src/modules/cim/cim_data_mapping.py

@@ -1,4 +1,4 @@
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 # TODO: should this really be a class? Could be a module, made it a class
 # in case we have configuration arguments at some point

stix_shifter/src/modules/csa/cloudsql_data_mapping.py → stix_shifter/stix_translation/src/modules/csa/cloudsql_data_mapping.py

@@ -2,7 +2,7 @@
 import json
 import re
 
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 def _fetch_mapping():
     try:

stix_shifter/src/modules/csa/cloudsql_query_constructor.py → stix_shifter/stix_translation/src/modules/csa/cloudsql_query_constructor.py

@@ -5,18 +5,18 @@
 
 logger = logging.getLogger(__name__)
 
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.patterns.errors import SearchFeatureNotSupportedError
+from stix_shifter.stix_translation.src.patterns.errors import SearchFeatureNotSupportedError
 
-from stix_shifter.src.transformers import TimestampToMilliseconds, ValueTransformer
+from stix_shifter.stix_translation.src.transformers import TimestampToMilliseconds, ValueTransformer
 
 
 def _fetch_network_protocol_mapping():
     try:
         map_file = open(
-            'stix_shifter/src/modules/qradar/json/network_protocol_map.json').read()
+            'stix_shifter/stix_translation/src/modules/qradar/json/network_protocol_map.json').read()
         map_data = json.loads(map_file)
         return map_data
     except Exception as ex:

stix_shifter/src/modules/csa/stix_to_cloudsql.py → stix_shifter/stix_translation/src/modules/csa/stix_to_cloudsql.py

@@ -19,7 +19,7 @@ def transform_query(self, data, options, mapping=None):
         Transforms STIX query into sql query format. Based on a mapping file
         :param data: STIX query string to transform into sql query format
         :type data: str
-        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into sql format. This defaults to the from_stix_map.json in the stix_shifter/src/modules/qradar/json/ directory
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into sql format. This defaults to the from_stix_map.json in the stix_shifter/stix_translation/src/modules/qradar/json/ directory
         :type mapping: str (filepath)
         :return: sql query string
         :rtype: str

stix_shifter/src/modules/dummy/dummy_data_mapping.py → stix_shifter/stix_translation/src/modules/dummy/dummy_data_mapping.py

@@ -1,6 +1,6 @@
 from os import path
 import json
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 
 def _fetch_mapping():

stix_shifter/src/modules/dummy/dummy_query_constructor.py → stix_shifter/stix_translation/src/modules/dummy/dummy_query_constructor.py

@@ -1,7 +1,7 @@
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.transformers import TimestampToMilliseconds
+from stix_shifter.stix_translation.src.transformers import TimestampToMilliseconds
 import logging
 import re
 

stix_shifter/src/modules/elastic/elastic_query_constructor.py → stix_shifter/stix_translation/src/modules/elastic/elastic_query_constructor.py

@@ -3,10 +3,10 @@
 
 logger = logging.getLogger(__name__)
 
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.patterns.errors import SearchFeatureNotSupportedError
+from stix_shifter.stix_translation.src.patterns.errors import SearchFeatureNotSupportedError
 
 class ElasticQueryStringPatternTranslator:
     comparator_lookup = {

stix_shifter/src/modules/elastic/stix_to_elastic.py → stix_shifter/stix_translation/src/modules/elastic/stix_to_elastic.py

@@ -15,7 +15,7 @@ def transform_query(self, data, options, mapping=None):
         Transforms STIX query into elastic query format. Based on a mapping file
         :param data: STIX query string to transform into elastic query format
         :type data: str
-        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into elastic format. This defaults to the from_stix_map.json in the stix_shifter/src/modules/qradar/json/ directory
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into elastic format. This defaults to the from_stix_map.json in the stix_shifter/stix_translation/src/modules/qradar/json/ directory
         :type mapping: str (filepath)
         :return: elastic query string
         :rtype: str
@@ -26,7 +26,7 @@ def transform_query(self, data, options, mapping=None):
         if not data_mapper:
             data_mapper = 'car'
 
-        data_mapper_module_name = ''.join(["stix_shifter.src.modules.", data_mapper, ".", data_mapper, "_data_mapping"])
+        data_mapper_module_name = ''.join(["stix_shifter.stix_translation.src.modules.", data_mapper, ".", data_mapper, "_data_mapping"])
 
         try:
             data_mapper_module = importlib.import_module(data_mapper_module_name)

stix_shifter/src/modules/qradar/aql_query_constructor.py → stix_shifter/stix_translation/src/modules/qradar/aql_query_constructor.py

@@ -1,7 +1,7 @@
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.transformers import TimestampToMilliseconds
+from stix_shifter.stix_translation.src.transformers import TimestampToMilliseconds
 import logging
 import json
 import re
@@ -15,7 +15,7 @@
 def _fetch_network_protocol_mapping():
     try:
         map_file = open(
-            'stix_shifter/src/modules/qradar/json/network_protocol_map.json').read()
+            'stix_shifter/stix_translation/src/modules/qradar/json/network_protocol_map.json').read()
         map_data = json.loads(map_file)
         return map_data
     except Exception as ex:

stix_shifter/src/modules/qradar/qradar_data_mapping.py → stix_shifter/stix_translation/src/modules/qradar/qradar_data_mapping.py

@@ -1,6 +1,6 @@
 from os import path
 import json
-from stix_shifter.src.exceptions import DataMappingException
+from stix_shifter.stix_translation.src.exceptions import DataMappingException
 
 
 def _fetch_mapping():

stix_shifter/src/modules/qradar/stix_to_aql.py → stix_shifter/stix_translation/src/modules/qradar/stix_to_aql.py

@@ -18,7 +18,7 @@ def transform_query(self, data, options, mapping=None):
         Transforms STIX query into aql query format. Based on a mapping file
         :param data: STIX query string to transform into aql query format
         :type data: str
-        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into aql format. This defaults to the from_stix_map.json in the stix_shifter/src/modules/qradar/json/ directory
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into aql format. This defaults to the from_stix_map.json in the stix_shifter/stix_translation/src/modules/qradar/json/ directory
         :type mapping: str (filepath)
         :return: aql query string
         :rtype: str

stix_shifter/src/modules/splunk/cim_to_stix/cim_to_stix.py → stix_shifter/stix_translation/src/modules/splunk/cim_to_stix/cim_to_stix.py

@@ -2,7 +2,7 @@
 import uuid
 from . import cim_to_stix_translator
 from ...base.base_result_translator import BaseResultTranslator
-from stix_shifter.src import transformers
+from stix_shifter.stix_translation.src import transformers
 
 class CIMToStix(BaseResultTranslator):
 

stix_shifter/src/modules/splunk/splunk_query_constructor.py → stix_shifter/stix_translation/src/modules/splunk/splunk_query_constructor.py

@@ -4,10 +4,10 @@
 
 logger = logging.getLogger(__name__)
 
-from stix_shifter.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
+from stix_shifter.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \
     ComparisonExpressionOperators, ComparisonComparators, Pattern, \
     CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators
-from stix_shifter.src.modules.car.car_data_mapping import CarDataMapper
+from stix_shifter.stix_translation.src.modules.car.car_data_mapping import CarDataMapper
 
 from . import encoders
 from . import object_scopers

stix_shifter/src/modules/splunk/stix_to_splunk.py → stix_shifter/stix_translation/src/modules/splunk/stix_to_splunk.py

@@ -18,7 +18,7 @@ def transform_query(self, data, options, mapping=None):
         Transforms STIX query into splunk query format. Based on a mapping file
         :param data: STIX query string to transform into splunk query format
         :type data: str
-        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into splunk format. This defaults to the from_stix_map.json in the stix_shifter/src/modules/qradar/json/ directory
+        :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into splunk format. This defaults to the from_stix_map.json in the stix_shifter/stix_translation/src/modules/qradar/json/ directory
         :type mapping: str (filepath)
         :return: splunk query string
         :rtype: str
@@ -33,7 +33,7 @@ def transform_query(self, data, options, mapping=None):
         if not data_mapper:
             data_mapper = 'cim'
 
-        data_mapper_module_name = ''.join(["stix_shifter.src.modules.", data_mapper, ".", data_mapper, "_data_mapping"])
+        data_mapper_module_name = ''.join(["stix_shifter.stix_translation.src.modules.", data_mapper, ".", data_mapper, "_data_mapping"])
 
         try:
             data_mapper_module = importlib.import_module(data_mapper_module_name)