Merge ccc8a29 into a0a284c

docs/source/scheduled-queries.rst

@@ -129,9 +129,43 @@ All scheduled queries are located in the ``scheduled_queries/`` directory, locat
 * ``name`` - (str) The name of this query. This name is published in the final result, and is useful when writing rules.
 * ``description`` - (str) Description of this query. This is published in the final result.
 * ``query`` - (str) A template SQL statement sent to Athena, with query parameters identified ``{like_this}``.
-* ``params`` - (list[str]) A list of query parameters to pass to the query string. These have special values that are calculated at runtime, and are interpolated into the template SQL string.
+* ``params`` - (list[str]|dict[str,callable]) Read on below...
 * ``tags`` - (list[str]) Tags required by this query to be run. The simplest way to use this is to put the **Query pack name** into this array.
 
+params
+``````
+The "params" option specifies how to calculate special query parameters. It supports two formats.
+
+The first format is a list of strings from a predefined set of strings. These have special values that are calculated at runtime,
+and are interpolated into the template SQL string. Here is a list of the supported strings:
+
+
+
+The second format is a dictionary mapping parameter names to functions, like so:
+
+.. code-block:: python
+
+    def func1(date):
+        return date.timestamp()
+
+    def func2(date):
+        return LookupTables.get('aaaa', 'bbbb')
+
+    QueryPackConfiguration(
+        ...
+        query="""
+    SELECT *
+    FROM stuff
+    WHERE
+      dt = '{my_param_1}'
+      AND p2 = '{my_param_2}'
+    """,
+        params={
+            'my_param_1': func1,
+            'my_param_2': func2,
+        }
+    )
+
 
 
 Writing Rules for StreamQuery

publishers/sample/sample_demisto.py

@@ -0,0 +1,55 @@
+"""
+Batch of example publishers usable with Demisto.
+"""
+from streamalert.shared.publisher import Register
+
+
+@Register
+def demisto_classification(alert, publication):
+    """
+    This publisher appropriately sets the demisto incident type and playbook.
+
+    It first looks into the alert's context for the "demisto" key, where individual rules can
+    explcitly specify the desired classification traits of the output alert.
+    """
+
+    # If a rule explicitly states Demisto information with the alert context, obey that
+    # The convention to follow is any key in this dict (example, "incident_type") is mapped
+    # directly onto the Demisto output magic keys (example, @demisto.incident_type)
+    if 'demisto' in alert.context:
+        for key, value in alert.context['demisto'].items():
+            output_key = '@demisto.{}'.format(key)
+            publication[output_key] = value
+
+        return publication
+
+    # If no context was explicitly declared, then we default to our global rules
+    for code in GLOBAL_CLASSIFIERS:
+        payload = code(alert)
+        if payload:
+            for key, value in payload:
+                output_key = '@demisto.{}'.format(key)
+                publication[output_key] = value
+
+            return publication
+
+    # Else, nothing
+    return publication
+
+
+def _any_rule_with_demisto(alert):
+    if alert.rule_name.contains('sample'):
+        return {
+            'incident_type': 'Sample Alert',
+            'playbook': 'Sample Playbook',
+        }
+
+    return False
+
+
+# The GLOBAL_CLASSIFIERS is an array of functions. Any function that returns a truthy value is
+# considered to be a "match". This value must be a dict, and the keys on the dict map directly
+# onto the Demisto output magic keys (e.g. "incident_type" -> "@demisto.incident_type")
+GLOBAL_CLASSIFIERS = [
+    _any_rule_with_demisto
+]

rules/sample/sample_demisto.json

@@ -0,0 +1,41 @@
+[
+  {
+    "data": {
+      "action": "added",
+      "calendarTime": "Wed Feb 12 21:38:11 2020 UTC",
+      "columns": {
+        "host": "10.0.2.2",
+        "pid": 12345,
+        "time": 1581542540,
+        "tty": "ttys001",
+        "type": "8",
+        "username": "runlevel"
+      },
+      "decorations": {
+        "envIdentifier": "fake-environment",
+        "roleIdentifier": "fake-role"
+      },
+      "epoch": "0",
+      "hostIdentifier": "sample_demisto",
+      "log_type": "result",
+      "name": "pack_incident-response_last",
+      "unixTime": "1581543491"
+    },
+    "description": "Just shows how to do Demisto stuff",
+    "log": "osquery:differential",
+    "service": "kinesis",
+    "source": "prefix_cluster1_streamalert",
+    "trigger_rules": [
+      "sample_demisto"
+    ],
+    "publisher_tests": {
+      "demisto:sample-integration": [
+        {
+          "jmespath_expression": "\"@demisto.incident_type\"",
+          "condition": "is",
+          "value": "My sample type"
+        }
+      ]
+    }
+  }
+]

rules/sample/sample_demisto.py

@@ -0,0 +1,25 @@
+"""
+Example for writing a Demisto role
+"""
+from publishers.sample.sample_demisto import demisto_classification
+from streamalert.shared.rule import rule
+
+
+@rule(
+    logs=['osquery:differential'],
+    outputs=['demisto:sample-integration'],
+    publishers=[demisto_classification],
+    context={
+        'demisto': {
+            'incident_type': 'My sample type',
+            'playbook': 'A Playbook',
+            'severity': 'informational'
+        },
+    }
+)
+def sample_demisto(record, _):
+    """
+    author:           Derek Wang
+    description:      An example of how to write a Demisto alert using publishers to classify
+    """
+    return record.get('hostIdentifier', '') == 'sample_demisto'

streamalert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '3.2.1'
+__version__ = '3.3.0'

streamalert/alert_processor/outputs/demisto.py

@@ -137,6 +137,7 @@ def send(self, request):
             'type': request.incident_type,
             'name': request.incident_name,
             'owner': request.owner,
+            'playbook': request.playbook,
             'severity': request.severity,
             'labels': request.labels,
             'customFields': request.custom_fields,
@@ -173,6 +174,7 @@ class DemistoCreateIncidentRequest:
     def __init__(self,
                  incident_name='Unnamed StreamAlert Alert',
                  incident_type='Unclassified',
+                 playbook='',
                  severity=SEVERITY_UNKNOWN,
                  owner='StreamAlert',
                  details='Details not specified.',
@@ -186,6 +188,9 @@ def __init__(self,
         # "Unclassified".
         self._incident_type = str(incident_type)
 
+        # The playbook to assign to the case.
+        self._playbook = playbook
+
         # Severity is an integer. Use the constants above.
         self._severity = severity
 
@@ -222,6 +227,10 @@ def incident_name(self):
     def incident_type(self):
         return self._incident_type
 
+    @property
+    def playbook(self):
+        return self._playbook
+
     @property
     def severity(self):
         return self._severity
@@ -282,13 +291,15 @@ def assemble(alert, alert_publication):
         # Default presentation values
         default_incident_name = alert.rule_name
         default_incident_type = 'Unclassified'
+        default_playbook = 'Unknown'
         default_severity = 'unknown'
         default_owner = 'StreamAlert'
         default_details = alert.rule_description
         default_label_data = alert_publication
 
         # Special keys that publishers can use to modify default presentation
         incident_type = alert_publication.get('@demisto.incident_type', default_incident_type)
+        playbook = alert_publication.get('@demisto.playbook', default_playbook)
         severity = DemistoCreateIncidentRequest.map_severity_string_to_severity_value(
             alert_publication.get('@demisto.severity', default_severity)
         )
@@ -303,6 +314,7 @@ def assemble(alert, alert_publication):
             severity=severity,
             owner=owner,
             details=details,
+            playbook=playbook,
             create_investigation=True  # Important: Trigger workbooks automatically
         )
 

streamalert/apps/_apps/aliyun.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from datetime import datetime
 import json
 import re
 
@@ -59,6 +60,11 @@ def __init__(self, event, context):
         self.request.set_MaxResults(self._MAX_RESULTS)
         self.request.set_StartTime(self._config.last_timestamp)
 
+        # Source code can be found here https://github.com/aliyun/aliyun-openapi-python-sdk/
+        # blob/master/aliyun-python-sdk-actiontrail/aliyunsdkactiontrail/request/v20171204/
+        # LookupEventsRequest.py
+        self.request.set_EndTime(datetime.utcnow().strftime(self.date_formatter()))
+
     @classmethod
     def _type(cls):
         return 'actiontrail'

streamalert/scheduled_queries/query_packs/configuration.py

@@ -52,7 +52,6 @@ def generate_query(self, **kwargs):
 '''.strip().format(name=self.name, error=e, kwargs=kwargs)
             raise KeyError(msg)
 
-
     @property
     def query_template(self):
         return self._query_template

streamalert/scheduled_queries/query_packs/manager.py

@@ -76,10 +76,22 @@ def __init__(self, query_pack_configuration, execution_context):
         self._query_execution_id = None
         self._query_result = None
 
-        self._query_parameters = {
-            param: self._execution_context.parameter_generator.generate(param)
-            for param in self._configuration.query_parameters
-        }
+        if isinstance(self._configuration.query_parameters, dict):
+            self._query_parameters = {
+                param: self._execution_context.parameter_generator.generate_advanced(
+                    param, configuration
+                )
+                for param, configuration in self._configuration.query_parameters.items()
+            }
+        elif isinstance(self._configuration.query_parameters, list):
+            self._query_parameters = {
+                param: self._execution_context.parameter_generator.generate(param)
+                for param in self._configuration.query_parameters
+            }
+        else:
+            # not intended to be reached
+            self._query_parameters = {}
+
         self._query_string = None
 
     @property

streamalert/scheduled_queries/query_packs/parameters.py

@@ -60,7 +60,17 @@ def generate(self, parameter):
         if parameter == 'utctimestamp':
             return str(round(self._clock.now.timestamp()))
 
+        if parameter == 'utcisotime':
+            return str(round(self._clock.now.timestamp()))
+
         self._logger.error(
             'Parameter generator does not know how to handle "{}"'.format(parameter)
         )
         return None
+
+    def generate_advanced(self, key, configuration):
+        if callable(configuration):
+            return configuration(self._clock.now)
+
+        # else, default to whatever generate returns
+        return self.generate(key)

streamalert_cli/_infrastructure/modules/tf_scheduled_queries/outputs.tf

@@ -0,0 +1,4 @@
+# Role id of the lambda function that runs scheduled queries
+output "lambda_function_role_id" {
+  value = module.scheduled_queries_lambda.role_id
+}

streamalert_cli/terraform/generate.py

@@ -548,6 +548,7 @@ def _generate_lookup_tables_settings(config):
         '${module.alert_processor_lambda.role_id}',
         '${module.alert_merger_lambda.role_id}',
         '${module.rules_engine_lambda.role_id}',
+        '${module.scheduled_queries.lambda_function_role_id}',
     }
 
     for cluster in config.clusters():

streamalert_cli/test/handler.py

@@ -134,9 +134,8 @@ def _add_default_test_args(test_parser):
             '-f',
             '--test-files',
             dest='files',
-            metavar='FILENAMES',
             nargs='+',
-            help='One or more file to test, separated by spaces',
+            help='Full path to one or more file(s) to test, separated by spaces',
             action=UniqueSortedFileListAction,
             type=argparse.FileType('r'),
             default=[]

tests/unit/streamalert/alert_processor/outputs/test_demisto.py

@@ -122,6 +122,7 @@ def test_dispatch(self, request_mock):
             'type': 'Unclassified',
             'name': 'cb_binarystore_file_added',
             'owner': 'StreamAlert',
+            'playbook': 'Unknown',
             'severity': 0,
             'labels': EXPECTED_LABELS_FOR_SAMPLE_ALERT,
             'customFields': {},

tests/unit/streamalert/scheduled_queries/query_packs/test_manager.py

@@ -261,6 +261,16 @@ def test_generate_unsupported(self):
             'Parameter generator does not know how to handle "unsupported"'
         )
 
+    def test_generate_advanced_function(self):
+        """StreamQuery - QueryParameterGenerator - generate_advanced - Function"""
+        def thing(date):
+            return date.strftime('%Y-%m-%d-%H-%I-%S')
+        assert_equals(self._generator.generate_advanced('thing', thing), '2019-01-01-01-01-01')
+
+    def test_generate_advanced_nothing(self):
+        """StreamQuery - QueryParameterGenerator - generate_advanced - Nothing"""
+        assert_equals(self._generator.generate_advanced('utctimestamp', None), '1546304461')
+
 
 @patch('streamalert.scheduled_queries.query_packs.manager.QueryPacksManager')
 def test_new_manager(constructor_spy):