diff --git a/conf/schemas/carbonblack.json b/conf/schemas/carbonblack.json index 83a408de1..c9d4e88de 100644 --- a/conf/schemas/carbonblack.json +++ b/conf/schemas/carbonblack.json @@ -22,7 +22,7 @@ }, "carbonblack:alert.watchlist.hit.feedsearch.bin": { "schema": { - "alert_severity": "float", + "alert_severity": "string", "alert_type": "string", "assigned_to": "string", "cb_server": "string", @@ -48,7 +48,7 @@ "segment_id": "integer", "sensor_criticality": "integer", "status": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "unique_id": "string", "watchlist_id": "string", @@ -71,7 +71,7 @@ }, "carbonblack:alert.watchlist.hit.ingress.binary": { "schema": { - "alert_severity": "float", + "alert_severity": "string", "alert_type": "string", "assigned_to": "string", "cb_server": "string", @@ -97,7 +97,7 @@ "segment_id": "integer", "sensor_criticality": "integer", "status": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "unique_id": "string", "watchlist_id": "string", @@ -107,7 +107,7 @@ }, "carbonblack:alert.watchlist.hit.ingress.process": { "schema": { - "alert_severity": "integer", + "alert_severity": "string", "alert_type": "string", "assigned_to": "string", "cb_server": "string", @@ -145,8 +145,9 @@ "segment_id": "string", "sensor_criticality": "integer", "sensor_id": "integer", + "sha256": "string", "status": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "unique_id": "string", "username": "string", @@ -166,7 +167,8 @@ "ioc_query_index", "ioc_query_string", "ioc_value_facet", - "resolved_time" + "resolved_time", + "sha256" ] } }, @@ -234,8 +236,9 @@ "segment_id": "string", "sensor_criticality": "integer", "sensor_id": "integer", + "sha256": "string", "status": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "unique_id": "string", "username": "string", @@ -280,7 +283,8 @@ "alliance_updated_virustotalconnector", "assigned_to", "ioc_attr", - "resolved_time" + "resolved_time", + "sha256" ] } }, @@ -298,7 +302,8 @@ "group": "string", "md5": "string", "scores": {}, - "timestamp": "float", + "sha256": "string", + "timestamp": "string", "type": "string", "watchlists": {} }, @@ -312,18 +317,25 @@ "md5": "string", "scores": {}, "sensor_id": "integer", - "timestamp": "float", + "sha256": "string", + "timestamp": "string", "type": "string", "watchlists": {} }, - "parser": "json" + "parser": "json", + "configuration": { + "optional_top_level_keys": [ + "sha256" + ] + } }, "carbonblack:binaryinfo.observed": { "schema": { "cb_server": "string", "md5": "string", "scores": {}, - "timestamp": "float", + "sha256": "string", + "timestamp": "string", "type": "string", "watchlists": {} }, @@ -361,7 +373,7 @@ "report_score": "integer", "sensor_id": "integer", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json" @@ -384,7 +396,7 @@ "report_score": "integer", "sensor_id": "integer", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json" @@ -410,7 +422,7 @@ "segment_id": "string", "sensor_id": "integer", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -475,7 +487,7 @@ "report_id": "string", "report_score": "integer", "sensor_id": "integer", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "json_path": "docs[*]", @@ -486,6 +498,7 @@ }, "carbonblack:feed.query.hit.process": { "schema": { + "alliance_data_attackframework": [], "alliance_data_bit9endpointvisibility": [], "alliance_data_bit9suspiciousindicators": [], "alliance_data_nvd": [], @@ -493,6 +506,7 @@ "alliance_data_srstrust": [], "alliance_data_virustotal": [], "alliance_data_virustotalconnector": [], + "alliance_link_attackframework": "string", "alliance_link_bit9endpointvisibility": "string", "alliance_link_bit9suspiciousindicators": "string", "alliance_link_nvd": "string", @@ -500,6 +514,7 @@ "alliance_link_srstrust": "string", "alliance_link_virustotal": "string", "alliance_link_virustotalconnector": "string", + "alliance_score_attackframework": "integer", "alliance_score_bit9endpointvisibility": "integer", "alliance_score_bit9suspiciousindicators": "integer", "alliance_score_nvd": "integer", @@ -507,6 +522,7 @@ "alliance_score_srstrust": "integer", "alliance_score_virustotal": "integer", "alliance_score_virustotalconnector": "integer", + "alliance_updated_attackframework": "string", "alliance_updated_bit9endpointvisibility": "string", "alliance_updated_bit9suspiciousindicators": "string", "alliance_updated_nvd": "string", @@ -539,6 +555,7 @@ "process_md5": "string", "process_name": "string", "process_pid": "integer", + "process_sha256": "string", "processblock_count": "integer", "regmod_count": "integer", "segment_id": "string", @@ -567,11 +584,12 @@ "report_score": "integer", "segment_id": "string", "sensor_id": "integer", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "json_path": "docs[*]", "optional_top_level_keys": [ + "alliance_data_attackframework", "alliance_data_bit9endpointvisibility", "alliance_data_bit9suspiciousindicators", "alliance_data_nvd", @@ -579,6 +597,7 @@ "alliance_data_srstrust", "alliance_data_virustotal", "alliance_data_virustotalconnector", + "alliance_link_attackframework", "alliance_link_bit9endpointvisibility", "alliance_link_bit9suspiciousindicators", "alliance_link_nvd", @@ -586,6 +605,7 @@ "alliance_link_srstrust", "alliance_link_virustotal", "alliance_link_virustotalconnector", + "alliance_score_attackframework", "alliance_score_bit9endpointvisibility", "alliance_score_bit9suspiciousindicators", "alliance_score_nvd", @@ -593,6 +613,7 @@ "alliance_score_srstrust", "alliance_score_virustotal", "alliance_score_virustotalconnector", + "alliance_updated_attackframework", "alliance_updated_bit9endpointvisibility", "alliance_updated_bit9suspiciousindicators", "alliance_updated_nvd", @@ -602,6 +623,7 @@ "alliance_updated_virustotalconnector", "interface_ip", "process_guid", + "process_sha256", "segment_id" ] } @@ -627,13 +649,15 @@ "report_score": "integer", "sensor_id": "integer", "server_name": "string", - "timestamp": "float", + "sha256": "string", + "timestamp": "string", "type": "string" }, "configuration": { "optional_top_level_keys": [ "ioc_query_index", - "ioc_query_string" + "ioc_query_string", + "sha256" ] }, "parser": "json" @@ -663,7 +687,7 @@ "segment_id": "string", "sensor_id": "integer", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -681,7 +705,7 @@ "feed_name": "string", "feed_update_time": "string", "scan_start_time": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json" @@ -703,17 +727,19 @@ "path": "string", "pid": "integer", "process_guid": "string", + "process_path": "string", "sensor_id": "integer", "sha256": "string", "tamper": "boolean", "tamper_sent": "boolean", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "configuration": { "optional_top_level_keys": [ "child_command_line", - "child_username" + "child_username", + "process_path" ] }, "parser": "json" @@ -738,7 +764,7 @@ "target_pid": "integer", "target_process_guid": "string", "target_sha256": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -769,7 +795,7 @@ "sha256": "string", "tamper": "boolean", "tamper_sent": "boolean", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -794,7 +820,7 @@ "sensor_id": "integer", "sha256": "string", "size": "integer", - "timestamp": "float", + "timestamp": "string", "type": "string", "utf8_comments": "string", "utf8_company_name": "string", @@ -827,7 +853,7 @@ "process_path": "string", "sensor_id": "integer", "sha256": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -868,7 +894,7 @@ "remote_port": "string", "sensor_id": "string", "sha256": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -912,7 +938,7 @@ "process_path": "string", "sensor_id": "integer", "sha256": "string", - "timestamp": "integer", + "timestamp": "string", "type": "string", "uid": "string", "username": "string" @@ -952,7 +978,7 @@ "process_create_time": "integer", "process_guid": "string", "sensor_id": "integer", - "timestamp": "float", + "timestamp": "string", "type": "string", "uid": "string", "username": "string" @@ -1036,7 +1062,7 @@ "sha256": "string", "tamper": "boolean", "tamper_sent": "boolean", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -1065,7 +1091,7 @@ "target_pid": "integer", "target_process_guid": "string", "target_sha256": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -1082,7 +1108,7 @@ "event_type": "string", "sensor_id": "integer", "tamper_type": "string", - "timestamp": "float", + "timestamp": "string", "type": "string" }, "parser": "json", @@ -1166,7 +1192,7 @@ "cb_version": "string", "highlights_by_doc": {}, "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "watchlist_id": "integer", "watchlist_name": "string" @@ -1214,7 +1240,7 @@ }, "carbonblack:watchlist.hit.ingress.binary": { "schema": { - "alert_severity": "float", + "alert_severity": "string", "alert_type": "string", "assigned_to": "string", "cb_server": "string", @@ -1238,7 +1264,7 @@ "report_score": "integer", "sensor_criticality": "integer", "status": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "unique_id": "string", "watchlist_id": "string", @@ -1258,6 +1284,7 @@ "alliance_data_nvd": [], "alliance_data_srsthreat": [], "alliance_data_srstrust": [], + "alliance_data_tor": [], "alliance_data_virustotal": [], "alliance_data_virustotalconnector": [], "alliance_link_bit9endpointvisibility": "string", @@ -1265,6 +1292,7 @@ "alliance_link_nvd": "string", "alliance_link_srsthreat": "string", "alliance_link_srstrust": "string", + "alliance_link_tor": "string", "alliance_link_virustotal": "string", "alliance_link_virustotalconnector": "string", "alliance_score_bit9endpointvisibility": "integer", @@ -1272,6 +1300,7 @@ "alliance_score_nvd": "integer", "alliance_score_srsthreat": "integer", "alliance_score_srstrust": "integer", + "alliance_score_tor": "integer", "alliance_score_virustotal": "integer", "alliance_score_virustotalconnector": "integer", "alliance_updated_bit9endpointvisibility": "string", @@ -1279,6 +1308,7 @@ "alliance_updated_nvd": "string", "alliance_updated_srsthreat": "string", "alliance_updated_srstrust": "string", + "alliance_updated_tor": "string", "alliance_updated_virustotal": "string", "alliance_updated_virustotalconnector": "string", "childproc_count": "integer", @@ -1311,6 +1341,7 @@ "process_md5": "string", "process_name": "string", "process_pid": "integer", + "process_sha256": "string", "processblock_count": "integer", "regmod_count": "integer", "segment_id": "integer", @@ -1326,7 +1357,7 @@ "cb_version": "string", "highlights_by_doc": {}, "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "watchlist_id": "integer", "watchlist_name": "string" @@ -1338,6 +1369,7 @@ "alliance_data_nvd", "alliance_data_srsthreat", "alliance_data_srstrust", + "alliance_data_tor", "alliance_data_virustotal", "alliance_data_virustotalconnector", "alliance_link_bit9endpointvisibility", @@ -1345,6 +1377,7 @@ "alliance_link_nvd", "alliance_link_srsthreat", "alliance_link_srstrust", + "alliance_link_tor", "alliance_link_virustotal", "alliance_link_virustotalconnector", "alliance_score_bit9endpointvisibility", @@ -1352,6 +1385,7 @@ "alliance_score_nvd", "alliance_score_srsthreat", "alliance_score_srstrust", + "alliance_score_tor", "alliance_score_virustotal", "alliance_score_virustotalconnector", "alliance_updated_bit9endpointvisibility", @@ -1359,9 +1393,11 @@ "alliance_updated_nvd", "alliance_updated_srsthreat", "alliance_updated_srstrust", + "alliance_updated_tor", "alliance_updated_virustotal", "alliance_updated_virustotalconnector", - "process_guid" + "process_guid", + "process_sha256" ] } }, @@ -1372,7 +1408,7 @@ "docs": [], "md5": "string", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "watchlist_id": "integer", "watchlist_name": "string" @@ -1395,7 +1431,7 @@ "process_id": "string", "segment_id": "string", "server_name": "string", - "timestamp": "float", + "timestamp": "string", "type": "string", "watchlist_id": "integer", "watchlist_name": "string" diff --git a/docs/source/historical-search.rst b/docs/source/historical-search.rst index e4de0bc1e..b221ddeb8 100644 --- a/docs/source/historical-search.rst +++ b/docs/source/historical-search.rst @@ -32,7 +32,7 @@ The pipeline is Alerts Search ************* -* Review alert Firehose configuration, see :ref:`alerts_Firehose_configuration` in ``CONFIGURATION`` session. Athena database and Athena alerts table are created automatically when you first deploy StreamAlert. +* Review alert Firehose configuration, see :ref:`alerts_firehose_configuration` in ``CONFIGURATION`` session. Athena database and Athena alerts table are created automatically when you first deploy StreamAlert. * If the ``file_format`` is set to ``parquet``, you can run ``MSCK REPAIR TABLE alerts`` command in the Athena to load all available partitions and then alerts can be searchable. However, using ``MSCK REPAIR`` command can not load new partitions automatically. * StreamAlert provides a lambda function ``athena_partition_refresh`` to load new partitions to Athena tables once the data arrives in the S3 buckets automatically. Update ``athena_partition_refresh_config`` if necessary. Open ``conf/lambda.json``. See more settings :ref:`configure_athena_partition_refresh_lambda` @@ -54,7 +54,7 @@ Alerts Search * Search alerts in `Athena Console `_ - * Choose your ``Database`` from the dropdown on the left. Database name is ``PREFIX_streamalert`` + * Choose your ``Database`` from the dropdown on the left. Database name is ``_streamalert`` * Write SQL query statement in the ``Query Editor`` on the right .. image:: ../images/athena-alerts-search.png @@ -65,7 +65,7 @@ Data Search It is optional to store data in S3 bucket and available for search in Athena tables. -* Enable Firehose in ``conf/global.json`` see :ref:`Firehose_configuration` +* Enable Firehose in ``conf/global.json`` see :ref:`firehose_configuration` * Build the Firehose and Athena tables .. code-block:: bash @@ -80,7 +80,7 @@ It is optional to store data in S3 bucket and available for search in Athena tab * Search data `Athena Console `_ - * Choose your ``Database`` from the dropdown on the left. Database name is ``PREFIX_streamalert`` + * Choose your ``Database`` from the dropdown on the left. Database name is ``_streamalert`` * Write SQL query statement in the ``Query Editor`` on the right .. image:: ../images/athena-data-search.png @@ -138,5 +138,5 @@ Athena References .. code-block:: sql - SELECT * FROM "PREFIX_streamalert"."alerts" + SELECT * FROM "_streamalert"."alerts" WHERE dt BETWEEN 2020-02-28-00 AND 2020-02-29-00 diff --git a/streamalert/classifier/clients/firehose.py b/streamalert/classifier/clients/firehose.py index 594303b94..f6c3c2443 100644 --- a/streamalert/classifier/clients/firehose.py +++ b/streamalert/classifier/clients/firehose.py @@ -331,7 +331,7 @@ def generate_firehose_name(cls, prefix, log_stream_name): # combine the base_name and first 8 chars of hash result together as new # stream name. return '{}{}'.format( - base_name, hashlib.md5(base_name.encode()).hexdigest() # nosec + base_name, hashlib.md5(stream_name.encode()).hexdigest() # nosec )[:cls.AWS_FIREHOSE_NAME_MAX_LEN] @classmethod diff --git a/streamalert/shared/utils.py b/streamalert/shared/utils.py index c30f174a0..30457a180 100644 --- a/streamalert/shared/utils.py +++ b/streamalert/shared/utils.py @@ -165,4 +165,4 @@ def get_data_file_format(config): """ athena_config = config['lambda'].get('athena_partition_refresh_config', {}) - return athena_config.get('file_format', None) + return athena_config.get('file_format') diff --git a/streamalert_cli/_infrastructure/modules/tf_globals/alerts_firehose/main.tf b/streamalert_cli/_infrastructure/modules/tf_globals/alerts_firehose/main.tf index f112ba833..cb5d4a26a 100644 --- a/streamalert_cli/_infrastructure/modules/tf_globals/alerts_firehose/main.tf +++ b/streamalert_cli/_infrastructure/modules/tf_globals/alerts_firehose/main.tf @@ -10,8 +10,8 @@ locals { locals { stream_name = "${var.prefix}_streamalert_alert_delivery" - bucket_arn = "arn:aws:s3:::${var.prefix}-streamalerts" - alerts_location = "s3://${var.prefix}-streamalerts/${local.s3_path_prefix}" + bucket_arn = "arn:aws:s3:::${var.bucket_name}" + alerts_location = "s3://${var.bucket_name}/${local.s3_path_prefix}" ser_de_params_key = var.file_format == "parquet" ? "serialization.format" : "ignore.malformed.json" ser_de_params_value = var.file_format == "parquet" ? "1" : "true" } diff --git a/streamalert_cli/athena/helpers.py b/streamalert_cli/athena/helpers.py index d4e03face..5c87937a6 100644 --- a/streamalert_cli/athena/helpers.py +++ b/streamalert_cli/athena/helpers.py @@ -159,20 +159,20 @@ def format_schema_tf(schema): formatted_schema (list(tuple)) """ # Construct the main Athena Schema - formated_schema = [] + formatted_schema = [] for key_name in sorted(schema.keys()): key_type = schema[key_name] if isinstance(key_type, str): - formated_schema.append((key_name.lower(), key_type)) + formatted_schema.append((key_name.lower(), key_type)) # Account for nested structs elif isinstance(key_type, dict): struct_schema = ','.join( '{0}:{1}'.format(sub_key.lower(), key_type[sub_key]) for sub_key in sorted(key_type.keys()) ) - formated_schema.append((key_name.lower(), 'struct<{}>'.format(struct_schema))) + formatted_schema.append((key_name.lower(), 'struct<{}>'.format(struct_schema))) - return formated_schema + return formatted_schema def generate_alerts_table_schema(): """Generate the schema for alerts table in terraform by using a fake alert diff --git a/tests/unit/streamalert/classifier/clients/test_firehose.py b/tests/unit/streamalert/classifier/clients/test_firehose.py index 824fa0af0..2b551f799 100644 --- a/tests/unit/streamalert/classifier/clients/test_firehose.py +++ b/tests/unit/streamalert/classifier/clients/test_firehose.py @@ -476,7 +476,7 @@ def test_send_long_log_name(self, send_batch_mock): client.send(self._sample_payloads_long_log_name) send_batch_mock.assert_called_with( - 'streamalert_very_very_very_long_log_stream_name_abcdefg_bbe68ccf', expected_batch + 'streamalert_very_very_very_long_log_stream_name_abcdefg_7c88167b', expected_batch ) def test_generate_firehose_name(self): @@ -492,7 +492,7 @@ def test_generate_firehose_name(self): 'streamalert_logstreamname', 'streamalert_log_stream_name', 'streamalert_very_very_long_log_stream_name_ab_52_characters_long', - 'streamalert_very_very_very_long_log_stream_name_abcdefg_bbe68ccf' + 'streamalert_very_very_very_long_log_stream_name_abcdefg_272fa762' ] results = [ self._client.generate_firehose_name('', log_name) @@ -513,8 +513,8 @@ def test_generate_firehose_name_prefix(self): expected_results = [ 'prefix_streamalert_logstreamname', 'prefix_streamalert_log_stream_name', - 'prefix_streamalert_very_very_long_log_stream_name_ab_52_4ccde648', - 'prefix_streamalert_very_very_very_long_log_stream_name_6fafde49a' + 'prefix_streamalert_very_very_long_log_stream_name_ab_52_63bd84dc', + 'prefix_streamalert_very_very_very_long_log_stream_name_a0c91e099' ] results = [ self._client.generate_firehose_name('prefix', log_name)