diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/logo.json new file mode 100644 index 00000000000000..ca61db7992083c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "logoSecurity" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/manifest.json new file mode 100644 index 00000000000000..b7afe8d2b158a5 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/manifest.json @@ -0,0 +1,64 @@ +{ + "id": "siem_cloudtrail", + "title": "SIEM Cloudtrail", + "description": "Detect suspicious activity recorded in your cloudtrail logs.", + "type": "Filebeat data", + "logoFile": "logo.json", + "defaultIndexPattern": "filebeat-*", + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}} + ] + } + }, + "jobs": [ + { + "id": "rare_method_for_a_city", + "file": "rare_method_for_a_city.json" + }, + { + "id": "rare_method_for_a_country", + "file": "rare_method_for_a_country.json" + }, + { + "id": "rare_method_for_a_username", + "file": "rare_method_for_a_username.json" + }, + { + "id": "high_distinct_count_error_message", + "file": "high_distinct_count_error_message.json" + }, + { + "id": "rare_error_code", + "file": "rare_error_code.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-rare_method_for_a_city", + "file": "datafeed_rare_method_for_a_city.json", + "job_id": "rare_method_for_a_city" + }, + { + "id": "datafeed-rare_method_for_a_country", + "file": "datafeed_rare_method_for_a_country.json", + "job_id": "rare_method_for_a_country" + }, + { + "id": "datafeed-rare_method_for_a_username", + "file": "datafeed_rare_method_for_a_username.json", + "job_id": "rare_method_for_a_username" + }, + { + "id": "datafeed-high_distinct_count_error_message", + "file": "datafeed_high_distinct_count_error_message.json", + "job_id": "high_distinct_count_error_message" + }, + { + "id": "datafeed-rare_error_code", + "file": "datafeed_rare_error_code.json", + "job_id": "rare_error_code" + } + ] + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_high_distinct_count_error_message.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_high_distinct_count_error_message.json new file mode 100644 index 00000000000000..269aac2ea72a17 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_high_distinct_count_error_message.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}}, + {"term": {"event.module": "aws"}}, + {"exists": {"field": "aws.cloudtrail.error_message"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_error_code.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_error_code.json new file mode 100644 index 00000000000000..4b463a4d109911 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_error_code.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}}, + {"term": {"event.module": "aws"}}, + {"exists": {"field": "aws.cloudtrail.error_code"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_city.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_city.json new file mode 100644 index 00000000000000..e436273a848e7c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_city.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}}, + {"term": {"event.module": "aws"}}, + {"exists": {"field": "source.geo.city_name"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_country.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_country.json new file mode 100644 index 00000000000000..f0e80174b87912 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_country.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}}, + {"term": {"event.module": "aws"}}, + {"exists": {"field": "source.geo.country_iso_code"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_username.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_username.json new file mode 100644 index 00000000000000..2fd3622ff81cee --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_username.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.dataset": "aws.cloudtrail"}}, + {"term": {"event.module": "aws"}}, + {"exists": {"field": "user.name"}} + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json new file mode 100644 index 00000000000000..fdabf66ac91b30 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json @@ -0,0 +1,33 @@ +{ + "job_type": "anomaly_detector", + "description": "Looks for a spike in the rate of an error message which may simply indicate an impending service failure but these can also be byproducts of attempted or successful persistence, privilege escalation, defense evasion, discovery, lateral movement, or collection activity by a threat actor.", + "groups": [ + "siem", + "cloudtrail" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high_distinct_count(\"aws.cloudtrail.error_message\")", + "function": "high_distinct_count", + "field_name": "aws.cloudtrail.error_message" + } + ], + "influencers": [ + "aws.cloudtrail.user_identity.arn", + "source.ip", + "source.geo.city_name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "16mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-cloudtrail" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_error_code.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_error_code.json new file mode 100644 index 00000000000000..0f8fa814ac60a6 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_error_code.json @@ -0,0 +1,33 @@ +{ + "job_type": "anomaly_detector", + "description": "Looks for unsual errors. Rare and unusual errors may simply indicate an impending service failure but they can also be byproducts of attempted or successful persistence, privilege escalation, defense evasion, discovery, lateral movement, or collection activity by a threat actor.", + "groups": [ + "siem", + "cloudtrail" + ], + "analysis_config": { + "bucket_span": "60m", + "detectors": [ + { + "detector_description": "rare by \"aws.cloudtrail.error_code\"", + "function": "rare", + "by_field_name": "aws.cloudtrail.error_code" + } + ], + "influencers": [ + "aws.cloudtrail.user_identity.arn", + "source.ip", + "source.geo.city_name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "16mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-cloudtrail" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_city.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_city.json new file mode 100644 index 00000000000000..eff4d4cdbb8892 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_city.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a geolocation (city) that is unusual. This can be the result of compromised credentials or keys.", + "groups": [ + "siem", + "cloudtrail" + ], + "analysis_config": { + "bucket_span": "60m", + "detectors": [ + { + "detector_description": "rare by \"event.action\" partition by \"source.geo.city_name\"", + "function": "rare", + "by_field_name": "event.action", + "partition_field_name": "source.geo.city_name" + } + ], + "influencers": [ + "aws.cloudtrail.user_identity.arn", + "source.ip", + "source.geo.city_name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-cloudtrail" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_country.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_country.json new file mode 100644 index 00000000000000..810822c30a5dd6 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_country.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a geolocation (country) that is unusual. This can be the result of compromised credentials or keys.", + "groups": [ + "siem", + "cloudtrail" + ], + "analysis_config": { + "bucket_span": "60m", + "detectors": [ + { + "detector_description": "rare by \"event.action\" partition by \"source.geo.country_iso_code\"", + "function": "rare", + "by_field_name": "event.action", + "partition_field_name": "source.geo.country_iso_code" + } + ], + "influencers": [ + "aws.cloudtrail.user_identity.arn", + "source.ip", + "source.geo.country_iso_code" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-cloudtrail" + } + } \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_username.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_username.json new file mode 100644 index 00000000000000..2edf52e8351ed6 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_username.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a user context that does not normally call the method. This can be the result of compromised credentials or keys as someone uses a valid account to persist, move laterally, or exfil data.", + "groups": [ + "siem", + "cloudtrail" + ], + "analysis_config": { + "bucket_span": "60m", + "detectors": [ + { + "detector_description": "rare by \"event.action\" partition by \"user.name\"", + "function": "rare", + "by_field_name": "event.action", + "partition_field_name": "user.name" + } + ], + "influencers": [ + "user.name", + "source.ip", + "source.geo.city_name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-cloudtrail" + } + } \ No newline at end of file diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index 5ca496a7a7fe9b..cfb3c17ac7f21d 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -25,6 +25,7 @@ const moduleIds = [ 'sample_data_weblogs', 'siem_auditbeat', 'siem_auditbeat_auth', + 'siem_cloudtrail', 'siem_packetbeat', 'siem_winlogbeat', 'siem_winlogbeat_auth',