Merge pull request #278 from splunk/CRL-1700_ES_macros

CRL_1700 ES Macros
splunk · Dec 16, 2019 · b59991e · b59991e
2 parents 1acf708 + f38a6b7
commit b59991e
Show file tree

Hide file tree

Showing 193 changed files with 607 additions and 606 deletions.
diff --git a/baselines/baseline_network_acl_modifications.yml b/baselines/baseline_network_acl_modifications.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: sourcetype=aws:cloudtrail `NetworkACLEvents` | spath output=arn path=userIdentity.arn
+    search: sourcetype=aws:cloudtrail `network_acl_events` | spath output=arn path=userIdentity.arn
       | bucket _time span=1h | stats count as apiCalls by _time, arn | stats count(apiCalls)
       as numDataPoints, latest(apiCalls) as latestCount, avg(apiCalls) as avgApiCalls,
       stdev(apiCalls) as stdevApiCalls by arn | table arn, latestCount, numDataPoints,
@@ -31,7 +31,7 @@ eli5: Use this search to create a baseline for API calls related to network ACLs
   This table is stored in a lookup file.
 how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later)
   and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail
-  inputs. To add or remove API event names for network ACLs, edit the macro `NetworkACLEvents`.
+  inputs. To add or remove API event names for network ACLs, edit the macro `network_acl_events`.
 id: fc0edd96-ff2b-4810-9f1f-63da3783fd63
 known_false_positives: ''
 maintainers:

diff --git a/baselines/baseline_security_group_activity_by_arn.yml b/baselines/baseline_security_group_activity_by_arn.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -90d@d
       latest_time: -10m@m
-    search: sourcetype=aws:cloudtrail `securityGroupAPIs` | spath output=arn path=userIdentity.arn
+    search: sourcetype=aws:cloudtrail `security_group_api_calls` | spath output=arn path=userIdentity.arn
       | bucket _time span=1h | stats count as apiCalls by _time, arn | stats count(apiCalls)
       as numDataPoints, latest(apiCalls) as latestCount, avg(apiCalls) as avgApiCalls,
       stdev(apiCalls) as stdevApiCalls by arn | table arn, latestCount, numDataPoints,
@@ -31,7 +31,7 @@ eli5: Use this search to create a baseline for API calls related to security gro
   of data points for each ARN. This table is stored in a lookup file.
 how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later)
   and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail
-  inputs. To add or remove API event names for security groups, edit the macro `securityGroupAPIs`.
+  inputs. To add or remove API event names for security groups, edit the macro `security_group_api_calls`.
 id: fc0edd96-ff2b-48b0-9f1f-63da3783fd63
 known_false_positives: ''
 maintainers:

diff --git a/baselines/discover_dns_records.yml b/baselines/discover_dns_records.yml
@@ -6,7 +6,7 @@ baseline:
       latest_time: -10m@m
     search: '| inputlookup cim_corporate_email_domains.csv | inputlookup append=T
       cim_corporate_web_domains.csv | inputlookup append=T cim_cloud_domains.csv |
-      eval domain = trim(replace(domain, "\*", "")) | join domain [|tstats summariesonly=true
+      eval domain = trim(replace(domain, "\*", "")) | join domain [|tstats `security_content_summariesonly`
       count values(DNS.record_type) as type, values(DNS.answer) as answer from datamodel=Network_Resolution
       where DNS.message_type=RESPONSE DNS.answer!="unknown" DNS.answer!="" by DNS.query
       | rename DNS.query as query | where query!="unknown" | rex field=query "(?<domain>\w+\.\w+?)(?:$|/)"]

diff --git a/baselines/identify_ports_on_network.yml b/baselines/identify_ports_on_network.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` count dc(All_Traffic.src) as numberOfUniqueHosts
+    search: '| tstats `security_content_summariesonly` count dc(All_Traffic.src) as numberOfUniqueHosts
       from datamodel=Network_Traffic by All_Traffic.dest_port | `drop_dm_object_name("All_Traffic")`
       | sort - count'
 creation_date: '2017-06-24'

diff --git a/baselines/identify_systems_creating_rdp_traffic.yml b/baselines/identify_systems_creating_rdp_traffic.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=3389
+    search: '| tstats `security_content_summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=3389
       by All_Traffic.src | `drop_dm_object_name("All_Traffic")` | sort - count'
 creation_date: '2017-04-24'
 data_metadata:

diff --git a/baselines/identify_systems_receiving_rdp_traffic.yml b/baselines/identify_systems_receiving_rdp_traffic.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=3389
+    search: '| tstats `security_content_summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=3389
       by All_Traffic.dest | `drop_dm_object_name("All_Traffic")` | sort - count'
 creation_date: '2017-04-24'
 data_metadata:

diff --git a/baselines/identify_systems_using_remote_desktop.yml b/baselines/identify_systems_using_remote_desktop.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` count from datamodel=Endpoint.Processes where
+    search: '| tstats `security_content_summariesonly` count from datamodel=Endpoint.Processes where
       Processes.process_name="*mstsc.exe*" by Processes.dest Processes.process_name
       | `drop_dm_object_name(Processes)` | sort - count'
 creation_date: '2017-04-18'

diff --git a/baselines/monitor_successful_windows_updates.yml b/baselines/monitor_successful_windows_updates.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` dc(Updates.dest) as count FROM datamodel=Updates
+    search: '| tstats `security_content_summariesonly` dc(Updates.dest) as count FROM datamodel=Updates
       where Updates.vendor_product="Microsoft Windows" AND Updates.status=installed
       by _time span=1d'
 creation_date: '2017-08-24'

diff --git a/baselines/monitor_unsuccessful_windows_updates.yml b/baselines/monitor_unsuccessful_windows_updates.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` dc(Updates.dest) as count FROM datamodel=Updates
+    search: '| tstats `security_content_summariesonly` dc(Updates.dest) as count FROM datamodel=Updates
       where Updates.vendor_product="Microsoft Windows" AND Updates.status=failure
       by _time span=1d'
 creation_date: '2017-08-24'

diff --git a/baselines/previously_seen_aws_users.yml b/baselines/previously_seen_aws_users.yml
@@ -6,7 +6,7 @@ baseline:
       latest_time: -10m@m
     search: sourcetype=aws:cloudtrail eventName=ConsoleLogin | rename userIdentity.arn
       as user | iplocation src | eval City=if(City LIKE "",src,City),Region=if(Region
-      LIKE "",src,Region) | stats earliest(_time) as earliest latest(_time) as latest
+      LIKE "",src,Region) | stats earliest(_time) as firstTime latest(_time) as lastTime
       by user src City Region Country | outputlookup previously_seen_users_console_logins.csv
       | stats count
 creation_date: '2018-02-23'

diff --git a/baselines/previously_seen_cmd_arguments.yml b/baselines/previously_seen_cmd_arguments.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -30d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` min(_time) as firstTime max(_time) as lastTime
+    search: '| tstats `security_content_summariesonly` min(_time) as firstTime max(_time) as lastTime
       from datamodel=Endpoint.Processes where Processes.process_name=cmd.exe AND Processes.process="*
       /c *" by Processes.process | `drop_dm_object_name(Processes)`'
 creation_date: '2018-04-09'

diff --git a/baselines/previously_seen_ec2_amis.yml b/baselines/previously_seen_ec2_amis.yml
@@ -6,7 +6,7 @@ baseline:
       latest_time: -10m@m
     search: sourcetype=aws:cloudtrail eventName=RunInstances errorCode=success | rename
       requestParameters.instancesSet.items{}.imageId as amiID | stats earliest(_time)
-      as earliest latest(_time) as latest by amiID | outputlookup previously_seen_ec2_amis.csv
+      as firstTime latest(_time) as lastTime by amiID | outputlookup previously_seen_ec2_amis.csv
       | stats count
 creation_date: '2018-03-12'
 data_metadata:

diff --git a/baselines/previously_seen_ec2_modifications.yml b/baselines/previously_seen_ec2_modifications.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -90d@d
       latest_time: -10m@m
-    search: sourcetype=aws:cloudtrail `ec2ModificationAPIs` errorCode=success | spath
+    search: sourcetype=aws:cloudtrail `ec2_modification_api_calls` errorCode=success | spath
       output=arn userIdentity.arn | stats earliest(_time) as firstTime latest(_time)
       as lastTime by arn | outputlookup previously_seen_ec2_modifications_by_user
       | stats count
@@ -20,11 +20,11 @@ description: This search builds a table of previously seen ARNs that have launch
   a EC2 instance.
 eli5: In this support search, we create a table of the earliest and latest times that
   an ARN has modified a EC2 instance. The list of APIs that modify an EC2 are defined
-  in the `ec2ModificationAPIs` macro for ease of use. This table is then outputted
+  in the `ec2_modification_api_calls` macro for ease of use. This table is then outputted
   to a file.
 how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later)
   and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail
-  inputs. To add or remove APIs that modify an EC2 instance, edit the macro `ec2ModificationAPIs`.
+  inputs. To add or remove APIs that modify an EC2 instance, edit the macro `ec2_modification_api_calls`.
 id: 4d69091b-d975-4267-85df-888bd41034eb
 known_false_positives: ''
 maintainers:

diff --git a/baselines/systems_ready_for_spectre_meltdown_patch.yml b/baselines/systems_ready_for_spectre_meltdown_patch.yml
@@ -4,11 +4,11 @@ baseline:
       cron_schedule: ''
       earliest_time: -1d@d
       latest_time: -10m@m
-    search: '| tstats `summariesonly` count min(_time) as firstTime max(_time) as
+    search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as
       lastTime FROM datamodel=Change_Analysis.All_Changes where All_Changes.object_category=registry
       AND (All_Changes.object_path="HKLM\Software\Microsoft\Windows\CurrentVersion\QualityCompat*")
       by All_Changes.dest, All_Changes.command, All_Changes.user, All_Changes.object,
-      All_Changes.object_path | `ctime(lastTime)` | `ctime(firstTime)` | `drop_dm_object_name("All_Changes")`'
+      All_Changes.object_path | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name("All_Changes")`'
 creation_date: '2018-01-08'
 data_metadata:
   data_models:

diff --git a/baselines/train_dns_query_length.yml b/baselines/train_dns_query_length.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -31d@d
       latest_time: -1d@d
-    search: '| tstats `summariesonly` count from datamodel=Network_Resolution by DNS.query
+    search: '| tstats `security_content_summariesonly` count from datamodel=Network_Resolution by DNS.query
       DNS.record_type | search DNS.record_type=* | `drop_dm_object_name("DNS")` |
       eval query_length = len(query) | fit DensityFunction query_length by record_type
       into dns_query_pdfmodel'

diff --git a/baselines/train_smb_traffic_spike.yml b/baselines/train_smb_traffic_spike.yml
@@ -4,7 +4,7 @@ baseline:
       cron_schedule: ''
       earliest_time: -31d@d
       latest_time: -1d@d
-    search: '| tstats `summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=139
+    search: '| tstats `security_content_summariesonly` count from datamodel=Network_Traffic where All_Traffic.dest_port=139
       OR All_Traffic.dest_port=445 OR All_Traffic.app=smb by _time span=10m, All_Traffic.src
       | eval HourOfDay=strftime(_time, "%H") | eval DayOfWeek=strftime(_time, "%A")
       | `drop_dm_object_name("All_Traffic")` | fit DensityFunction count by "HourOfDay,DayOfWeek"

diff --git a/baselines/train_unusually_long_commandlines.yml b/baselines/train_unusually_long_commandlines.yml
@@ -4,10 +4,10 @@ baseline:
       cron_schedule: ''
       earliest_time: -31d@d
       latest_time: -1d@d
-    search: '| tstats `summariesonly` count min(_time) as start_time max(_time) as
+    search: '| tstats `security_content_summariesonly` count min(_time) as start_time max(_time) as
       end_time FROM datamodel=Endpoint.Processes by Processes.user Processes.dest
       Processes.process_name Processes.process | `drop_dm_object_name(Processes)`
-      | search user!=unknown | `ctime(start_time)`| `ctime(end_time)`| eval processlen=len(process)
+      | search user!=unknown | `security_content_ctime(start_time)`| `security_content_ctime(end_time)`| eval processlen=len(process)
       | fit DensityFunction processlen by user into cmdline_pdfmodel'
 creation_date: '2019-05-08'
 data_metadata:

diff --git a/baselines/update_previously_seen_aws_users.yml b/baselines/update_previously_seen_aws_users.yml
@@ -6,9 +6,9 @@ baseline:
       latest_time: m@m
     search: sourcetype=aws:cloudtrail eventName=ConsoleLogin | rename userIdentity.arn
       as user | iplocation src | eval City=if(City LIKE "",src,City),Region=if(Region
-      LIKE "",src,Region) | stats earliest(_time) AS earliest latest(_time) AS latest
+      LIKE "",src,Region) | stats earliest(_time) AS firstTime latest(_time) AS firstTime
       by user src City Region Country | inputlookup append=t previously_seen_users_console_logins.csv
-      | stats min(earliest) as earliest max(latest) as latest by user src City Region
+      | stats min(firstTime) as firstTime max(firstTime) as firstTime by user src City Region
       Country | outputlookup previously_seen_users_console_logins.csv
 creation_date: '2019-04-25'
 data_metadata:

diff --git a/bin/jinja2_templates/macros.j2 b/bin/jinja2_templates/macros.j2
@@ -14,7 +14,7 @@ args = {% for arg in macro.arguments %}{{ arg }}{{ ", " if not loop.last }}
 {% if macro.definition is defined %}
 definition = {{ macro.definition }}
 {% else %}
-definition = `comment({{ macro.description }})`
+definition = 
 {% endif %}
 description = {{ macro.description }}
 

diff --git a/detections/add_to_untrust_cert_store.yml b/detections/add_to_untrust_cert_store.yml
@@ -30,11 +30,11 @@ detect:
         cron_schedule: 0 * * * *
         earliest_time: -70m@m
         latest_time: -10m@m
-      search: '| tstats `summariesonly` count min(_time) values(Processes.process)
+      search: '| tstats `security_content_summariesonly` count min(_time) as firstTime values(Processes.process)
         as process max(_time) as lastTime from datamodel=Endpoint.Processes where
         Processes.process_name=certutil.exe (Processes.process=*-addstore* AND Processes.process=*disallowed*
         ) by Processes.parent_process Processes.process_name Processes.user | `drop_dm_object_name("Processes")`
-        | `ctime(firstTime)`|`ctime(lastTime)`'
+        | `security_content_ctime(firstTime)`|`security_content_ctime(lastTime)`'
       suppress:
         suppress_fields: process, dest
         suppress_period: 86400s

diff --git a/detections/api_acitivity_from_previously_unseen_user_role.yml b/detections/api_acitivity_from_previously_unseen_user_role.yml
@@ -38,10 +38,10 @@ detect:
         |  inputlookup append=t previously_seen_api_calls_from_user_roles | stats
         min(earliest) as earliest, max(latest) as latest by userName eventName | outputlookup
         previously_seen_api_calls_from_user_roles| eval newApiCallfromUserRole=if(earliest>=relative_time(now(),
-        "-70m@m"), 1, 0) | where newApiCallfromUserRole=1 | `ctime(earliest)` | `ctime(latest)`
+        "-70m@m"), 1, 0) | where newApiCallfromUserRole=1 | `security_content_ctime(earliest)` | `security_content_ctime(latest)`
         | table eventName userName]  |rename userName as user| stats values(eventName)
-        earliest(_time) as earliest latest(_time) as latest by user | `ctime(earliest)`
-        | `ctime(latest)`
+        earliest(_time) as earliest latest(_time) as latest by user | `security_content_ctime(earliest)`
+        | `security_content_ctime(latest)`
       suppress:
         suppress_fields: user
         suppress_period: 86400s

diff --git a/detections/attackers_scanning_for_vulnerable_jboss_servers.yml b/detections/attackers_scanning_for_vulnerable_jboss_servers.yml
@@ -33,11 +33,11 @@ detect:
         cron_schedule: 0 * * * *
         earliest_time: -70m@m
         latest_time: -10m@m
-      search: '| tstats `summariesonly` count min(_time) as firstTime max(_time) as
+      search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as
         lastTime from datamodel=Web where (Web.http_method="GET" OR Web.http_method="HEAD")
         AND (Web.url="*/web-console/ServerInfo.jsp*" OR Web.url="*web-console*" OR
         Web.url="*jmx-console*" OR Web.url = "*invoker*") by Web.http_method, Web.url,
-        Web.src, Web.dest | `drop_dm_object_name("Web")` | `ctime(firstTime)` | `ctime(lastTime)`'
+        Web.src, Web.dest | `drop_dm_object_name("Web")` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`'
       suppress:
         suppress_fields: dest,url
         suppress_period: 86400s

diff --git a/detections/attrib_to_hide_files.yml b/detections/attrib_to_hide_files.yml
@@ -34,11 +34,11 @@ detect:
         cron_schedule: 30 * * * *
         earliest_time: -70m@m
         latest_time: -10m@m
-      search: '| tstats `summariesonly` count min(_time) values(Processes.process)
+      search: '| tstats `security_content_summariesonly` count min(_time) values(Processes.process)
         as process max(_time) as lastTime from datamodel=Endpoint.Processes where
         Processes.process_name=attrib.exe (Processes.process=*+h*) by Processes.parent_process
         Processes.process_name Processes.user | `drop_dm_object_name("Processes")`
-        | `ctime(firstTime)`|`ctime(lastTime)`'
+        | `security_content_ctime(firstTime)`|`security_content_ctime(lastTime)`'
       suppress:
         suppress_fields: dest, process
         suppress_period: 86400s

diff --git a/detections/aws_activity_from_non_approved_accounts.yml b/detections/aws_activity_from_non_approved_accounts.yml
@@ -37,7 +37,7 @@ detect:
         | search NOT [| inputlookup identity_lookup_expanded | fields identity] |
         search NOT [| inputlookup aws_service_accounts | fields identity] | rename
         identity as user | stats count min(_time) as firstTime max(_time) as lastTime
-        values(eventName) as eventName by user | `ctime(firstTime)` | `ctime(lastTime)`
+        values(eventName) as eventName by user | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
       suppress:
         suppress_fields: user
         suppress_period: 14400s

diff --git a/detections/aws_activity_in_new_region.yml b/detections/aws_activity_in_new_region.yml
@@ -37,7 +37,7 @@ detect:
         | stats min(earliest) as earliest max(latest) as latest by awsRegion | outputlookup
         previously_seen_aws_regions.csv | eval regionStatus=if(earliest >= relative_time(now(),
         "-1d@d"), "Instance Started in a New Region","Previously Seen Region") | convert
-        ctime(earliest) ctime(latest) | where regionStatus="Instance Started in a
+        security_content_ctime(earliest) security_content_ctime(latest) | where regionStatus="Instance Started in a
         New Region"
       suppress:
         suppress_fields: awsRegion

diff --git a/detections/aws_spike_acl_activity.yml b/detections/aws_spike_acl_activity.yml
@@ -31,8 +31,8 @@ detect:
         cron_schedule: 10 * * * *
         earliest_time: -70m@m
         latest_time: -10m@m
-      search: sourcetype=aws:cloudtrail `NetworkACLEvents` [search sourcetype=aws:cloudtrail
-        `NetworkACLEvents` | spath output=arn path=userIdentity.arn | stats count
+      search: sourcetype=aws:cloudtrail `network_acl_events` [search sourcetype=aws:cloudtrail
+        `network_acl_events` | spath output=arn path=userIdentity.arn | stats count
         as apiCalls by arn | inputlookup network_acl_activity_baseline append=t |
         fields - latestCount | stats values(*) as * by arn | rename apiCalls as latestCount
         | eval newAvgApiCalls=avgApiCalls + (latestCount-avgApiCalls)/720 | eval newStdevApiCalls=sqrt(((pow(stdevApiCalls,
@@ -98,7 +98,7 @@ how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or late
   the mean that the value must be to be considered a spike. This search works best
   when you run the "Baseline of Network ACL Activity by ARN" support search once to
   create a lookup file of previously seen Network ACL Activity. To add or remove API
-  event names related to network ACLs, edit the macro `NetworkACLEvents`.
+  event names related to network ACLs, edit the macro `network_acl_events`.
 id: ada0f478-84a8-4641-a1f1-e32372d4bd53
 investigations:
   - id: bc91a8cd-35e7-4bb2-6140-e756cc46fd71

diff --git a/detections/badrabbit_schtasks.yml b/detections/badrabbit_schtasks.yml
@@ -33,11 +33,11 @@ detect:
         cron_schedule: 0 * * * *
         earliest_time: -70m@m
         latest_time: -10m@m
-      search: '| tstats `summariesonly` count min(_time) as firstTime max(_time) as
+      search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as
         lastTime values(Processes.process) as process  from datamodel=Endpoint.Processes
         where Processes.process_name=schtasks.exe (Processes.process= "*create*"  OR
         Processes.process= "*delete*") by Processes.parent_process Processes.process_name
-        Processes.user | `drop_dm_object_name("Processes")` | `ctime(firstTime)`|`ctime(lastTime)`
+        Processes.user | `drop_dm_object_name("Processes")` | `security_content_ctime(firstTime)`|`security_content_ctime(lastTime)`
         | search (process=*rhaegal* OR process=*drogon* OR *viserion_*)'
       suppress:
         suppress_fields: dest, process_name