Work in progress 1.2.21

- Change: Time to live definition for scheduled reports (dispatch.ttl) to reduce overhead in the dispatch directory
- Change: Automatically affect a 1 minute time window when creating Elastic dedicated trackers

docs/releasenotes.rst

@@ -21,6 +21,8 @@ https://trackme.readthedocs.io/en/latest/configuration.html
 - Fix: Limitation of the list function used in stats limits the number for Elastic shared data sources to 99 sources maximum, fixed by alternative improved syntax
 - Fix: For Elastic shared sources, if the first source is a raw search, the addition of the "search" key word in the first pipeline fails under some conditions
 - Change: Automatically join the acknowledgement comment in the acknowledgement screen
+- Change: Time to live definition for scheduled reports (dispatch.ttl) to reduce overhead in the dispatch directory
+- Change: Automatically affect a 1 minute time window when creating Elastic dedicated trackers
 
 Version 1.2.20
 ==============

trackme/default/data/ui/html/TrackMe.html

@@ -1634,7 +1634,7 @@ <h3 style="color: steelblue;">Acting on a data sampling and events format recogn
                         </div>
                     </div>
 
-                    <div style="margin-top: 60px;" class="modal-footer">
+                    <div style="margin-top: 20px;" class="modal-footer">
                         <div class="btn_data_sampling_back" style="margin-right: 15px; float: left;">
                             <button type="submit" class="btn btn-default btn-primary" data-dismiss="modal"><span class="glyphicon glyphicon-remove"></span> Back</button>
                         </div>
@@ -24993,7 +24993,8 @@ <h2 class="panel-title">Search for audit changes:</h2>
                         "dispatch.earliest_time": tk_input_elastic_source_earliest,
                         "dispatch.latest_time": tk_input_elastic_source_latest,
                         "is_scheduled": "1",
-                        "cron_schedule": "*/5 * * * *"
+                        "cron_schedule": "*/5 * * * *",
+                        "schedule_window": "1"
                     };
 
                     // Create a saved search/report as an alert.

trackme/default/macros.conf

@@ -1359,11 +1359,11 @@ iseval = 0
 # Data sampling - detect raw events formats via regular expression for most common use cases
 [trackme_data_sampling_detect_event_format(1)]
 definition = eval $dest_field$=case(\
-match(raw_sample, "^\{") AND match(raw_sample, "\}$"), "json",\
+match(raw_sample, "^\{"), "json",\
 match(raw_sample, "^\<\d*\>\w{3}\s\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}\s"), "syslog_rfc3164",\
 match(raw_sample, "^\<\d*\>\d*\s\d{4}\-\d{1,2}\-\d{1,2}T\d{2}:\d{2}:\d{2}\."), "syslog_rfc5424",\
 match(raw_sample, "^\[\w*]\s*\d{4}-\d{1,2}-\d{1,2}\s*\d{1,2}:\d{1,2}:\d{1,2}\,\d{1,3}"), "log4j",\
-match(raw_sample, "^\<") AND match(raw_sample, "\>$"), "xml",\
+match(raw_sample, "^\<[^\s]*\sxmlns="), "xml",\
 match(raw_sample, "^type=[^\s]*\s*msg=\w*\(\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}\.\d{6}\)"), "auditd",\
 match(raw_sample, "^[^\:]*:\[timestamp=\d{1,2}-\d{1,2}-\d{4}\s*\d{1,2}\:\d{1,2}\:\d{1,2}\.\d{3}"), "linux_syslog",\
 match(raw_sample, "\[\d{2}\/\w{3}\/\d{4}\s\d{2}:\d{2}:\d{2}:\d+\]"), "access_log1",\

trackme/default/savedsearches.conf

@@ -9,6 +9,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search = | savedsearch runSPL [\
 \
 | inputlookup trackme_data_source_monitoring where data_monitored_state="enabled" | eval key=_key | where data_last_time_seen>relative_time(now(), "-24h") | sort limit=0 data_sample_lastrun | head [ `trackme_data_sampling_algo_entities_to_process` ]\
@@ -116,6 +117,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search= | savedsearch runSPL [\
 \
 `comment("#### Load the KVstore collection ####")`\
@@ -202,6 +204,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 \
 `comment("#### Exclude Elastic sources which are managed by the Elastic shared tracker or dedicated Elastic trackers ####")`\
@@ -224,6 +227,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 \
 `comment("#### Exclude Elastic sources which are managed by the Elastic shared tracker or dedicated Elastic trackers ####")`\
@@ -303,6 +307,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 `comment("#### output flipping change status if changes ####")`\
 | `trackme_get_flip(data_host_state, data_previous_host_state, data_host, trackme_audit_flip_temp_data_host)`\
@@ -321,6 +326,7 @@ dispatch.latest_time = +4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 | where data_last_time_seen>relative_time(now(), "-4h-5m")\
 `comment("#### output flipping change status if changes ####")`\
@@ -562,6 +568,7 @@ dispatch.latest_time = now
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search = | savedsearch "TrackMe - metric hosts abstract root tracker" host=*\
 \
 `comment("#### output flipping change status if changes ####")`\
@@ -669,6 +676,7 @@ enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 schedule_window = 15
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | `trackme_summary_investigator_mstats(-24h)`\
 \
 `comment("#### Call the trackme_summary_investigator_define_bound_abstract macro ####")`\
@@ -688,6 +696,7 @@ enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 schedule_window = 15
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | `trackme_summary_investigator_mstats(-48h)`\
 \
 `comment("#### Call the trackme_summary_investigator_define_bound_abstract macro ####")`\
@@ -707,6 +716,7 @@ enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 schedule_window = 15
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | `trackme_summary_investigator_mstats(-7d)`\
 \
 `comment("#### Call the trackme_summary_investigator_define_bound_abstract macro ####")`\
@@ -726,6 +736,7 @@ enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 schedule_window = 15
+dispatch.ttl = 900 # 15m ttl for this artefact
 search  = | `trackme_summary_investigator_mstats(-30d)`\
 \
 `comment("#### Call the trackme_summary_investigator_define_bound_abstract macro ####")`\
@@ -807,6 +818,7 @@ dispatch.latest_time = now
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search = | inputlookup trackme_alerts_ack | eval keyid=_key\
 | eval limit_expiration=ack_expiration-300\
 | eval ack_state=if(now()>=limit_expiration, "inactive", ack_state)\
@@ -844,6 +856,7 @@ quantity = 0
 relation = greater than
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
+dispatch.ttl = 300 # 5m ttl for this artefact
 search = | inputlookup trackme_audit_changes\
 | sort limit=0 - time\
 | eval _time=time/1000 | eval time=strftime(_time, "%c") | addinfo | where _time>=info_min_time AND (_time<=info_max_time OR info_max_time="+Infinity")\
@@ -920,6 +933,7 @@ request.ui_dispatch_app = trackme
 request.ui_dispatch_view = search
 schedule_window = 5
 run_on_startup = true
+dispatch.ttl = 300 # 5m ttl for this artefact
 search  = | makeresults\
 | appendcols [ | inputlookup trackme_maintenance_mode ]\
 | eval maintenance_mode=if(isnull(maintenance_mode), "disabled", maintenance_mode), time_updated=if(isnull(time_updated), now(), time_updated)\