Merge pull request #298 from guilhemmarchand/testing

Version 1.2.40
guilhemmarchand · Mar 30, 2021 · b030180 · b030180
2 parents 1189f25 + b0421e7
commit b030180
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 55 deletions.
diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst
@@ -1,6 +1,22 @@
 Release notes
 #############
 
+Version 1.2.40
+==============
+
+**CAUTION:**
+
+This is a new main release branch, TrackMe 1.2.x requires the deployment of the following dependencies:
+
+- Semicircle Donut Chart Viz, Splunk Base: https://splunkbase.splunk.com/app/4378
+- Splunk Machine Learning Toolkit, Splunk Base: https://splunkbase.splunk.com/app/2890
+- Splunk Timeline - Custom Visualization, Splunk Base: https://splunkbase.splunk.com/app/3120
+
+TrackMe requires a summary index (defaults to trackme_summary) and a metric index (defaults to trackme_metrics):
+https://trackme.readthedocs.io/en/latest/configuration.html
+
+- Enhancement - Issue #297 - Performances - Long term tracker improvements
+
 Version 1.2.39
 ==============
 

diff --git a/trackme/app.manifest b/trackme/app.manifest
@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "trackme",
-      "version": "1.2.39"
+      "version": "1.2.40"
     },
     "author": [
       {

diff --git a/trackme/default/app.conf b/trackme/default/app.conf
@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Data tracking system for Splunk
-version = 1.2.39
+version = 1.2.40
diff --git a/trackme/default/macros.conf b/trackme/default/macros.conf
@@ -147,25 +147,6 @@ iseval = 0
 definition = index, sourcetype, cribl_pipe
 iseval = 0
 
-# used to optimize long term data source tracker run time by excluding green data sources as much as possible
-[trackme_optimize_data_source_long_term]
-definition = inputlookup trackme_data_source_monitoring where data_source_state="green"\
-| lookup trackme_elastic_sources data_name OUTPUTNEW data_name as elastic_source\
-| lookup trackme_elastic_sources_dedicated data_name OUTPUTNEW data_name as elastic_source\
-| where isnull(elastic_source)\
-| eval data_source_mode=case(match(data_name, ":all$"), "merged_mode", match(data_name, "\|cribl:"), "cribl_mode", match(data_name, ".*"), "split_mode")\
-| rex field=data_name "\|cribl:(?<cribl_pipe>.*)"\
-| eval filter=case(\
-data_source_mode="split_mode", "index=\"" . data_index . "\" sourcetype=\"" . data_sourcetype . "\"",\
-data_source_mode="merged_mode", "index=\"" . data_index . "\"",\
-data_source_mode="cribl_mode", "index=\"" . data_index . "\" sourcetype=\"" . data_sourcetype . "\" cribl_pipe=\"" . cribl_pipe . "\""\
-)\
-| table filter | format | fields - filter\
-| rex mode=sed field="search" "s/filter=\"//g"\
-| rex mode=sed field="search" "s/\\\\\"/\"/g"\
-| rex mode=sed field="search" "s/\"\"/\"/g"
-iseval = 0
-
 # used by the metrics tracker report, can be customised at large scale to reduce computing costs
 [trackme_mstats_span]
 definition = span=1s

diff --git a/trackme/default/savedsearches.conf b/trackme/default/savedsearches.conf
@@ -220,21 +220,6 @@ search = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as d
 `comment("#### call the abstract macro ####")`\
 `trackme_data_source_tracker_abstract`
 
-[TrackMe - Data sources abstract root tracker for long term]
-request.ui_dispatch_app = trackme
-request.ui_dispatch_view = trackme
-is_visible = false
-search = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host where index=* sourcetype=* `trackme_tstats_main_filter` `trackme_get_idx_whitelist(trackme_data_source_monitoring_whitelist_index, data_index)` `apply_data_source_blacklists_data_retrieve` NOT [ | `trackme_optimize_data_source_long_term` ] by `trackme_data_source_tstats_root_splitby`\
-\
-`comment("#### tstats result table is loaded ####")`\
-| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
-\
-`comment("#### intermediate calculation ####")`\
-| `trackme_default_data_source_mode`\
-\
-`comment("#### call the abstract macro ####")`\
-`trackme_data_source_tracker_abstract`
-
 [TrackMe - Data sources availability short term tracker]
 cron_schedule = */5 * * * *
 description = This scheduled report tracks and updates the data source availability KVstore based lookup
@@ -262,13 +247,13 @@ search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 [TrackMe - Data sources availability long term tracker]
 cron_schedule = 1 * * * *
 description = This scheduled report tracks and updates the data source availability KVstore based lookup
-dispatch.earliest_time = -7d
-dispatch.latest_time = +4h
+dispatch.earliest_time = -24h
+dispatch.latest_time = -4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 dispatch.ttl = 900 # 15m ttl for this artefact
-search  = | savedsearch "TrackMe - Data sources abstract root tracker for long term"\
+search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 \
 `comment("#### Exclude Elastic sources which are managed by the Elastic shared tracker or dedicated Elastic trackers ####")`\
 | search NOT [ | inputlookup append=t trackme_elastic_sources | inputlookup append=t trackme_elastic_sources_dedicated | fields data_name | format | fields search ]\
@@ -341,19 +326,6 @@ search = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as d
 `comment("#### Cal the asbtract macro ####")`\
 | `trackme_data_host_tracker_abstract`
 
-[TrackMe - Data hosts abstract root tracker for long term]
-request.ui_dispatch_app = trackme
-request.ui_dispatch_view = trackme
-is_visible = false
-search = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount where sourcetype=* host=* host!="" `trackme_tstats_main_filter` ( ( `trackme_get_idx_whitelist(trackme_data_host_monitoring_whitelist_index, data_index)` `apply_data_host_blacklists_data_retrieve` ) OR `trackme_tstats_main_filter_for_host` ) NOT [ | inputlookup trackme_host_monitoring where data_host_state="green" | fields data_host | rename data_host as host ] by index, sourcetype, host\
-`comment("#### tstats result table is loaded ####")`\
-\
-`comment("#### define a value for the last seen ingestion lag for that source ####")`\
-| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
-\
-`comment("#### Cal the asbtract macro ####")`\
-| `trackme_data_host_tracker_abstract`
-
 [TrackMe - hosts availability short term tracker]
 cron_schedule = */5 * * * *
 description = This scheduled report tracks and updates the data source availability KVstore based lookup
@@ -378,13 +350,13 @@ search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 [TrackMe - hosts availability long term tracker]
 cron_schedule = 1 * * * *
 description = This scheduled report tracks and updates the data source availability KVstore based lookup
-dispatch.earliest_time = -7d
-dispatch.latest_time = +4h
+dispatch.earliest_time = -24h
+dispatch.latest_time = -4h
 enableSched = 1
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 dispatch.ttl = 900 # 15m ttl for this artefact
-search  = | savedsearch "TrackMe - Data hosts abstract root tracker for long term"\
+search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 \
 `comment("#### collects latest collection state into the summary index ####")`\
 | `trackme_collect_state("current_state_tracking:data_host", "data_host")`\