Fix Issue #285

guilhemmarchand · Mar 14, 2021 · 6a06c0b · 6a06c0b
1 parent 432997b
commit 6a06c0b
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 23 deletions.
diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst
@@ -17,6 +17,7 @@ https://trackme.readthedocs.io/en/latest/configuration.html
 
 - Enhancement - Issue #279 - Decomission of the getlistdef custom command in favor of a simpler and cleaner pure SPL approach
 - Enhancement - Issue #280 - Add new REST endpoint to manage logical group associations
+- Enhancement - Issue #285 - Flipping statuses workflow improvements
 - Change - Issue #275 - permissions - provides a builtin trackme_user role to handle the minimal non admin access for TrackMe
 - Change - Issue #276 - User Interface - Migration of Ajax javascript REST calls made within the UI from splunkd to TrackMe based API endpoints
 - Change - Issue #278 - Upgrade of splunklib Python SDK to latest release 1.6.15

diff --git a/trackme/default/collections.conf b/trackme/default/collections.conf
@@ -124,6 +124,10 @@ replicate = false
 [kv_trackme_audit_changes]
 replicate = false
 
+#############
+# Deprecated: this macro is not used anymore, but left for retro compatibility for user that would have created custom sources prior to the deprecation
+#############
+
 #
 # Audit flip temp collections
 #
@@ -146,6 +150,10 @@ replicate = false
 [kv_trackme_audit_flip_temp_manual_refresh]
 replicate = false
 
+#############
+# End of deprecated
+#############
+
 #
 # Documentation knowledge base
 #

diff --git a/trackme/default/macros.conf b/trackme/default/macros.conf
@@ -380,12 +380,16 @@ definition = eval hasFlipped=if($previous_state$!=$state$ AND isnull(simulation)
 | eval latest_flip_state=if(hasFlipped=0, $state$, latest_flip_state)\
 | eval object_state=$state$, object_previous_state=$previous_state$\
 | eval object=$key$, object_category=case(isnotnull(data_name), "data_source", isnotnull(data_host), "data_host", isnotnull(metric_host), "metric_host")\
-| appendpipe [ | where hasFlipped==0 | eval time=now(), result = strftime(now(), "%d/%m/%Y %H:%M:%S") . ", object=" . $key$ . " has flipped from previous_state=" . $previous_state$ . " to state=" . $state$ | outputlookup $collection$ append=t | eval rectype="0"]\
+| appendpipe [ | where hasFlipped==0 | eval time=now(), result = strftime(now(), "%d/%m/%Y %H:%M:%S") . ", object=" . $key$ . " has flipped from previous_state=" . $previous_state$ . " to state=" . $state$ | `trackme_sumarycollect("flip_state_change_tracking")` | eval rectype="0"]\
 | appendpipe [ | where hasFlipped==1 | eval time=now(), rectype="1"]\
 | where isnotnull(rectype) | fields - hasFlipped, rectype, time
 args = state, previous_state, key, collection
 iseval = 0
 
+#############
+# Deprecated: this macro is not used anymore, but left for retro compatibility for users that would have created custom sources prior to the deprecation
+#############
+
 # manage flip temp collection and collect
 [trackme_collect_flip(1)]
 definition = inputlookup append=t $collection$ | where isnotnull(object)\
@@ -396,6 +400,10 @@ definition = inputlookup append=t $collection$ | where isnotnull(object)\
 args = collection
 iseval = 0
 
+#############
+# End of deprecated
+#############
+
 # generates summary events of latest statuses from the KVstore collections
 [trackme_collect_state(2)]
 definition = eval current_state=case(\
@@ -1495,8 +1503,7 @@ definition = where isnotnull(data_name) AND data_eventcount>0\
 `comment("#### run collect and updates the KVstore ####")`\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
 | `trackme_mcollect(data_name, data_source, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.hostcount_4h=dcount_host, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_source_dedicated)`
+| stats c
 iseval = 0
 
 [trackme_elastic_dedicated_tracker(1)]
@@ -1522,8 +1529,7 @@ definition = where isnotnull(data_name) AND data_eventcount>0\
 `comment("#### run collect and updates the KVstore ####")`\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
 | `trackme_mcollect(data_name, data_source, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_source_dedicated)`
+| stats c
 args = tracker_name
 iseval = 0
 

diff --git a/trackme/default/savedsearches.conf b/trackme/default/savedsearches.conf
@@ -201,8 +201,6 @@ search_mode="rest_mstats", "| rest " . rest_target . " /servicesNS/admin/search/
 `comment("#### run collect and updates the KVstore ####")`\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
 | `trackme_mcollect(data_name, data_source, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.hostcount_4h=dcount_host, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_source_shared)`\
 | stats c
 
 # Monitoring of data sources
@@ -244,8 +242,6 @@ search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
 | where data_source_is_online="true"\
 | `trackme_mcollect(data_name, data_source, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.hostcount_4h=dcount_host, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_source)`\
 | stats c
 
 [TrackMe - Data sources availability long term tracker]
@@ -270,8 +266,6 @@ search  = | savedsearch "TrackMe - Data sources abstract root tracker"\
 `comment("#### output flipping change status if changes ####")`\
 | `trackme_get_flip(data_source_state, data_previous_source_state, data_name, trackme_audit_flip_temp_data_source)`\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_source)`\
 | stats c
 
 [TrackMe - Alert on data source availability]
@@ -351,8 +345,6 @@ search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 | `trackme_outputlookup(trackme_host_monitoring, key)`\
 | where data_host_is_online="true"\
 | `trackme_mcollect(data_host, data_host, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.hostcount_4h=dcount_host, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_host)`\
 | stats c
 
 [TrackMe - hosts availability long term tracker]
@@ -373,8 +365,6 @@ search  = | savedsearch "TrackMe - Data hosts abstract root tracker"\
 `comment("#### output flipping change status if changes ####")`\
 | `trackme_get_flip(data_host_state, data_previous_host_state, data_host, trackme_audit_flip_temp_data_host)`\
 | `trackme_outputlookup(trackme_host_monitoring, key)`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_host)`\
 | stats c
 
 [TrackMe - Alert on data host availability]
@@ -624,8 +614,6 @@ search = | savedsearch "TrackMe - metric hosts abstract root tracker" host=*\
 | search NOT [ | inputlookup trackme_audit_changes | where action="success" AND change_type="delete permanent" AND object_category="metric_host" | eval _time=time/1000 | where _time>relative_time(now(), "-7d") | table object | dedup object | sort limit=0 object | rename object as metric_host ]\
 | eval metric_monitored_state=if(metric_last_time_seen<=`trackme_auto_disablement_period`, "disabled", metric_monitored_state)\
 | `trackme_outputlookup(trackme_metric_host_monitoring, key)`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_metric_host)`\
 | stats c
 
 [trackme_update_metric_host_target_by_metric_host]
@@ -646,8 +634,6 @@ search = | savedsearch "TrackMe - metric hosts abstract root tracker" host="$hos
 | search NOT [ | inputlookup trackme_audit_changes where (object_type="metric_host" AND object="$host$") | where action="success" AND change_type="delete permanent" AND object_category="metric_host" | eval _time=time/1000 | where _time>relative_time(now(), "-7d") | table object | dedup object | sort limit=0 object | rename object as metric_host ]\
 | eval metric_monitored_state=if(metric_last_time_seen<=`trackme_auto_disablement_period`, "disabled", metric_monitored_state)\
 | `trackme_outputlookup(trackme_metric_host_monitoring, key)`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_metric_host)`\
 | stats c
 
 [trackMe - metric host table report]
@@ -845,8 +831,6 @@ search  = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as
 `comment("#### run collect and updates the KVstore ####")`\
 | `trackme_outputlookup(trackme_data_source_monitoring, key)`\
 | `trackme_mcollect(data_name, data_source, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.hostcount_4h=dcount_host, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_manual_refresh)`\
 | stats c
 
 [TrackMe - Data host entity refresh]
@@ -860,8 +844,6 @@ search  = | `trackme_tstats` max(_indextime) as data_last_ingest, min(_time) as
 | `trackme_get_flip(data_host_state, data_previous_host_state, data_host, trackme_audit_flip_temp_data_host)`\
 | `trackme_outputlookup(trackme_host_monitoring, key)`\
 | `trackme_mcollect(data_host, data_host, "metric_name:trackme.eventcount_4h=data_eventcount, metric_name:trackme.lag_event_sec=data_last_lag_seen, metric_name:trackme.lag_ingestion_sec=data_last_ingestion_lag_seen", "object_category, object, OutlierTimePeriod, enable_behaviour_analytic")`\
-| stats c\
-| `trackme_collect_flip(trackme_audit_flip_temp_data_host)`\
 | stats c
 
 # Various

diff --git a/trackme/default/transforms.conf b/trackme/default/transforms.conf
@@ -174,6 +174,10 @@ external_type = kvstore
 collection = kv_trackme_audit_changes
 fields_list = _key, time, action, change_type, object, object_category, object_attrs, user, result, comment
 
+#############
+# Deprecated: this macro is not used anymore, but left for retro compatibility for user that would have created custom sources prior to the deprecation
+#############
+
 #
 # Audit flip
 #
@@ -208,6 +212,10 @@ external_type = kvstore
 collection = kv_trackme_audit_flip_temp_manual_refresh
 fields_list = _key, time, object, object_category, result, object_previous_state, object_state, priority
 
+#############
+# End of deprecated
+#############
+
 #
 # Documentation knowledge base
 #