Merge pull request #61 from guilhemmarchand/version_112

Version 1.1.2
guilhemmarchand · Dec 30, 2019 · 08201fd · 08201fd
2 parents 12fb1ab + 78cf8ed
commit 08201fd
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 8 deletions.
diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst
@@ -1,6 +1,14 @@
 Release notes
 #############
 
+Version 1.1.2
+=============
+
+- fix: Under some circumstances, the last flipping status and date fail to be properly updated in the collections due to a weakness in the merging process
+- fix: data_source modal window embedded chart should split by host in a first pipeline level for better lagging calculation accuracy
+- fix: modal window embedded chart searches should refer to the tstats macro for consistency
+- feature: Add audit view for KVstore collections
+
 Version 1.1.1
 =============
 

diff --git a/trackme/default/app.conf b/trackme/default/app.conf
@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Easy data tracking system for Splunk admins
-version = 1.1.1
+version = 1.1.2
diff --git a/trackme/default/data/ui/html/TrackMe.html b/trackme/default/data/ui/html/TrackMe.html
@@ -5564,7 +5564,7 @@ <h2 class="panel-title">Search for audit changes:</h2>
         var searchChartLag = new SearchManager({
             "id": "searchChartLag",
             "sample_ratio": 1,
-            "search": "| tstats count latest(_indextime) as indextime where index=\"$tk_data_index$\" sourcetype=\"$tk_data_sourcetype$\" by _time, index, sourcetype span=1s | eval delta=(indextime-_time) | timechart span=$modalTime.span$ sum(count) as events_count, avg(delta) as avg_lag_sec",
+            "search": "| `tstats` count max(_indextime) as indextime where index=\"$tk_data_index$\" sourcetype=\"$tk_data_sourcetype$\" by _time, index, sourcetype, host span=1s | eval delta=(indextime-_time) | timechart span=$modalTime.span$ sum(count) as events_count, avg(delta) as avg_lag_sec",
             "status_buckets": 0,
             "earliest_time": "$modalTime.earliest$",
             "cancelOnUnload": true,
@@ -5586,7 +5586,7 @@ <h2 class="panel-title">Search for audit changes:</h2>
         var searchSingleLag = new SearchManager({
             "id": "searchSingleLag",
             "sample_ratio": 1,
-            "search": "| tstats count latest(_indextime) as indextime where index=\"$tk_data_index$\" sourcetype=\"$tk_data_sourcetype$\" by _time, index, sourcetype span=1s | eval delta=(indextime-_time) | stats perc95(delta) as perc95_lag, avg(delta) as avg_lag",
+            "search": "| `tstats` count max(_indextime) as indextime where index=\"$tk_data_index$\" sourcetype=\"$tk_data_sourcetype$\" by _time, index, sourcetype, host span=1s | eval delta=(indextime-_time) | stats perc95(delta) as perc95_lag, avg(delta) as avg_lag",
             "status_buckets": 0,
             "earliest_time": "$modalTime.earliest$",
             "cancelOnUnload": true,
@@ -5711,7 +5711,7 @@ <h2 class="panel-title">Search for audit changes:</h2>
             "cancelOnUnload": true,
             "latest_time": "$modalTimeHost.latest$",
             "sample_ratio": 1,
-            "search": "| tstats count latest(_indextime) as indextime where index=* host=\"$tk_data_host$\" `trackme_tstats_main_filter` `trackme_get_idx_whitelist(trackme_data_host_monitoring_whitelist_index, data_index)` `apply_data_host_blacklists_data_retrieve` by _time, index, sourcetype span=1s | eval delta=(indextime-_time) | timechart span=$modalTimeHost.span$ sum(count) as events_count, avg(delta) as avg_lag_sec",
+            "search": "| tstats count max(_indextime) as indextime where index=* host=\"$tk_data_host$\" `trackme_tstats_main_filter` `trackme_get_idx_whitelist(trackme_data_host_monitoring_whitelist_index, data_index)` `apply_data_host_blacklists_data_retrieve` by _time, index, sourcetype span=1s | eval delta=(indextime-_time) | timechart span=$modalTimeHost.span$ sum(count) as events_count, avg(delta) as avg_lag_sec",
             "status_buckets": 0,
             "app": utils.getCurrentApp(),
             "auto_cancel": 90,

diff --git a/trackme/default/data/ui/nav/default.xml b/trackme/default/data/ui/nav/default.xml
@@ -44,6 +44,7 @@
   </collection>
   <collection label="Audit">
     <a href="trackMe_internal_scheduling">Audit scheduling</a>
+    <a href="trackMe_kvstore">KVstore collections</a>
   </collection>
   <a href="search">Run a search</a>
 </nav>
diff --git a/trackme/default/data/ui/views/trackMe_kvstore.xml b/trackme/default/data/ui/views/trackMe_kvstore.xml
@@ -0,0 +1,70 @@
+<form>
+  <label>TrackMe - KVstore collections audit</label>
+  <search id="kvstore_mainsearch">
+    <query>| rest splunk_server=local /services/server/introspection/kvstore/collectionstats
+| mvexpand data
+| spath input=data
+| rex field=ns "(?&lt;App&gt;.*)\.(?&lt;Collection&gt;.*)"
+| eval dbsize=round(size/1024/1024, 2)
+| eval indexsize=round(totalIndexSize/1024/1024, 2)
+| stats first(count) AS "Number of Objects" first(nindexes) AS Accelerations first(indexsize) AS "Acceleration Size (MB)" first(dbsize) AS "Collection Size (MB)" by App, Collection
+| sort - "Collection Size (MB)"
+| where App="trackme"</query>
+  </search>
+  <fieldset submitButton="false"></fieldset>
+  <row>
+    <panel>
+      <title>KVstore usage summary</title>
+      <single>
+        <search base="kvstore_mainsearch">
+          <query>stats count</query>
+        </search>
+        <option name="rangeColors">["0x53a051","0x0877a6","0xf8be34","0xf1813f","0xdc4e41"]</option>
+        <option name="underLabel">Number of KVstore collections</option>
+      </single>
+      <single>
+        <search base="kvstore_mainsearch">
+          <query>stats sum("Number of Objects")</query>
+        </search>
+        <option name="rangeColors">["0x53a051","0x0877a6","0xf8be34","0xf1813f","0xdc4e41"]</option>
+        <option name="underLabel">Total number of entries in the KVstore</option>
+      </single>
+      <single>
+        <search base="kvstore_mainsearch">
+          <query>stats sum("Collection Size (MB)")</query>
+        </search>
+        <option name="rangeColors">["0x53a051","0x0877a6","0xf8be34","0xf1813f","0xdc4e41"]</option>
+        <option name="underLabel">Total size used by KVstore collections</option>
+        <option name="unit">MB</option>
+      </single>
+      <single>
+        <search base="kvstore_mainsearch">
+          <query>stats sum("Acceleration Size (MB)")</query>
+        </search>
+        <option name="rangeColors">["0x53a051","0x0877a6","0xf8be34","0xf1813f","0xdc4e41"]</option>
+        <option name="underLabel">Total acceleration size used by KVstore collections</option>
+        <option name="unit">MB</option>
+      </single>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <html>
+        <div style="text-align: center;">
+          <h1>KVstore collections auditing</h1>
+        </div>
+      </html>
+    </panel>
+  </row>
+  <row>
+    <panel>
+      <title>KVstore collections details</title>
+      <table>
+        <search base="kvstore_mainsearch">
+          <query>sort - "Collection Size (MB)"</query>
+        </search>
+        <option name="count">100</option>
+      </table>
+    </panel>
+  </row>
+</form>
diff --git a/trackme/default/savedsearches.conf b/trackme/default/savedsearches.conf
@@ -35,8 +35,11 @@ search = | `tstats` max(_indextime) as current_data_last_ingest, min(_time) as c
 | `trackme_data_sources_filtering`\
 | `trackme_eval_data_source_state` | eval data_tracker_runtime=now()\
 | lookup local=t trackme_data_source_monitoring data_name OUTPUT data_source_state as data_previous_source_state, data_tracker_runtime as data_previous_tracker_runtime\
-| append [ | inputlookup trackme_audit_flip | where object_category="data_source" | eval _time=time | stats max(_time) as latest_flip_time, latest(object_state) as latest_flip_state by object | rename object as data_name ]\
+| append [ | inputlookup trackme_audit_flip | where object_category="data_source" | eval _time=time | stats max(_time) as current_latest_flip_time, latest(object_state) as current_latest_flip_state by object | rename object as data_name ]\
 | stats first(_key) as _key, first(*) as "*" by data_name\
+| eval latest_flip_time=if(isnotnull(current_latest_flip_time), current_latest_flip_time, latest_flip_time)\
+| eval latest_flip_state=if(isnotnull(current_latest_flip_state), current_latest_flip_state, latest_flip_state)\
+| fields - current_latest_flip_time, current_latest_flip_state\
 | where isnotnull(data_last_time_seen)\
 | eval data_last_lag_seen=if(data_source_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
 | outputlookup trackme_data_source_monitoring append=t key_field=_key\
@@ -160,8 +163,11 @@ search = | `tstats` max(_indextime) as current_data_last_ingest, min(_time) as c
 | `trackme_data_host_group_lookup`\
 | eval data_tracker_runtime=now()\
 | lookup local=t trackme_host_monitoring data_host OUTPUT data_host_state as data_previous_host_state, data_tracker_runtime as data_previous_tracker_runtime\
-| append [ | inputlookup trackme_audit_flip | where object_category="data_host" | eval _time=time | stats max(_time) as latest_flip_time, latest(object_state) as latest_flip_state by object | rename object as data_host ]\
+| append [ | inputlookup trackme_audit_flip | where object_category="data_host" | eval _time=time | stats max(_time) as current_latest_flip_time, latest(object_state) as current_latest_flip_state by object | rename object as data_host ]\
 | stats first(_key) as _key, first(*) as "*" by data_host\
+| eval latest_flip_time=if(isnotnull(current_latest_flip_time), current_latest_flip_time, latest_flip_time)\
+| eval latest_flip_state=if(isnotnull(current_latest_flip_state), current_latest_flip_state, latest_flip_state)\
+| fields - current_latest_flip_time, current_latest_flip_state\
 | where isnotnull(data_last_time_seen)\
 | eval data_last_lag_seen=if(data_host_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
 | `trackme_data_host_rule_filter(data_host)`\
@@ -292,8 +298,11 @@ search = | mstats latest(_value) as value where index=* `trackme_mstats_main_fil
 | `trackme_default_priority`\
 | search `trackme_get_idx_whitelist_searchtime(trackme_metric_host_monitoring_whitelist_index, metric_index)`\
 | lookup local=t trackme_metric_host_monitoring metric_host OUTPUT metric_host_state as metric_previous_host_state, metric_tracker_runtime as metric_previous_tracker_runtime\
-| append [ | inputlookup trackme_audit_flip | where object_category="metric_host" | eval _time=time | stats max(_time) as latest_flip_time, latest(object_state) as latest_flip_state by object | rename object as metric_host ]\
-| stats first(_key) as keyid, first(latest_flip_time) as latest_flip_time, first(latest_flip_state) as latest_flip_state, values(*) as "*" by metric_host\
+| append [ | inputlookup trackme_audit_flip | where object_category="metric_host" | eval _time=time | stats max(_time) as current_latest_flip_time, latest(object_state) as current_latest_flip_state by object | rename object as metric_host ]\
+| stats first(_key) as keyid, first(latest_flip_time) as latest_flip_time, first(latest_flip_state) as latest_flip_state, first(current_latest_flip_time) as current_latest_flip_time, first(current_latest_flip_state) as current_latest_flip_state, values(*) as "*" by metric_host\
+| eval latest_flip_time=if(isnotnull(current_latest_flip_time), current_latest_flip_time, latest_flip_time)\
+| eval latest_flip_state=if(isnotnull(current_latest_flip_state), current_latest_flip_state, latest_flip_state)\
+| fields - current_latest_flip_time, current_latest_flip_state\
 | where isnotnull(metric_category)\
 | `trackme_default_metric_host_monitored_state`
 

diff --git a/trackme_111.tgz → trackme_112.tgz b/trackme_111.tgz → trackme_112.tgz