Merge pull request #49 from guilhemmarchand/version_1034

Version 1034
guilhemmarchand · Dec 9, 2019 · fe227aa · fe227aa
2 parents 1add670 + 040eb46
commit fe227aa
Show file tree

Hide file tree

Showing 14 changed files with 3,075 additions and 1,593 deletions.
diff --git a/docs/releasenotes.rst b/docs/releasenotes.rst
@@ -1,6 +1,30 @@
 Release notes
 #############
 
+Version 1.0.34
+==============
+
+- feature: introduction of the concept of source identity card, allows defining and store a documentation link and note for data sources, which
+identity cards are made available automatically via the UI and via the OOTB alert.
+Identity cards records can be created, maintained and delete via the UI.
+- feature: increase default size of modal windows
+- feature: fixed charts color for data sources and data hosts events vs lag embedded charts
+- feature: add last 48 hours in link time selectors
+
+Version 1.0.33
+==============
+
+- fix: Avoids post processed searches in the Mobile dashboard, better single form placement for Apple TV rendering
+
+Version 1.0.32
+==============
+
+- fix: Performance issues with TrackMe mobile dashboard on mobile devices
+- fix: TrackMe does not honour indexes whitelisting for metric hosts
+- fix: Add metric host lookup in initial configuration load check operation
+- fix: Wrong message for flush of metric KVstore collection
+- feature: Remove management features from main UI to be transferred to a second management UI available from the nav menu
+
 Version 1.0.31
 ==============
 

diff --git a/trackme/appserver/static/icons/identity_card.png b/trackme/appserver/static/icons/identity_card.png
diff --git a/trackme/default/app.conf b/trackme/default/app.conf
@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Easy data tracking system for Splunk admins
-version = 1.0.31
+version = 1.0.34
diff --git a/trackme/default/collections.conf b/trackme/default/collections.conf
@@ -108,3 +108,10 @@ replicate = false
 
 [kv_trackme_audit_flip]
 replicate = false
+
+#
+# Documentation knowledge base
+#
+
+[kv_trackme_sources_knowledge]
+replicate = false
diff --git a/trackme/default/data/ui/html/TrackMe.html b/trackme/default/data/ui/html/TrackMe.html
diff --git a/trackme/default/data/ui/html/TrackMe_manage.html b/trackme/default/data/ui/html/TrackMe_manage.html
diff --git a/trackme/default/data/ui/nav/default.xml b/trackme/default/data/ui/nav/default.xml
@@ -1,6 +1,7 @@
 <nav search_view="search" color="#404040">
   <view name="TrackMe" default="true" />
   <view name="trackMe_summary" />
+  <a href="TrackMe_manage">TrackMe manage and configure</a>
   <collection label="Search">
     <view name="search" />
     <view name="reports" />

diff --git a/trackme/default/data/ui/views/trackMe_summary.xml b/trackme/default/data/ui/views/trackMe_summary.xml
@@ -2,39 +2,17 @@
   <label>TrackMe Mobile</label>
   <description>Connected Experience tracking review of data sources availability</description>
 
-  <search id="baseMainDataSources">
-    <query>| `tstats` max(_indextime) as current_data_last_ingest, max(_time) as current_data_last_time_seen count as current_data_last_count_seen where index=* sourcetype=* `trackme_tstats_main_filter` `trackme_get_idx_whitelist(trackme_data_source_monitoring_whitelist_index, data_index)` `apply_data_source_blacklists_data_retrieve` by index, sourcetype | rename index as data_index, sourcetype as data_sourcetype | lookup trackme_data_source_monitoring data_index, data_sourcetype OUTPUT data_name | append [ | inputlookup trackme_data_source_monitoring | eval keyid=_key | search `trackme_get_idx_whitelist_searchtime(trackme_data_source_monitoring_whitelist_index, data_index)` ] | stats first(*) as "*" by data_name, data_index, data_sourcetype | eval data_last_ingest=if(isnotnull(current_data_last_ingest), current_data_last_ingest, data_last_ingest), data_last_time_seen=if(isnotnull(current_data_last_time_seen), current_data_last_time_seen, data_last_time_seen) | eval current_data_last_count_seen=if(isnotnull(current_data_last_count_seen), current_data_last_count_seen, data_last_count_seen) | fields - current_* | eventstats max(data_last_time_seen) as data_last_time_seen_idx, min(data_last_lag_seen) as data_last_lag_seen_idx by data_index | `trackme_eval_data_source_state` | `trackme_default_priority` | `trackme_date_format(data_last_time_seen)` | `trackme_date_format(data_last_time_seen_idx)` | `trackme_date_format(data_last_ingest)` | fillnull value="red" data_source_state | sort 0 data_source | `trackme_eval_icons` | search `trackme_get_idx_whitelist_searchtime(trackme_data_source_monitoring_whitelist_index, data_index)` | `apply_data_source_blacklists`</query>
-      <earliest>-5m</earliest>
-      <latest>now</latest>
-      <sampleRatio>1</sampleRatio>
-      <refresh>30s</refresh>
-      <refreshType>delay</refreshType>
-  </search>
-
-  <search id="baseMainDataHosts">
-    <query>| `tstats` max(_indextime) as current_data_last_ingest, max(_time) as current_data_last_time_seen count as current_data_last_count_seen values(sourcetype) as sourcetype where index=* sourcetype=* host=* host!="" `trackme_tstats_main_filter` `trackme_get_idx_whitelist(trackme_data_host_monitoring_whitelist_index, data_index)` `apply_data_host_blacklists_data_retrieve` by index, host | stats max(current_data_last_ingest) as current_data_last_ingest, max(current_data_last_time_seen) as current_data_last_time_seen, sum(current_data_last_count_seen) as current_data_last_count_seen, values(index) as index, values(sourcetype) as sourcetype by host | rename index as data_index, host as data_host, sourcetype as data_sourcetype | lookup trackme_host_monitoring data_host | append [ | inputlookup trackme_host_monitoring | eval keyid=_key ] | stats values(data_index) as data_index, values(data_sourcetype) as data_sourcetype, first(*) as "*" by data_host | eval data_last_ingest=if(isnotnull(current_data_last_ingest), current_data_last_ingest, data_last_ingest), data_last_time_seen=if(isnotnull(current_data_last_time_seen), current_data_last_time_seen, data_last_time_seen) | eval current_data_last_count_seen=if(isnotnull(current_data_last_count_seen), current_data_last_count_seen, data_last_count_seen) | fields - current_* | `trackme_eval_data_host_state` | `trackme_default_priority` | `trackme_date_format(data_last_time_seen)` | `trackme_date_format(data_last_time_seen_idx)` | `trackme_date_format(data_last_ingest)` | fillnull value="red" data_host_state | sort 0 data_host | `trackme_eval_icons_host` | search data_host=* | search data_host=* data_monitored_state=* | fields keyid, data_host, data_index, data_sourcetype, data_last_ingest, "data_last_ingest (translated)", "data_last_time_seen (translated)", data_last_time_seen, state, data_last_lag_seen, data_max_lag_allowed, monitoring, data_monitored_state, data_monitoring_wdays, data_host_state, data_override_lagging_class, priority | `apply_data_host_blacklists` | rename "data_last_ingest (translated)" as "last ingest", "data_last_time_seen (translated)" as "last time" | eval data_index_raw=data_index, data_sourcetype_raw=data_sourcetype | makemv data_index delim="," | makemv data_sourcetype delim="," | eval data_tracker_runtime=now() | lookup local=t trackme_host_monitoring data_host OUTPUT data_host_state as data_previous_host_state, data_tracker_runtime as data_previous_tracker_runtime, latest_flip_state, latest_flip_time | `trackme_date_format("latest_flip_time")` | fillnull value="N/A" latest_flip_state, latest_flip_time, "latest_flip_time (translated)" | search `trackme_get_idx_whitelist_searchtime(trackme_data_host_monitoring_whitelist_index, data_index)`</query>
-      <earliest>-5m</earliest>
-      <latest>now</latest>
-      <sampleRatio>1</sampleRatio>
-      <refresh>30s</refresh>
-      <refreshType>delay</refreshType>
-  </search>
-
-  <search id="baseMainMetricHosts">
-    <query>| savedsearch "trackMe - metric host table report" | `trackme_eval_icons_metric_host` | eval metric_index_raw=metric_index, metric_category_raw=metric_category, metric_details_raw=metric_details | lookup local=t trackme_metric_host_monitoring metric_host OUTPUT metric_host_state as metric_previous_host_state, metric_tracker_runtime as metric_previous_tracker_runtime, latest_flip_state, latest_flip_time | where isnotnull(metric_previous_tracker_runtime) | `trackme_date_format("latest_flip_time")` | fillnull value="N/A" latest_flip_state, latest_flip_time, "latest_flip_time (translated)"</query>
-      <earliest>-5m</earliest>
-      <latest>now</latest>
-      <sampleRatio>1</sampleRatio>
-      <refresh>30s</refresh>
-      <refreshType>delay</refreshType>
-  </search>
-
   <row>
     <panel>
-      <title>Any priority data sources in SLA alert</title>
+      <title>Any priority data sources in alert</title>
       <single>
-        <search base="baseMainDataSources">
-          <query>| where data_monitored_state="enabled" AND data_source_state="red" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_data_source_monitoring
+| `apply_data_source_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_source_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_source_state="red" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -55,10 +33,15 @@
       </single>
     </panel>
     <panel>
-      <title>High priority data sources in SLA alert</title>
+      <title>High priority data sources in alert</title>
       <single>
-        <search base="baseMainDataSources">
-          <query>| where data_monitored_state="enabled" AND data_source_state="red" AND priority="high" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_data_source_monitoring
+| `apply_data_source_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_source_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_source_state="red" AND priority="high" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -78,13 +61,17 @@
         <option name="useThousandSeparators">1</option>
       </single>
     </panel>
-  </row>
-  <row>
     <panel>
-      <title>Any priority data hosts in SLA alert</title>
+      <title>Any priority data hosts in alert</title>
       <single>
-        <search base="baseMainDataHosts">
-          <query>| where data_monitored_state="enabled" AND data_host_state="red" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_host_monitoring
+| makemv delim="," data_index
+| `apply_data_host_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_host_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_host_state="red" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -105,10 +92,16 @@
       </single>
     </panel>
     <panel>
-      <title>High priority data hosts in SLA alert</title>
+      <title>High priority data hosts in alert</title>
       <single>
-        <search base="baseMainDataHosts">
-          <query>| where data_monitored_state="enabled" AND data_host_state="red" AND priority="high" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_host_monitoring
+| makemv delim="," data_index
+| `apply_data_host_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_host_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_host_state="red" AND priority="high" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -128,13 +121,19 @@
         <option name="useThousandSeparators">1</option>
       </single>
     </panel>
-  </row>
-  <row>
     <panel>
-      <title>Any priority metric hosts in SLA alert</title>
+      <title>Any priority metric hosts in alert</title>
       <single>
-        <search base="baseMainMetricHosts">
-          <query>| where metric_monitored_state="enabled" AND metric_host_state="red" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_metric_host_monitoring
+| makemv delim="," metric_category
+| makemv delim="," metric_details
+| makemv delim="," metric_index
+| `apply_metric_host_blacklists_detail_metric_category`
+| search `trackme_get_idx_whitelist_searchtime(trackme_metric_host_monitoring_whitelist_index, metric_index)`
+| where metric_monitored_state="enabled" AND metric_host_state="red" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -155,10 +154,18 @@
       </single>
     </panel>
     <panel>
-      <title>High priority metric hosts in SLA alert</title>
+      <title>High priority metric hosts in alert</title>
       <single>
-        <search base="baseMainMetricHosts">
-          <query>| where metric_monitored_state="enabled" AND metric_host_state="red" AND priority="high" | stats count</query>
+        <search>
+          <query>| inputlookup trackme_metric_host_monitoring
+| makemv delim="," metric_category
+| makemv delim="," metric_details
+| makemv delim="," metric_index
+| `apply_metric_host_blacklists_detail_metric_category`
+| search `trackme_get_idx_whitelist_searchtime(trackme_metric_host_monitoring_whitelist_index, metric_index)`
+| where metric_monitored_state="enabled" AND metric_host_state="red" AND priority="high" | stats count</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="colorBy">value</option>
         <option name="colorMode">none</option>
@@ -182,15 +189,20 @@
   <row>
     <panel>
       <table>
-        <title>Data Sources in SLA alert by priority</title>
-        <search base="baseMainDataSources">
-          <query>| where data_monitored_state="enabled" AND data_source_state="red"
+        <title>Data Sources in alert by priority</title>
+        <search>
+          <query>| inputlookup trackme_data_source_monitoring
+| `apply_data_source_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_source_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_source_state="red"
 | eval priority_num=case(priority="low", 2, priority="medium", 1, priority="high", 0)
 | sort limit=0 priority_num, data_host
 | `trackme_date_format(data_last_ingest)`
 | `trackme_date_format(data_last_time_seen)`
 | fields data_name "data_last_ingest (translated)" "data_last_time_seen (translated)" data_last_lag_seen "data_last_time_seen (translated)" data_max_lag_allowed priority
 | rename "* (translated)" as "*"</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="drilldown">none</option>
       </table>
@@ -199,15 +211,21 @@
   <row>
     <panel>
       <table>
-        <title>Data hosts in SLA alert by priority</title>
-        <search base="baseMainDataHosts">
-          <query>| where data_monitored_state="enabled" AND data_host_state="red"
+        <title>Data hosts in alert by priority</title>
+        <search>
+          <query>| inputlookup trackme_host_monitoring
+| makemv delim="," data_index
+| `apply_data_host_blacklists`
+| search `trackme_get_idx_whitelist_searchtime(trackme_data_host_monitoring_whitelist_index, data_index)`
+| where data_monitored_state="enabled" AND data_host_state="red"
 | eval priority_num=case(priority="low", 2, priority="medium", 1, priority="high", 0)
 | sort limit=0 priority_num, data_host
 | `trackme_date_format(data_last_ingest)`
 | `trackme_date_format(data_last_time_seen)`
 | fields data_host "data_last_ingest (translated)" "data_last_time_seen (translated)" data_last_lag_seen "data_last_time_seen (translated)" data_max_lag_allowed priority
 | rename "* (translated)" as "*"</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="drilldown">none</option>
       </table>
@@ -216,9 +234,15 @@
   <row>
     <panel>
       <table>
-        <title>Metric hosts in SLA alert by priority</title>
-        <search base="baseMainMetricHosts">
-          <query>| where metric_monitored_state="enabled" AND metric_host_state="red"
+        <title>Metric hosts in alert by priority</title>
+        <search>
+          <query>| inputlookup trackme_metric_host_monitoring
+| makemv delim="," metric_category
+| makemv delim="," metric_details
+| makemv delim="," metric_index
+| `apply_metric_host_blacklists_detail_metric_category`
+| search `trackme_get_idx_whitelist_searchtime(trackme_metric_host_monitoring_whitelist_index, metric_index)`
+| where metric_monitored_state="enabled" AND metric_host_state="red"
 | eval priority_num=case(priority="low", 2, priority="medium", 1, priority="high", 0)
 | sort limit=0 priority_num, metric_host
 | `trackme_date_format(metric_last_time_seen)`
@@ -227,6 +251,8 @@
 | stats first(priority) as priority, first(metric_last_time_seen) as metric_last_time_seen, dc(metric_category) as count_metric_category, count(eval(match(metric_details, "metric_host_state\=green"))) as count_green, count(eval(match(metric_details, "metric_host_state=green"))) as count_non_green by metric_host
 | eval summary=" metric categories: " . count_metric_category .  ", " . "green state: " . count_green . ", " . "red state: " . count_non_green
 | fields metric_host, metric_last_time_seen, priority, summary</query>
+          <refresh>30s</refresh>
+          <refreshType>delay</refreshType>
         </search>
         <option name="drilldown">none</option>
       </table>

diff --git a/trackme/default/macros.conf b/trackme/default/macros.conf
@@ -191,4 +191,10 @@ args = lookup, outname
 definition = [ | inputlookup $lookup$\
 | getidxwhitelist fieldname=word_count pattern="\\w+" outname=$outname$ $outname$\
 | rex max_match=0 "\(\'$outname$\'\, \'(?<index>[^\']*)\'\)" | fields - _raw | mvexpand index | rename index as $outname$ ]
-iseval = 0
+iseval = 0
+
+# TrackMe data source identity card
+[trackme_get_identity_card(1)]
+args = key
+definition = lookup trackme_sources_knowledge object as $key$ OUTPUT doc_link, doc_note
+iseval = 0
diff --git a/trackme/default/savedsearches.conf b/trackme/default/savedsearches.conf
@@ -110,7 +110,8 @@ search = | inputlookup trackme_data_source_monitoring\
 | `trackme_date_format(data_last_time_seen)`\
 | `trackme_date_format(data_tracker_runtime)`\
 | `trackme_date_format(data_previous_tracker_runtime)`\
-| rename "* (translated)" as "*"
+| rename "* (translated)" as "*"\
+| `trackme_get_identity_card(data_name)`
 
 # Monitoring of hosts
 
@@ -238,7 +239,7 @@ search = | inputlookup trackme_host_monitoring\
 request.ui_dispatch_app = trackme
 request.ui_dispatch_view = trackme
 is_visible = false
-search = | mstats latest(_value) as value where index=* `trackme_mstats_main_filter` host="$host$" by metric_name, index, host span=1s\
+search = | mstats latest(_value) as value where index=* `trackme_mstats_main_filter` `trackme_get_idx_whitelist(trackme_metric_host_monitoring_whitelist_index, metric_index)` host="$host$" by metric_name, index, host span=1s\
 | stats max(_time) as _time by metric_name, index, host\
 | rex field=metric_name "(?<metric_category>[^\.]*)\.{0,1}"\
 | `apply_metric_host_blacklists_metric_category`\
@@ -274,6 +275,7 @@ search = | mstats latest(_value) as value where index=* `trackme_mstats_main_fil
 | rename key as _key\
 | `trackme_eval_metric_host_state`\
 | `trackme_default_priority`\
+| search `trackme_get_idx_whitelist_searchtime(trackme_metric_host_monitoring_whitelist_index, metric_index)`\
 | lookup local=t trackme_metric_host_monitoring metric_host OUTPUT metric_host_state as metric_previous_host_state, metric_tracker_runtime as metric_previous_tracker_runtime\
 | append [ | inputlookup trackme_audit_flip | where object_category="metric_host" | eval _time=time | stats max(_time) as latest_flip_time, latest(object_state) as latest_flip_state by object | rename object as metric_host ]\
 | stats first(_key) as keyid, first(latest_flip_time) as latest_flip_time, first(latest_flip_state) as latest_flip_state, values(*) as "*" by metric_host\

diff --git a/trackme/default/transforms.conf b/trackme/default/transforms.conf
@@ -142,3 +142,12 @@ fields_list = _key, time, action, change_type, object, object_category, object_a
 external_type = kvstore
 collection = kv_trackme_audit_flip
 fields_list = _key, time, object, object_category, result, object_previous_state, object_state
+
+#
+# Documentation knowledge base
+#
+
+[trackme_sources_knowledge]
+external_type = kvstore
+collection = kv_trackme_sources_knowledge
+fields_list = _key, object, doc_link, doc_note