Merge pull request #330 from guilhemmarchand/testing

Version 1.2.47

docs/releasenotes.rst

@@ -1,6 +1,26 @@
 Release notes
 #############
 
+Version 1.2.47
+==============
+
+**CAUTION:**
+
+This is a new main release branch, TrackMe 1.2.x requires the deployment of the following dependencies:
+
+- Semicircle Donut Chart Viz, Splunk Base: https://splunkbase.splunk.com/app/4378
+- Splunk Machine Learning Toolkit, Splunk Base: https://splunkbase.splunk.com/app/2890
+- Splunk Timeline - Custom Visualization, Splunk Base: https://splunkbase.splunk.com/app/3120
+- Splunk SA CIM - Splunk Common Information Model, Splunk Base: https://splunkbase.splunk.com/app/1621
+
+TrackMe requires a summary index (defaults to trackme_summary) and a metric index (defaults to trackme_metrics):
+https://trackme.readthedocs.io/en/latest/configuration.html
+
+- Fix - Issue #328 - Data host - Regex based block lists are not honored as documented
+- Fix - Issue #329 - Data host - Splunk 8.2 regression with multivalue aggregation caused by a change in behaviour
+- Change: Update splunktaucclib to 4.2.0
+- Change: Update splunktalib to 1.2.1
+
 Version 1.2.46
 ==============
 

trackme/app.manifest

@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "trackme",
-      "version": "1.2.46"
+      "version": "1.2.47"
     },
     "author": [
       {

trackme/default/app.conf

@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Data tracking system for Splunk
-version = 1.2.46
+version = 1.2.47

trackme/default/macros.conf

@@ -583,6 +583,7 @@ definition = eval host=upper(host)\
 \
 `comment("#### Apply blocklists ####")`\
 | search `apply_data_host_blacklists_summary_data_retrieve`\
+| `apply_data_host_blacklists_summary_rex`\
 \
 `comment("#### lagging policies per index and sourcetype ####")`\
 \
@@ -681,7 +682,38 @@ first(latest_flip_state) as latest_flip_state, first(latest_flip_time) as latest
 | inputlookup append=t trackme_summary_investigator_volume_outliers\
 | eval data_host=if(isnull(data_host) AND object_category="data_host", object, data_host)\
 | where isnotnull(data_host)\
-| stats values(data_host_st_summary) as data_host_st_summary, first(*) as "*" by data_host\
+| stats values(data_host_st_summary) as data_host_st_summary,\
+first(OutlierAlertOnUpper) as OutlierAlertOnUpper,\
+first(OutlierLowerThresholdMultiplier) as OutlierLowerThresholdMultiplier,\
+first(OutlierMinEventCount) as OutlierMinEventCount,\
+first(OutlierSpan) as OutlierSpan,\
+first(OutlierTimePeriod) as OutlierTimePeriod,\
+first(OutlierUpperThresholdMultiplier) as OutlierUpperThresholdMultiplier,\
+first(data_eventcount) as data_eventcount,\
+first(data_first_time_seen) as data_first_time_seen,\
+first(data_host_alerting_policy) as data_host_alerting_policy,\
+first(data_host_is_online) as data_host_is_online,\
+first(data_index) as data_index,\
+first(data_lag_alert_kpis) as data_lag_alert_kpis,\
+first(data_last_ingest) as data_last_ingest,\
+first(data_last_ingestion_lag_seen) as data_last_ingestion_lag_seen,\
+first(data_last_lag_seen) as data_last_lag_seen,\
+first(data_last_time_seen) as data_last_time_seen,\
+first(data_max_lag_allowed) as data_max_lag_allowed,\
+first(data_monitored_state) as data_monitored_state,\
+first(data_monitoring_wdays) as data_monitoring_wdays,\
+first(data_override_lagging_class) as data_override_lagging_class,\
+first(data_previous_host_state) as data_previous_host_state,\
+first(data_previous_tracker_runtime) as data_previous_tracker_runtime,\
+first(data_sourcetype) as data_sourcetype,\
+first(enable_behaviour_analytic) as enable_behaviour_analytic,\
+first(isOutlier) as isOutlier,\
+first(key) as key,\
+first(latest_flip_state) as latest_flip_state,\
+first(latest_flip_time) as latest_flip_time,\
+first(object_category) as object_category,\
+first(priority) as priority,\
+first(summary_max_allowed) as summary_max_allowed by data_host\
 | where isnotnull(key)\
 \
 `comment("#### fillnull for OutlierMinEventCount, isOutlier ####")`\
@@ -816,6 +848,12 @@ definition = [ | inputlookup trackme_data_host_monitoring_blacklist_index\
 | `detect_rex(summary_st)` | where is_rex="false" | fields - is_rex | where NOT match(summary_st, "^\*$") | format | eval search=if(match(search, "NOT \(\)"), "(summary_st=*)", "NOT " . search) ]
 iseval = 0
 
+[apply_data_host_blacklists_summary_rex]
+definition = where NOT [ `apply_blacklist_rex(trackme_data_host_monitoring_blacklist_host, data_host, host)` ]\
+| where NOT [ `apply_blacklist_rex(trackme_data_host_monitoring_blacklist_sourcetype, data_sourcetype, summary_st)` ]\
+| where NOT [ `apply_blacklist_rex(trackme_data_host_monitoring_blacklist_index, data_index, summary_idx)` ]
+iseval = 0
+
 [apply_metric_host_blacklists_data_retrieve]
 definition = [ | inputlookup trackme_metric_host_monitoring_blacklist_index\
 | stats values(metric_index) as metric_index | mvexpand metric_index\

trackme/lib/splunktalib/__init__.py

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-__version__ = "1.1.7"
+__version__ = "1.2.1"

trackme/lib/splunktalib/common/xml_dom_parser.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import re
-from xml.etree import cElementTree as et
+from defusedxml import cElementTree as et
 
 
 def parse_conf_xml_dom(xml_content):

trackme/lib/splunktalib/credentials.py

@@ -10,7 +10,7 @@
 from builtins import range
 from builtins import object
 import re
-import xml.dom.minidom as xdm
+import defusedxml.minidom as xdm
 
 import splunktalib.common.util as util
 import splunktalib.common.xml_dom_parser as xdp

trackme/lib/splunktalib/kv_client.py

@@ -6,7 +6,7 @@
 from builtins import object
 import re
 import json
-from xml.etree import cElementTree as et
+from defusedxml import cElementTree as et
 
 import splunktalib.rest as rest
 

trackme/lib/splunktalib/modinput.py

@@ -83,7 +83,7 @@ def parse_modinput_configs(config_str):
     @return: meta_config and stanza_config
     """
 
-    import xml.dom.minidom as xdm
+    import defusedxml.minidom as xdm
 
     meta_configs = {
         "server_host": None,

trackme/lib/splunktaucclib/rest_handler/endpoint/validator.py

@@ -269,13 +269,7 @@ def __init__(self, min_val=None, max_val=None, is_int=False):
         :param is_int: the value should be integer or not
         """
 
-        def check(val):
-            try:
-                return val is None or isinstance(val, (int, long, float))
-            except NameError:
-                return val is None or isinstance(val, (int, float))
-
-        assert check(min_val) and check(
+        assert self._check(min_val) and self._check(
             max_val
         ), "%(min_val)s & %(max_val)s should be numbers" % {
             "min_val": min_val,
@@ -287,6 +281,12 @@ def check(val):
         self._max_val = max_val
         self._is_int = is_int
 
+    def _check(self, val):
+        try:
+            return val is None or isinstance(val, (int, long, float))
+        except NameError:
+            return val is None or isinstance(val, (int, float))
+
     def validate(self, value, data):
         try:
             try:
@@ -337,15 +337,7 @@ def __init__(self, min_len=None, max_len=None):
             it should be longer than ``max_len``
         """
 
-        def check(val):
-            if val is None:
-                return True
-            try:
-                return isinstance(val, (int, long)) and val >= 0
-            except NameError:
-                return isinstance(val, (int)) and val >= 0
-
-        assert check(min_len) and check(
+        assert self._check(min_len) and self._check(
             max_len
         ), "%(min_len)s & %(max_len)s should be numbers" % {
             "min_len": min_len,
@@ -355,6 +347,14 @@ def check(val):
         super(String, self).__init__()
         self._min_len, self._max_len = min_len, max_len
 
+    def _check(self, val):
+        if val is None:
+            return True
+        try:
+            return isinstance(val, (int, long)) and val >= 0
+        except NameError:
+            return isinstance(val, (int)) and val >= 0
+
     def validate(self, value, data):
         if not isinstance(value, basestring):
             self.put_msg("Input value should be string")