Merge pull request #346 from AntoineBlaud/fix_match_query_key_struct

#340 fix match["query_key"] malformation

CHANGELOG.md

@@ -15,18 +15,24 @@
 ## Breaking changes
 - [VictorOps] Changed `state_message` and `entity_display_name` values to be taken from an alert rule. - [#329](https://github.com/jertel/elastalert2/pull/329) - @ChristophShyper
   - Potentially a breaking change if the alert subject changes due to the new default behavior.
+- Change metric/percentage rule types to store query_key as dict, instead of string, for consistency with other rule types. [#340](https://github.com/jertel/elastalert2/issues/340) - @AntoineBlaud
 
 ## New features
 - None
 
 ## Other changes
+- [Tests] Improve test code coverage - [#331](https://github.com/jertel/elastalert2/pull/331) - @nsano-rururu
+- [Docs] Upgrade Sphinx from 4.0.2 to 4.1.1 - [#344](https://github.com/jertel/elastalert2/pull/344) - @nsano-rururu
+- [Tests] Upgrade Tox from 3.23.1 to 3.24.0 - [#345](https://github.com/jertel/elastalert2/pull/345) - @nsano-rururu
+- Upgrade Jinja from 2.11.3 to 3.0.1 - [#350](https://github.com/jertel/elastalert2/pull/350) - @mrfroggg
 - [Tests] Add test code. Changed ubuntu version of Dockerfile-test from latest to 21.10. - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
 - [Tests] Improved test coverage for opsgenie.py 96% to 100% - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu
 - Remove Python 2.x compatibility code - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
 - [Docs] Added Chatwork proxy settings to documentation - [#360](https://github.com/jertel/elastalert2/pull/360) - @nsano-rururu
+- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu
 - [Docs] Tidy Twilio alerter documentation - [#363](https://github.com/jertel/elastalert2/pull/363) - @ferozsalam
+- [Tests] Improve opsgenie test code coverage - [#364](https://github.com/jertel/elastalert2/pull/364) - @nsano-rururu
 - [Docs] Update mentions of JIRA to Jira - [#365](https://github.com/jertel/elastalert2/pull/365) - @ferozsalam
-- Add settings to schema.yaml(Chatwork proxy, Dingtalk proxy) - [#361](https://github.com/jertel/elastalert2/pull/361) - @nsano-rururu
 
 # 2.1.2
 ## Breaking changes

elastalert/ruletypes.py

@@ -7,7 +7,7 @@
 
 from elastalert.util import (add_raw_postfix, dt_to_ts, EAException, elastalert_logger, elasticsearch_client,
                              format_index, hashable, lookup_es_key, new_get_event_ts, pretty_ts, total_seconds,
-                             ts_now, ts_to_dt)
+                             ts_now, ts_to_dt, expand_string_into_dict)
 
 
 class RuleType(object):
@@ -1096,7 +1096,7 @@ def check_matches(self, timestamp, query_key, aggregation_data):
                 match = {self.rules['timestamp_field']: timestamp,
                          self.metric_key: metric_val}
                 if query_key is not None:
-                    match[self.rules['query_key']] = query_key
+                    match = expand_string_into_dict(match, self.rules['query_key'], query_key)
                 self.add_match(match)
 
     def check_matches_recursive(self, timestamp, query_key, aggregation_data, compound_keys, match_data):
@@ -1286,7 +1286,7 @@ def check_matches(self, timestamp, query_key, aggregation_data):
                 if self.percentage_violation(match_percentage):
                     match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count}
                     if query_key is not None:
-                        match[self.rules['query_key']] = query_key
+                        match = expand_string_into_dict(match, self.rules['query_key'], query_key)
                     self.add_match(match)
 
     def percentage_violation(self, match_percentage):

elastalert/util.py

@@ -481,3 +481,40 @@ def should_scrolling_continue(rule_conf):
     stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle')
 
     return not stop_the_scroll
+
+
+def _expand_string_into_dict(string, value,  sep='.'):
+    """
+    Converts a encapsulated string-dict to a sequence of dict. Use separator (default '.') to split the string.
+    Example: 
+        string1.string2.stringN : value  -> {string1: {string2: {string3: value}}
+ 
+    :param string: The encapsulated "string-dict"
+    :param value: Value associated to the last field of the "string-dict"
+    :param sep: Separator character. Default: '.'
+    :rtype: dict
+    """
+    if sep not in string:
+        return {string : value}
+    key, val = string.split(sep, 1)
+    return {key: _expand_string_into_dict(val, value)}
+
+
+def expand_string_into_dict(dictionary, string , value, sep='.'):
+    """
+    Useful function to "compile" a string-dict string used in metric and percentage rules into a dictionary sequence.
+ 
+    :param dictionary: The dictionary dict
+    :param string:  String Key 
+    :param value: String Value
+    :param sep: Separator character. Default: '.'
+    :rtype: dict
+    """
+
+    if sep not in string:
+        dictionary[string] = value
+        return dictionary
+    else:
+        field1, new_string = string.split(sep, 1)
+        dictionary[field1] = _expand_string_into_dict(new_string, value)
+    return dictionary

tests/rules_test.py

@@ -1163,10 +1163,15 @@ def test_metric_aggregation():
     rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}})
     assert len(rule.matches) == 2
 
-    rules['query_key'] = 'qk'
+    rules['query_key'] = 'subdict'
     rule = MetricAggregationRule(rules)
     rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}})
-    assert rule.matches[0]['qk'] == 'qk_val'
+    assert rule.matches[0]['subdict'] == 'qk_val'
+
+    rules['query_key'] = 'subdict1.subdict2.subdict3'
+    rule = MetricAggregationRule(rules)
+    rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}})
+    assert rule.matches[0]['subdict1']['subdict2']['subdict3'] == 'qk_val'
 
 
 def test_metric_aggregation_complex_query_key():
@@ -1279,6 +1284,12 @@ def test_percentage_match():
     rule = PercentageMatchRule(rules)
     rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24))
     assert rule.matches[0]['qk'] == 'qk_val'
+
+    rules['query_key'] = 'subdict1.subdict2'
+    rule = PercentageMatchRule(rules)
+    rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24))
+    assert rule.matches[0]['subdict1']['subdict2'] == 'qk_val'
+
     assert '76.1589403974' in rule.get_match_str(rule.matches[0])
     rules['percentage_format_string'] = '%.2f'
     assert '76.16' in rule.get_match_str(rule.matches[0])

tests/util_test.py

@@ -33,6 +33,7 @@
 from elastalert.util import total_seconds
 from elastalert.util import ts_to_dt_with_format
 from elastalert.util import ts_utc_to_tz
+from elastalert.util import expand_string_into_dict
 from elastalert.util import unixms_to_dt
 
 
@@ -460,6 +461,15 @@ def test_elasticsearch_client(es_host, es_port, es_bearer, es_api_key):
     assert None is not acutual
 
 
+def test_expand_string_into_dict():
+    dictionnary = {'@timestamp': '2021-07-06 01:00:00', 'metric_netfilter.ipv4_dst_cardinality': 401}
+    string = 'metadata.source.ip'
+    value = '0.0.0.0'
+
+    expand_string_into_dict(dictionnary, string, value)
+    assert dictionnary['metadata']['source']['ip'] == value
+
+
 def test_inc_ts():
     dt = datetime(2021, 7, 6, hour=0, minute=0, second=0)
     actual = inc_ts(dt)