add NOT for condition: regex match scenario

aliyun/log/etl_core/__init__.py

@@ -2,3 +2,4 @@
 from .transform import *
 from .runner import Runner
 from .settings import *
+from .etl_util import NOT

aliyun/log/etl_core/etl_util.py

@@ -21,16 +21,21 @@ def _wrapped(*args, **kwargs):
 
 
 def re_full_match(pattern, string, *args, **kwargs):
+    is_not = isinstance(pattern, NOT)
     if six.PY2 and isinstance(pattern, six.binary_type):
         pattern = pattern.decode('utf8', 'ignore')
     if six.PY2 and isinstance(string, six.binary_type):
         string = string.decode('utf8', 'ignore')
 
+    ret = None
     if hasattr(re, 'fullmatch'):
-        return re.fullmatch(pattern, string, *args, **kwargs)
-    m = re.match(pattern, string, *args, **kwargs)
-    if m and m.span()[1] == len(string):
-        return m
+        ret = re.fullmatch(pattern, string, *args, **kwargs)
+    else:
+        m = re.match(pattern, string, *args, **kwargs)
+        if m and m.span()[1] == len(string):
+            ret = m
+
+    return not ret if is_not else ret
 
 
 # this function is used to bypass lambda trap
@@ -156,3 +161,31 @@ def u(d):
 
     return d
 
+
+class NOT(object):
+    def __new__(self, v):
+        if isinstance(v, six.binary_type):
+            return _NOT_B(v)
+        elif isinstance(v, six.text_type):
+            return _NOT_U(v)
+        else:
+            raise ValueError("NOT can only be used with string. ")
+
+
+class _NOT_B(six.binary_type, NOT):
+    def __init__(self, v):
+        super(_NOT_B, self).__init__(v)
+        self.v = v
+
+    def decode(self, *args, **kwargs):
+        return _NOT_B(self.v.decode(*args, **kwargs))
+
+
+class _NOT_U(six.text_type, NOT):
+    def __init__(self, v):
+        if six.PY2:
+            super(_NOT_U, self).__init__(v)
+        self.v = v
+
+    def encode(self, *args, **kwargs):
+        return _NOT_B(self.v.encode(*args, **kwargs))

aliyun/log/etl_core/transform/condition_list.py

@@ -19,7 +19,7 @@
 import functools
 import six
 
-from ..etl_util import re_full_match, get_re_full_match, u
+from ..etl_util import re_full_match, get_re_full_match, u, NOT
 from ..exceptions import SettingError
 
 logger = logging.getLogger(__name__)
@@ -83,6 +83,8 @@ def __init__(self, cond, pass_meta=None, restore_meta=None):
             ck = _get_check(c)
             if ck is not None:
                 self.check_list.append(ck)
+            else:
+                raise SettingError(msg="Invalid condition", settings=c)
 
     DEFAULT_META_KEYS = set(("__time__", "__topic__", "__source__"))
     tag_meta_check = staticmethod(get_re_full_match(r"__tag__:.+"))

aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.6.40'
+__version__ = '0.6.41'
 USER_AGENT = 'log-python-sdk-v-' + __version__
 LOGGING_HANDLER_USER_AGENT = 'logging-handler, ' + USER_AGENT
 ES_MIGRATION_USER_AGENT = 'es-migration, ' + USER_AGENT

tests/etl_test/test_case.py

@@ -59,6 +59,25 @@ def test_condition():
     assert condition(lambda e: 'k1' in e and e['k1'].isdigit())(event)
     assert not condition(lambda e: 'k5' in e)(event)
 
+
+def test_condition_not():
+    event = {'k1': '123', 'k2': 'abc', 'k3': "abc123"}
+
+    # dict - string
+    assert not condition({'k1': NOT(r'\d+')})(event)
+    assert not condition([{'k1': NOT(r'\d+')}])(event)
+    assert not condition({'k2': NOT(r'\w+')})(event)
+    assert condition({'k3': NOT(r'\d+')})(event)
+
+    # dict - or
+    assert condition([{'k1': NOT(r'\d+')}, {'k2': NOT(r'\d+')}])(event)
+    assert not condition([{'k1': NOT(r'\d+')}, {'k4': r'\w+'}])(event)
+
+    # dict - and
+    assert not condition([{'k1': NOT(r'\d+'), 'k2': r'\w+'}])(event)
+    assert condition([{'k1': NOT(r'[a-z]+'), 'k3': NOT(r'\d+')}])(event)
+
+
 def test_regex():
     """
 
@@ -873,6 +892,7 @@ def test_zip():
 
 
 test_condition()
+test_condition_not()
 test_regex()
 test_csv()
 test_lookup_dict()