add test cases and passed in py2.7/3.7 for restrict config parser

.travis.yml

@@ -22,6 +22,7 @@ install:
 script:
   - python tests/ut/test_util.py
   - python tests/etl_test/test_case.py
+  - python tests/cli_config_check/test_case_safe.py
   - python tests/sample.py
   - python tests/integration_test/test_logtail_config.py
   - python tests/integration_test/test_entity.py

aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.6.39'
+__version__ = '0.6.40'
 USER_AGENT = 'log-python-sdk-v-' + __version__
 LOGGING_HANDLER_USER_AGENT = 'logging-handler, ' + USER_AGENT
 ES_MIGRATION_USER_AGENT = 'es-migration, ' + USER_AGENT

tests/cli_config_check/restrict_config_test_data/config1_fail.py

@@ -0,0 +1,32 @@
+from aliyun.log.etl_core import *
+
+
+KEEP_EVENT_default = True
+
+
+DROP_EVENT_useless = {"data": "useless code.+|useless2 code.+", "raw": "bad data.+"}
+
+KEEP_FIELDS_all = '.+'
+
+DROP_FIELDS_none = ""
+
+RENAME_FIELDS_simple = {"f1": "f1_new", "f2": "f2_new"}
+
+
+DISPATCH_EVENT_data = [
+    ({"data": "^ETL_Information .+"}, {"__topic__": "etl_info"}),
+    ({"data": "^Status .+"}, {"__topic__": "machine_status"}),
+    ({"data": "^System Reboot .+"}, {"__topic__": "reboot_event"}),
+    ({"data": "^Provision Firmware Download start .+"}, {"__topic__": "download"}),
+    (True, {"__topic__": "unknown"})]
+
+
+TRANSFORM_EVENT_data = [
+    ({"__topic__": "etl_info"}, {"__topic__": "etl_info"}),
+    ({"__topic__": "machine_status"}, {"__topic__": "machine_status"}),
+    ({"__topic__": "reboot_event"}, {"__topic__": "reboot_event"}),
+    ({"__topic__": "download"}, {"__topic__": "download"}),
+    (True, {"__topic__": "unknown"})]
+
+KEEP_EVENT_pass_or_failure = [{"result": r"(?i)ok|pass"}, {"status": lambda v: int(v) == 200},
+                                    lambda e: (('status' in e and int(e['status']) > 200) or ('__raw__' in e and 'error' in e['__raw__']))]

tests/cli_config_check/restrict_config_test_data/config1_safe.py

@@ -0,0 +1,30 @@
+from aliyun.log.etl_core import *
+
+
+KEEP_EVENT_default = True
+
+
+DROP_EVENT_useless = {"data": "useless code.+|useless2 code.+", "raw": "bad data.+"}
+
+KEEP_FIELDS_all = '.+'
+
+DROP_FIELDS_none = ""
+
+RENAME_FIELDS_simple = {"f1": "f1_new", "f2": "f2_new"}
+
+
+DISPATCH_EVENT_data = [
+    ({"data": "^ETL_Information .+"}, {"__topic__": "etl_info"}),
+    ({"data": "^Status .+"}, {"__topic__": "machine_status"}),
+    ({"data": "^System Reboot .+"}, {"__topic__": "reboot_event"}),
+    ({"data": "^Provision Firmware Download start .+"}, {"__topic__": "download"}),
+    (True, {"__topic__": "unknown"})]
+
+
+TRANSFORM_EVENT_data = [
+    ({"__topic__": "etl_info"}, {"__topic__": "etl_info"}),
+    ({"__topic__": "machine_status"}, {"__topic__": "machine_status"}),
+    ({"__topic__": "reboot_event"}, {"__topic__": "reboot_event"}),
+    ({"__topic__": "download"}, {"__topic__": "download"}),
+    (True, {"__topic__": "unknown"})]
+

tests/cli_config_check/restrict_config_test_data/config2_fail1.py

@@ -0,0 +1,4 @@
+
+from aliyun.log.etl_core import *
+
+a = 10

tests/cli_config_check/restrict_config_test_data/config2_fail2.py

@@ -0,0 +1,7 @@
+
+from aliyun.log.etl_core import *
+
+def sls_eu_my_logic(event):
+    event["hello"] = "world"
+    return event
+

tests/cli_config_check/restrict_config_test_data/config2_fail3.py

@@ -0,0 +1,5 @@
+
+from aliyun.log.etl_core import *
+
+def p(fn): pass
+

tests/cli_config_check/restrict_config_test_data/config3_fail.py

@@ -0,0 +1,25 @@
+from aliyun.log.etl_core import *
+
+# keep events, AND in dict (string, regex)
+KEEP_EVENT_a = {"__topic__": "ddos_access_log", "cc_phase": r".+"}
+
+
+# drop condition, OR in outer list  (built-in condition: EMPTY)
+DROP_EVENT_b = [EMPTY("status"), EMPTY("http_user_agent"),
+                           {"http_user_agent": "-"}, {"status": r"2\d+"}]
+
+# keep fields, field list
+KEEP_FIELDS_a = ["http_user_agent", "real_client_ip", "remote_addr", "status", "cc_phase"]
+
+
+# transform event:
+#     built-in condition (EMPTY): dict-built-in function (V)
+#     built-in condition (ALL=ANY=True): tuple-field, built-in transform (V)
+#     dict condition (field-regex): tuple-field,
+TRANSFORM_EVENT_v = [
+    ( EMPTY("real_client_ip"), {"real_client_ip": V('remote_addr')} ),
+    ( ANY2, ("ip", V('real_client_ip') ) ),
+    ( {"http_user_agent": r".+\bWindows\b.+"}, {"is_windows": "1"} ),
+    ( {"http_user_agent": r".+\bLinux\b.+"}, {"is_linux": "1"} ),
+    ( NO_EMPTY("real_client_ip"), {"real_client_ip": "xyz" } ) ]
+

tests/cli_config_check/restrict_config_test_data/config3_safe.py

@@ -0,0 +1,30 @@
+from aliyun.log.etl_core import *
+
+# keep events, AND in dict (string, regex)
+KEEP_EVENT_a = {"__topic__": "ddos_access_log", "cc_phase": r".+"}
+
+
+# drop condition, OR in outer list  (built-in condition: EMPTY)
+DROP_EVENT_b = [EMPTY("status"), EMPTY("http_user_agent"),
+                           {"http_user_agent": "-"}, {"status": r"2\d+"}]
+
+# keep fields, field list
+KEEP_FIELDS_a = ["http_user_agent", "real_client_ip", "remote_addr", "status", "cc_phase"]
+
+
+# transform event:
+#     built-in condition (EMPTY): dict-built-in function (V)
+#     built-in condition (ALL=ANY=True): tuple-field, built-in transform (V)
+#     dict condition (field-regex): tuple-field,
+TRANSFORM_EVENT_v = [
+    ( EMPTY("real_client_ip"), {"real_client_ip": V('remote_addr')} ),
+    ( ANY, ("ip", V('real_client_ip') ) ),
+    ( {"http_user_agent": r".+\bWindows\b.+"}, {"is_windows": "1"} ),
+    ( {"http_user_agent": r".+\bLinux\b.+"}, {"is_linux": "1"} ),
+    ( NO_EMPTY("real_client_ip"), DROP_F("abc") ) ]
+
+DROP_FIELDS_a = ["remote_addr", "http_user_agent"]
+
+ALIAS_x = {"real_client_ip": "client_ip"}
+
+

tests/cli_config_check/restrict_config_test_data/config4_safe.py

@@ -0,0 +1,4 @@
+
+from aliyun.log.etl_core import *
+
+DROP_FIELDS_f1 = [F_TAGS, F_META]

tests/cli_config_check/restrict_config_test_data/config5_safe.py

@@ -0,0 +1,23 @@
+
+from aliyun.log.etl_core import *
+
+# drop fields - regex
+DROP_FIELDS_meta = "__.+__|time|imei"
+
+DISPATCH_EVENT_data = [
+    ({"data": "^LTE_Information .+"}, {"__topic__": "let_info"}),
+    ({"data": "^Status,.+"}, {"__topic__": "machine_status"}),
+    ({"data": "^System Reboot .+"}, {"__topic__": "reboot_event"}),
+    ({"data": "^Provision Firmware Download start .+"}, {"__topic__": "download"}),
+    (True, {"__topic__": "default"})]
+
+# transform
+#     lambda : DROP_F
+TRANSFORM_EVENT_data = [
+    ({"__topic__": "let_info"}, ("data", r"LTE_Information (?P<RSPR>[^,]+),(?P<SINR>[^,]+),(?P<global_cell_id>[^,]+),")),
+    ({"__topic__": "machine_status"}, ("data", r"Status,(?P<cpu_usage_usr>[\d\.]+)% usr (?P<cpu_usage_sys>[\d\.]+)% sys,(?P<memory_free>\d+)(?P<memory_free_unit>[a-zA-Z]+),")),
+    ({"__topic__": "reboot_event"}, ("data", r"System Reboot \[(?P<reboot_code>\d+)\]")),
+    ({"__topic__": "download"}, ("data", r"Provision Firmware Download start \[(?P<firmware_version>[\w\.]+)\]")),
+    ( ANY, DROP_F("data") )
+    ]
+

tests/cli_config_check/restrict_config_test_data/config6_safe.py

@@ -0,0 +1,156 @@
+from aliyun.log.etl_core import *
+
+DROP_FIELDS_v1 = True
+DROP_FIELDS_v2 = False
+DROP_FIELDS_v3 = [True]
+DROP_FIELDS_v4 = [False]
+DROP_FIELDS_v5 = {'k1': r'\d+'}
+DROP_FIELDS_v6 = [{'k1': r'\d+'}]
+DROP_FIELDS_v9 = [{'k1': r'\d+'}, {'k2': r'\d+'}]
+DROP_FIELDS_v10 = [{'k1': r'\d+', 'k2': r'\w+'}]
+DROP_FIELDS_v7 = {'k4': None}
+DROP_FIELDS_v11 = {'k1': True}
+DROP_FIELDS_v12 = {'k1': False}
+
+TRANSFORM_EVENT_v1 = (ANY, {'k3': 'abc123'})
+TRANSFORM_EVENT_v2 = (ANY, ("k1", r"hello (?P<name>\w+)"))
+TRANSFORM_EVENT_v3 = (ANY, ("k1", r"(?i)(?P<word>[a-z]+)(?P<num>\d+)"))
+TRANSFORM_EVENT_v4 = (ANY, ("k1", r"(?P<abc>\d+)"))
+TRANSFORM_EVENT_v5 = (ANY, ("k1", r".*?(?P<abc>\d+)") )
+TRANSFORM_EVENT_v7 = (ANY, (["k1", 'k2'], r"(?P<abc>\d+)"))
+TRANSFORM_EVENT_v9 = (ANY, ("k1", REGEX(r"^(?P<abc>\d+)$") ))
+TRANSFORM_EVENT_v10 = (ANY, ("k1", REGEX(r"(?P<abc>\d+)") ))
+TRANSFORM_EVENT_v11 = (ANY, ("k1", REGEX(r"\d+", "f1")) )
+TRANSFORM_EVENT_v12 = (ANY, ("k1", r"\d+", ["f1", "f2"]))
+TRANSFORM_EVENT_v13 = (ANY, ("k1", r"(\w+) (\d+)", ["f1", "f2"]) )
+TRANSFORM_EVENT_v14 = (ANY, ("k1", REGEX(r"(?P<abc>\d+)") ))
+TRANSFORM_EVENT_v15 = (ANY, ("k1", r"(\d+) (\d+)", ["f1", "f2"]))
+TRANSFORM_EVENT_v16 = (ANY, ("k1", r"(\w+):(\d+)", {r"k_\1": r"v_\2"}))
+
+
+TRANSFORM_EVENT_v17 = (ANY,  ("data", CSV(r"city,pop,province")) )
+TRANSFORM_EVENT_v18 = (ANY,  ("data", CSV(['city', 'pop', 'province'])) )
+TRANSFORM_EVENT_v19 = (ANY,  ("data", CSV(r"city, pop, province", sep='#')) )
+TRANSFORM_EVENT_v20 = (ANY,  ("data", CSV(r"city, pop, province", lstrip=False)) )
+TRANSFORM_EVENT_v21 = (ANY,  ("data", CSV(r"city, pop, province", quote='|')) )
+TRANSFORM_EVENT_v22 = (ANY,  ("data", CSV(r"city, pop, province", restrict=True)) )
+TRANSFORM_EVENT_v23 = (ANY,  ("data", TSV(r"city,pop,province")) )
+TRANSFORM_EVENT_v24 = (ANY,  ("data", PSV(r"city,pop,province")) )
+
+TRANSFORM_EVENT_v25 = (ANY,  ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP", "*": "Unknown"}, "protocol")) )
+TRANSFORM_EVENT_v26 = (ANY,  ("pro", LOOKUP({"1": "TCP", "2": "UDP", "3": "HTTP"}, "protocol")) )
+TRANSFORM_EVENT_v27 = (ANY,  ("pro", LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")) )
+TRANSFORM_EVENT_v28 = (ANY,  ("pro", LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp", "*": "Unknown"}, "type", case_insensitive=False)) )
+TRANSFORM_EVENT_v29 = (ANY,  (["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")) )
+
+TRANSFORM_EVENT_v30 = (ANY,  ("city", LOOKUP('abc.csv', "province")) )
+TRANSFORM_EVENT_v31 = (ANY,  ("city", LOOKUP('abc.csv', ["province", "pop"])) )
+TRANSFORM_EVENT_v32 = (ANY,  ("city", LOOKUP('abc.csv', ["province", "pop"], lstrip=False)) )
+TRANSFORM_EVENT_v35 = (ANY,  ("city", LOOKUP('abc.csv', ["city,province", "city,pop"])) )
+TRANSFORM_EVENT_v37 = (ANY,  ("city", LOOKUP('abc.csv', ["province", "pop"], quote='|')) )
+TRANSFORM_EVENT_v39 = (ANY,  ("ct", LOOKUP('abc.csv', "pro", headers=["ct", "pop", "pro"])) )
+TRANSFORM_EVENT_v40 = (ANY,  ("ct2", LOOKUP('abc.csv', "pro", headers="ct2, pop , pro")) )
+
+TRANSFORM_EVENT_v51 = (ANY,   ("city", LOOKUP("abc.csv", "province") )  )
+TRANSFORM_EVENT_v52 = (ANY,   (["city", "province"], LOOKUP("abc.csv", "pop") )  )
+TRANSFORM_EVENT_v53 = (ANY,   ("city", LOOKUP("abc.csv", "province") )  )
+TRANSFORM_EVENT_v54 = (ANY,   ("pro", LOOKUP("abc.csv", "type") )  )
+TRANSFORM_EVENT_v55 = (ANY,   ("pro", LOOKUP("abc.csv", "type") )  )
+TRANSFORM_EVENT_v56 = (ANY,  ("pro", LOOKUP("abc.csv", "type", case_insensitive=False)) )
+TRANSFORM_EVENT_v57 = (ANY,  ("pro", LOOKUP("abc.csv", "type", case_insensitive=False)) )
+TRANSFORM_EVENT_v58 = (ANY,   (["city", "province"], LOOKUP("abc.csv", "pop") )  )
+TRANSFORM_EVENT_v59 = (ANY,   (["city", "province"], LOOKUP("abc.csv", ["pop", "type"]) )  )
+TRANSFORM_EVENT_v60 = (ANY,   ([("ct", "city"), "province"], LOOKUP("abc.csv", "pop") )  )
+TRANSFORM_EVENT_v61 = (ANY,   ([("ct", "city"), ("prv", "province")], LOOKUP("abc.csv", "pop") )  )
+TRANSFORM_EVENT_v62 = (ANY,   (["city", "province"], LOOKUP("abc.csv", [("pop", "population"), ("type", "city_type") ]) )  )
+TRANSFORM_EVENT_v63 = (ANY,   ([("ct", "city"), ("prv", "province")], LOOKUP("abc.csv", [("pop", "population")] ) ) )
+TRANSFORM_EVENT_v64 = (ANY,   (["c1", "c2"], LOOKUP("abc.csv", ["d1", "d2"]) )  )
+TRANSFORM_EVENT_v65 = (ANY,   (["c1", "c2"], LOOKUP("abc.csv", ["d1", "d2"]) )  )
+TRANSFORM_EVENT_v66 = (ANY,   (["c1", "c2"], LOOKUP("abc.csv", ["d1", "d2"]) )  )
+TRANSFORM_EVENT_v67 = (ANY,   ("data", KV)  )
+TRANSFORM_EVENT_v68 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v69 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v70 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v71 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v72 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v73 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v74 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v75 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v76 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v77 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v78 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v79 = (ANY,  ("data", KV) )
+TRANSFORM_EVENT_v80 = (ANY,  ("data", KV(sep='(?::|=)')) )
+TRANSFORM_EVENT_v81 = (ANY,  ("data", KV(quote="'")) )
+TRANSFORM_EVENT_v82 = (ANY,   ("data", KV(prefix="d_", suffix="_e"))  )
+TRANSFORM_EVENT_v83 = (ANY,   (["data1", "data2"], KV)  )
+KV_FIELDS_x1 = (ANY,   (["data1", "data2"]) )
+KV_FIELDS_x2 = (ANY,   (r'data2') )
+TRANSFORM_EVENT_v84 = (ANY,   ("data", KV)  )
+TRANSFORM_EVENT_v85 = (ANY,   ("data", SPLIT)  )
+TRANSFORM_EVENT_v86 = (ANY,   ("data", SPLIT)  )
+TRANSFORM_EVENT_v87 = (ANY,   ("data", SPLIT)  )
+TRANSFORM_EVENT_v88 = (ANY,  ("data", SPLIT(sep=' )') ) )
+TRANSFORM_EVENT_v89 = (ANY,   ("data", SPLIT ) )
+TRANSFORM_EVENT_v90 = (ANY,   ("data", SPLIT(lstrip=False) ) )
+TRANSFORM_EVENT_v91 = (ANY,   ("data", SPLIT ) )
+TRANSFORM_EVENT_v92 = (ANY,   ("data", SPLIT ) )
+TRANSFORM_EVENT_v93 = (ANY,   ("data", SPLIT ) )
+TRANSFORM_EVENT_v94 = (ANY,   ("data", SPLIT(quote='|') ) )
+TRANSFORM_EVENT_v95 = (ANY,   ("data", SPLIT(output="v"))  )
+TRANSFORM_EVENT_v96 = (ANY,   ("data", SPLIT(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v97 = (ANY,   ("data", SPLIT(jmes="jmes filter", sep='.', output='data'))  )
+TRANSFORM_EVENT_v98 = (ANY,  ("data", SPLIT(jmes="jmes filter", output='data')) )
+TRANSFORM_EVENT_v99 = (ANY,  ("data", SPLIT(jmes="jmes filter", output='data')) )
+TRANSFORM_EVENT_v110 = (ANY,   ("data", JSON(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v111 = (ANY,   ("data", JSON(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v112 = (ANY,   ("data", JSON(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v113 = (ANY,  ("data", JSON(jmes="jmes filter", output='data', jmes_ignore_none=False)) )
+TRANSFORM_EVENT_v114 = (ANY,   ("data", JSON(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v115 = (ANY,  ("data", JSON(jmes="jmes filter", output='data', jmes_ignore_none=False)) )
+TRANSFORM_EVENT_v116 = (ANY,   ("data", JSON(jmes='k2', output='k3'))  )
+TRANSFORM_EVENT_v117 = (ANY,   ("data", JSON)  )
+TRANSFORM_EVENT_v118 = (ANY,  ("data", JSON) )
+TRANSFORM_EVENT_v119 = (ANY,   ("data", JSON(depth=1))  )
+TRANSFORM_EVENT_v120 = (ANY,  ("data", JSON(depth=2)) )
+TRANSFORM_EVENT_v121 = (ANY,   ("data", JSON(prefix="data_", suffix="_end"))  )
+TRANSFORM_EVENT_v122 = (ANY,  ("data", JSON(fmt='full')) )
+TRANSFORM_EVENT_v123 = (ANY,  ("data", JSON(fmt='parent')) )
+TRANSFORM_EVENT_v124 = (ANY,  ("data", JSON(fmt='root')) )
+TRANSFORM_EVENT_v125 = (ANY,  ("data", JSON(fmt='full', sep="_")) )
+TRANSFORM_EVENT_v126 = (ANY,  ("data", JSON(fmt='parent', sep="_")) )
+TRANSFORM_EVENT_v127 = (ANY,  ("data", JSON(fmt='full', sep=" )")) )
+TRANSFORM_EVENT_v128 = (ANY,  ("data", JSON(fmt='parent', sep="@", prefix="__", suffix="__")) )
+TRANSFORM_EVENT_v129 = (ANY,  ("data", JSON(fmt='{parent_rlist[1]}-{parent_rlist[0]}')) )
+TRANSFORM_EVENT_v130 = (ANY,  ("data", JSON(fmt='{parent_list[1]}{sep}{prefix}{current}{suffix}')) )
+TRANSFORM_EVENT_v131 = (ANY,  ("data", JSON(fmt="abc")) )
+TRANSFORM_EVENT_v132 = (ANY,   ("data", JSON(expand_array=False))  )
+TRANSFORM_EVENT_v133 = (ANY,   ("data", JSON)  )
+TRANSFORM_EVENT_v134 = (ANY,  ("people", JSON(fmt='parent')) )
+TRANSFORM_EVENT_v135 = (ANY,  ("people", JSON(fmt='parent', fmt_array="{parent_rlist[0]}-{index}")) )
+TRANSFORM_EVENT_v136 = (ANY,  ("data", JSON(include_node='k5')) )
+TRANSFORM_EVENT_v137 = (ANY,  ("data", JSON(include_node='k2')) )
+TRANSFORM_EVENT_v138 = (ANY,  ("data", JSON(include_node=r'k\d{2,}')) )
+TRANSFORM_EVENT_v139 = (ANY,  ("data", JSON(exclude_node=r'k55')) )
+TRANSFORM_EVENT_v140 = (ANY,  ("data", JSON(exclude_node=r'k2')) )
+TRANSFORM_EVENT_v141 = (ANY,  ("data", JSON(exclude_node=r'k\d{2,}')) )
+TRANSFORM_EVENT_v142 = (ANY,  ("data", JSON(include_path='data.k1')) )
+TRANSFORM_EVENT_v143 = (ANY,  ("data", JSON(include_path='data.k2')) )
+TRANSFORM_EVENT_v144 = (ANY,  ("data", JSON(include_path='k2')) )
+TRANSFORM_EVENT_v145 = (ANY,  ("data", JSON(exclude_path='data.k1')) )
+TRANSFORM_EVENT_v146 = (ANY,  ("data", JSON(exclude_path='data.k2')) )
+TRANSFORM_EVENT_v147 = (ANY,  ("data", JSON(exclude_path='k2')) )
+TRANSFORM_EVENT_v148 = (ANY,   ("data", JSON(jmes="jmes filter", output='data', expand=True))  )
+TRANSFORM_EVENT_v149 = (ANY,   ("data", JSON(jmes="jmes filter", output='data'))  )
+TRANSFORM_EVENT_v150 = (ANY,   [("data", JSON(jmes="jmes filter", output='data', expand=True, exclude_path=r'.+version')), DROP_F('data')]  )
+TRANSFORM_EVENT_v151 = (ANY,   [("data", JSON(jmes="jmes filter", fmt='full')), DROP_F('data')]  )
+TRANSFORM_EVENT_v152 = (ANY,   {"combine": ZIP("f1", "f2")} )
+TRANSFORM_EVENT_v153 = (ANY,   {"combine": ZIP("f1", "f2")} )
+TRANSFORM_EVENT_v154 = (ANY,   {"combine": ZIP("f1", "f2")} )
+TRANSFORM_EVENT_v155 = (ANY,   {"combine": ZIP("f1", "f2", sep="|")} )
+TRANSFORM_EVENT_v156 = (ANY,   {"combine": ZIP("f1", "f2", quote='|')} )
+TRANSFORM_EVENT_v157 = (ANY,   {"combine": ZIP("f1", "f2", combine_sep="|")} )
+TRANSFORM_EVENT_v158 = (ANY,   {"combine": ZIP("f1", "f2")} )
+TRANSFORM_EVENT_v159 = (ANY,   {"combine": ZIP("f1", "f2")} )
+TRANSFORM_EVENT_v160 = (ANY,  {"combine": ZIP("f1", "f2", lparse=(",", '|'), rparse=(",", '"'))} )
+

tests/cli_config_check/restrict_config_test_data/config7_safe.py

@@ -0,0 +1,40 @@
+
+
+TRANSFORM_EVENT_topic = True, {"__topic__": "saf_access_log"}
+
+KEEP_EVENT_t1 = {"instance_id": "account_abuse_pro|coupon_abuse_pro|account_takeover_pro"}
+
+
+RENAME_FIELDS_t1 = {"userId": "user_id", "requestParams": "request_param",
+                    "deviceInfo": "device_info"}
+
+KEEP_FIELDS_test1 = ["__topic__", "__time__",
+                     "product_code", "instance_id", "timestamp", "req_id",
+                     "uid", "user_id", "region", "score",
+                     "request_param", "device_info", "tags", "code"]
+
+
+from aliyun.log.etl_core import *
+#import re
+
+KEEP_FIELDS_f1 = [F_TIME, "acl_action", "acl_blocks",
+                  "aliwaf_action", "aliwaf_rule_type", "body_bytes_sent",
+                  "content_type", "eagleeye_traceid", "host$", "http_cookie",
+                  "http_referer", "http_user_agent", "http_x_forwarded_for",
+                  "https", "matched_host", "real_client_ip", "region",
+                  "remote_addr", "remote_port", "request_length", "request_method",
+                  "request_time_msec", "server_name", "server_protocol",
+                  "status", "time", "tmd_action", "tmd_blocks", "tmd_phase",
+                  r"ua_\w+", "upstream_addr", "upstream_ip", "upstream_response_time",
+                  "upstream_status", "user_id", "block_action", "eagleeye_traceid",
+                  "antibot", "antibot_action", "request_path", "querystring"]
+
+RENAME_FIELDS_r1 = {"aliwaf_action": "waf_action",
+                    "aliwaf_rule_type": "web_attack_type",
+                    "tmd_action": "cc_action",
+                    "tmd_blocks": "cc_blocks",
+                    "tmd_phase": "cc_phase",
+                    "eagleeye_traceid": "request_traceid"
+                    }
+
+TRANSFORM_EVENT_t1 = (ANY, {'__topic__': 'waf_access_log', '__source__': "log_service"})