add test case for csv/lookup in module level and pass it

aliyun · Dec 4, 2018 · dfb6270 · dfb6270
1 parent 66d4bf8
commit dfb6270
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 2 deletions.
diff --git a/tests/etl_test/data4.txt b/tests/etl_test/data4.txt
@@ -0,0 +1,5 @@
+{"csv_field": "v1,v2,v3", "dsv_field": "d1#d2#d3"}
+{"csv_field": "v1, v2, \"v3,v3\"", "dsv_field": "|d1#d1|#d2#d3"}
+{"data": "123", "f1":"a", "f2":"b"}
+{"data": "123", "f1":"A", "f2":"C"}
+{"data": "123", "f1":"x", "f2":"y"}
diff --git a/tests/etl_test/data4_lookup_csv1.txt b/tests/etl_test/data4_lookup_csv1.txt
@@ -0,0 +1,5 @@
+c1,c2,d1,d2
+a,b,1,2
+c,d,3,4
+a,*,5,6
+*,*,7,8
diff --git a/tests/etl_test/data4_test1.py b/tests/etl_test/data4_test1.py
@@ -0,0 +1,11 @@
+
+from aliyun.log.etl_core import *
+
+TRANSFORM_EVENT_lookup = [
+    ( NO_EMPTY('csv_field'), ('csv_field', CSV("f_v1,f_v2,f_v3")) ),
+    ( NO_EMPTY('dsv_field'), ('dsv_field', CSV("f_d1,f_d2,f_d3", sep='#', quote='|')) ),
+    ( ANY, ([("f1", "c1"), ("f2", "c2")], LOOKUP('./data4_lookup_csv1.txt', ["d1", "d2"]) ) ),
+    ( ANY, ("f1",LOOKUP({'a': "a_new", '*': "unknown"}, "f1_new")) )
+]
+
+DROP_FIELDS_origin = ["csv_field", 'dsv_field', 'data', 'f1', 'f2']
diff --git a/tests/etl_test/data4_test1_result.txt b/tests/etl_test/data4_test1_result.txt
@@ -0,0 +1,5 @@
+{"f_v3": "v3", "f_v2": "v2", "f_v1": "v1", "f_d1": "d1", "f_d3": "d3", "f_d2": "d2"}
+{"f_v3": "v3,v3", "f_v2": "v2", "f_v1": "v1", "f_d1": "d1#d1", "f_d3": "d3", "f_d2": "d2"}
+{"d2": "2", "d1": "1"}
+{"d2": "6", "d1": "5"}
+{"d2": "8", "d1": "7"}
diff --git a/tests/etl_test/test_case.py b/tests/etl_test/test_case.py
@@ -270,6 +270,9 @@ def test_module():
     import data3_test1
     verify_case(data3_test1, './data3.txt', './data3_test1_result.txt')
 
+    import data4_test1
+    verify_case(data4_test1, './data4.txt', './data4_test1_result.txt')
+
 
 def test_csv():
     # sep
@@ -323,14 +326,29 @@ def test_lookup_dict():
     assert t((["pro", "protocol"], LOOKUP({"http": "tcp", "dns": "udp", "https": "tcp"}, "type")))({'data': '123', "pro": "dns", "protocol": "http"}) == {'data': '123', "pro": "dns", "protocol": "http", "type": "tcp"}
 
 
+import atexit
+_tmp_files = set()
+
+
+def _del_csv():
+    for x in _tmp_files:
+        os.unlink(x)
+
+
 def _pre_csv(content, suffix=None):
     suffix = suffix or "{}_{}".format(time(), randint(1, 1000000))
     file_path = './tmp_test_lookup_csv_{0}.csv'.format(suffix)
+
     with open(file_path, "w") as f:
         f.write(content)
+
+    _tmp_files.add(file_path)
     return file_path
 
 
+atexit.register(_del_csv)
+
+
 def test_lookup_load_csv():
     #########
     # CSV standard reading/mapping
@@ -423,8 +441,6 @@ def test_lookup_mapping():
     assert t( (["c1", "c2"], LOOKUP(csv_path, ["d1", "d2"]) ) )({'data': '123', 'c1': 'c', 'c2': 'v'})  == {'data': '123', 'c1': 'c', 'c2': 'v', 'd1': '0', 'd2': '0'}
 
 
-
-
 test_condition()
 test_regex()
 test_csv()