add more test case for json expand formatting

aliyun · Dec 17, 2018 · c6c8c09 · c6c8c09
1 parent 125858c
commit c6c8c09
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 9 deletions.
diff --git a/aliyun/log/etl_core/trans_comp/trans_json.py b/aliyun/log/etl_core/trans_comp/trans_json.py
@@ -35,7 +35,7 @@ class json_transformer(trans_comp_base):
     FMT_MAP = {
         "simple": "{prefix}{current}{suffix}",
         "full":  "{parent_list_str}{sep}{prefix}{current}{suffix}",  # "{sep.join(parent_path)}{sep}{prefix}{current}{suffix}",
-        "parent": "{parent_list[-1]}{sep}{prefix}{current}{suffix}",
+        "parent": "{parent}{sep}{prefix}{current}{suffix}",
         "root": "{parent_list[0]}{sep}{prefix}{current}{suffix}"
         # could also be custom formatting string using up to five placehodler: parent_list, current, sep, prefix, suffix
         # could also be formatting function accepting the two parameters: parrent_list, current key, value
@@ -94,10 +94,11 @@ def format_add_kv(event, fmt, current, value, parent_list, sep, prefix, suffix):
         if isinstance(fmt, (six.text_type, six.binary_type)):
             fmt = json_transformer.FMT_MAP.get(fmt.strip().lower(), fmt)
             try:
-                ret = fmt.format(parent_list_str=sep.join(parent_list), parent_list=parent_list, current=current, sep=sep, prefix=prefix, suffix=suffix), \
+                ret = fmt.format(parent=parent_list[-1], parent_list_str=sep.join(parent_list), parent_list=parent_list, current=current, sep=sep, prefix=prefix, suffix=suffix), \
                    json_transformer._n(value)
-            except ValueError as ex:
-                logger.info("json_transformer")
+            except Exception as ex:
+                logger.info("json_transformer: fail to format with settings: '{0}'".format( (fmt, current, value,
+                                                                                             parent_list, sep, prefix, suffix) ))
         elif isinstance(fmt, inspect.isfunction):
             ret = fmt(parent_list, current, value)
 

diff --git a/tests/etl_test/test_case.py b/tests/etl_test/test_case.py
@@ -621,7 +621,7 @@ def test_json():
     d1 = {"data": """{"k1": 100, "k2": 200}"""}
     assert t( ("data", JSON(jmes='k2', output='k3')) )(d1) == {"data": """{"k1": 100, "k2": 200}""", 'k3': '200'}
 
-    # extract
+    # auto expand
     d1 = {"data": """{"k1": 100, "k2": 200}"""}
     assert t( ("data", JSON) )(d1) == {"data": """{"k1": 100, "k2": 200}""", "k1": "100", "k2": "200"}
 
@@ -641,12 +641,17 @@ def test_json():
     d2 = {"data": """{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }"""}
     assert t(("data", JSON(fmt='full')))(d2) == {'data': '{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }', 'data.k1': '100', 'data.k2.k3': '200', 'data.k2.k4.k5': '300'}
 
+    # extract - deep - format - parent
+    d2 = {"data": """{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }"""}
+    assert t(("data", JSON(fmt='parent')))(d2) == {'data': '{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }', 'data.k1': '100', 'k2.k3': '200', 'k4.k5': '300'}
 
-    # # extract - deep - format - parent
-    # d2 = {"data": """{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }"""}
-    # print(t(("data", JSON(fmt='parent')))(d2))
-    # assert t(("data", JSON(fmt='parent')))(d2) == {'data': '{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }', 'data.k1': '100', 'data.k2.k3': '200', 'data.k2.k4.k5': '300'}
+    # extract - deep - format - root
+    d2 = {"data": """{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }"""}
+    assert t(("data", JSON(fmt='root')))(d2) == {'data': '{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }', 'data.k1': '100', 'data.k3': '200', 'data.k5': '300'}
 
+    # extract - deep - format - custom
+    d2 = {"data": """{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }"""}
+    assert t(("data", JSON(fmt='root')))(d2) == {'data': '{"k1": 100, "k2": {"k3": 200, "k4": {"k5": 300} } }', 'data.k1': '100', 'data.k3': '200', 'data.k5': '300'}
 
     # extract - array
     d3 = {"data": """[1,2,3]"""}
@@ -667,6 +672,12 @@ def test_json():
     assert t( ("data", JSON(prefix="data_", suffix="_end")) )(d1) == {'data': '{"k1": 100, "k2": 200}', 'data_k1_end': '100', 'data_k2_end': '200'}
 
 
+
+
+
+# test_json()
+# exit(1)
+
 test_condition()
 test_regex()
 test_csv()