fix: exception when parsing certain lateral view SQL

sqllineage/__init__.py

@@ -16,10 +16,23 @@ def _patch_adding_builtin_type() -> None:
     KEYWORDS["DATETIME"] = tokens.Name.Builtin
 
 
+def _patch_updating_lateral_view_lexeme() -> None:
+    import re
+    from sqlparse.keywords import SQL_REGEX
+
+    for i, (regex, lexeme) in enumerate(SQL_REGEX):
+        if regex("LATERAL VIEW EXPLODE(col)"):
+            new_regex = r"(LATERAL\s+VIEW\s+)(OUTER\s+)?(EXPLODE|INLINE|PARSE_URL_TUPLE|POSEXPLODE|STACK|JSON_TUPLE)\b"
+            new_compile = re.compile(new_regex, re.IGNORECASE | re.UNICODE).match
+            SQL_REGEX[i] = (new_compile, lexeme)
+            break
+
+
 def _monkey_patch() -> None:
     try:
         _patch_adding_window_function_token()
         _patch_adding_builtin_type()
+        _patch_updating_lateral_view_lexeme()
     except ImportError:
         # when imported by setup.py for constant variables, dependency is not ready yet
         pass

tests/test_others.py

@@ -225,6 +225,22 @@ def test_delete_from_table():
     assert_table_lineage_equal("delete from table tab1", None, None)
 
 
+def test_lateral_view_using_json_tuple():
+    sql = """INSERT OVERWRITE TABLE foo
+SELECT sc.id, q.item0, q.item1
+FROM bar sc
+LATERAL VIEW json_tuple(sc.json, 'key1', 'key2') q AS item0, item1"""
+    assert_table_lineage_equal(sql, {"bar"}, {"foo"})
+
+
+def test_lateral_view_outer():
+    sql = """INSERT OVERWRITE TABLE foo
+SELECT sc.id, q.col1
+FROM bar sc
+LATERAL VIEW OUTER explode(sc.json_array) q AS col1"""
+    assert_table_lineage_equal(sql, {"bar"}, {"foo"})
+
+
 def test_show_create_table():
     assert_table_lineage_equal("show create table tab1", None, None)