From fbf6c98aa0ff91059bfd1d7f5fba0360a0a04bab Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 11 Oct 2023 01:52:56 -0700 Subject: [PATCH 1/5] fix error occurs when non-select SQL statement is executed and tableWithUTFCharacter is True --- .../org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 5 +++++ .../org/apache/zeppelin/livy/LivySQLInterpreterTest.java | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 40aa36349a8..337f2e2f701 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -180,6 +180,11 @@ protected List parseSQLJsonOutput(String output) { List rows = new ArrayList<>(); String[] rowsOutput = output.split("(? cells = new ArrayList<>(Arrays.asList(header)); rows.add(StringUtils.join(cells, "\t")); diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java index d9ba3f69214..e8cb76aafff 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java @@ -174,9 +174,14 @@ void testParseSQLOutput() { @Test void parseSQLJsonOutput() { + + // Empty output + List rows = sqlInterpreter.parseSQLJsonOutput("\n"); + assertEquals(0, rows.size()); + // Empty sql output // id name - List rows = sqlInterpreter.parseSQLJsonOutput("\nid\tname\n"); + rows = sqlInterpreter.parseSQLJsonOutput("\nid\tname\n"); assertEquals(1, rows.size()); assertEquals("id\tname", rows.get(0)); From 48ee95e1c69a7c87778056c3f1daf44c002e5643 Mon Sep 17 00:00:00 2001 From: Levi Date: Thu, 12 Oct 2023 02:16:31 -0700 Subject: [PATCH 2/5] fix error occurs when non-primitive type of data is within json response such as array and tableWithUTFCharacter is True --- .../livy/LivySparkSQLInterpreter.java | 30 +++++++++++++++---- .../zeppelin/livy/LivySQLInterpreterTest.java | 7 +++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 337f2e2f701..f6cbca7f96c 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -18,7 +18,8 @@ package org.apache.zeppelin.livy; import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import java.io.IOException; import java.io.StringWriter; @@ -190,10 +191,10 @@ protected List parseSQLJsonOutput(String output) { rows.add(StringUtils.join(cells, "\t")); for (int i = 2; i < rowsOutput.length; i++) { - Map retMap = new Gson().fromJson( - rowsOutput[i], new TypeToken>() { - }.getType() - ); + // one-by-one serialization to handle the case when + // the value is non-primitive such as: {"lang": ["java", "NodeJS"]}. + Map retMap = serialize(rowsOutput[i]); + cells = new ArrayList<>(); for (String s : header) { cells.add(retMap.getOrDefault(s, "null") @@ -201,10 +202,29 @@ protected List parseSQLJsonOutput(String output) { .replace("\t", "\\t")); } rows.add(StringUtils.join(cells, "\t")); + } return rows; } + private Map serialize(String jsonString) { + Map map = new HashMap<>(); + Gson gson = new Gson(); + JsonElement jsonElement = gson.fromJson(jsonString, JsonElement.class); + JsonObject jsonObject = jsonElement.getAsJsonObject(); + for (String key : jsonObject.keySet()) { + JsonElement value = jsonObject.get(key); + + if (value.isJsonPrimitive()) { + map.put(key, value.getAsString()); + } else { + map.put(key, value.toString()); + } + + } + return map; + } + protected List parseSQLOutput(String str) { // the regex is referred to org.apache.spark.util.Utils#fullWidthRegex // for spark every chinese character has two placeholder(one placeholder is one char) diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java index e8cb76aafff..6294473371f 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java @@ -279,5 +279,12 @@ void parseSQLJsonOutput() { assertEquals("1\t1a", rows.get(1)); assertEquals("2\tみんく", rows.get(2)); assertEquals("3\t3a", rows.get(3)); + + + rows = sqlInterpreter.parseSQLJsonOutput("\nid\tarray\tname\n" + + "{\"id\":1,\"array\":[\"1a\",\"2a\"],\"name\":\"1b\"}\n"); + assertEquals(2, rows.size()); + assertEquals("id\tarray\tname", rows.get(0)); + assertEquals("1\t[\"1a\",\"2a\"]\t1b", rows.get(1)); } } From 45526b8e3bae93b0c9479416bcd4caf50d3c0faa Mon Sep 17 00:00:00 2001 From: levi0090 <147389858+levi0090@users.noreply.github.com> Date: Sun, 15 Oct 2023 17:38:11 +1300 Subject: [PATCH 3/5] rename custom function --- .../java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index f6cbca7f96c..81f952f381b 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -207,7 +207,7 @@ protected List parseSQLJsonOutput(String output) { return rows; } - private Map serialize(String jsonString) { + private Map deserialize(String jsonString) { Map map = new HashMap<>(); Gson gson = new Gson(); JsonElement jsonElement = gson.fromJson(jsonString, JsonElement.class); From 526718e189f12c5fee829aee465cf6510dbd0021 Mon Sep 17 00:00:00 2001 From: Levi Date: Wed, 18 Oct 2023 20:50:28 -0700 Subject: [PATCH 4/5] fix bug and improve the performance to iterate over the entrySet. --- .../org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 81f952f381b..ec404796a1d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -193,7 +193,7 @@ protected List parseSQLJsonOutput(String output) { for (int i = 2; i < rowsOutput.length; i++) { // one-by-one serialization to handle the case when // the value is non-primitive such as: {"lang": ["java", "NodeJS"]}. - Map retMap = serialize(rowsOutput[i]); + Map retMap = deserialize(rowsOutput[i]); cells = new ArrayList<>(); for (String s : header) { @@ -212,8 +212,10 @@ private Map deserialize(String jsonString) { Gson gson = new Gson(); JsonElement jsonElement = gson.fromJson(jsonString, JsonElement.class); JsonObject jsonObject = jsonElement.getAsJsonObject(); - for (String key : jsonObject.keySet()) { - JsonElement value = jsonObject.get(key); + + for (Map.Entry entry : jsonObject.entrySet()) { + String key = entry.getKey(); + JsonElement value = entry.getValue(); if (value.isJsonPrimitive()) { map.put(key, value.getAsString()); From 0b6918038251dd7a7d1b906cfe70917c7758684e Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Fri, 17 Nov 2023 11:26:42 +0100 Subject: [PATCH 5/5] remove nit --- .../java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index ec404796a1d..aec81464991 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -202,7 +202,6 @@ protected List parseSQLJsonOutput(String output) { .replace("\t", "\\t")); } rows.add(StringUtils.join(cells, "\t")); - } return rows; }