diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 40aa36349a8..aec81464991 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -18,7 +18,8 @@ package org.apache.zeppelin.livy; import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import java.io.IOException; import java.io.StringWriter; @@ -180,15 +181,20 @@ protected List parseSQLJsonOutput(String output) { List rows = new ArrayList<>(); String[] rowsOutput = output.split("(? cells = new ArrayList<>(Arrays.asList(header)); rows.add(StringUtils.join(cells, "\t")); for (int i = 2; i < rowsOutput.length; i++) { - Map retMap = new Gson().fromJson( - rowsOutput[i], new TypeToken>() { - }.getType() - ); + // one-by-one serialization to handle the case when + // the value is non-primitive such as: {"lang": ["java", "NodeJS"]}. + Map retMap = deserialize(rowsOutput[i]); + cells = new ArrayList<>(); for (String s : header) { cells.add(retMap.getOrDefault(s, "null") @@ -200,6 +206,26 @@ protected List parseSQLJsonOutput(String output) { return rows; } + private Map deserialize(String jsonString) { + Map map = new HashMap<>(); + Gson gson = new Gson(); + JsonElement jsonElement = gson.fromJson(jsonString, JsonElement.class); + JsonObject jsonObject = jsonElement.getAsJsonObject(); + + for (Map.Entry entry : jsonObject.entrySet()) { + String key = entry.getKey(); + JsonElement value = entry.getValue(); + + if (value.isJsonPrimitive()) { + map.put(key, value.getAsString()); + } else { + map.put(key, value.toString()); + } + + } + return map; + } + protected List parseSQLOutput(String str) { // the regex is referred to org.apache.spark.util.Utils#fullWidthRegex // for spark every chinese character has two placeholder(one placeholder is one char) diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java index d9ba3f69214..6294473371f 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java @@ -174,9 +174,14 @@ void testParseSQLOutput() { @Test void parseSQLJsonOutput() { + + // Empty output + List rows = sqlInterpreter.parseSQLJsonOutput("\n"); + assertEquals(0, rows.size()); + // Empty sql output // id name - List rows = sqlInterpreter.parseSQLJsonOutput("\nid\tname\n"); + rows = sqlInterpreter.parseSQLJsonOutput("\nid\tname\n"); assertEquals(1, rows.size()); assertEquals("id\tname", rows.get(0)); @@ -274,5 +279,12 @@ void parseSQLJsonOutput() { assertEquals("1\t1a", rows.get(1)); assertEquals("2\tみんく", rows.get(2)); assertEquals("3\t3a", rows.get(3)); + + + rows = sqlInterpreter.parseSQLJsonOutput("\nid\tarray\tname\n" + + "{\"id\":1,\"array\":[\"1a\",\"2a\"],\"name\":\"1b\"}\n"); + assertEquals(2, rows.size()); + assertEquals("id\tarray\tname", rows.get(0)); + assertEquals("1\t[\"1a\",\"2a\"]\t1b", rows.get(1)); } }