Skip to content

Commit

Permalink
[SPARK-12028] [SQL] get_json_object returns an incorrect result when …
Browse files Browse the repository at this point in the history
…the value is null literals

When calling `get_json_object` for the following two cases, both results are `"null"`:

```scala
    val tuple: Seq[(String, String)] = ("5", """{"f1": null}""") :: Nil
    val df: DataFrame = tuple.toDF("key", "jstring")
    val res = df.select(functions.get_json_object($"jstring", "$.f1")).collect()
```
```scala
    val tuple2: Seq[(String, String)] = ("5", """{"f1": "null"}""") :: Nil
    val df2: DataFrame = tuple2.toDF("key", "jstring")
    val res3 = df2.select(functions.get_json_object($"jstring", "$.f1")).collect()
```

Fixed the problem and also added a test case.

Author: gatorsmile <gatorsmile@gmail.com>

Closes #10018 from gatorsmile/get_json_object.

(cherry picked from commit 149cd69)
Signed-off-by: Davies Liu <davies.liu@gmail.com>
  • Loading branch information
gatorsmile authored and davies committed Nov 28, 2015
1 parent 47c2c8c commit 2ecc0f2
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,11 @@ case class GetJsonObject(json: Expression, path: Expression)

case (FIELD_NAME, Named(name) :: xs) if p.getCurrentName == name =>
// exact field match
p.nextToken()
evaluatePath(p, g, style, xs)
if (p.nextToken() != JsonToken.VALUE_NULL) {
evaluatePath(p, g, style, xs)
} else {
false
}

case (FIELD_NAME, Wildcard :: xs) =>
// wildcard field match
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,26 @@ class JsonFunctionsSuite extends QueryTest with SharedSQLContext {
("6", "[invalid JSON string]") ::
Nil

test("function get_json_object - null") {
val df: DataFrame = tuples.toDF("key", "jstring")
val expected =
Row("1", "value1", "value2", "3", null, "5.23") ::
Row("2", "value12", "2", "value3", "4.01", null) ::
Row("3", "value13", "2", "value33", "value44", "5.01") ::
Row("4", null, null, null, null, null) ::
Row("5", "", null, null, null, null) ::
Row("6", null, null, null, null, null) ::
Nil

checkAnswer(
df.select($"key", functions.get_json_object($"jstring", "$.f1"),
functions.get_json_object($"jstring", "$.f2"),
functions.get_json_object($"jstring", "$.f3"),
functions.get_json_object($"jstring", "$.f4"),
functions.get_json_object($"jstring", "$.f5")),
expected)
}

test("json_tuple select") {
val df: DataFrame = tuples.toDF("key", "jstring")
val expected =
Expand Down

0 comments on commit 2ecc0f2

Please sign in to comment.