Skip to content
Permalink
Browse files

[SPARK-23094] Fix invalid character handling in JsonDataSource

## What changes were proposed in this pull request?

There were two related fixes regarding `from_json`, `get_json_object` and `json_tuple` ([Fix #1](c8803c0),
 [Fix #2](86174ea)), but they weren't comprehensive it seems. I wanted to extend those fixes to all the parsers, and add tests for each case.

## How was this patch tested?

Regression tests

Author: Burak Yavuz <brkyvz@gmail.com>

Closes #20302 from brkyvz/json-invfix.

(cherry picked from commit e01919e)
Signed-off-by: hyukjinkwon <gurwls223@gmail.com>
  • Loading branch information...
brkyvz authored and HyukjinKwon committed Jan 18, 2018
1 parent b8c6d93 commit a295034da6178f8654c3977903435384b3765b5e
@@ -40,10 +40,11 @@ private[sql] object CreateJacksonParser extends Serializable {
}

def text(jsonFactory: JsonFactory, record: Text): JsonParser = {
jsonFactory.createParser(record.getBytes, 0, record.getLength)
val bain = new ByteArrayInputStream(record.getBytes, 0, record.getLength)
jsonFactory.createParser(new InputStreamReader(bain, "UTF-8"))
}

def inputStream(jsonFactory: JsonFactory, record: InputStream): JsonParser = {
jsonFactory.createParser(record)
jsonFactory.createParser(new InputStreamReader(record, "UTF-8"))
}
}
@@ -28,6 +28,8 @@ import org.apache.spark.sql.types._
class JsonHadoopFsRelationSuite extends HadoopFsRelationTest {
override val dataSourceName: String = "json"

private val badJson = "\u0000\u0000\u0000A\u0001AAA"

// JSON does not write data of NullType and does not play well with BinaryType.
override protected def supportsDataType(dataType: DataType): Boolean = dataType match {
case _: NullType => false
@@ -105,4 +107,36 @@ class JsonHadoopFsRelationSuite extends HadoopFsRelationTest {
)
}
}

test("invalid json with leading nulls - from file (multiLine=true)") {
import testImplicits._
withTempDir { tempDir =>
val path = tempDir.getAbsolutePath
Seq(badJson, """{"a":1}""").toDS().write.mode("overwrite").text(path)
val expected = s"""$badJson\n{"a":1}\n"""
val schema = new StructType().add("a", IntegerType).add("_corrupt_record", StringType)
val df =
spark.read.format(dataSourceName).option("multiLine", true).schema(schema).load(path)
checkAnswer(df, Row(null, expected))
}
}

test("invalid json with leading nulls - from file (multiLine=false)") {
import testImplicits._
withTempDir { tempDir =>
val path = tempDir.getAbsolutePath
Seq(badJson, """{"a":1}""").toDS().write.mode("overwrite").text(path)
val schema = new StructType().add("a", IntegerType).add("_corrupt_record", StringType)
val df =
spark.read.format(dataSourceName).option("multiLine", false).schema(schema).load(path)
checkAnswer(df, Seq(Row(1, null), Row(null, badJson)))
}
}

test("invalid json with leading nulls - from dataset") {
import testImplicits._
checkAnswer(
spark.read.json(Seq(badJson).toDS()),
Row(badJson))
}
}

0 comments on commit a295034

Please sign in to comment.
You can’t perform that action at this time.