Skip to content

Commit

Permalink
[SPARK-35220][SQL] DayTimeIntervalType/YearMonthIntervalType show dif…
Browse files Browse the repository at this point in the history
…ferent between Hive SerDe and row format delimited

### What changes were proposed in this pull request?
DayTimeIntervalType/YearMonthIntervalString show different between Hive SerDe and row format delimited.
Create this pr to add a test and  have disscuss.

For this problem I think we have two direction:

1. leave it as current and add a item t explain this  in migration guide docs.
2. Since we should not change hive serde's behavior, so we can cast spark row format delimited's behavior to use cast  DayTimeIntervalType/YearMonthIntervalType as HIVE_STYLE

### Why are the changes needed?
Add UT

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
added ut

Closes #32335 from AngersZhuuuu/SPARK-35220.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: hyukjinkwon <gurwls223@apache.org>
  • Loading branch information
AngersZhuuuu authored and HyukjinKwon committed Apr 26, 2021
1 parent 5b1353f commit 6f782ef
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.sql.execution

import java.sql.{Date, Timestamp}
import java.time.{Duration, Period}

import org.json4s.DefaultFormats
import org.json4s.JsonDSL._
Expand All @@ -43,6 +44,8 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU
import testImplicits._
import ScriptTransformationIOSchema._

protected def defaultSerDe(): String

protected val uncaughtExceptionHandler = new TestUncaughtExceptionHandler

private var defaultUncaughtExceptionHandler: Thread.UncaughtExceptionHandler = _
Expand Down Expand Up @@ -599,6 +602,37 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU
'e.cast("string")).collect())
}
}

test("SPARK-35220: DayTimeIntervalType/YearMonthIntervalType show different " +
"between hive serde and row format delimited\t") {
assume(TestUtils.testCommandAvailable("/bin/bash"))
withTempView("v") {
val df = Seq(
(Duration.ofDays(1), Period.ofMonths(10))
).toDF("a", "b")
df.createTempView("v")

if (defaultSerDe == "hive-serde") {
checkAnswer(sql(
"""
|SELECT TRANSFORM(a, b)
| USING 'cat' AS (a, b)
|FROM v
|""".stripMargin),
identity,
Row("1 00:00:00.000000000", "0-10") :: Nil)
} else {
checkAnswer(sql(
"""
|SELECT TRANSFORM(a, b)
| USING 'cat' AS (a, b)
|FROM v
|""".stripMargin),
identity,
Row("INTERVAL '1 00:00:00' DAY TO SECOND", "INTERVAL '0-10' YEAR TO MONTH") :: Nil)
}
}
}
}

case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import org.apache.spark.sql.test.SharedSparkSession
class SparkScriptTransformationSuite extends BaseScriptTransformationSuite with SharedSparkSession {
import testImplicits._

override protected def defaultSerDe(): String = "row-format-delimited"

override def createScriptTransformationExec(
script: String,
output: Seq[Attribute],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T

import ScriptTransformationIOSchema._

override protected def defaultSerDe(): String = "hive-serde"

override def createScriptTransformationExec(
script: String,
output: Seq[Attribute],
Expand Down

0 comments on commit 6f782ef

Please sign in to comment.