[KYUUBI #2312] Spark data type TimestampNTZ supported version changes…

… as 3.4.0 ### _Why are the changes needed?_ fix #2312 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2313 from Nick-0723/2312. Closes #2312 91e607a [Nick Song] Spark data type TimestampNTZ supported version changes as 3.4.0 Authored-by: Nick Song <chun2184@163.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
apache · Apr 10, 2022 · e1e0b35 · e1e0b35
1 parent e5834ae
commit e1e0b35
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 8 deletions.
diff --git a/...-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala b/...-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.types._
 object SchemaHelper {
 
   /**
-   * Spark 3.3.0 DataType TimestampNTZType's class name.
+   * Spark 3.4.0 DataType TimestampNTZType's class name.
    */
   final val TIMESTAMP_NTZ = "TimestampNTZType$"
 

diff --git a/...-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala b/...-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
@@ -80,9 +80,13 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with
     // since spark3.3.0
     if (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 > 3 ||
       (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3 && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 >= 3)) {
-      schema = schema.add("c18", "timestamp_ntz", nullable = true, "18")
-        .add("c19", "interval day", nullable = true, "19")
-        .add("c20", "interval year", nullable = true, "20")
+      schema = schema.add("c18", "interval day", nullable = true, "18")
+        .add("c19", "interval year", nullable = true, "19")
+    }
+    // since spark3.4.0
+    if (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 > 3 ||
+      (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3 && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 >= 4)) {
+      schema = schema.add("c20", "timestamp_ntz", nullable = true, "20")
     }
 
     val ddl =
@@ -120,9 +124,9 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with
           STRUCT,
           BINARY,
           STRUCT,
-          TIMESTAMP,
           OTHER,
-          OTHER)
+          OTHER,
+          TIMESTAMP)
 
         var pos = 0
 

diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
@@ -175,8 +175,8 @@ trait SparkQueryTests extends HiveJDBCTestHelper {
   }
 
   test("execute statement - select timestamp_ntz") {
-    assume(SPARK_ENGINE_MAJOR_MINOR_VERSION._1 >= 3
-      && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 > 2)
+    assume(SPARK_ENGINE_MAJOR_MINOR_VERSION._1 > 3 ||
+      (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3 && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 >= 4))
     withJdbcStatement() { statement =>
       val resultSet = statement.executeQuery(
         "SELECT make_timestamp_ntz(2022, 03, 24, 18, 08, 31.800) AS col")