Skip to content

Commit 47276ab

Browse files
gengliangwangHyukjinKwon
authored andcommitted
[SPARK-37990][SQL] Support TimestampNTZ in RowToColumnConverter
### What changes were proposed in this pull request? Support TimestampNTZ in RowToColumnConverter ### Why are the changes needed? Support converting `InternalRow` with TimestampNTZ type column to `WritableColumnVector`, as all the other data type does. ### Does this PR introduce _any_ user-facing change? No, the TimestampNTZ type is not released yet. ### How was this patch tested? Unit test Closes #35288 from gengliangwang/RowToColumnConverterNTZ. Authored-by: Gengliang Wang <gengliang@apache.org> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
1 parent f8ff786 commit 47276ab

File tree

2 files changed

+27
-9
lines changed

2 files changed

+27
-9
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ private object RowToColumnConverter {
264264
case ShortType => ShortConverter
265265
case IntegerType | DateType | _: YearMonthIntervalType => IntConverter
266266
case FloatType => FloatConverter
267-
case LongType | TimestampType | _: DayTimeIntervalType => LongConverter
267+
case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType => LongConverter
268268
case DoubleType => DoubleConverter
269269
case StringType => StringConverter
270270
case CalendarIntervalType => CalendarConverter

sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.vectorized
2020
import java.nio.ByteBuffer
2121
import java.nio.ByteOrder
2222
import java.nio.charset.StandardCharsets
23+
import java.time.LocalDateTime
2324
import java.util
2425
import java.util.NoSuchElementException
2526

@@ -1591,10 +1592,21 @@ class ColumnarBatchSuite extends SparkFunSuite {
15911592
)) ::
15921593
StructField("int_to_int", MapType(IntegerType, IntegerType)) ::
15931594
StructField("binary", BinaryType) ::
1595+
StructField("ts_ntz", TimestampNTZType) ::
15941596
Nil)
15951597
var mapBuilder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
15961598
mapBuilder.put(1, 10)
15971599
mapBuilder.put(20, null)
1600+
1601+
val tsString1 = "2015-01-01 23:50:59.123"
1602+
val ts1 = DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(tsString1))
1603+
val tsNTZ1 =
1604+
DateTimeUtils.localDateTimeToMicros(LocalDateTime.parse(tsString1.replace(" ", "T")))
1605+
val tsString2 = "1880-01-05 12:45:21.321"
1606+
val ts2 = DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(tsString2))
1607+
val tsNTZ2 =
1608+
DateTimeUtils.localDateTimeToMicros(LocalDateTime.parse(tsString2.replace(" ", "T")))
1609+
15981610
val row1 = new GenericInternalRow(Array[Any](
15991611
UTF8String.fromString("a string"),
16001612
true,
@@ -1606,12 +1618,13 @@ class ColumnarBatchSuite extends SparkFunSuite {
16061618
0.75D,
16071619
Decimal("1234.23456"),
16081620
DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("2015-01-01")),
1609-
DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123")),
1621+
ts1,
16101622
new CalendarInterval(1, 0, 0),
16111623
new GenericArrayData(Array(1, 2, 3, 4, null)),
16121624
new GenericInternalRow(Array[Any](5.asInstanceOf[Any], 10)),
16131625
mapBuilder.build(),
1614-
"Spark SQL".getBytes()
1626+
"Spark SQL".getBytes(),
1627+
tsNTZ1
16151628
))
16161629

16171630
mapBuilder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
@@ -1628,12 +1641,13 @@ class ColumnarBatchSuite extends SparkFunSuite {
16281641
Double.PositiveInfinity,
16291642
Decimal("0.01000"),
16301643
DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("1875-12-12")),
1631-
DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("1880-01-05 12:45:21.321")),
1644+
ts2,
16321645
new CalendarInterval(-10, -50, -100),
16331646
new GenericArrayData(Array(5, 10, -100)),
16341647
new GenericInternalRow(Array[Any](20.asInstanceOf[Any], null)),
16351648
mapBuilder.build(),
1636-
"Parquet".getBytes()
1649+
"Parquet".getBytes(),
1650+
tsNTZ2
16371651
))
16381652

16391653
val row3 = new GenericInternalRow(Array[Any](
@@ -1652,6 +1666,7 @@ class ColumnarBatchSuite extends SparkFunSuite {
16521666
null,
16531667
null,
16541668
null,
1669+
null,
16551670
null
16561671
))
16571672

@@ -1716,10 +1731,8 @@ class ColumnarBatchSuite extends SparkFunSuite {
17161731
assert(columns(9).isNullAt(2))
17171732

17181733
assert(columns(10).dataType() == TimestampType)
1719-
assert(columns(10).getLong(0) ==
1720-
DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123")))
1721-
assert(columns(10).getLong(1) ==
1722-
DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("1880-01-05 12:45:21.321")))
1734+
assert(columns(10).getLong(0) == ts1)
1735+
assert(columns(10).getLong(1) == ts2)
17231736
assert(columns(10).isNullAt(2))
17241737

17251738
assert(columns(11).dataType() == CalendarIntervalType)
@@ -1777,6 +1790,11 @@ class ColumnarBatchSuite extends SparkFunSuite {
17771790
assert(new String(columns(15).getBinary(0)) == "Spark SQL")
17781791
assert(new String(columns(15).getBinary(1)) == "Parquet")
17791792
assert(columns(15).isNullAt(2))
1793+
1794+
assert(columns(16).dataType() == TimestampNTZType)
1795+
assert(columns(16).getLong(0) == tsNTZ1)
1796+
assert(columns(16).getLong(1) == tsNTZ2)
1797+
assert(columns(16).isNullAt(2))
17801798
} finally {
17811799
batch.close()
17821800
}

0 commit comments

Comments
 (0)