From 169f4230fc4e71c3fa16df5268de9469f7a26cab Mon Sep 17 00:00:00 2001 From: Cheng Su Date: Mon, 24 Jan 2022 23:14:25 -0800 Subject: [PATCH] Fix ColumnVectorUtils.populate to handle CalendarIntervalType correctly --- .../sql/execution/vectorized/ColumnVectorUtils.java | 3 ++- .../sql/execution/vectorized/ColumnVectorSuite.scala | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java index 37c348cf4ed66..353a128254412 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java @@ -91,7 +91,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field } else if (t instanceof CalendarIntervalType) { CalendarInterval c = (CalendarInterval)row.get(fieldIdx, t); col.getChild(0).putInts(0, capacity, c.months); - col.getChild(1).putLongs(0, capacity, c.microseconds); + col.getChild(1).putInts(0, capacity, c.days); + col.getChild(2).putLongs(0, capacity, c.microseconds); } else if (t instanceof DateType || t instanceof YearMonthIntervalType) { col.putInts(0, capacity, row.getInt(fieldIdx)); } else if (t instanceof TimestampType || t instanceof TimestampNTZType || diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala index cdf41ed651d4e..4cf2376a3fccd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.columnar.ColumnAccessor import org.apache.spark.sql.execution.columnar.compression.ColumnBuilderHelper import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarArray -import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach { private def withVector( @@ -605,5 +605,14 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach { } } } + + test("SPARK-38018: ColumnVectorUtils.populate to handle CalendarIntervalType correctly") { + val vector = new OnHeapColumnVector(5, CalendarIntervalType) + val row = new SpecificInternalRow(Array(CalendarIntervalType)) + val interval = new CalendarInterval(3, 5, 1000000) + row.setInterval(0, interval) + ColumnVectorUtils.populate(vector, row, 0) + assert(vector.getInterval(0) === interval) + } }