From c0b98590116734ba5ca82ec9914ea29ad36652c3 Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Wed, 29 Jul 2026 17:40:25 +0000 Subject: [PATCH 1/2] [SPARK-57825][SQL] Add +/- ANSI year-month interval support for TIMESTAMP_NTZ/LTZ(p) ### What changes were proposed in this pull request? This PR adds support for `TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)` (`p in [7, 9]`) in `+/- ANSI year-month interval` arithmetic. Concretely: - Extends `TimestampAddYMInterval` input typing to accept nanos timestamp types alongside the existing microsecond timestamp types. - Adds nanos-aware execution/codegen paths that apply the month shift on epoch micros while carrying the `nanosWithinMicro` remainder through unchanged. - Routes nanos timestamps into `TimestampAddYMInterval` from `BinaryArithmeticWithDatetimeResolver` for both `Add` and `Subtract` (the year-month branches previously matched only `TimestampType | TimestampNTZType`, so nanos timestamps fell through to an unresolved `Add` / `Subtract` and failed analysis). - Adds a `DateTimeUtils.timestampNanosAddMonths` helper. - Adds catalyst and SQL test coverage for NTZ/LTZ nanos year-month interval arithmetic parity, and regenerates the impacted nanos SQL golden files. ### Why are the changes needed? Spark already supports timestamp +/- ANSI year-month interval for the microsecond timestamp families, and SPARK-57501 added the day-time-interval parity for the nanos families, but year-month intervals were still unsupported for `TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)` (`p in [7, 9]`). This left valid datetime arithmetic unsupported for nanos types. These changes close that parity gap while preserving nanos precision semantics and existing LTZ/NTZ timezone behavior. ### Does this PR introduce _any_ user-facing change? Yes. `TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)` (`p in [7, 9]`) now support `+/- ANSI year-month interval` arithmetic. Examples: - `TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR` - `TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1-2' YEAR TO MONTH` ### How was this patch tested? - `build/sbt 'catalyst/testOnly org.apache.spark.sql.catalyst.expressions.DateExpressionsSuite org.apache.spark.sql.catalyst.util.DateTimeUtilsSuite'` - `SPARK_GENERATE_GOLDEN_FILES=1 build/sbt 'sql/testOnly org.apache.spark.sql.SQLQueryTestSuite -- -z "timestamp-ntz-nanos" -z "timestamp-ltz-nanos" -z "interval"'` - `build/sbt 'sql/testOnly org.apache.spark.sql.SQLQueryTestSuite -- -z "timestamp-ntz-nanos" -z "timestamp-ltz-nanos"'` - `build/sbt catalyst/scalastyle catalyst/Test/scalastyle` ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 4.8) Co-authored-by: Isaac --- ...BinaryArithmeticWithDatetimeResolver.scala | 8 +++ .../expressions/datetimeExpressions.scala | 22 +++++-- .../sql/catalyst/util/DateTimeUtils.scala | 12 ++++ .../expressions/DateExpressionsSuite.scala | 66 +++++++++++++++++++ .../catalyst/util/DateTimeUtilsSuite.scala | 43 ++++++++++++ .../timestamp-ltz-nanos.sql.out | 50 +++++++------- .../timestamp-ntz-nanos.sql.out | 50 +++++++------- .../sql-tests/inputs/timestamp-ltz-nanos.sql | 14 ++-- .../sql-tests/inputs/timestamp-ntz-nanos.sql | 14 ++-- .../results/timestamp-ltz-nanos.sql.out | 56 +++++++++------- .../results/timestamp-ntz-nanos.sql.out | 56 +++++++++------- 11 files changed, 285 insertions(+), 106 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/BinaryArithmeticWithDatetimeResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/BinaryArithmeticWithDatetimeResolver.scala index cdfd942ca09af..72e11082d52ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/BinaryArithmeticWithDatetimeResolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/BinaryArithmeticWithDatetimeResolver.scala @@ -47,6 +47,7 @@ import org.apache.spark.sql.catalyst.expressions.{ } import org.apache.spark.sql.types.{ AnsiIntervalType, + AnyTimestampNanoType, AnyTimestampTypeExpression, CalendarIntervalType, DatetimeType, @@ -75,6 +76,10 @@ object BinaryArithmeticWithDatetimeResolver { TimestampAddYMInterval(l, r) case (_: YearMonthIntervalType, TimestampType | TimestampNTZType) => TimestampAddYMInterval(r, l) + case (_: AnyTimestampNanoType, _: YearMonthIntervalType) => + TimestampAddYMInterval(l, r) + case (_: YearMonthIntervalType, _: AnyTimestampNanoType) => + TimestampAddYMInterval(r, l) case (CalendarIntervalType, CalendarIntervalType) | (_: DayTimeIntervalType, _: DayTimeIntervalType) => a @@ -113,6 +118,9 @@ object BinaryArithmeticWithDatetimeResolver { case (TimestampType | TimestampNTZType, _: YearMonthIntervalType) => DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, context.evalMode == EvalMode.ANSI))) + case (_: AnyTimestampNanoType, _: YearMonthIntervalType) => + DatetimeSub(l, r, TimestampAddYMInterval(l, + UnaryMinus(r, context.evalMode == EvalMode.ANSI))) case (CalendarIntervalType, CalendarIntervalType) | (_: DayTimeIntervalType, _: DayTimeIntervalType) => s diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 0a73945bd4f61..df9b4571440f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -2749,7 +2749,8 @@ case class TimestampAddYMInterval( override def toString: String = s"$left + $right" override def sql: String = s"${left.sql} + ${right.sql}" - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimestampType, YearMonthIntervalType) + override def inputTypes: Seq[AbstractDataType] = + Seq(TypeCollection(AnyTimestampType, AnyTimestampNanoType), YearMonthIntervalType) override def dataType: DataType = timestamp.dataType @@ -2758,16 +2759,25 @@ case class TimestampAddYMInterval( @transient private lazy val zoneIdInEval: ZoneId = zoneIdForType(left.dataType) - override def nullSafeEval(micros: Any, months: Any): Any = { - timestampAddMonths(micros.asInstanceOf[Long], months.asInstanceOf[Int], zoneIdInEval) + override def nullSafeEval(start: Any, months: Any): Any = left.dataType match { + case _: AnyTimestampNanoType => + timestampNanosAddMonths( + start.asInstanceOf[TimestampNanosVal], months.asInstanceOf[Int], zoneIdInEval) + case _ => + timestampAddMonths(start.asInstanceOf[Long], months.asInstanceOf[Int], zoneIdInEval) } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val zid = ctx.addReferenceObj("zoneId", zoneIdInEval, classOf[ZoneId].getName) val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - defineCodeGen(ctx, ev, (micros, months) => { - s"""$dtu.timestampAddMonths($micros, $months, $zid)""" - }) + left.dataType match { + case _: AnyTimestampNanoType => + defineCodeGen(ctx, ev, (sd, months) => + s"""$dtu.timestampNanosAddMonths($sd, $months, $zid)""") + case _ => + defineCodeGen(ctx, ev, (micros, months) => + s"""$dtu.timestampAddMonths($micros, $months, $zid)""") + } } override protected def withNewChildrenInternal( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 24aabdc988fb5..d605cbae7bb44 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -283,6 +283,18 @@ object DateTimeUtils extends SparkDateTimeUtils { instantToMicros(microsToInstant(micros).atZone(zoneId).plusMonths(months).toInstant) } + /** + * Adds a year-month interval expressed in months to a nanosecond-precision timestamp value while + * preserving the `nanosWithinMicro` remainder. + */ + def timestampNanosAddMonths( + start: TimestampNanosVal, + months: Int, + zoneId: ZoneId): TimestampNanosVal = { + val epochMicros = timestampAddMonths(start.epochMicros, months, zoneId) + TimestampNanosVal.fromParts(epochMicros, start.nanosWithinMicro) + } + /** * Adds a day-time interval expressed in microseconds to a timestamp at the given time zone. * It converts the input timestamp to a local timestamp, and adds the interval by: diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 226165b7b36f5..5feecc86bdfc1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -2341,6 +2341,72 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { assert(ltzMismatch.errorSubClass == "UNEXPECTED_INPUT_TYPE") } + test("SPARK-57825: add/subtract ANSI year-month interval on nanos timestamps") { + val interval = Period.ofYears(1).plusMonths(2) + val minusInterval = Period.ofMonths(-1) + + // A year-month shift moves only the month field: the whole fraction (including the + // sub-microsecond `789`) and the time of day are carried through unchanged. + val ntzType = TimestampNTZNanosType(9) + val ntzStart = DateTimeUtils.localDateTimeToTimestampNanos( + LocalDateTime.parse("2020-01-02T03:04:05.123456789"), precision = 9) + val ntzExpectedAdd = DateTimeUtils.localDateTimeToTimestampNanos( + LocalDateTime.parse("2021-03-02T03:04:05.123456789"), precision = 9) + val ntzExpectedSub = DateTimeUtils.localDateTimeToTimestampNanos( + LocalDateTime.parse("2019-12-02T03:04:05.123456789"), precision = 9) + + checkEvaluation( + TimestampAddYMInterval(Literal.create(ntzStart, ntzType), Literal(interval), Some("UTC")), + ntzExpectedAdd) + checkEvaluation( + TimestampAddYMInterval( + Literal.create(ntzStart, ntzType), + UnaryMinus(Literal(interval)), + Some("UTC")), + DateTimeUtils.localDateTimeToTimestampNanos( + LocalDateTime.parse("2018-11-02T03:04:05.123456789"), precision = 9)) + checkEvaluation( + TimestampAddYMInterval( + Literal.create(ntzStart, ntzType), Literal(minusInterval), Some("UTC")), + ntzExpectedSub) + assert(ntzExpectedAdd.nanosWithinMicro == ntzStart.nanosWithinMicro) + assert(ntzExpectedSub.nanosWithinMicro == ntzStart.nanosWithinMicro) + + val ltzType = TimestampLTZNanosType(9) + val ltzStart = DateTimeUtils.instantToTimestampNanos( + Instant.parse("2020-01-02T03:04:05.123456789Z"), precision = 9) + val ltzExpectedAdd = DateTimeUtils.instantToTimestampNanos( + Instant.parse("2021-03-02T03:04:05.123456789Z"), precision = 9) + val ltzExpectedSub = DateTimeUtils.instantToTimestampNanos( + Instant.parse("2019-12-02T03:04:05.123456789Z"), precision = 9) + + checkEvaluation( + TimestampAddYMInterval(Literal.create(ltzStart, ltzType), Literal(interval), Some("UTC")), + ltzExpectedAdd) + checkEvaluation( + TimestampAddYMInterval( + Literal.create(ltzStart, ltzType), + UnaryMinus(Literal(interval)), + Some("UTC")), + DateTimeUtils.instantToTimestampNanos( + Instant.parse("2018-11-02T03:04:05.123456789Z"), precision = 9)) + checkEvaluation( + TimestampAddYMInterval( + Literal.create(ltzStart, ltzType), Literal(minusInterval), Some("UTC")), + ltzExpectedSub) + assert(ltzExpectedAdd.nanosWithinMicro == ltzStart.nanosWithinMicro) + assert(ltzExpectedSub.nanosWithinMicro == ltzStart.nanosWithinMicro) + + yearMonthIntervalTypes.foreach { it => + checkConsistencyBetweenInterpretedAndCodegen( + (ts: Expression, ym: Expression) => TimestampAddYMInterval(ts, ym, Some("UTC")), + ntzType, it) + checkConsistencyBetweenInterpretedAndCodegen( + (ts: Expression, ym: Expression) => TimestampAddYMInterval(ts, ym, Some("UTC")), + ltzType, it) + } + } + test("SPARK-37552: convert a timestamp_ntz to another time zone") { checkEvaluation( ConvertTimezone( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 196d32e1a4f9d..c46daf8035989 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -1199,6 +1199,49 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { } } + test("SPARK-57825: timestamp nanos add year-month interval preserves nanosWithinMicro") { + def nanos(epochMicros: Long, nanosWithinMicro: Int): TimestampNanosVal = + TimestampNanosVal.fromParts(epochMicros, nanosWithinMicro.toShort) + + // The epoch-micros part follows the micro `timestampAddMonths` (including the Jan-31 -> Feb-29 + // day clamp in a leap year) while the sub-microsecond remainder is carried through unchanged. + assert(timestampNanosAddMonths( + nanos(date(2020, 1, 31, 12, 0, 0, 123000, LA), 789), 1, LA) === + nanos(date(2020, 2, 29, 12, 0, 0, 123000, LA), 789)) + + outstandingZoneIds.foreach { zid => + // The sub-microsecond remainder is preserved for the boundary values 0, 1 and 999. + Seq(0, 1, 999).foreach { rem => + // Zero interval is a no-op on both the epoch-micros and the remainder. + assert(timestampNanosAddMonths( + nanos(date(2021, 3, 18, 19, 44, 1, 123456, zid), rem), 0, zid) === + nanos(date(2021, 3, 18, 19, 44, 1, 123456, zid), rem)) + // Adding whole years/months shifts only the month field, never the fraction. + assert(timestampNanosAddMonths( + nanos(date(2020, 1, 2, 3, 4, 5, 123456, zid), rem), 14, zid) === + nanos(date(2021, 3, 2, 3, 4, 5, 123456, zid), rem)) + // Subtracting months is symmetric. + assert(timestampNanosAddMonths( + nanos(date(2020, 1, 2, 3, 4, 5, 123456, zid), rem), -1, zid) === + nanos(date(2019, 12, 2, 3, 4, 5, 123456, zid), rem)) + // Pre-epoch (negative epochMicros) value. + assert(timestampNanosAddMonths( + nanos(date(1960, 1, 2, 3, 4, 5, 123456, zid), rem), 1, zid) === + nanos(date(1960, 2, 2, 3, 4, 5, 123456, zid), rem)) + } + } + + // Consistency with the micro helper: epochMicros matches `timestampAddMonths` exactly and the + // remainder is independent of the interval amount. + outstandingZoneIds.foreach { zid => + val start = nanos(date(2020, 1, 2, 3, 4, 5, 123456, zid), 789) + val months = 15 + val result = timestampNanosAddMonths(start, months, zid) + assert(result.epochMicros === timestampAddMonths(start.epochMicros, months, zid)) + assert(result.nanosWithinMicro === start.nanosWithinMicro) + } + } + test("SPARK-57159: timestampNanosToEpochNanos packs into int64 epoch-nanoseconds") { def nanos(epochMicros: Long, nanosWithinMicro: Int): TimestampNanosVal = TimestampNanosVal.fromParts(epochMicros, nanosWithinMicro.toShort) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index cdfb8a78bb22c..006d7156115e1 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -603,6 +603,34 @@ Project [cast(1960-01-01 19:04:05.123456789 + INTERVAL '0 00:00:00.000001' DAY T +- OneRowRelation +-- !query +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' YEAR +-- !query analysis +Project [2020-01-01 19:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' + INTERVAL '1' YEAR#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH +-- !query analysis +Project [2020-01-01 19:04:05.123456789 + INTERVAL '1' MONTH AS TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' + INTERVAL '1' MONTH#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1-2' YEAR TO MONTH +-- !query analysis +Project [2020-01-01 19:04:05.123456789 - INTERVAL '1-2' YEAR TO MONTH AS TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_LTZ '1960-01-31 03:04:05.123456789 UTC' + INTERVAL '1' MONTH +-- !query analysis +Project [1960-01-30 19:04:05.123456789 + INTERVAL '1' MONTH AS TIMESTAMP_LTZ '1960-01-30 19:04:05.123456789' + INTERVAL '1' MONTH#x] ++- OneRowRelation + + -- !query SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + make_interval(0, 1, 0, 2, 0, 0, 0) -- !query analysis @@ -627,28 +655,6 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } --- !query -SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH --- !query analysis -org.apache.spark.sql.catalyst.ExtendedAnalysisException -{ - "errorClass" : "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES", - "sqlState" : "42K09", - "messageParameters" : { - "left" : "\"TIMESTAMP_LTZ(9)\"", - "right" : "\"INTERVAL MONTH\"", - "sqlExpr" : "\"(TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' + INTERVAL '1' MONTH)\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 8, - "stopIndex" : 77, - "fragment" : "TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH" - } ] -} - - -- !query SELECT max(c), min(c) FROM VALUES (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index 7b03461ea2ac1..7af9b314a1ec0 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -530,6 +530,34 @@ Project [cast(1960-01-02 03:04:05.123456789 + INTERVAL '0 00:00:00.000001' DAY T +- OneRowRelation +-- !query +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR +-- !query analysis +Project [2020-01-02 03:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH +-- !query analysis +Project [2020-01-02 03:04:05.123456789 + INTERVAL '1' MONTH AS TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH +-- !query analysis +Project [2020-01-02 03:04:05.123456789 - INTERVAL '1-2' YEAR TO MONTH AS TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH#x] ++- OneRowRelation + + +-- !query +SELECT TIMESTAMP_NTZ '1960-01-31 03:04:05.123456789' + INTERVAL '1' MONTH +-- !query analysis +Project [1960-01-31 03:04:05.123456789 + INTERVAL '1' MONTH AS TIMESTAMP_NTZ '1960-01-31 03:04:05.123456789' + INTERVAL '1' MONTH#x] ++- OneRowRelation + + -- !query SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + make_interval(0, 1, 0, 2, 0, 0, 0) -- !query analysis @@ -554,28 +582,6 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } --- !query -SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH --- !query analysis -org.apache.spark.sql.catalyst.ExtendedAnalysisException -{ - "errorClass" : "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES", - "sqlState" : "42K09", - "messageParameters" : { - "left" : "\"TIMESTAMP_NTZ(9)\"", - "right" : "\"INTERVAL MONTH\"", - "sqlExpr" : "\"(TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH)\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 8, - "stopIndex" : 73, - "fragment" : "TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH" - } ] -} - - -- !query SELECT max(c), min(c) FROM VALUES (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index 62e9b8eb93b5f..2ac2809481681 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -168,11 +168,17 @@ SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1 00:04:00.000321' DAY TO SECOND; SELECT TIMESTAMP_LTZ '1960-01-02 03:04:05.123456789 UTC' + INTERVAL '0 00:00:00.000001' DAY TO SECOND; --- SPARK-57501: nanos timestamps support only ANSI day-time intervals. A (legacy) calendar interval --- is rejected by TimestampAddInterval's type check, and a year-month interval has no supported --- operator overload. -SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + make_interval(0, 1, 0, 2, 0, 0, 0); +-- SPARK-57825: TIMESTAMP_LTZ(p) +/- ANSI year-month interval keeps the nanos type/precision and +-- carries the whole fraction (including the sub-microsecond digits) through unchanged; the month +-- shift is applied on the session-local wall clock. +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' YEAR; SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH; +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1-2' YEAR TO MONTH; +-- Jan-31 -> Feb-29 day clamp on a pre-epoch (leap-year) value. +SELECT TIMESTAMP_LTZ '1960-01-31 03:04:05.123456789 UTC' + INTERVAL '1' MONTH; +-- SPARK-57501, SPARK-57825: nanos timestamps support ANSI day-time and year-month intervals; the +-- legacy calendar interval is still rejected by TimestampAddInterval's type check. +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + make_interval(0, 1, 0, 2, 0, 0, 0); -- SPARK-57103: MAX / MIN over nanosecond-precision TIMESTAMP_LTZ. The aggregate preserves the -- nanosecond type and orders by the sub-microsecond remainder; NULLs are ignored. Values are diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index 971a2f8e148fd..52a1fd8e5178a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -144,11 +144,17 @@ SELECT named_struct('f', DATE '2020-01-01') :: struct; SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '2 00:03:00.000456' DAY TO SECOND; SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1 00:04:00.000321' DAY TO SECOND; SELECT TIMESTAMP_NTZ '1960-01-02 03:04:05.123456789' + INTERVAL '0 00:00:00.000001' DAY TO SECOND; --- SPARK-57501: nanos timestamps support only ANSI day-time intervals. A (legacy) calendar interval --- is rejected by TimestampAddInterval's type check, and a year-month interval has no supported --- operator overload. -SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + make_interval(0, 1, 0, 2, 0, 0, 0); +-- SPARK-57825: TIMESTAMP_NTZ(p) +/- ANSI year-month interval keeps the nanos type/precision and +-- carries the whole fraction (including the sub-microsecond digits) through unchanged; a month +-- shift never touches the time of day. +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR; SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH; +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH; +-- Jan-31 -> Feb-29 day clamp on a pre-epoch (leap-year) value. +SELECT TIMESTAMP_NTZ '1960-01-31 03:04:05.123456789' + INTERVAL '1' MONTH; +-- SPARK-57501, SPARK-57825: nanos timestamps support ANSI day-time and year-month intervals; the +-- legacy calendar interval is still rejected by TimestampAddInterval's type check. +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + make_interval(0, 1, 0, 2, 0, 0, 0); -- SPARK-57103: MAX / MIN over nanosecond-precision TIMESTAMP_NTZ. The aggregate preserves the -- nanosecond type and orders by the sub-microsecond remainder (two values share the same diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index 388ce96f73967..cba4ca855fbf8 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -678,6 +678,38 @@ struct +-- !query output +2021-01-01 19:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH +-- !query schema +struct +-- !query output +2020-02-01 19:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1-2' YEAR TO MONTH +-- !query schema +struct +-- !query output +2018-11-01 19:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_LTZ '1960-01-31 03:04:05.123456789 UTC' + INTERVAL '1' MONTH +-- !query schema +struct +-- !query output +1960-02-29 19:04:05.123456789 + + -- !query SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + make_interval(0, 1, 0, 2, 0, 0, 0) -- !query schema @@ -704,30 +736,6 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } --- !query -SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH --- !query schema -struct<> --- !query output -org.apache.spark.sql.catalyst.ExtendedAnalysisException -{ - "errorClass" : "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES", - "sqlState" : "42K09", - "messageParameters" : { - "left" : "\"TIMESTAMP_LTZ(9)\"", - "right" : "\"INTERVAL MONTH\"", - "sqlExpr" : "\"(TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' + INTERVAL '1' MONTH)\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 8, - "stopIndex" : 77, - "fragment" : "TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH" - } ] -} - - -- !query SELECT max(c), min(c) FROM VALUES (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'), diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index 821aad995bc65..c4206ed2ea380 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -596,6 +596,38 @@ struct +-- !query output +2021-01-02 03:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH +-- !query schema +struct +-- !query output +2020-02-02 03:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH +-- !query schema +struct +-- !query output +2018-11-02 03:04:05.123456789 + + +-- !query +SELECT TIMESTAMP_NTZ '1960-01-31 03:04:05.123456789' + INTERVAL '1' MONTH +-- !query schema +struct +-- !query output +1960-02-29 03:04:05.123456789 + + -- !query SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + make_interval(0, 1, 0, 2, 0, 0, 0) -- !query schema @@ -622,30 +654,6 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } --- !query -SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH --- !query schema -struct<> --- !query output -org.apache.spark.sql.catalyst.ExtendedAnalysisException -{ - "errorClass" : "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES", - "sqlState" : "42K09", - "messageParameters" : { - "left" : "\"TIMESTAMP_NTZ(9)\"", - "right" : "\"INTERVAL MONTH\"", - "sqlExpr" : "\"(TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH)\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 8, - "stopIndex" : 73, - "fragment" : "TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH" - } ] -} - - -- !query SELECT max(c), min(c) FROM VALUES (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'), From 9d5f079084eeb48c3977cf219f3c29f3f2b3692e Mon Sep 17 00:00:00 2001 From: Stevo Mitric Date: Thu, 30 Jul 2026 14:22:35 +0000 Subject: [PATCH 2/2] update tests --- .../analyzer-results/timestamp-ltz-nanos.sql.out | 7 +++++++ .../analyzer-results/timestamp-ntz-nanos.sql.out | 7 +++++++ .../resources/sql-tests/inputs/timestamp-ltz-nanos.sql | 2 ++ .../resources/sql-tests/inputs/timestamp-ntz-nanos.sql | 2 ++ .../sql-tests/results/timestamp-ltz-nanos.sql.out | 8 ++++++++ .../sql-tests/results/timestamp-ntz-nanos.sql.out | 8 ++++++++ 6 files changed, 34 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index 006d7156115e1..478292ef43e4c 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -610,6 +610,13 @@ Project [2020-01-01 19:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_LTZ '202 +- OneRowRelation +-- !query +SELECT INTERVAL '1' YEAR + TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' +-- !query analysis +Project [2020-01-01 19:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_LTZ '2020-01-01 19:04:05.123456789' + INTERVAL '1' YEAR#x] ++- OneRowRelation + + -- !query SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index 7af9b314a1ec0..6d9c87540181d 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -537,6 +537,13 @@ Project [2020-01-02 03:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_NTZ '202 +- OneRowRelation +-- !query +SELECT INTERVAL '1' YEAR + TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' +-- !query analysis +Project [2020-01-02 03:04:05.123456789 + INTERVAL '1' YEAR AS TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR#x] ++- OneRowRelation + + -- !query SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index 2ac2809481681..332d61e5ecae5 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -172,6 +172,8 @@ SELECT TIMESTAMP_LTZ '1960-01-02 03:04:05.123456789 UTC' + -- carries the whole fraction (including the sub-microsecond digits) through unchanged; the month -- shift is applied on the session-local wall clock. SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' YEAR; +-- The interval-first operand order resolves to the same addition. +SELECT INTERVAL '1' YEAR + TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC'; SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH; SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' - INTERVAL '1-2' YEAR TO MONTH; -- Jan-31 -> Feb-29 day clamp on a pre-epoch (leap-year) value. diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index 52a1fd8e5178a..57288983acbd6 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -148,6 +148,8 @@ SELECT TIMESTAMP_NTZ '1960-01-02 03:04:05.123456789' + INTERVAL '0 00:00:00.0000 -- carries the whole fraction (including the sub-microsecond digits) through unchanged; a month -- shift never touches the time of day. SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' YEAR; +-- The interval-first operand order resolves to the same addition. +SELECT INTERVAL '1' YEAR + TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789'; SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH; SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' - INTERVAL '1-2' YEAR TO MONTH; -- Jan-31 -> Feb-29 day clamp on a pre-epoch (leap-year) value. diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index cba4ca855fbf8..3815142d55d00 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -686,6 +686,14 @@ struct +-- !query output +2021-01-01 19:04:05.123456789 + + -- !query SELECT TIMESTAMP_LTZ '2020-01-02 03:04:05.123456789 UTC' + INTERVAL '1' MONTH -- !query schema diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index c4206ed2ea380..69fc1c4a33a30 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -604,6 +604,14 @@ struct +-- !query output +2021-01-02 03:04:05.123456789 + + -- !query SELECT TIMESTAMP_NTZ '2020-01-02 03:04:05.123456789' + INTERVAL '1' MONTH -- !query schema