Skip to content

Commit

Permalink
[SPARK-36222][SQL] Step by days in the Sequence expression for dates
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
The current implement of `Sequence` expression not support step by days for dates.
```
spark-sql> select sequence(date'2021-07-01', date'2021-07-10', interval '3' day);
Error in query: cannot resolve 'sequence(DATE '2021-07-01', DATE '2021-07-10', INTERVAL '3' DAY)' due to data type mismatch:
sequence uses the wrong parameter type. The parameter type must conform to:
1. The start and stop expressions must resolve to the same type.
2. If start and stop expressions resolve to the 'date' or 'timestamp' type
then the step expression must resolve to the 'interval' or
'interval year to month' or 'interval day to second' type,
otherwise to the same type as the start and stop expressions.
         ; line 1 pos 7;
'Project [unresolvedalias(sequence(2021-07-01, 2021-07-10, Some(INTERVAL '3' DAY), Some(Europe/Moscow)), None)]
+- OneRowRelation
```

### Why are the changes needed?
`DayTimeInterval` has day granularity should as step for dates.

### Does this PR introduce _any_ user-facing change?
'Yes'.
Sequence expression will supports step by `DayTimeInterval` has day granularity for dates.

### How was this patch tested?
New tests.

Closes #33439 from beliefer/SPARK-36222.

Authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
  • Loading branch information
beliefer authored and MaxGekk committed Jul 20, 2021
1 parent bf680bf commit c0d84e6
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2574,7 +2574,8 @@ case class Sequence(
DayTimeIntervalType.acceptsType(stepType)
case DateType =>
stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepType) ||
YearMonthIntervalType.acceptsType(stepType)
YearMonthIntervalType.acceptsType(stepType) ||
DayTimeIntervalType.acceptsType(stepType)
case _: IntegralType =>
stepOpt.isEmpty || stepType.sameType(startType)
case _ => false
Expand Down Expand Up @@ -2626,8 +2627,10 @@ case class Sequence(
case DateType =>
if (stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepOpt.get.dataType)) {
new TemporalSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
} else {
} else if (YearMonthIntervalType.acceptsType(stepOpt.get.dataType)) {
new PeriodSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
} else {
new DurationSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
}
}

Expand Down Expand Up @@ -2807,15 +2810,19 @@ object Sequence {
val intervalType: DataType = DayTimeIntervalType()

def splitStep(input: Any): (Int, Int, Long) = {
(0, 0, input.asInstanceOf[Long])
val duration = input.asInstanceOf[Long]
val days = IntervalUtils.getDays(duration)
val micros = duration - days * MICROS_PER_DAY
(0, days, micros)
}

def stepSplitCode(
stepMonths: String, stepDays: String, stepMicros: String, step: String): String = {
s"""
|final int $stepMonths = 0;
|final int $stepDays = 0;
|final long $stepMicros = $step;
|final int $stepDays =
| (int) org.apache.spark.sql.catalyst.util.IntervalUtils.getDays($step);
|final long $stepMicros = $step - $stepDays * ${MICROS_PER_DAY}L;
""".stripMargin
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -919,6 +919,16 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Date.valueOf("2020-11-01"),
Date.valueOf("2022-04-01")))

checkEvaluation(new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(fromDayTimeString("3 0:0:0"))),
Seq(
Date.valueOf("2021-07-01"),
Date.valueOf("2021-07-04"),
Date.valueOf("2021-07-07"),
Date.valueOf("2021-07-10")))

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("1970-01-02")),
Expand Down Expand Up @@ -1092,6 +1102,52 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Date.valueOf("2020-11-01"),
Date.valueOf("2022-04-01")))

checkEvaluation(new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofDays(3))),
Seq(
Date.valueOf("2021-07-01"),
Date.valueOf("2021-07-04"),
Date.valueOf("2021-07-07"),
Date.valueOf("2021-07-10")))

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofHours(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofMinutes(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofSeconds(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofMillis(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")

checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2018-01-01")),
Expand All @@ -1108,11 +1164,6 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Literal(Period.ofMonths(-1))),
EmptyRow,
s"sequence boundaries: 0 to 2678400000000 by -1")

assert(Sequence(
Cast(Literal("2011-03-01"), DateType),
Cast(Literal("2011-04-01"), DateType),
Option(Literal(Duration.ofHours(1)))).checkInputDataTypes().isFailure)
}
}

Expand Down

0 comments on commit c0d84e6

Please sign in to comment.