Skip to content

[SPARK-36049][SQL] Remove IntervalUnit #33265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AngersZhuuuu wants to merge 4 commits into from
Closed

[SPARK-36049][SQL] Remove IntervalUnit #33265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bf2f4df
[SPARK-36049][SQL] Remove IntervalUnit
AngersZhuuuu Jul 8, 2021
e70a954
Update IntervalUtils.scala
AngersZhuuuu Jul 8, 2021
01d346c
follow comment
AngersZhuuuu Jul 8, 2021
5428d0c
Re-trigger build
MaxGekk Jul 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 47 additions & 73 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,6 @@ object IntervalStringStyles extends Enumeration {

object IntervalUtils {

object IntervalUnit extends Enumeration {
type IntervalUnit = Value

val NANOSECOND = Value(0, "nanosecond")
val MICROSECOND = Value(1, "microsecond")
val MILLISECOND = Value(2, "millisecond")
val SECOND = Value(3, "second")
val MINUTE = Value(4, "minute")
val HOUR = Value(5, "hour")
val DAY = Value(6, "day")
val WEEK = Value(7, "week")
val MONTH = Value(8, "month")
val YEAR = Value(9, "year")
}
import IntervalUnit._

private val MAX_DAY = Long.MaxValue / MICROS_PER_DAY
private val MAX_HOUR = Long.MaxValue / MICROS_PER_HOUR
private val MAX_MINUTE = Long.MaxValue / MICROS_PER_MINUTE
Expand Down Expand Up @@ -97,7 +81,7 @@ object IntervalUtils {
def getSeconds(interval: CalendarInterval): Decimal = getSeconds(interval.microseconds)

private def toLongWithRange(
fieldName: IntervalUnit,
fieldName: UTF8String,
s: String,
minValue: Long,
maxValue: Long): Long = {
Expand Down Expand Up @@ -250,10 +234,11 @@ object IntervalUtils {
}
}

private def toYMInterval(yearStr: String, monthStr: String, sign: Int): Int = {
private def toYMInterval(year: String, month: String, sign: Int): Int = {
safeToInterval("year-month") {
val years = toLongWithRange(YEAR, yearStr, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR)
val totalMonths = sign * (years * MONTHS_PER_YEAR + toLongWithRange(MONTH, monthStr, 0, 11))
val years = toLongWithRange(yearStr, year, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR)
val totalMonths =
sign * (years * MONTHS_PER_YEAR + toLongWithRange(monthStr, month, 0, 11))
Math.toIntExact(totalMonths)
}
}
Expand Down Expand Up @@ -402,45 +387,33 @@ object IntervalUtils {
}
}

def toDTInterval(
dayStr: String,
hourStr: String,
minuteStr: String,
secondStr: String,
sign: Int): Long = {
def toDTInterval(day: String, hour: String, minute: String, second: String, sign: Int): Long = {
var micros = 0L
val days = toLongWithRange(DAY, dayStr, 0, MAX_DAY).toInt
val days = toLongWithRange(dayStr, day, 0, MAX_DAY).toInt
micros = Math.addExact(micros, sign * days * MICROS_PER_DAY)
val hours = toLongWithRange(HOUR, hourStr, 0, 23)
val hours = toLongWithRange(hourStr, hour, 0, 23)
micros = Math.addExact(micros, sign * hours * MICROS_PER_HOUR)
val minutes = toLongWithRange(MINUTE, minuteStr, 0, 59)
val minutes = toLongWithRange(minuteStr, minute, 0, 59)
micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE)
micros = Math.addExact(micros, sign * parseSecondNano(secondStr))
micros = Math.addExact(micros, sign * parseSecondNano(second))
micros
}

def toDTInterval(
hourStr: String,
minuteStr: String,
secondStr: String,
sign: Int): Long = {
def toDTInterval(hour: String, minute: String, second: String, sign: Int): Long = {
var micros = 0L
val hours = toLongWithRange(HOUR, hourStr, 0, MAX_HOUR)
val hours = toLongWithRange(hourStr, hour, 0, MAX_HOUR)
micros = Math.addExact(micros, sign * hours * MICROS_PER_HOUR)
val minutes = toLongWithRange(MINUTE, minuteStr, 0, 59)
val minutes = toLongWithRange(minuteStr, minute, 0, 59)
micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE)
micros = Math.addExact(micros, sign * parseSecondNano(secondStr))
micros = Math.addExact(micros, sign * parseSecondNano(second))
micros
}

def toDTInterval(
minuteStr: String,
secondStr: String,
sign: Int): Long = {
def toDTInterval(minute: String, second: String, sign: Int): Long = {
var micros = 0L
val minutes = toLongWithRange(MINUTE, minuteStr, 0, MAX_MINUTE)
val minutes = toLongWithRange(minuteStr, minute, 0, MAX_MINUTE)
micros = Math.addExact(micros, sign * minutes * MICROS_PER_MINUTE)
micros = Math.addExact(micros, sign * parseSecondNano(secondStr))
micros = Math.addExact(micros, sign * parseSecondNano(second))
micros
}

Expand Down Expand Up @@ -511,21 +484,21 @@ object IntervalUtils {
val days = if (m.group(2) == null) {
0
} else {
toLongWithRange(DAY, m.group(3), 0, Integer.MAX_VALUE).toInt
toLongWithRange(dayStr, m.group(3), 0, Integer.MAX_VALUE).toInt
}
var hours: Long = 0L
var minutes: Long = 0L
var seconds: Long = 0L
if (m.group(5) != null || from == DT.MINUTE) { // 'HH:mm:ss' or 'mm:ss minute'
hours = toLongWithRange(HOUR, m.group(5), 0, 23)
minutes = toLongWithRange(MINUTE, m.group(6), 0, 59)
seconds = toLongWithRange(SECOND, m.group(7), 0, 59)
hours = toLongWithRange(hourStr, m.group(5), 0, 23)
minutes = toLongWithRange(minuteStr, m.group(6), 0, 59)
seconds = toLongWithRange(secondStr, m.group(7), 0, 59)
} else if (m.group(8) != null) { // 'mm:ss.nn'
minutes = toLongWithRange(MINUTE, m.group(6), 0, 59)
seconds = toLongWithRange(SECOND, m.group(7), 0, 59)
minutes = toLongWithRange(minuteStr, m.group(6), 0, 59)
seconds = toLongWithRange(secondStr, m.group(7), 0, 59)
} else { // 'HH:mm'
hours = toLongWithRange(HOUR, m.group(6), 0, 23)
minutes = toLongWithRange(SECOND, m.group(7), 0, 59)
hours = toLongWithRange(hourStr, m.group(6), 0, 23)
minutes = toLongWithRange(secondStr, m.group(7), 0, 59)
}
// Hive allow nanosecond precision interval
var secondsFraction = parseNanos(m.group(9), seconds < 0)
Expand Down Expand Up @@ -555,15 +528,15 @@ object IntervalUtils {
}

// Parses a string with nanoseconds, truncates the result and returns microseconds
private def parseNanos(nanosStr: String, isNegative: Boolean): Long = {
if (nanosStr != null) {
private def parseNanos(nanos: String, isNegative: Boolean): Long = {
if (nanos != null) {
val maxNanosLen = 9
val alignedStr = if (nanosStr.length < maxNanosLen) {
(nanosStr + "000000000").substring(0, maxNanosLen)
} else nanosStr
val nanos = toLongWithRange(NANOSECOND, alignedStr, 0L, 999999999L)
val micros = nanos / NANOS_PER_MICROS
if (isNegative) -micros else micros
val alignedStr = if (nanos.length < maxNanosLen) {
(nanos + "000000000").substring(0, maxNanosLen)
} else nanos
val nanoSecond = toLongWithRange(nanosStr, alignedStr, 0L, 999999999L)
val microSecond = nanoSecond / NANOS_PER_MICROS
if (isNegative) -microSecond else microSecond
} else {
0L
}
Expand All @@ -574,7 +547,7 @@ object IntervalUtils {
*/
private def parseSecondNano(secondNano: String): Long = {
def parseSeconds(secondsStr: String): Long = {
toLongWithRange(SECOND, secondsStr, MIN_SECOND, MAX_SECOND) * MICROS_PER_SECOND
toLongWithRange(secondStr, secondsStr, MIN_SECOND, MAX_SECOND) * MICROS_PER_SECOND
}

secondNano.split("\\.") match {
Expand Down Expand Up @@ -774,19 +747,20 @@ object IntervalUtils {
UNIT_SUFFIX,
UNIT_END = Value
}
private final val intervalStr = UTF8String.fromString("interval")
private def unitToUtf8(unit: IntervalUnit): UTF8String = {
UTF8String.fromString(unit.toString)
private final val intervalStr = unitToUtf8("interval")
private def unitToUtf8(unit: String): UTF8String = {
UTF8String.fromString(unit)
}
private final val yearStr = unitToUtf8(YEAR)
private final val monthStr = unitToUtf8(MONTH)
private final val weekStr = unitToUtf8(WEEK)
private final val dayStr = unitToUtf8(DAY)
private final val hourStr = unitToUtf8(HOUR)
private final val minuteStr = unitToUtf8(MINUTE)
private final val secondStr = unitToUtf8(SECOND)
private final val millisStr = unitToUtf8(MILLISECOND)
private final val microsStr = unitToUtf8(MICROSECOND)
private final val yearStr = unitToUtf8("year")
private final val monthStr = unitToUtf8("month")
private final val weekStr = unitToUtf8("week")
private final val dayStr = unitToUtf8("day")
private final val hourStr = unitToUtf8("hour")
private final val minuteStr = unitToUtf8("minute")
private final val secondStr = unitToUtf8("second")
private final val millisStr = unitToUtf8("millisecond")
private final val microsStr = unitToUtf8("microsecond")
private final val nanosStr = unitToUtf8("nanosecond")

/**
* A safe version of `stringToInterval`. It returns null for invalid input string.
Expand Down
33 changes: 11 additions & 22 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last}
import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils}
import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.TimestampTypes
import org.apache.spark.sql.types._
Expand Down Expand Up @@ -679,18 +678,7 @@ class ExpressionParserSuite extends AnalysisTest {
}
}

val intervalUnits = Seq(
YEAR,
MONTH,
WEEK,
DAY,
HOUR,
MINUTE,
SECOND,
MILLISECOND,
MICROSECOND)

def intervalLiteral(u: IntervalUnit, s: String): Literal = {
def intervalLiteral(u: UTF8String, s: String): Literal = {
Literal(IntervalUtils.stringToInterval(s + " " + u.toString))
}

Expand All @@ -710,18 +698,19 @@ class ExpressionParserSuite extends AnalysisTest {
// Single Intervals.
val forms = Seq("", "s")
val values = Seq("0", "10", "-7", "21")
intervalUnits.foreach { unit =>
forms.foreach { form =>
values.foreach { value =>
val expected = intervalLiteral(unit, value)
checkIntervals(s"$value $unit$form", expected)
checkIntervals(s"'$value' $unit$form", expected)
}
Seq("year", "month", "week", "day", "hour", "minute", "second", "millisecond", "microsecond")
.foreach { unit =>
forms.foreach { form =>
values.foreach { value =>
val expected = intervalLiteral(unit, value)
checkIntervals(s"$value $unit$form", expected)
checkIntervals(s"'$value' $unit$form", expected)
}
}
}
}

// Hive nanosecond notation.
checkIntervals("13.123456789 seconds", intervalLiteral(SECOND, "13.123456789"))
checkIntervals("13.123456789 seconds", intervalLiteral("second", "13.123456789"))
checkIntervals(
"-13.123456789 second",
Literal(new CalendarInterval(
Expand Down