Skip to content

Commit

Permalink
[SPARK-34392][SQL] Support ZoneOffset +h:mm in DateTimeUtils. getZoneId
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
To support +8:00 in Spark3 when execute sql
`select to_utc_timestamp("2020-02-07 16:00:00", "GMT+8:00")`

### Why are the changes needed?
+8:00 this format is supported in PostgreSQL,hive, presto, but not supported in Spark3
https://issues.apache.org/jira/browse/SPARK-34392

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
unit test

Closes #31624 from Karl-WangSK/zone.

Lead-authored-by: ShiKai Wang <wskqing@gmail.com>
Co-authored-by: Karl-WangSK <shikai.wang@linkflowtech.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
  • Loading branch information
2 people authored and srowen committed Feb 26, 2021
1 parent 8d68f3f commit 56e664c
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 4 deletions.
Expand Up @@ -50,7 +50,10 @@ object DateTimeUtils {

val TIMEZONE_OPTION = "timeZone"

def getZoneId(timeZoneId: String): ZoneId = ZoneId.of(timeZoneId, ZoneId.SHORT_IDS)
def getZoneId(timeZoneId: String): ZoneId = {
// To support the (+|-)h:mm format because it was supported before Spark 3.0.
ZoneId.of(timeZoneId.replaceFirst("(\\+|\\-)(\\d):", "$10$2:"), ZoneId.SHORT_IDS)
}
def getTimeZone(timeZoneId: String): TimeZone = TimeZone.getTimeZone(getZoneId(timeZoneId))

/**
Expand Down
Expand Up @@ -471,6 +471,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
test("2011-12-25 09:00:00.123456", JST.getId, "2011-12-25 18:00:00.123456")
test("2011-12-25 09:00:00.123456", LA.getId, "2011-12-25 01:00:00.123456")
test("2011-12-25 09:00:00.123456", "Asia/Shanghai", "2011-12-25 17:00:00.123456")
test("2011-12-25 09:00:00.123456", "-7", "2011-12-25 02:00:00.123456")
test("2011-12-25 09:00:00.123456", "+8:00", "2011-12-25 17:00:00.123456")
test("2011-12-25 09:00:00.123456", "+8:00:00", "2011-12-25 17:00:00.123456")
test("2011-12-25 09:00:00.123456", "+0800", "2011-12-25 17:00:00.123456")
test("2011-12-25 09:00:00.123456", "-071020", "2011-12-25 01:49:40.123456")
test("2011-12-25 09:00:00.123456", "-07:10:20", "2011-12-25 01:49:40.123456")

}
}

Expand All @@ -496,6 +503,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456")
test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456")
test("2011-12-25 17:00:00.123456", "Asia/Shanghai", "2011-12-25 09:00:00.123456")
test("2011-12-25 02:00:00.123456", "-7", "2011-12-25 09:00:00.123456")
test("2011-12-25 17:00:00.123456", "+8:00", "2011-12-25 09:00:00.123456")
test("2011-12-25 17:00:00.123456", "+8:00:00", "2011-12-25 09:00:00.123456")
test("2011-12-25 17:00:00.123456", "+0800", "2011-12-25 09:00:00.123456")
test("2011-12-25 01:49:40.123456", "-071020", "2011-12-25 09:00:00.123456")
test("2011-12-25 01:49:40.123456", "-07:10:20", "2011-12-25 09:00:00.123456")
}
}

Expand Down
Expand Up @@ -414,13 +414,12 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago")
assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) ===
"America/Chicago")
spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === "GMT+8:00")

intercept[IllegalArgumentException] {
spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "pst")
}
intercept[IllegalArgumentException] {
spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
}
val e = intercept[IllegalArgumentException] {
spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "Asia/shanghai")
}
Expand Down

0 comments on commit 56e664c

Please sign in to comment.