[SPARK-31879][SQL] Using GB as default Locale for datetime formatters

apache · Jun 1, 2020 · ab0517d · ab0517d
1 parent b806fc4
commit ab0517d
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 5 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -117,7 +117,16 @@ class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyD
 object DateFormatter {
   import LegacyDateFormats._
 
-  val defaultLocale: Locale = Locale.US
+  /**
+   * This is change from Locale.US to GB, because:
+   * The first day-of-week varies by culture.
+   * For example, the US uses Sunday, while the United Kingdom and the ISO-8601 standard use Monday.
+   *
+   * Using `US` makes functions which rely on the Locale to express the first day of week
+   * inconsistent with Spark 2.4
+   * see https://issues.apache.org/jira/browse/SPARK-31879
+   */
+  val defaultLocale: Locale = new Locale("en", "GB")
 
   val defaultPattern: String = "yyyy-MM-dd"
 

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -278,7 +278,16 @@ object LegacyDateFormats extends Enumeration {
 object TimestampFormatter {
   import LegacyDateFormats._
 
-  val defaultLocale: Locale = Locale.US
+  /**
+   * This is change from Locale.US to GB, because:
+   * The first day-of-week varies by culture.
+   * For example, the US uses Sunday, while the United Kingdom and the ISO-8601 standard use Monday.
+   *
+   * Using `US` makes functions which rely on the Locale to express the first day of week
+   * inconsistent with Spark 2.4
+   * see https://issues.apache.org/jira/browse/SPARK-31879
+   */
+  val defaultLocale: Locale = new Locale("en", "GB")
 
   def defaultPattern(): String = s"${DateFormatter.defaultPattern} HH:mm:ss"
 

diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -164,3 +164,5 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy
 select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
 select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss');
 select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd');
+
+select to_timestamp('2020-01-01', 'YYYY-ww-uu');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 119
+-- Number of queries: 120
 
 
 -- !query
@@ -1025,3 +1025,11 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('2020-01-01', 'YYYY-ww-uu')
+-- !query schema
+struct<to_timestamp(2020-01-01, YYYY-ww-uu):timestamp>
+-- !query output
+2019-12-30 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 119
+-- Number of queries: 120
 
 
 -- !query
@@ -980,3 +980,11 @@ select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
 struct<date_format(CAST(DATE '2018-11-17' AS TIMESTAMP), yyyyyyyyyyy-MM-dd):string>
 -- !query output
 00000002018-11-17
+
+
+-- !query
+select to_timestamp('2020-01-01', 'YYYY-ww-uu')
+-- !query schema
+struct<to_timestamp(2020-01-01, YYYY-ww-uu):timestamp>
+-- !query output
+2019-12-30 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 119
+-- Number of queries: 120
 
 
 -- !query
@@ -997,3 +997,11 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('2020-01-01', 'YYYY-ww-uu')
+-- !query schema
+struct<to_timestamp(2020-01-01, YYYY-ww-uu):timestamp>
+-- !query output
+2019-12-30 00:00:00