From 26c7dd183b41c0fd7da44433afa1be3b68ac5ab1 Mon Sep 17 00:00:00 2001 From: Daniel Pires Date: Tue, 3 Jul 2018 16:25:03 +0200 Subject: [PATCH 1/4] Adding CalendarInterval in sql.catalyst.parser; it is a supported data type, but wasn't parseable hence casting to an interval was throwing a ParseException --- .../scala/org/apache/spark/sql/catalyst/dsl/package.scala | 4 ++++ .../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 1 + .../spark/sql/catalyst/parser/DataTypeParserSuite.scala | 1 + 3 files changed, 6 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index efb2eba655e15..f7cc61d6c6435 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -229,6 +229,10 @@ package object dsl { /** Creates a new AttributeReference of type date */ def date: AttributeReference = AttributeReference(s, DateType, nullable = true)() + /** Creates a new AttributeReference of type calendar interval */ + def calendarinterval: AttributeReference = + AttributeReference(s, CalendarIntervalType, nullable = true)() + /** Creates a new AttributeReference of type decimal */ def decimal: AttributeReference = AttributeReference(s, DecimalType.SYSTEM_DEFAULT, nullable = true)() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 383ebde3229d6..62f3b4a9704c2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1708,6 +1708,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case ("float", Nil) => FloatType case ("double", Nil) => DoubleType case ("date", Nil) => DateType + case ("calendarinterval", Nil) => CalendarIntervalType case ("timestamp", Nil) => TimestampType case ("string", Nil) => StringType case ("char", length :: Nil) => CharType(length.getText.toInt) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala index 2c491cd376edc..2c068bcc01eb7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala @@ -58,6 +58,7 @@ class DataTypeParserSuite extends SparkFunSuite { checkDataType("varchAr(20)", StringType) checkDataType("cHaR(27)", StringType) checkDataType("BINARY", BinaryType) + checkDataType("calendarinterval", CalendarIntervalType) checkDataType("array", ArrayType(DoubleType, true)) checkDataType("Array>", ArrayType(MapType(IntegerType, ByteType, true), true)) From 3709504d8da4b0b260ffdd91c0030b331bd208b2 Mon Sep 17 00:00:00 2001 From: Daniel Pires Date: Tue, 3 Jul 2018 18:07:42 +0200 Subject: [PATCH 2/4] added cast to FunctionRegistry --- .../apache/spark/sql/catalyst/analysis/FunctionRegistry.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 3700c63d817ea..8f086f932426a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -516,6 +516,7 @@ object FunctionRegistry { castAlias("decimal", DecimalType.USER_DEFAULT), castAlias("date", DateType), castAlias("timestamp", TimestampType), + castAlias("calendarinterval", CalendarIntervalType), castAlias("binary", BinaryType), castAlias("string", StringType) ) From 80ac507b0941f83400e7b077efba53c6095f7774 Mon Sep 17 00:00:00 2001 From: Daniel Pires Date: Wed, 4 Jul 2018 10:57:05 +0200 Subject: [PATCH 3/4] adding sql-tests --- .../test/resources/sql-tests/inputs/cast.sql | 19 +++++ .../resources/sql-tests/results/cast.sql.out | 84 ++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql index 629df59cff8b3..0433a4eea0253 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql @@ -42,4 +42,23 @@ SELECT CAST('9223372036854775808' AS long); DESC FUNCTION boolean; DESC FUNCTION EXTENDED boolean; + +-- cast null to calendar interval should return null +SELECT CAST(NULL as calendarinterval); + +-- cast invalid strings to calendar interval should return null +SELECT CAST('5 minutes' as calendarinterval); +SELECT CAST('10' as calendarinterval); + +-- cast valid strings to calendar interval should return calendar interval +SELECT CAST('interval 5 minutes' as calendarinterval); +SELECT CAST('interval 10 hours' as calendarinterval); +SELECT CAST('interval 1 second' as calendarinterval); + +-- casting to calendar interval using the function should also work +SELECT CALENDARINTERVAL('interval 5 minutes'); +SELECT CALENDARINTERVAL('interval 10 hours'); +SELECT CALENDARINTERVAL('interval 1 second'); + +DESC FUNCTION calendarinterval; -- TODO: migrate all cast tests here. diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 9c5f4554d9fe9..472bca0cdd10b 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 34 -- !query 0 @@ -199,3 +199,85 @@ Extended Usage: Function: boolean Usage: boolean(expr) - Casts the value `expr` to the target data type `boolean`. + + +-- !query 24 +SELECT CAST(NULL as calendarinterval) +-- !query 24 schema +struct +-- !query 24 output +NULL + + +-- !query 25 +SELECT CAST('5 minutes' as calendarinterval) +-- !query 25 schema +struct +-- !query 25 output +NULL + + +-- !query 26 +SELECT CAST('10' as calendarinterval) +-- !query 26 schema +struct +-- !query 26 output +NULL + + +-- !query 27 +SELECT CAST('interval 5 minutes' as calendarinterval) +-- !query 27 schema +struct +-- !query 27 output +interval 5 minutes + + +-- !query 28 +SELECT CAST('interval 10 hours' as calendarinterval) +-- !query 28 schema +struct +-- !query 28 output +interval 10 hours + + +-- !query 29 +SELECT CAST('interval 1 second' as calendarinterval) +-- !query 29 schema +struct +-- !query 29 output +interval 1 seconds + + +-- !query 30 +SELECT CALENDARINTERVAL('interval 5 minutes') +-- !query 30 schema +struct +-- !query 30 output +interval 5 minutes + + +-- !query 31 +SELECT CALENDARINTERVAL('interval 10 hours') +-- !query 31 schema +struct +-- !query 31 output +interval 10 hours + + +-- !query 32 +SELECT CALENDARINTERVAL('interval 1 second') +-- !query 32 schema +struct +-- !query 32 output +interval 1 seconds + + +-- !query 33 +DESC FUNCTION calendarinterval +-- !query 33 schema +struct +-- !query 33 output +Class: org.apache.spark.sql.catalyst.expressions.Cast +Function: calendarinterval +Usage: calendarinterval(expr) - Casts the value `expr` to the target data type `calendarinterval`. From 2ccdd300a3a19121ad7f8c5064ac59cd4eae0d5e Mon Sep 17 00:00:00 2001 From: Daniel Pires Date: Mon, 9 Jul 2018 11:47:25 +0100 Subject: [PATCH 4/4] adding test cases --- .../test/resources/sql-tests/inputs/cast.sql | 27 ++- .../resources/sql-tests/results/cast.sql.out | 154 +++++++++++++++--- 2 files changed, 150 insertions(+), 31 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql index 0433a4eea0253..a43d1dea29633 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql @@ -45,20 +45,35 @@ DESC FUNCTION EXTENDED boolean; -- cast null to calendar interval should return null SELECT CAST(NULL as calendarinterval); +SELECT CALENDARINTERVAL(NULL); -- cast invalid strings to calendar interval should return null -SELECT CAST('5 minutes' as calendarinterval); -SELECT CAST('10' as calendarinterval); +SELECT CAST('interval 10' as calendarinterval); +SELECT CAST('interval 100 nanoseconds' as calendarinterval); +SELECT CAST('interval 1 second 10 years -10 months 1 minute' as calendarinterval); +SELECT CAST('interval 60 hours + 1 minute' as calendarinterval); +SELECT CAST('interval 1 day +5 minutes' as calendarinterval); -- cast valid strings to calendar interval should return calendar interval SELECT CAST('interval 5 minutes' as calendarinterval); SELECT CAST('interval 10 hours' as calendarinterval); SELECT CAST('interval 1 second' as calendarinterval); +SELECT CAST('interval 3 years -3 month 7 week 123 microseconds' as calendarinterval); +SELECT CAST('interval 100 years 15 months -24 weeks 66 seconds' as calendarinterval); --- casting to calendar interval using the function should also work -SELECT CALENDARINTERVAL('interval 5 minutes'); -SELECT CALENDARINTERVAL('interval 10 hours'); -SELECT CALENDARINTERVAL('interval 1 second'); +-- casting invalid strings to calendar interval using the function should return null +SELECT CALENDARINTERVAL('interval 2 months 1 year 3 weeks 4 days 5 minutes 6 seconds'); +SELECT CALENDARINTERVAL('interval 1 seconds + 10 milliseconds'); +SELECT CALENDARINTERVAL('interval 1 day -1 day'); +SELECT CALENDARINTERVAL('interval 10 microseconds _ 1 microsecond'); +SELECT CALENDARINTERVAL('5 weeks 5 days 23 hours'); + +-- casting to calendar interval using the function should return calendar interval +SELECT CALENDARINTERVAL('interval 1 year 2 months 3 weeks 4 days 5 minutes 6 seconds'); +SELECT CALENDARINTERVAL('interval 10 hours -5 minutes'); +SELECT CALENDARINTERVAL('interval 15 second 1000 milliseconds'); +SELECT CALENDARINTERVAL('interval 10 hours 5 minutes'); +SELECT CALENDARINTERVAL('interval 1 minute -60 seconds'); DESC FUNCTION calendarinterval; -- TODO: migrate all cast tests here. diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 472bca0cdd10b..abbc13f213ae4 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 34 +-- Number of queries: 47 -- !query 0 @@ -210,74 +210,178 @@ NULL -- !query 25 -SELECT CAST('5 minutes' as calendarinterval) +SELECT CALENDARINTERVAL(NULL) -- !query 25 schema -struct +struct -- !query 25 output NULL -- !query 26 -SELECT CAST('10' as calendarinterval) +SELECT CAST('interval 10' as calendarinterval) -- !query 26 schema -struct +struct -- !query 26 output NULL -- !query 27 -SELECT CAST('interval 5 minutes' as calendarinterval) +SELECT CAST('interval 100 nanoseconds' as calendarinterval) -- !query 27 schema -struct +struct -- !query 27 output -interval 5 minutes +NULL -- !query 28 -SELECT CAST('interval 10 hours' as calendarinterval) +SELECT CAST('interval 1 second 10 years -10 months 1 minute' as calendarinterval) -- !query 28 schema -struct +struct -- !query 28 output -interval 10 hours +NULL -- !query 29 -SELECT CAST('interval 1 second' as calendarinterval) +SELECT CAST('interval 60 hours + 1 minute' as calendarinterval) -- !query 29 schema -struct +struct -- !query 29 output -interval 1 seconds +NULL -- !query 30 -SELECT CALENDARINTERVAL('interval 5 minutes') +SELECT CAST('interval 1 day +5 minutes' as calendarinterval) -- !query 30 schema -struct +struct -- !query 30 output -interval 5 minutes +NULL -- !query 31 -SELECT CALENDARINTERVAL('interval 10 hours') +SELECT CAST('interval 5 minutes' as calendarinterval) -- !query 31 schema -struct +struct -- !query 31 output -interval 10 hours +interval 5 minutes -- !query 32 -SELECT CALENDARINTERVAL('interval 1 second') +SELECT CAST('interval 10 hours' as calendarinterval) -- !query 32 schema -struct +struct -- !query 32 output -interval 1 seconds +interval 10 hours -- !query 33 -DESC FUNCTION calendarinterval +SELECT CAST('interval 1 second' as calendarinterval) -- !query 33 schema -struct +struct -- !query 33 output +interval 1 seconds + + +-- !query 34 +SELECT CAST('interval 3 years -3 month 7 week 123 microseconds' as calendarinterval) +-- !query 34 schema +struct +-- !query 34 output +interval 2 years 9 months 7 weeks 123 microseconds + + +-- !query 35 +SELECT CAST('interval 100 years 15 months -24 weeks 66 seconds' as calendarinterval) +-- !query 35 schema +struct +-- !query 35 output +interval 101 years 3 months -23 weeks -6 days -23 hours -58 minutes -54 seconds + + +-- !query 36 +SELECT CALENDARINTERVAL('interval 2 months 1 year 3 weeks 4 days 5 minutes 6 seconds') +-- !query 36 schema +struct +-- !query 36 output +NULL + + +-- !query 37 +SELECT CALENDARINTERVAL('interval 1 seconds + 10 milliseconds') +-- !query 37 schema +struct +-- !query 37 output +NULL + + +-- !query 38 +SELECT CALENDARINTERVAL('interval 1 day -1 day') +-- !query 38 schema +struct +-- !query 38 output +NULL + + +-- !query 39 +SELECT CALENDARINTERVAL('interval 10 microseconds _ 1 microsecond') +-- !query 39 schema +struct +-- !query 39 output +NULL + + +-- !query 40 +SELECT CALENDARINTERVAL('5 weeks 5 days 23 hours') +-- !query 40 schema +struct +-- !query 40 output +NULL + + +-- !query 41 +SELECT CALENDARINTERVAL('interval 1 year 2 months 3 weeks 4 days 5 minutes 6 seconds') +-- !query 41 schema +struct +-- !query 41 output +interval 1 years 2 months 3 weeks 4 days 5 minutes 6 seconds + + +-- !query 42 +SELECT CALENDARINTERVAL('interval 10 hours -5 minutes') +-- !query 42 schema +struct +-- !query 42 output +interval 9 hours 55 minutes + + +-- !query 43 +SELECT CALENDARINTERVAL('interval 15 second 1000 milliseconds') +-- !query 43 schema +struct +-- !query 43 output +interval 16 seconds + + +-- !query 44 +SELECT CALENDARINTERVAL('interval 10 hours 5 minutes') +-- !query 44 schema +struct +-- !query 44 output +interval 10 hours 5 minutes + + +-- !query 45 +SELECT CALENDARINTERVAL('interval 1 minute -60 seconds') +-- !query 45 schema +struct +-- !query 45 output +interval + + +-- !query 46 +DESC FUNCTION calendarinterval +-- !query 46 schema +struct +-- !query 46 output Class: org.apache.spark.sql.catalyst.expressions.Cast Function: calendarinterval Usage: calendarinterval(expr) - Casts the value `expr` to the target data type `calendarinterval`.