From 397102bb9ddca30f70dfd2004036ba73be950c99 Mon Sep 17 00:00:00 2001 From: Huaxin Gao Date: Tue, 16 Nov 2021 09:08:22 -0800 Subject: [PATCH 1/2] [SPARK-37219][SQL] Make AS OF syntax more flexible --- .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index e11f737590e95..426f529c93055 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -600,8 +600,8 @@ fromClause ; temporalClause - : ((FOR SYSTEM_VERSION) | VERSION) AS OF version=(INTEGER_VALUE | STRING) - | ((FOR SYSTEM_TIME) | TIMESTAMP) AS OF timestamp=STRING + : FOR? (SYSTEM_VERSION | VERSION) AS OF version=(INTEGER_VALUE | STRING) + | FOR? (SYSTEM_TIME | TIMESTAMP) AS OF timestamp=STRING ; aggregationClause From 93cafdd11687cf7df2a5880740780e72244eb6f8 Mon Sep 17 00:00:00 2001 From: Huaxin Gao Date: Tue, 16 Nov 2021 10:05:39 -0800 Subject: [PATCH 2/2] add tests --- .../sql/catalyst/parser/DDLParserSuite.scala | 50 +++++++++++++++++++ .../sql/connector/DataSourceV2SQLSuite.scala | 18 +++++++ 2 files changed, 68 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index cea1fdfd43932..741fe1eac6c8a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -2453,6 +2453,14 @@ class DDLParserSuite extends AnalysisTest { new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + comparePlans( + parsePlan("SELECT * FROM a.b.c FOR VERSION AS OF 'Snapshot123456789'"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + timeTravel = TimeTravelSpec.create(None, Some("123456789")) comparePlans( parsePlan("SELECT * FROM a.b.c FOR SYSTEM_VERSION AS OF 123456789"), @@ -2462,6 +2470,14 @@ class DDLParserSuite extends AnalysisTest { new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + comparePlans( + parsePlan("SELECT * FROM a.b.c SYSTEM_VERSION AS OF 123456789"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + timeTravel = TimeTravelSpec.create(Some("2019-01-29 00:37:58"), None) comparePlans( parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-29 00:37:58'"), @@ -2470,6 +2486,15 @@ class DDLParserSuite extends AnalysisTest { Seq("a", "b", "c"), new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + + comparePlans( + parsePlan("SELECT * FROM a.b.c FOR TIMESTAMP AS OF '2019-01-29 00:37:58'"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + comparePlans( parsePlan("SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'"), Project(Seq(UnresolvedStar(None)), @@ -2478,6 +2503,14 @@ class DDLParserSuite extends AnalysisTest { new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + comparePlans( + parsePlan("SELECT * FROM a.b.c SYSTEM_TIME AS OF '2019-01-29 00:37:58'"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + timeTravel = TimeTravelSpec.create(Some("2019-01-29"), None) comparePlans( parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-29'"), @@ -2486,6 +2519,15 @@ class DDLParserSuite extends AnalysisTest { Seq("a", "b", "c"), new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + + comparePlans( + parsePlan("SELECT * FROM a.b.c FOR TIMESTAMP AS OF '2019-01-29'"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + comparePlans( parsePlan("SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29'"), Project(Seq(UnresolvedStar(None)), @@ -2494,6 +2536,14 @@ class DDLParserSuite extends AnalysisTest { new CaseInsensitiveStringMap(properties), timeTravelSpec = timeTravel))) + comparePlans( + parsePlan("SELECT * FROM a.b.c SYSTEM_TIME AS OF '2019-01-29'"), + Project(Seq(UnresolvedStar(None)), + UnresolvedRelation( + Seq("a", "b", "c"), + new CaseInsensitiveStringMap(properties), + timeTravelSpec = timeTravel))) + val e1 = intercept[DateTimeException] { parsePlan("SELECT * FROM a.b.c TIMESTAMP AS OF '2019-01-11111'") }.getMessage diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 7bdcad00e8931..aede3e0af46f2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -2966,10 +2966,19 @@ class DataSourceV2SQLSuite === Array(Row(1), Row(2))) assert(sql("SELECT * FROM t VERSION AS OF 2345678910").collect === Array(Row(3), Row(4))) + assert(sql("SELECT * FROM t FOR VERSION AS OF 'Snapshot123456789'").collect + === Array(Row(1), Row(2))) + assert(sql("SELECT * FROM t FOR VERSION AS OF 2345678910").collect + === Array(Row(3), Row(4))) + assert(sql("SELECT * FROM t FOR SYSTEM_VERSION AS OF 'Snapshot123456789'").collect === Array(Row(1), Row(2))) assert(sql("SELECT * FROM t FOR SYSTEM_VERSION AS OF 2345678910").collect === Array(Row(3), Row(4))) + assert(sql("SELECT * FROM t SYSTEM_VERSION AS OF 'Snapshot123456789'").collect + === Array(Row(1), Row(2))) + assert(sql("SELECT * FROM t SYSTEM_VERSION AS OF 2345678910").collect + === Array(Row(3), Row(4))) } val ts1 = DateTimeUtils.stringToTimestampAnsi( @@ -2994,10 +3003,19 @@ class DataSourceV2SQLSuite === Array(Row(5), Row(6))) assert(sql("SELECT * FROM t TIMESTAMP AS OF '2021-01-29 00:37:58'").collect === Array(Row(7), Row(8))) + assert(sql("SELECT * FROM t FOR TIMESTAMP AS OF '2019-01-29 00:37:58'").collect + === Array(Row(5), Row(6))) + assert(sql("SELECT * FROM t FOR TIMESTAMP AS OF '2021-01-29 00:37:58'").collect + === Array(Row(7), Row(8))) + assert(sql("SELECT * FROM t FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'").collect === Array(Row(5), Row(6))) assert(sql("SELECT * FROM t FOR SYSTEM_TIME AS OF '2021-01-29 00:37:58'").collect === Array(Row(7), Row(8))) + assert(sql("SELECT * FROM t SYSTEM_TIME AS OF '2019-01-29 00:37:58'").collect + === Array(Row(5), Row(6))) + assert(sql("SELECT * FROM t SYSTEM_TIME AS OF '2021-01-29 00:37:58'").collect + === Array(Row(7), Row(8))) } }