From 1ce7fdcaa059536b2b467bdbf5219a50e2535400 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Wed, 1 Jun 2016 18:04:14 -0700 Subject: [PATCH 1/2] Ban CTAS with schema --- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 6 ++++++ .../org/apache/spark/sql/hive/HiveDDLCommandSuite.scala | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 01409c6a77c1e..3e67cfe6e8b71 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -937,6 +937,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { selectQuery match { case Some(q) => + // Just use whatever is projected in the select statement as our schema + if (schema.nonEmpty) { + throw operationNotAllowed( + "Schema may not be specified in a Create Table As Select (CTAS) statement", + ctx) + } // Hive does not allow to use a CTAS statement to create a partitioned table. if (tableDesc.partitionColumnNames.nonEmpty) { val errorMessage = "A Create Table As Select (CTAS) statement is not allowed to " + diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala index ba9fe54db86ee..0abaeeaa44dec 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala @@ -190,6 +190,11 @@ class HiveDDLCommandSuite extends PlanTest { " AS SELECT key, value FROM (SELECT 1 as key, 2 as value) tmp") } + test("CTAS statement with schema is not allowed") { + assertUnsupported(s"CREATE TABLE ctas1 (age INT, name STRING) " + + "AS SELECT key, value FROM (SELECT 1 as key, 2 as value) tmp") + } + test("unsupported operations") { intercept[ParseException] { parser.parsePlan( From 28910ab1ed7ccffbe63a92fed40227f9e8901eab Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Wed, 1 Jun 2016 18:08:45 -0700 Subject: [PATCH 2/2] Fix tests --- .../apache/spark/sql/hive/HiveDDLCommandSuite.scala | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala index 0abaeeaa44dec..f8da1b20e13df 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala @@ -51,12 +51,6 @@ class HiveDDLCommandSuite extends PlanTest { test("Test CTAS #1") { val s1 = """CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view - |(viewTime INT, - |userid BIGINT, - |page_url STRING, - |referrer_url STRING, - |ip STRING COMMENT 'IP Address of the User', - |country STRING COMMENT 'country of origination') |COMMENT 'This is the staging page view table' |STORED AS RCFILE |LOCATION '/user/external/page_view' @@ -91,12 +85,6 @@ class HiveDDLCommandSuite extends PlanTest { test("Test CTAS #2") { val s2 = """CREATE EXTERNAL TABLE IF NOT EXISTS mydb.page_view - |(viewTime INT, - |userid BIGINT, - |page_url STRING, - |referrer_url STRING, - |ip STRING COMMENT 'IP Address of the User', - |country STRING COMMENT 'country of origination') |COMMENT 'This is the staging page view table' |ROW FORMAT SERDE 'parquet.hive.serde.ParquetHiveSerDe' | STORED AS