From 2a0a4743a0502dc727ff7aaaa0b5ccd3582e2545 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Apr 2016 10:45:13 +0000 Subject: [PATCH 1/7] init import. --- .../spark/sql/catalyst/parser/SqlBase.g4 | 6 +- .../spark/sql/execution/command/tables.scala | 63 +++++++++++++++++++ .../sql/hive/execution/HiveSqlParser.scala | 29 +++++++++ 3 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 0e2cd39448bba..73bc2c354363d 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -55,6 +55,7 @@ statement rowFormat? createFileFormat? locationSpec? (TBLPROPERTIES tablePropertyList)? (AS? query)? #createTable + | createTableHeader LIKE source=tableIdentifier #createTableLike | ANALYZE TABLE tableIdentifier partitionSpec? COMPUTE STATISTICS (identifier | FOR COLUMNS identifierSeq?)? #analyze | ALTER (TABLE | VIEW) from=tableIdentifier @@ -136,10 +137,7 @@ statement ; hiveNativeCommands - : createTableHeader LIKE tableIdentifier - rowFormat? createFileFormat? locationSpec? - (TBLPROPERTIES tablePropertyList)? - | DELETE FROM tableIdentifier (WHERE booleanExpression)? + : DELETE FROM tableIdentifier (WHERE booleanExpression)? | TRUNCATE TABLE tableIdentifier partitionSpec? (COLUMNS identifierList)? | SHOW COLUMNS (FROM | IN) tableIdentifier ((FROM|IN) identifier)? diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala new file mode 100644 index 0000000000000..4c94a485eb6c7 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.{Row, SQLContext} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +/** + * A command to create a table with the same definition of the given existing table. + * + * The syntax of using this command in SQL is: + * {{{ + * CREATE TABLE [db_name.]table_name + * LIKE [other_db_name.]existing_table_name + * }}} + */ +case class CreateTableLike( + table: CatalogTable, + sourceTable: TableIdentifier) extends RunnableCommand { + + override def run(sqlContext: SQLContext): Seq[Row] = { + sqlContext.sessionState.catalog.createTable(table, ifNotExists) + + val catalog = sqlContext.sessionState.catalog + if (!catalog.tableExists(sourceTable)) { + throw new AnalysisException( + s"Cannot create table like on not existing table ${sourceTable.identifier}.") + } + + val sourceTableMetadata = catalog.getTableMetadata(sourceTable) + val tableToCreate = CatalogTable( + identifier = table.identifier, + tableType = table.tableType, + storage = sourceTableMetadata.storage, + schema = sourceTableMetadata.schema, + partitionColumns = sourceTableMetadata.partitionColumns, + sortColumns = sourceTableMetadata.sortColumns, + numBuckets = sourceTableMetadata.numBuckets, + properties = sourceTableMetadata.properties, + viewOriginalText = sourceTableMetadata.viewOriginalText, + viewText = sourceTableMetadata.viewText + ) + + catalog.createTable(tableToCreate, false) + Seq.empty[Row] + } +} diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala index 7a435117e7a82..638388abafbaa 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala @@ -202,6 +202,35 @@ class HiveSqlAstBuilder extends SparkSqlAstBuilder { } } + /** + * Create a [[CreateTableLike]] command. + */ + override def visitCreateTableLike(ctx: CreateTableContext): LogicalPlan = { + // Get the table header. + val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader) + val sourceTable = visitTableIdentifier(ctx.source) + + val tableType = if (external) { + throw new ParseException("Unsupported operation: EXTERNAL clause.", ctx) + } else { + CatalogTableType.MANAGED_TABLE + } + + // Unsupported clauses. + if (temp) { + throw new ParseException(s"Unsupported operation: TEMPORARY clause.", ctx) + } + if (ifNotExists) { + throw new ParseException("Unsupported operation: IF NOT EXISTS clause.", ctx) + } + val tableDesc = CatalogTable( + identifier = table, + tableType = tableType, + schema = Seq[CatalogColumn].empty, + storage = EmptyStorageFormat) + CreateTableLike(tableDesc, sourceTable) + } + /** * Create or replace a view. This creates a [[CreateViewAsSelect]] command. * From 6525695c4f7edad066a54c1f9dd17c9f4ad918db Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Apr 2016 15:17:36 +0000 Subject: [PATCH 2/7] Make it work. --- .../spark/sql/execution/command/tables.scala | 11 +++-- .../sql/hive/execution/HiveSqlParser.scala | 10 ++--- .../spark/sql/hive/HiveDDLCommandSuite.scala | 42 +++++++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 4c94a485eb6c7..f786c52857af4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command -import org.apache.spark.sql.{Row, SQLContext} +import org.apache.spark.sql.{AnalysisException, Row, SQLContext} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable @@ -32,16 +32,19 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable */ case class CreateTableLike( table: CatalogTable, + ifNotExists: Boolean, sourceTable: TableIdentifier) extends RunnableCommand { override def run(sqlContext: SQLContext): Seq[Row] = { - sqlContext.sessionState.catalog.createTable(table, ifNotExists) - val catalog = sqlContext.sessionState.catalog if (!catalog.tableExists(sourceTable)) { throw new AnalysisException( s"Cannot create table like on not existing table ${sourceTable.identifier}.") } + if (catalog.isTemporaryTable(sourceTable)) { + throw new AnalysisException( + s"Cannot create table like on a temporary table ${sourceTable.identifier}.") + } val sourceTableMetadata = catalog.getTableMetadata(sourceTable) val tableToCreate = CatalogTable( @@ -57,7 +60,7 @@ case class CreateTableLike( viewText = sourceTableMetadata.viewText ) - catalog.createTable(tableToCreate, false) + catalog.createTable(tableToCreate, ifNotExists) Seq.empty[Row] } } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala index 638388abafbaa..04f025c4f827c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.parser._ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.SparkSqlAstBuilder +import org.apache.spark.sql.execution.command.CreateTableLike import org.apache.spark.sql.hive.{CreateTableAsSelect => CTAS, CreateViewAsSelect => CreateView} import org.apache.spark.sql.hive.{HiveGenericUDTF, HiveSerDe} import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper @@ -205,7 +206,7 @@ class HiveSqlAstBuilder extends SparkSqlAstBuilder { /** * Create a [[CreateTableLike]] command. */ - override def visitCreateTableLike(ctx: CreateTableContext): LogicalPlan = { + override def visitCreateTableLike(ctx: CreateTableLikeContext): LogicalPlan = withOrigin(ctx) { // Get the table header. val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader) val sourceTable = visitTableIdentifier(ctx.source) @@ -220,15 +221,12 @@ class HiveSqlAstBuilder extends SparkSqlAstBuilder { if (temp) { throw new ParseException(s"Unsupported operation: TEMPORARY clause.", ctx) } - if (ifNotExists) { - throw new ParseException("Unsupported operation: IF NOT EXISTS clause.", ctx) - } val tableDesc = CatalogTable( identifier = table, tableType = tableType, - schema = Seq[CatalogColumn].empty, + schema = Seq.empty, storage = EmptyStorageFormat) - CreateTableLike(tableDesc, sourceTable) + CreateTableLike(tableDesc, ifNotExists, sourceTable) } /** diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala index e8086aec327bf..7487e02775f3b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.JsonTuple import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical.{Generate, ScriptTransformation} +import org.apache.spark.sql.execution.command.CreateTableLike import org.apache.spark.sql.hive.execution.{HiveNativeCommand, HiveSqlParser} class HiveDDLCommandSuite extends PlanTest { @@ -38,6 +39,7 @@ class HiveDDLCommandSuite extends PlanTest { parser.parsePlan(sql).collect { case CreateTableAsSelect(desc, _, allowExisting) => (desc, allowExisting) case CreateViewAsSelect(desc, _, allowExisting, _, _) => (desc, allowExisting) + case CreateTableLike(desc, allowExisting, _) => (desc, allowExisting) }.head } @@ -379,4 +381,44 @@ class HiveDDLCommandSuite extends PlanTest { assertUnsupported("MSCK REPAIR TABLE tab1") } + test("create table like") { + val v1 = "CREATE TABLE table1 LIKE table2" + val (desc, exists) = extractTableDesc(v1) + assert(exists == false) + assert(desc.identifier.database.isEmpty) + assert(desc.identifier.table == "table1") + assert(desc.tableType == CatalogTableType.MANAGED_TABLE) + assert(desc.storage.locationUri.isEmpty) + assert(desc.storage.serdeProperties == Map()) + assert(desc.storage.inputFormat.isEmpty) + assert(desc.storage.outputFormat.isEmpty) + assert(desc.storage.serde.isEmpty) + assert(desc.schema.isEmpty) + assert(desc.viewText == None) + assert(desc.viewOriginalText == None) + assert(desc.properties == Map()) + assert(desc.partitionColumns.isEmpty) + assert(desc.numBuckets == 0) + assert(desc.sortColumns.isEmpty) + + val v2 = "CREATE TABLE IF NOT EXISTS table1 LIKE table2" + val (desc2, exists2) = extractTableDesc(v2) + assert(exists2) + assert(desc2.identifier.database.isEmpty) + assert(desc2.identifier.table == "table1") + assert(desc2.tableType == CatalogTableType.MANAGED_TABLE) + assert(desc2.storage.locationUri.isEmpty) + assert(desc2.storage.serdeProperties == Map()) + assert(desc2.storage.inputFormat.isEmpty) + assert(desc2.storage.outputFormat.isEmpty) + assert(desc2.storage.serde.isEmpty) + assert(desc2.schema.isEmpty) + assert(desc2.viewText == None) + assert(desc2.viewOriginalText == None) + assert(desc2.properties == Map()) + assert(desc2.partitionColumns.isEmpty) + assert(desc2.numBuckets == 0) + assert(desc2.sortColumns.isEmpty) + } + } From 82a3ee864f3a7a6b5f46801e3e49a714e6e03f1e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Apr 2016 15:28:00 +0000 Subject: [PATCH 3/7] Reset storage location. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index f786c52857af4..c56e592e0743f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -58,7 +58,7 @@ case class CreateTableLike( properties = sourceTableMetadata.properties, viewOriginalText = sourceTableMetadata.viewOriginalText, viewText = sourceTableMetadata.viewText - ) + ).withNewStorage(locationUri = None) catalog.createTable(tableToCreate, ifNotExists) Seq.empty[Row] From a23559d0df74df30f08615489dbaa67aaa87e097 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Apr 2016 15:31:47 +0000 Subject: [PATCH 4/7] Modify comment. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index c56e592e0743f..c549b27aec25c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable * * The syntax of using this command in SQL is: * {{{ - * CREATE TABLE [db_name.]table_name + * CREATE TABLE [IF NOT EXISTS] [db_name.]table_name * LIKE [other_db_name.]existing_table_name * }}} */ From def90392f7a0dac9e6e69b9735acbc24c05c037e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Apr 2016 00:51:00 +0000 Subject: [PATCH 5/7] Fix it. --- .../apache/spark/sql/execution/command/tables.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 9d024d523d703..c652fd966bdd8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.command import org.apache.spark.sql.{AnalysisException, Row, SQLContext} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} /** * A command to create a table with the same definition of the given existing table. @@ -46,12 +46,9 @@ case class CreateTableLike( s"Source table in CREATE TABLE LIKE cannot be temporary: '$sourceTable'") } - if (catalog.tableExists(targetTable)) { - throw new AnalysisException( - s"Target table in CREATE TABLE LIKE already exists: '$targetTable'") - } - - val tableToCreate = catalog.getTableMetadata(sourceTable).copy( + val tableToCreate = catalog.getTableMetadata(sourceTable).copy( + identifier = targetTable, + tableType = CatalogTableType.MANAGED_TABLE, createTime = System.currentTimeMillis, lastAccessTime = -1).withNewStorage(locationUri = None) From 0bcafcd669e7f7b556c1523575c07c113d431c5a Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Apr 2016 00:51:39 +0000 Subject: [PATCH 6/7] Fix scala style. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index c652fd966bdd8..f7c51644fdc4a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -46,7 +46,7 @@ case class CreateTableLike( s"Source table in CREATE TABLE LIKE cannot be temporary: '$sourceTable'") } - val tableToCreate = catalog.getTableMetadata(sourceTable).copy( + val tableToCreate = catalog.getTableMetadata(sourceTable).copy( identifier = targetTable, tableType = CatalogTableType.MANAGED_TABLE, createTime = System.currentTimeMillis, From ec589b87c68343409d657d1a7c6ed82692a79ac6 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Apr 2016 02:38:39 +0000 Subject: [PATCH 7/7] Disable create_like_tbl_props. S --- .../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index a45d180464602..989e68aebed9b 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -416,6 +416,9 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "skewjoinopt18", "skewjoinopt9", + // This test tries to create a table like with TBLPROPERTIES clause, which we don't support. + "create_like_tbl_props", + // Index commands are not supported "drop_index", "drop_index_removes_partition_dirs", @@ -537,7 +540,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "count", "cp_mj_rc", "create_insert_outputformat", - "create_like_tbl_props", "create_nested_type", "create_struct_table", "create_view_translate",