From 40f28965f48c2de2cb02eb5a26a52a4ca27bda5b Mon Sep 17 00:00:00 2001 From: windpiger Date: Tue, 7 Feb 2017 14:00:46 +0800 Subject: [PATCH 1/8] [SPARK-19484][SQL]continue work to create hive table with an empty schema --- .../spark/sql/hive/client/HiveClientImpl.scala | 5 +++-- ...veExternalCatalogBackwardCompatibilitySuite.scala | 12 +----------- .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 8 +++++--- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index bf703a5ab6e60..1d72df530be8f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -46,7 +46,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException} import org.apache.spark.sql.execution.QueryExecutionException -import org.apache.spark.sql.hive.HiveUtils +import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} import org.apache.spark.util.{CircularBuffer, Utils} @@ -832,7 +832,8 @@ private[hive] class HiveClientImpl( val (partCols, schema) = table.schema.map(toHiveColumn).partition { c => table.partitionColumnNames.contains(c.getName) } - if (schema.isEmpty) { + if (schema.isEmpty&& table.properties.getOrElse( + HiveExternalCatalog.DATASOURCE_SCHEMA_NUMPARTS, "0").toInt != 0) { // This is a hack to preserve existing behavior. Before Spark 2.0, we do not // set a default serde here (this was done in Hive), and so if the user provides // an empty schema Hive would automatically populate the schema with a single diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala index 00fdfbcebbe85..990d293311dfb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala @@ -187,15 +187,6 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest "spark.sql.sources.schema.numParts" -> "1", "spark.sql.sources.schema.part.0" -> simpleSchemaJson)) - lazy val dataSourceTableWithoutSchema = CatalogTable( - identifier = TableIdentifier("tbl9", Some("test_db")), - tableType = CatalogTableType.EXTERNAL, - storage = CatalogStorageFormat.empty.copy( - locationUri = Some(defaultTableURI("tbl9").toString + "-__PLACEHOLDER__"), - properties = Map("path" -> tempDirUri)), - schema = new StructType(), - properties = Map("spark.sql.sources.provider" -> "json")) - // A list of all raw tables we want to test, with their expected schema. lazy val rawTablesAndExpectations = Seq( hiveTable -> simpleSchema, @@ -205,8 +196,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest hiveCompatibleDataSourceTable -> simpleSchema, partitionedDataSourceTable -> partitionedSchema, externalDataSourceTable -> simpleSchema, - hiveCompatibleExternalDataSourceTable -> simpleSchema, - dataSourceTableWithoutSchema -> new StructType()) + hiveCompatibleExternalDataSourceTable -> simpleSchema) test("make sure we can read table created by old version of Spark") { for ((tbl, expectedSchema) <- rawTablesAndExpectations) { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index c262095df65b4..c28a1206df80d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -46,6 +46,8 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv var jsonFilePath: String = _ + val simpleSchema = new StructType().add("i", "int") + override def beforeAll(): Unit = { super.beforeAll() jsonFilePath = Utils.getSparkClassLoader.getResource("sample.json").getFile @@ -746,7 +748,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier(tableName, Some("default")), tableType = CatalogTableType.MANAGED, - schema = new StructType, + schema = simpleSchema, storage = CatalogStorageFormat( locationUri = None, inputFormat = None, @@ -1274,7 +1276,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier("t", Some("default")), tableType = CatalogTableType.MANAGED, - schema = new StructType, + schema = simpleSchema, storage = CatalogStorageFormat.empty, properties = Map( DATASOURCE_PROVIDER -> "json", @@ -1328,7 +1330,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv storage = CatalogStorageFormat.empty.copy( properties = Map("path" -> path.getAbsolutePath) ), - schema = new StructType(), + schema = simpleSchema, properties = Map( HiveExternalCatalog.DATASOURCE_PROVIDER -> "parquet")) hiveClient.createTable(tableDesc, ignoreIfExists = false) From 56a01e71bfe845077a48e8a00f7a7e1c304301fe Mon Sep 17 00:00:00 2001 From: windpiger Date: Tue, 7 Feb 2017 17:41:30 +0800 Subject: [PATCH 2/8] add a new property to indicate to create an old spark source table --- .../apache/spark/sql/hive/HiveExternalCatalog.scala | 3 +++ .../spark/sql/hive/client/HiveClientImpl.scala | 9 +++++++-- ...eExternalCatalogBackwardCompatibilitySuite.scala | 13 ++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 208c8c9d5d0cf..3466844ebe030 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -1057,6 +1057,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } object HiveExternalCatalog { + // a property for test, to create an old spark source table + val SPARK_TEST_OLD_SOURCE_TABLE_CREATE = "spark.test.old.source.table.create" + val SPARK_SQL_PREFIX = "spark.sql." val DATASOURCE_PREFIX = SPARK_SQL_PREFIX + "sources." diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 1d72df530be8f..c2211039b4a29 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -832,8 +832,13 @@ private[hive] class HiveClientImpl( val (partCols, schema) = table.schema.map(toHiveColumn).partition { c => table.partitionColumnNames.contains(c.getName) } - if (schema.isEmpty&& table.properties.getOrElse( - HiveExternalCatalog.DATASOURCE_SCHEMA_NUMPARTS, "0").toInt != 0) { + + // after SPARK-19279, it is not allowed to create a table with an empty schema, but we should + // support it when test some HiveExternalCatalog BackwardCompatibility cases to create an old + // spark table. the SPARK_TEST_OLD_SOURCE_TABLE_CREATE property is used to resolve this. + if (schema.isEmpty&& (table.properties.getOrElse( + HiveExternalCatalog.DATASOURCE_SCHEMA_NUMPARTS, "0").toInt != 0) || table.properties + .getOrElse(HiveExternalCatalog.SPARK_TEST_OLD_SOURCE_TABLE_CREATE, "false").toBoolean) { // This is a hack to preserve existing behavior. Before Spark 2.0, we do not // set a default serde here (this was done in Hive), and so if the user provides // an empty schema Hive would automatically populate the schema with a single diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala index 990d293311dfb..bd90bd9c3b57e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala @@ -187,6 +187,16 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest "spark.sql.sources.schema.numParts" -> "1", "spark.sql.sources.schema.part.0" -> simpleSchemaJson)) + lazy val dataSourceTableWithoutSchema = CatalogTable( + identifier = TableIdentifier("tbl9", Some("test_db")), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat.empty.copy( + locationUri = Some(defaultTableURI("tbl9").toString + "-__PLACEHOLDER__"), + properties = Map("path" -> tempDirUri)), + schema = new StructType(), + properties = Map("spark.sql.sources.provider" -> "json", + HiveExternalCatalog.SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) + // A list of all raw tables we want to test, with their expected schema. lazy val rawTablesAndExpectations = Seq( hiveTable -> simpleSchema, @@ -196,7 +206,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest hiveCompatibleDataSourceTable -> simpleSchema, partitionedDataSourceTable -> partitionedSchema, externalDataSourceTable -> simpleSchema, - hiveCompatibleExternalDataSourceTable -> simpleSchema) + hiveCompatibleExternalDataSourceTable -> simpleSchema, + dataSourceTableWithoutSchema -> new StructType()) test("make sure we can read table created by old version of Spark") { for ((tbl, expectedSchema) <- rawTablesAndExpectations) { From 02f3147f68171ca87a616a1393aa78984428173c Mon Sep 17 00:00:00 2001 From: windpiger Date: Tue, 7 Feb 2017 19:37:44 +0800 Subject: [PATCH 3/8] fix a code style --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index c2211039b4a29..385a062294ae8 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -836,7 +836,7 @@ private[hive] class HiveClientImpl( // after SPARK-19279, it is not allowed to create a table with an empty schema, but we should // support it when test some HiveExternalCatalog BackwardCompatibility cases to create an old // spark table. the SPARK_TEST_OLD_SOURCE_TABLE_CREATE property is used to resolve this. - if (schema.isEmpty&& (table.properties.getOrElse( + if (schema.isEmpty && (table.properties.getOrElse( HiveExternalCatalog.DATASOURCE_SCHEMA_NUMPARTS, "0").toInt != 0) || table.properties .getOrElse(HiveExternalCatalog.SPARK_TEST_OLD_SOURCE_TABLE_CREATE, "false").toBoolean) { // This is a hack to preserve existing behavior. Before Spark 2.0, we do not From c4e54c1e6119640023da87a82c7313dd8479c8b9 Mon Sep 17 00:00:00 2001 From: windpiger Date: Tue, 7 Feb 2017 23:59:43 +0800 Subject: [PATCH 4/8] modify test case --- .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index c28a1206df80d..afee4f409c458 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -748,7 +748,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier(tableName, Some("default")), tableType = CatalogTableType.MANAGED, - schema = simpleSchema, + schema = new StructType(), storage = CatalogStorageFormat( locationUri = None, inputFormat = None, @@ -761,6 +761,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv properties = Map( DATASOURCE_PROVIDER -> "json", DATASOURCE_SCHEMA -> schema.json, + SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true", "EXTERNAL" -> "FALSE")) hiveClient.createTable(hiveTable, ignoreIfExists = false) @@ -1276,12 +1277,13 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier("t", Some("default")), tableType = CatalogTableType.MANAGED, - schema = simpleSchema, + schema = new StructType(), storage = CatalogStorageFormat.empty, properties = Map( DATASOURCE_PROVIDER -> "json", // no DATASOURCE_SCHEMA_NUMPARTS - DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json)) + DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json, + SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) hiveClient.createTable(hiveTable, ignoreIfExists = false) @@ -1330,9 +1332,10 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv storage = CatalogStorageFormat.empty.copy( properties = Map("path" -> path.getAbsolutePath) ), - schema = simpleSchema, + schema = new StructType(), properties = Map( - HiveExternalCatalog.DATASOURCE_PROVIDER -> "parquet")) + DATASOURCE_PROVIDER -> "parquet", + SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) hiveClient.createTable(tableDesc, ignoreIfExists = false) checkAnswer(spark.table("old"), Row(1, "a")) From 58d1b1fe8b29e5aea313c54d0d0a08ca70466624 Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 8 Feb 2017 00:01:54 +0800 Subject: [PATCH 5/8] fix code style --- .../apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index afee4f409c458..76c420bbf03c1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -46,8 +46,6 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv var jsonFilePath: String = _ - val simpleSchema = new StructType().add("i", "int") - override def beforeAll(): Unit = { super.beforeAll() jsonFilePath = Utils.getSparkClassLoader.getResource("sample.json").getFile @@ -748,7 +746,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier(tableName, Some("default")), tableType = CatalogTableType.MANAGED, - schema = new StructType(), + schema = new StructType, storage = CatalogStorageFormat( locationUri = None, inputFormat = None, @@ -1277,7 +1275,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv val hiveTable = CatalogTable( identifier = TableIdentifier("t", Some("default")), tableType = CatalogTableType.MANAGED, - schema = new StructType(), + schema = new StructType, storage = CatalogStorageFormat.empty, properties = Map( DATASOURCE_PROVIDER -> "json", @@ -1332,7 +1330,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv storage = CatalogStorageFormat.empty.copy( properties = Map("path" -> path.getAbsolutePath) ), - schema = new StructType(), + schema = new StructType, properties = Map( DATASOURCE_PROVIDER -> "parquet", SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) From 8fba58d9f764babf717493923cd634f3d479a4cd Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 8 Feb 2017 00:04:27 +0800 Subject: [PATCH 6/8] fix code style --- .../org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 76c420bbf03c1..50f36e59e4acc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -1330,7 +1330,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv storage = CatalogStorageFormat.empty.copy( properties = Map("path" -> path.getAbsolutePath) ), - schema = new StructType, + schema = new StructType(), properties = Map( DATASOURCE_PROVIDER -> "parquet", SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) From cf2c38252c2c8515d92a7b4a70ec312170cdb8d1 Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 8 Feb 2017 10:56:46 +0800 Subject: [PATCH 7/8] just for hive serde table to check empty schema --- .../apache/spark/sql/hive/HiveExternalCatalog.scala | 3 --- .../spark/sql/hive/client/HiveClientImpl.scala | 12 +++++------- ...veExternalCatalogBackwardCompatibilitySuite.scala | 3 +-- .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 7 ++----- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 3466844ebe030..208c8c9d5d0cf 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -1057,9 +1057,6 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } object HiveExternalCatalog { - // a property for test, to create an old spark source table - val SPARK_TEST_OLD_SOURCE_TABLE_CREATE = "spark.test.old.source.table.create" - val SPARK_SQL_PREFIX = "spark.sql." val DATASOURCE_PREFIX = SPARK_SQL_PREFIX + "sources." diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 385a062294ae8..a1fbe04d57d94 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -46,7 +46,8 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException} import org.apache.spark.sql.execution.QueryExecutionException -import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} +import org.apache.spark.sql.execution.command.DDLUtils +import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType} import org.apache.spark.util.{CircularBuffer, Utils} @@ -833,12 +834,9 @@ private[hive] class HiveClientImpl( table.partitionColumnNames.contains(c.getName) } - // after SPARK-19279, it is not allowed to create a table with an empty schema, but we should - // support it when test some HiveExternalCatalog BackwardCompatibility cases to create an old - // spark table. the SPARK_TEST_OLD_SOURCE_TABLE_CREATE property is used to resolve this. - if (schema.isEmpty && (table.properties.getOrElse( - HiveExternalCatalog.DATASOURCE_SCHEMA_NUMPARTS, "0").toInt != 0) || table.properties - .getOrElse(HiveExternalCatalog.SPARK_TEST_OLD_SOURCE_TABLE_CREATE, "false").toBoolean) { + // after SPARK-19279, it is not allowed to create a hive table with an empty schema, + // so here we should not add a default col schema + if (schema.isEmpty && !DDLUtils.isHiveTable(table)) { // This is a hack to preserve existing behavior. Before Spark 2.0, we do not // set a default serde here (this was done in Hive), and so if the user provides // an empty schema Hive would automatically populate the schema with a single diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala index bd90bd9c3b57e..00fdfbcebbe85 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala @@ -194,8 +194,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest locationUri = Some(defaultTableURI("tbl9").toString + "-__PLACEHOLDER__"), properties = Map("path" -> tempDirUri)), schema = new StructType(), - properties = Map("spark.sql.sources.provider" -> "json", - HiveExternalCatalog.SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) + properties = Map("spark.sql.sources.provider" -> "json")) // A list of all raw tables we want to test, with their expected schema. lazy val rawTablesAndExpectations = Seq( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 50f36e59e4acc..3132207a4a9f4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -759,7 +759,6 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv properties = Map( DATASOURCE_PROVIDER -> "json", DATASOURCE_SCHEMA -> schema.json, - SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true", "EXTERNAL" -> "FALSE")) hiveClient.createTable(hiveTable, ignoreIfExists = false) @@ -1280,8 +1279,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv properties = Map( DATASOURCE_PROVIDER -> "json", // no DATASOURCE_SCHEMA_NUMPARTS - DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json, - SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) + DATASOURCE_SCHEMA_PART_PREFIX + 0 -> schema.json)) hiveClient.createTable(hiveTable, ignoreIfExists = false) @@ -1332,8 +1330,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv ), schema = new StructType(), properties = Map( - DATASOURCE_PROVIDER -> "parquet", - SPARK_TEST_OLD_SOURCE_TABLE_CREATE -> "true")) + DATASOURCE_PROVIDER -> "parquet")) hiveClient.createTable(tableDesc, ignoreIfExists = false) checkAnswer(spark.table("old"), Row(1, "a")) From cc5141a67ee9c0884c089e309746025e95ce1e0d Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 8 Feb 2017 12:49:08 +0800 Subject: [PATCH 8/8] change isHiveTable to isDataSourceTable --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index a1fbe04d57d94..b89e45fd4cec9 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -836,7 +836,7 @@ private[hive] class HiveClientImpl( // after SPARK-19279, it is not allowed to create a hive table with an empty schema, // so here we should not add a default col schema - if (schema.isEmpty && !DDLUtils.isHiveTable(table)) { + if (schema.isEmpty && DDLUtils.isDatasourceTable(table)) { // This is a hack to preserve existing behavior. Before Spark 2.0, we do not // set a default serde here (this was done in Hive), and so if the user provides // an empty schema Hive would automatically populate the schema with a single