From 9f8b889cade55fbc8ee4068413e319f97739a939 Mon Sep 17 00:00:00 2001 From: Chris Zhao Date: Tue, 18 Sep 2018 16:34:57 -0700 Subject: [PATCH 1/4] add viewOriginalText back to CatalogTable --- .../spark/sql/catalyst/catalog/interface.scala | 4 +++- .../apache/spark/sql/execution/command/views.scala | 2 ++ .../resources/sql-tests/results/describe.sql.out | 2 ++ .../results/higher-order-functions.sql.out | 1 + .../spark/sql/hive/client/HiveClientImpl.scala | 6 +++--- .../spark/sql/hive/execution/HiveDDLSuite.scala | 13 +++++++++++++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 30ded13410f7c..c09df6485efaa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -244,7 +244,8 @@ case class CatalogTable( unsupportedFeatures: Seq[String] = Seq.empty, tracksPartitionsInCatalog: Boolean = false, schemaPreservesCase: Boolean = true, - ignoredProperties: Map[String, String] = Map.empty) { + ignoredProperties: Map[String, String] = Map.empty, + viewOriginalText: Option[String] = None) { import CatalogTable._ @@ -331,6 +332,7 @@ case class CatalogTable( comment.foreach(map.put("Comment", _)) if (tableType == CatalogTableType.VIEW) { viewText.foreach(map.put("View Text", _)) + viewOriginalText.foreach(map.put("View Original Text:", _)) viewDefaultDatabase.foreach(map.put("View Default Database", _)) if (viewQueryColumnNames.nonEmpty) { map.put("View Query Output Columns", viewQueryColumnNames.mkString("[", ", ", "]")) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala index 5172f32ec7b9c..cd34dfafd1320 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala @@ -242,6 +242,7 @@ case class CreateViewCommand( storage = CatalogStorageFormat.empty, schema = aliasPlan(session, analyzedPlan).schema, properties = newProperties, + viewOriginalText = originalText, viewText = originalText, comment = comment ) @@ -299,6 +300,7 @@ case class AlterViewAsCommand( val updatedViewMeta = viewMeta.copy( schema = analyzedPlan.schema, properties = newProperties, + viewOriginalText = Some(originalText), viewText = Some(originalText)) session.sessionState.catalog.alterTable(updatedViewMeta) diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 79390cb424444..a5c0dd70c7e18 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -474,6 +474,7 @@ Last Access [not included in comparison] Created By [not included in comparison] Type VIEW View Text SELECT * FROM t +View Original Text: SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] @@ -497,6 +498,7 @@ Last Access [not included in comparison] Created By [not included in comparison] Type VIEW View Text SELECT * FROM t +View Original Text: SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] diff --git a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out index 32d20d1b73415..1b7c6f4f76250 100644 --- a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out @@ -201,6 +201,7 @@ struct<> -- !query 20 output + -- !query 21 select transform_keys(ys, (k, v) -> k) as v from nested -- !query 21 schema diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 02c1ed93eb2f8..ca91b63473e3c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -467,9 +467,9 @@ private[hive] class HiveClientImpl( properties = filteredProperties, stats = readHiveStats(properties), comment = comment, - // In older versions of Spark(before 2.2.0), we expand the view original text and store - // that into `viewExpandedText`, and that should be used in view resolution. So we get - // `viewExpandedText` instead of `viewOriginalText` for viewText here. + // We need to add `viewOriginalText` back to display the original view DDL + // in `DESC [EXTENDED|FORMATTED] table` command. + viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), unsupportedFeatures = unsupportedFeatures, ignoredProperties = ignoredProperties.toMap) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index be1aa83d682b2..f191d4665e1dc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -2348,4 +2348,17 @@ class HiveDDLSuite } } } + + test("desc formatted table should also show viewOriginalText for views") { + withView("v1") { + sql("CREATE VIEW v1 AS SELECT 1 AS value") + assert(sql("DESC FORMATTED v1").collect().containsSlice( + Seq( + Row("Type", "VIEW", ""), + Row("View Text", "SELECT 1 AS value", ""), + Row("View Original Text:", "SELECT 1 AS value", "") + ) + )) + } + } } From f3d3100399be442da9fd5e417aeefb9662903c49 Mon Sep 17 00:00:00 2001 From: Chris Zhao Date: Thu, 20 Sep 2018 22:51:13 -0700 Subject: [PATCH 2/4] nit and combine comments --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index ca91b63473e3c..5e9b324a168e0 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -467,8 +467,11 @@ private[hive] class HiveClientImpl( properties = filteredProperties, stats = readHiveStats(properties), comment = comment, - // We need to add `viewOriginalText` back to display the original view DDL - // in `DESC [EXTENDED|FORMATTED] table` command. + // In older versions of Spark(before 2.2.0), we expand the view original text and + // store that into `viewExpandedText`, that should be used in view resolution. + // We get `viewExpandedText` as viewText, and also get `viewOriginalText` in order to + // display the original view text in `DESC [EXTENDED|FORMATTED] table` command for views + // that created by older versions of Spark. viewOriginalText = Option(h.getViewOriginalText), viewText = Option(h.getViewExpandedText), unsupportedFeatures = unsupportedFeatures, From ee5fec6872b4a81bc44abbdd9d4cde9d3b575f83 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 27 Sep 2018 13:09:51 -0700 Subject: [PATCH 3/4] Add a test case for Hive created view. --- .../spark/sql/hive/execution/HiveDDLSuite.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index f191d4665e1dc..d01d98df08299 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -2350,7 +2350,7 @@ class HiveDDLSuite } test("desc formatted table should also show viewOriginalText for views") { - withView("v1") { + withView("v1", "v2") { sql("CREATE VIEW v1 AS SELECT 1 AS value") assert(sql("DESC FORMATTED v1").collect().containsSlice( Seq( @@ -2359,6 +2359,15 @@ class HiveDDLSuite Row("View Original Text:", "SELECT 1 AS value", "") ) )) + + hiveClient.runSqlHive("CREATE VIEW v2 AS SELECT * FROM (SELECT 1) T") + assert(sql("DESC FORMATTED v2").collect().containsSlice( + Seq( + Row("Type", "VIEW", ""), + Row("View Text", "SELECT `t`.`_c0` FROM (SELECT 1) `T`", ""), + Row("View Original Text:", "SELECT * FROM (SELECT 1) T", "") + ) + )) } } } From ed2d786f071f8ce12bad6876b8e3411eaa6a291b Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 27 Sep 2018 14:49:59 -0700 Subject: [PATCH 4/4] Remove colon --- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 2 +- .../src/test/resources/sql-tests/results/describe.sql.out | 4 ++-- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index c09df6485efaa..817abebd72ac0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -332,7 +332,7 @@ case class CatalogTable( comment.foreach(map.put("Comment", _)) if (tableType == CatalogTableType.VIEW) { viewText.foreach(map.put("View Text", _)) - viewOriginalText.foreach(map.put("View Original Text:", _)) + viewOriginalText.foreach(map.put("View Original Text", _)) viewDefaultDatabase.foreach(map.put("View Default Database", _)) if (viewQueryColumnNames.nonEmpty) { map.put("View Query Output Columns", viewQueryColumnNames.mkString("[", ", ", "]")) diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index a5c0dd70c7e18..9c4b70d1b1ab7 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -474,7 +474,7 @@ Last Access [not included in comparison] Created By [not included in comparison] Type VIEW View Text SELECT * FROM t -View Original Text: SELECT * FROM t +View Original Text SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] @@ -498,7 +498,7 @@ Last Access [not included in comparison] Created By [not included in comparison] Type VIEW View Text SELECT * FROM t -View Original Text: SELECT * FROM t +View Original Text SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index d01d98df08299..fd38944a5dd2e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -2356,7 +2356,7 @@ class HiveDDLSuite Seq( Row("Type", "VIEW", ""), Row("View Text", "SELECT 1 AS value", ""), - Row("View Original Text:", "SELECT 1 AS value", "") + Row("View Original Text", "SELECT 1 AS value", "") ) )) @@ -2365,7 +2365,7 @@ class HiveDDLSuite Seq( Row("Type", "VIEW", ""), Row("View Text", "SELECT `t`.`_c0` FROM (SELECT 1) `T`", ""), - Row("View Original Text:", "SELECT * FROM (SELECT 1) T", "") + Row("View Original Text", "SELECT * FROM (SELECT 1) T", "") ) )) }