From f7e4675a77a0d220e4b73dd5e54e4dba751c781e Mon Sep 17 00:00:00 2001 From: Alex Wiss-Wolferding Date: Mon, 30 Jan 2023 12:49:32 -0600 Subject: [PATCH 1/4] Using DB and table name when checking Delta table schema. --- .../org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala index 5977cd415b0..1aa322d3154 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala @@ -141,7 +141,7 @@ class CatalogShim_v2_4 extends SparkCatalogShim { catalog.getTablesByName(identifiers).flatMap { t => val tableSchema = if (t.provider.getOrElse("").equalsIgnoreCase("delta")) { - spark.table(t.identifier.table).schema + spark.table(f"${db}.${t.identifier.table}").schema } else { t.schema } From 20337dc968e1c9c751eb68d10303c3631c5fe49b Mon Sep 17 00:00:00 2001 From: Alex Wiss-Wolferding Date: Sat, 4 Feb 2023 13:39:33 -0600 Subject: [PATCH 2/4] Revert "Using DB and table name when checking Delta table schema." This reverts commit f7e4675a77a0d220e4b73dd5e54e4dba751c781e. --- .../org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala index 1aa322d3154..5977cd415b0 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala @@ -141,7 +141,7 @@ class CatalogShim_v2_4 extends SparkCatalogShim { catalog.getTablesByName(identifiers).flatMap { t => val tableSchema = if (t.provider.getOrElse("").equalsIgnoreCase("delta")) { - spark.table(f"${db}.${t.identifier.table}").schema + spark.table(t.identifier.table).schema } else { t.schema } From a6d973a3e5b1533c952a07b71115b82b2fa6295a Mon Sep 17 00:00:00 2001 From: Alex Wiss-Wolferding Date: Sat, 4 Feb 2023 13:42:54 -0600 Subject: [PATCH 3/4] Revert "[KYUUBI #1458] Delta lake table columns won't show up in DBeaver." This reverts commit b87d206ffb5da16b7d7b7d401be053b0e78c7a99. --- .../kyuubi/engine/spark/shim/CatalogShim_v2_4.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala index 5977cd415b0..3478abc6639 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala @@ -139,13 +139,7 @@ class CatalogShim_v2_4 extends SparkCatalogShim { databases.flatMap { db => val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = true) catalog.getTablesByName(identifiers).flatMap { t => - val tableSchema = - if (t.provider.getOrElse("").equalsIgnoreCase("delta")) { - spark.table(t.identifier.table).schema - } else { - t.schema - } - tableSchema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) + t.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) .map { case (f, i) => toColumnResult(catalogName, t.database, t.identifier.table, f, i) } } } From 569843213673b7221bb7b6483f8a53af8f6f8f4f Mon Sep 17 00:00:00 2001 From: Alex Wiss-Wolferding Date: Sat, 4 Feb 2023 13:58:43 -0600 Subject: [PATCH 4/4] Reversing match order in getColumnsByCatalog. --- .../engine/spark/shim/CatalogShim_v3_0.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala index d60f94ac755..50e641b59b2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala @@ -188,14 +188,6 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 { val catalog = getCatalog(spark, catalogName) catalog match { - case builtin if builtin.name() == SESSION_CATALOG => - super.getColumnsByCatalog( - spark, - SESSION_CATALOG, - schemaPattern, - tablePattern, - columnPattern) - case tc: TableCatalog => val namespaces = listNamespacesWithPattern(catalog, schemaPattern) val tp = tablePattern.r.pattern @@ -210,6 +202,14 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 { table.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) .map { case (f, i) => toColumnResult(tc.name(), namespace, tableName, f, i) } } + + case builtin if builtin.name() == SESSION_CATALOG => + super.getColumnsByCatalog( + spark, + SESSION_CATALOG, + schemaPattern, + tablePattern, + columnPattern) } } }