From c3937dbdf264484dc4cd85ea5b68b6ef526a8ea2 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Tue, 24 Mar 2026 19:53:10 +0900 Subject: [PATCH 1/3] Revert "[SPARK-55957][SQL] Add 'DATA_SOURCE_NOT_FOUND' in Catalog.ERROR_HANDLING_RULES" This reverts commit 305cdc326f602d1109dececd863258c6f4b22217. --- .../apache/spark/sql/classic/Catalog.scala | 3 +-- .../spark/sql/internal/CatalogSuite.scala | 20 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 251877842bcee..0b41c47cbe31f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -979,8 +979,7 @@ private[sql] object Catalog { val ERROR_HANDLING_RULES: Map[String, ErrorHandlingAction] = Map( "UNSUPPORTED_FEATURE.HIVE_TABLE_TYPE" -> ReturnPartialResults, - "TABLE_OR_VIEW_NOT_FOUND" -> Skip, - "DATA_SOURCE_NOT_FOUND" -> ReturnPartialResults + "TABLE_OR_VIEW_NOT_FOUND" -> Skip ) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala index 7e064a7dab902..ebfffc14b0144 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala @@ -1136,26 +1136,6 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf } } - test("SPARK-55957: listTables returns partial results when a table has DATA_SOURCE_NOT_FOUND") { - // Create a normal table (resolvable) - createTable("good_table") - // Create a table with a non-existent provider so resolution throws DATA_SOURCE_NOT_FOUND - val badTableMeta = utils.newTable("bad_table", None).copy( - provider = Some("non.existent.ProviderClass")) - sessionCatalog.createTable(badTableMeta, ignoreIfExists = false) - // Without ERROR_HANDLING_RULES for DATA_SOURCE_NOT_FOUND, listTables() would throw. - // With the fix, we get partial results: good_table fully resolved, bad_table as placeholder. - val tables = spark.catalog.listTables().collect() - assert(tables.length == 2, s"Expected 2 tables, got: ${tables.map(_.name).toList}") - val names = tables.map(_.name).toSet - assert(names == Set("good_table", "bad_table"), - s"Expected good_table and bad_table, got: $names") - val goodTable = tables.find(_.name == "good_table").get - val badTable = tables.find(_.name == "bad_table").get - assert(goodTable.tableType != null, "good_table should be fully resolved") - assert(badTable.tableType == null, "bad_table should be partial result (tableType null)") - } - private def getConstructorParameterValues(obj: DefinedByConstructorParams): Seq[AnyRef] = { ScalaReflection.getConstructorParameterNames(obj.getClass).map { name => obj.getClass.getMethod(name).invoke(obj) From c704b4046463cf5c2a7025b20caf71ffa4f5e966 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Tue, 24 Mar 2026 19:53:28 +0900 Subject: [PATCH 2/3] Revert "[SPARK-51899][SQL] Implement error handling rules for spark.catalog.listTables()" This reverts commit 439153819e3a5a586f5bccc28f676b08f7204f05. --- .../apache/spark/sql/classic/Catalog.scala | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 0b41c47cbe31f..7abebe65d8038 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -20,7 +20,8 @@ package org.apache.spark.sql.classic import scala.reflect.runtime.universe.TypeTag import scala.util.control.NonFatal -import org.apache.spark.SparkThrowable +import org.apache.spark.internal.{Logging, MDC} +import org.apache.spark.internal.LogKeys.{CATALOG_NAME, DATABASE_NAME, TABLE_NAME} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalog import org.apache.spark.sql.catalog.{CatalogMetadata, Column, Database, Function, Table} @@ -51,7 +52,7 @@ import org.apache.spark.util.ArrayImplicits._ /** * Internal implementation of the user-facing `Catalog`. */ -class Catalog(sparkSession: SparkSession) extends catalog.Catalog { +class Catalog(sparkSession: SparkSession) extends catalog.Catalog with Logging { private def sessionCatalog: SessionCatalog = sparkSession.sessionState.catalog @@ -185,20 +186,25 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog { try { Some(makeTable(nameParts)) } catch { - case e: SparkThrowable with Throwable => - Catalog.ListTable.ERROR_HANDLING_RULES.get(e.getCondition) match { - case Some(Catalog.ListTable.Skip) => None - case Some(Catalog.ListTable.ReturnPartialResults) if !isTempView => - Some(new Table( - name = tableName, - catalog = catalogName, - namespace = ns.toArray, - description = null, - tableType = null, - isTemporary = false - )) - case _ => throw e - } + case e: AnalysisException if e.getCondition == "TABLE_OR_VIEW_NOT_FOUND" => None + // Swallow non-fatal throwables when resolving a table or view and + // return a table or view with partial results to + // prevent listTables from breaking easily due to a broken table or view. + case NonFatal(e) if !isTempView => + val table = new Table( + name = tableName, + catalog = catalogName, + namespace = ns.toArray, + description = null, + tableType = null, + isTemporary = false + ) + logWarning(log"Unable to resolve the table or view [" + + log"catalog=${MDC(CATALOG_NAME, table.catalog)}, " + + log"database=${MDC(DATABASE_NAME, table.database)}, " + + log"name=${MDC(TABLE_NAME, table.name)}" + + log"]; partial results will be returned.", e) + Some(table) } } @@ -968,18 +974,4 @@ private[sql] object Catalog { } private val FUNCTION_EXISTS_COMMAND_NAME = "Catalog.functionExists" - - private object ListTable { - - sealed trait ErrorHandlingAction - - case object Skip extends ErrorHandlingAction - - case object ReturnPartialResults extends ErrorHandlingAction - - val ERROR_HANDLING_RULES: Map[String, ErrorHandlingAction] = Map( - "UNSUPPORTED_FEATURE.HIVE_TABLE_TYPE" -> ReturnPartialResults, - "TABLE_OR_VIEW_NOT_FOUND" -> Skip - ) - } } From e151035350d1c418c749340f2e175f6e7375db0e Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 25 Mar 2026 07:05:41 +0900 Subject: [PATCH 3/3] Compile error --- .../src/main/scala/org/apache/spark/sql/classic/Catalog.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 7abebe65d8038..c84f3c15ed36b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.classic import scala.reflect.runtime.universe.TypeTag import scala.util.control.NonFatal -import org.apache.spark.internal.{Logging, MDC} +import org.apache.spark.internal.Logging import org.apache.spark.internal.LogKeys.{CATALOG_NAME, DATABASE_NAME, TABLE_NAME} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalog