From 7c930ab895a70bef692796ab52df6b8bf50eee7a Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Wed, 15 Apr 2026 10:58:19 +0000 Subject: [PATCH 1/2] [SPARK-43752][SQL] Remove dead code for column DEFAULT in V2 write commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code added by SPARK-43752 to resolve column DEFAULT in V2WriteCommand (AppendData, OverwriteByExpression, OverwritePartitionsDynamic) is dead code. For `INSERT INTO v2_table VALUES (1, DEFAULT)`, the resolution flow is: 1. The parser produces `InsertIntoStatement` (for both V1 and V2 tables). 2. `ResolveReferences` resolves DEFAULT via `resolveColumnDefaultInCommandInputQuery` while still in the `InsertIntoStatement` stage. 3. `ResolveInsertInto` only converts to `V2WriteCommand` after the query is fully resolved — by which point DEFAULT is already gone. No code path creates a V2WriteCommand with unresolved DEFAULT attributes. This reverts the code changes from eca57ea955b and moves the test to InsertIntoTests where it runs across all V2 INSERT test suites. Co-authored-by: Isaac --- .../sql/catalyst/analysis/Analyzer.scala | 6 ----- ...olveColumnDefaultInCommandInputQuery.scala | 6 ----- .../sql/ResolveDefaultColumnsSuite.scala | 24 ------------------- .../spark/sql/connector/InsertIntoTests.scala | 17 +++++++++++++ 4 files changed, 17 insertions(+), 36 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 3d04dcb274ba2..3b4d725840935 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -1418,12 +1418,6 @@ class Analyzer( // to resolve column "DEFAULT" in the child plans so that they must be unresolved. case s: SetVariable => resolveColumnDefaultInCommandInputQuery(s) - // SPARK-43752: resolve column "DEFAULT" in V2 write commands before the - // query is fully resolved, matching the InsertIntoStatement behavior above. - case v2: V2WriteCommand - if v2.table.resolved && v2.query.containsPattern(UNRESOLVED_ATTRIBUTE) => - resolveColumnDefaultInCommandInputQuery(v2) - // Skip FetchCursor - let ResolveFetchCursor handle variable resolution // This prevents ResolveReferences from trying to resolve target variables as columns case s: SingleStatement if s.parsedPlan.isInstanceOf[FetchCursor] => s diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala index 03db3fae2fc85..39fc6028ccba7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala @@ -89,12 +89,6 @@ class ResolveColumnDefaultInCommandInputQuery(val catalogManager: CatalogManager // defined for the n-th variable of the SET. s.withNewChildren(Seq(resolveColumnDefault(s.sourceQuery, expectedQuerySchema))) - // SPARK-43752: resolve column "DEFAULT" in V2 write commands. - case v2: V2WriteCommand if conf.enableDefaultColumns && v2.table.resolved && - v2.query.containsPattern(UNRESOLVED_ATTRIBUTE) => - val expectedQuerySchema = v2.table.schema - v2.withNewQuery(resolveColumnDefault(v2.query, expectedQuerySchema)) - case _ => plan } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala index 5ad945dd75b82..cb9d0909554b4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala @@ -308,28 +308,4 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession { checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user))) } } - - test("SPARK-43752: column DEFAULT in V2 write commands") { - withSQLConf( - "spark.sql.catalog.testcat" -> - classOf[connector.catalog.InMemoryTableCatalog].getName) { - sql("CREATE TABLE testcat.t(c1 INT DEFAULT 42, c2 STRING DEFAULT 'hello')") - // INSERT INTO (AppendData) with DEFAULT - sql("INSERT INTO testcat.t VALUES (1, DEFAULT)") - checkAnswer( - sql("SELECT * FROM testcat.t"), - Row(1, "hello")) - // INSERT INTO with all DEFAULT - sql("INSERT INTO testcat.t VALUES (DEFAULT, DEFAULT)") - checkAnswer( - sql("SELECT * FROM testcat.t ORDER BY c1"), - Seq(Row(1, "hello"), Row(42, "hello"))) - // INSERT OVERWRITE (OverwriteByExpression) with DEFAULT - sql("INSERT OVERWRITE testcat.t VALUES (100, DEFAULT)") - checkAnswer( - sql("SELECT * FROM testcat.t"), - Row(100, "hello")) - sql("DROP TABLE testcat.t") - } - } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala index dd0c8f0a29e9c..eda8bf9876c03 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala @@ -505,6 +505,23 @@ trait InsertIntoSQLOnlyTests verifyTable(t1, Seq("a" -> "b").toDF("id", "data")) } } + + test("SPARK-43752: InsertInto: column DEFAULT values") { + val t1 = s"${catalogAndNamespace}tbl" + withTable(t1) { + sql(s"CREATE TABLE $t1 (c1 INT DEFAULT 42, c2 STRING DEFAULT 'hello') USING $v2Format") + sql(s"INSERT INTO $t1 VALUES (1, DEFAULT)") + checkAnswer(sql(s"SELECT * FROM $t1"), Row(1, "hello")) + + sql(s"INSERT INTO $t1 VALUES (DEFAULT, DEFAULT)") + checkAnswer( + sql(s"SELECT * FROM $t1 ORDER BY c1"), + Seq(Row(1, "hello"), Row(42, "hello"))) + + sql(s"INSERT OVERWRITE $t1 VALUES (100, DEFAULT)") + checkAnswer(sql(s"SELECT * FROM $t1"), Row(100, "hello")) + } + } } } From 5d6e4bde94f142dce9b24031086e46c7fe0a6140 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 16 Apr 2026 03:30:27 +0000 Subject: [PATCH 2/2] Address review comment: remove SPARK-43752 prefix from test name Co-authored-by: Isaac --- .../scala/org/apache/spark/sql/connector/InsertIntoTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala index eda8bf9876c03..1ad8e7e1c6835 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala @@ -506,7 +506,7 @@ trait InsertIntoSQLOnlyTests } } - test("SPARK-43752: InsertInto: column DEFAULT values") { + test("InsertInto: column DEFAULT values") { val t1 = s"${catalogAndNamespace}tbl" withTable(t1) { sql(s"CREATE TABLE $t1 (c1 INT DEFAULT 42, c2 STRING DEFAULT 'hello') USING $v2Format")