diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala index 818cff843b45..d78216a48e8b 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala @@ -162,7 +162,7 @@ trait SparkAdapter extends Serializable { * Extract condition in [[DeleteFromTable]] * SPARK-38626 condition is no longer Option in Spark 3.3 */ - def extractCondition(deleteFromTable: Command): Expression + def extractDeleteCondition(deleteFromTable: Command): Expression /** * Get parseQuery from ExtendedSqlParser, only for Spark 3.3+ diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala index 9d5f38066154..6afa8681339a 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala @@ -435,7 +435,7 @@ case class HoodieResolveReferences(sparkSession: SparkSession) extends Rule[Logi // Resolve Delete Table case dft @ DeleteFromTable(table, condition) if sparkAdapter.isHoodieTable(table, sparkSession) && table.resolved => - val resolveExpression = resolveExpressionFrom(table, None)_ + val resolveExpression = resolveExpressionFrom(table, None)(_) sparkAdapter.resolveDeleteFromTable(dft, resolveExpression) // Append the meta field to the insert query to walk through the validate for the diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala index 82f2ae29fa77..e1dc8daa4ca9 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala @@ -37,8 +37,7 @@ case class DeleteHoodieTableCommand(deleteTable: DeleteFromTable) extends Hoodie // Remove meta fields from the data frame var df = removeMetaFields(Dataset.ofRows(sparkSession, table)) - // SPARK-38626 DeleteFromTable.condition is changed from Option[Expression] to Expression in Spark 3.3 - val condition = sparkAdapter.extractCondition(deleteTable) + val condition = sparkAdapter.extractDeleteCondition(deleteTable) if (condition != null) df = df.filter(Column(condition)) val hoodieCatalogTable = HoodieCatalogTable(sparkSession, tableId) diff --git a/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/adapter/Spark2Adapter.scala b/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/adapter/Spark2Adapter.scala index d1ba1e36bbcf..eabdd337ee5d 100644 --- a/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/adapter/Spark2Adapter.scala +++ b/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/adapter/Spark2Adapter.scala @@ -138,7 +138,7 @@ class Spark2Adapter extends SparkAdapter { DeleteFromTable(deleteFromTableCommand.table, resolvedCondition) } - override def extractCondition(deleteFromTable: Command): Expression = { + override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition.getOrElse(null) } diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala index 7ccf51cbb40b..2697c6658ab1 100644 --- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala @@ -73,7 +73,7 @@ class Spark3_1Adapter extends BaseSpark3Adapter { DeleteFromTable(deleteFromTableCommand.table, resolvedCondition) } - override def extractCondition(deleteFromTable: Command): Expression = { + override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition.getOrElse(null) } } diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala index ce3912317115..bd2d6e4f7bff 100644 --- a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala @@ -70,7 +70,7 @@ class Spark3_2Adapter extends BaseSpark3Adapter { DeleteFromTable(deleteFromTableCommand.table, resolvedCondition) } - override def extractCondition(deleteFromTable: Command): Expression = { + override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition.getOrElse(null) } } diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala index e1a97a4646b1..1cbd6be31f01 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala @@ -69,7 +69,7 @@ class Spark3_3Adapter extends BaseSpark3Adapter { DeleteFromTable(deleteFromTableCommand.table, resolveExpression(deleteFromTableCommand.condition)) } - override def extractCondition(deleteFromTable: Command): Expression = { + override def extractDeleteCondition(deleteFromTable: Command): Expression = { deleteFromTable.asInstanceOf[DeleteFromTable].condition }