-
Notifications
You must be signed in to change notification settings - Fork 28.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-39551][SQL][3.2] Add AQE invalid plan check
### What changes were proposed in this pull request? This is a backport of #36953 This PR adds a check for invalid plans in AQE replanning process. The check will throw exceptions when it detects an invalid plan, causing AQE to void the current replanning result and keep using the latest valid plan. ### Why are the changes needed? AQE logical optimization rules can lead to invalid physical plans and cause runtime exceptions as certain physical plan nodes are not compatible with others. E.g., `BroadcastExchangeExec` can only work as a direct child of broadcast join nodes, but it could appear under other incompatible physical plan nodes because of empty relation propagation. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Added UT. Closes #37108 from dongjoon-hyun/SPARK-39551. Authored-by: Maryann Xue <maryann.xue@gmail.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
- Loading branch information
1 parent
1c0bd4c
commit be891ad
Showing
4 changed files
with
163 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
...core/src/main/scala/org/apache/spark/sql/execution/adaptive/InvalidAQEPlanException.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.sql.execution.adaptive | ||
|
|
||
| import org.apache.spark.sql.catalyst.plans.QueryPlan | ||
|
|
||
| /** | ||
| * Exception thrown when an invalid query plan is detected in AQE replanning, | ||
| * in which case AQE will stop the current replanning process and keep using the latest valid plan. | ||
| * | ||
| * @param message The reason why the plan is considered invalid. | ||
| * @param plan The invalid plan/sub-plan. | ||
| */ | ||
| case class InvalidAQEPlanException[QueryType <: QueryPlan[_]](message: String, plan: QueryType) | ||
| extends Exception(message) |
68 changes: 68 additions & 0 deletions
68
sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ValidateSparkPlan.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.spark.sql.execution.adaptive | ||
|
|
||
| import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} | ||
| import org.apache.spark.sql.catalyst.rules.Rule | ||
| import org.apache.spark.sql.execution.SparkPlan | ||
| import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} | ||
|
|
||
| /** | ||
| * Detects invalid physical plans generated by AQE replanning and throws `InvalidAQEPlanException` | ||
| * if such plans are detected. This rule should be called after EnsureRequirements where all | ||
| * necessary Exchange nodes are added. | ||
| */ | ||
| object ValidateSparkPlan extends Rule[SparkPlan] { | ||
|
|
||
| def apply(plan: SparkPlan): SparkPlan = { | ||
| validate(plan) | ||
| plan | ||
| } | ||
|
|
||
| /** | ||
| * Validate that the plan satisfies the following condition: | ||
| * - BroadcastQueryStage only appears as the immediate child and the build side of a broadcast | ||
| * hash join or broadcast nested loop join. | ||
| */ | ||
| private def validate(plan: SparkPlan): Unit = plan match { | ||
| case b: BroadcastHashJoinExec => | ||
| val (buildPlan, probePlan) = b.buildSide match { | ||
| case BuildLeft => (b.left, b.right) | ||
| case BuildRight => (b.right, b.left) | ||
| } | ||
| if (!buildPlan.isInstanceOf[BroadcastQueryStageExec]) { | ||
| validate(buildPlan) | ||
| } | ||
| validate(probePlan) | ||
| case b: BroadcastNestedLoopJoinExec => | ||
| val (buildPlan, probePlan) = b.buildSide match { | ||
| case BuildLeft => (b.left, b.right) | ||
| case BuildRight => (b.right, b.left) | ||
| } | ||
| if (!buildPlan.isInstanceOf[BroadcastQueryStageExec]) { | ||
| validate(buildPlan) | ||
| } | ||
| validate(probePlan) | ||
| case q: BroadcastQueryStageExec => errorOnInvalidBroadcastQueryStage(q) | ||
| case _ => plan.children.foreach(validate) | ||
| } | ||
|
|
||
| private def errorOnInvalidBroadcastQueryStage(plan: SparkPlan): Unit = { | ||
| throw InvalidAQEPlanException("Invalid broadcast query stage", plan) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters