Skip to content

Commit

Permalink
code style
Browse files Browse the repository at this point in the history
  • Loading branch information
wForget committed Oct 20, 2023
1 parent 0bf572f commit f77d1d6
Showing 1 changed file with 2 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import org.apache.spark.sql.internal.SQLConf._
case class DynamicShufflePartitions(spark: SparkSession) extends Rule[SparkPlan] {

override def apply(plan: SparkPlan): SparkPlan = {

def collectScanSizes(plan: SparkPlan): Seq[Long] = plan match {
case FileSourceScanExec(relation, _, _, _, _, _, _, _, _) =>
Seq(relation.location.sizeInBytes)
Expand All @@ -49,8 +48,8 @@ case class DynamicShufflePartitions(spark: SparkSession) extends Rule[SparkPlan]

val targetSize = conf.getConf(ADVISORY_PARTITION_SIZE_IN_BYTES)
val maxScanSizes = collectScanSizes(plan) match {
case Nil => targetSize
case sizes => sizes.max
case sizes if sizes.nonEmpty => sizes.max
case _ => targetSize
}
val targetShufflePartitions =
Math.max(maxScanSizes / targetSize + 1, conf.numShufflePartitions).toInt
Expand Down

0 comments on commit f77d1d6

Please sign in to comment.