From 6bfeb094248269920df8b107c86f0982404935cd Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 28 Jun 2024 15:16:48 -0700 Subject: [PATCH] [SPARK-48757][CORE] Make `IndexShuffleBlockResolver` have explicit constructors ### What changes were proposed in this pull request? This PR aims to make `IndexShuffleBlockResolver` have explicit constructors from Apache Spark 4.0.0. ### Why are the changes needed? Although `IndexShuffleBlockResolver` is `private` and there is no contract to keep class constructor signatures, from Apache Spark 4.0.0, we had better reduce the following situations with the old libraries built against old Spark versions. ``` Cause: java.lang.NoSuchMethodError: 'void org.apache.spark.shuffle.IndexShuffleBlockResolver.(org.apache.spark.SparkConf, org.apache.spark.storage.BlockManager)' [info] at org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager.(CometShuffleManager.scala:64) ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #47148 from dongjoon-hyun/SPARK-48757. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../shuffle/IndexShuffleBlockResolver.scala | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala index 7a76c2f97e8d7..30bc1382fb021 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala @@ -57,11 +57,23 @@ import org.apache.spark.util.collection.OpenHashSet private[spark] class IndexShuffleBlockResolver( conf: SparkConf, // var for testing - var _blockManager: BlockManager = null, - val taskIdMapsForShuffle: JMap[Int, OpenHashSet[Long]] = Collections.emptyMap()) + var _blockManager: BlockManager, + val taskIdMapsForShuffle: JMap[Int, OpenHashSet[Long]]) extends ShuffleBlockResolver with Logging with MigratableResolver { + def this(conf: SparkConf) = { + this(conf, null, Collections.emptyMap()) + } + + def this(conf: SparkConf, _blockManager: BlockManager) = { + this(conf, _blockManager, Collections.emptyMap()) + } + + def this(conf: SparkConf, taskIdMapsForShuffle: JMap[Int, OpenHashSet[Long]]) = { + this(conf, null, taskIdMapsForShuffle) + } + private lazy val blockManager = Option(_blockManager).getOrElse(SparkEnv.get.blockManager) private val transportConf = {