From abd10166d66d848ca5895a7b82b10dbeea2763ba Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 19 May 2026 16:09:00 +0000 Subject: [PATCH] [SPARK-56948][SQL][TESTS] Make TPCDSQueryBenchmark heap/broadcast configurable Switch hardcoded .set(...) to .setIfMissing(...) for spark.driver.memory, spark.executor.memory, and spark.sql.autoBroadcastJoinThreshold so they can be overridden via -Dspark.* without recompiling. Also unify spark.sql.shuffle.partitions to use setIfMissing for consistency. Defaults unchanged. --- .../sql/execution/benchmark/TPCDSQueryBenchmark.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala index c79f9f26d60df..c1ff0eb8458db 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala @@ -51,10 +51,10 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark with Logging { val conf = new SparkConf() .setMaster(System.getProperty("spark.sql.test.master", "local[1]")) .setAppName("test-sql-context") - .set("spark.sql.shuffle.partitions", System.getProperty("spark.sql.shuffle.partitions", "4")) - .set("spark.driver.memory", "3g") - .set("spark.executor.memory", "3g") - .set("spark.sql.autoBroadcastJoinThreshold", (20 * 1024 * 1024).toString) + .setIfMissing("spark.sql.shuffle.partitions", "4") + .setIfMissing("spark.driver.memory", "3g") + .setIfMissing("spark.executor.memory", "3g") + .setIfMissing("spark.sql.autoBroadcastJoinThreshold", (20 * 1024 * 1024).toString) .set("spark.sql.crossJoin.enabled", "true") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrationRequired", "true")