[SPARK-52308][CORE] Fix the bug that the actual configs of the driver are overwritten

core/src/main/scala/org/apache/spark/SparkConf.scala

@@ -288,6 +288,8 @@ class SparkConf(loadDefaults: Boolean)
 
   private val settings = new ConcurrentHashMap[String, String]()
 
+  private val driverJvmStartupConfList = new LinkedHashSet[String]()
+
   @transient private lazy val reader: ConfigReader = {
     val _reader = new ConfigReader(new SparkConfigProvider(settings))
     _reader.bindEnv((key: String) => Option(getenv(key)))
@@ -299,8 +301,24 @@ class SparkConf(loadDefaults: Boolean)
   }
 
   private[spark] def loadFromSystemProperties(silent: Boolean): SparkConf = {
+    val systemProperties = Utils.getSystemProperties
+    // Some configs of the spark driver cannot be modified by the user
+    // after the driver is started and `settings` contains it
+    if (systemProperties.contains(SPARK_DRIVER_JVM_CONFIG_BLACKLIST.key)) {
+      systemProperties.get(SPARK_DRIVER_JVM_CONFIG_BLACKLIST.key).foreach { conf =>
+        conf.split(",").map(_.trim).foreach { key =>
+          driverJvmStartupConfList.add(key)
+        }
+      }
+    } else {
+      SPARK_DRIVER_JVM_CONFIG_BLACKLIST.defaultValue.get.foreach { conf =>
+        conf.split(",").map(_.trim).foreach { key =>
+          driverJvmStartupConfList.add(key)
+        }
+      }
+    }
     // Load any spark.* system properties
-    for ((key, value) <- Utils.getSystemProperties if key.startsWith("spark.")) {
+    for ((key, value) <- systemProperties if key.startsWith("spark.")) {
       set(key, value, silent)
     }
     this
@@ -321,6 +339,14 @@ class SparkConf(loadDefaults: Boolean)
     if (!silent) {
       logDeprecationWarning(key)
     }
+    if (driverJvmStartupConfList.contains(key) && settings.containsKey(key)) {
+      logWarning(
+        log"Spark user should not overwrite spark driver JVM startup parameters in code:\n" +
+          log"key = ${MDC(LogKeys.CONFIG, key)} , " +
+          log"effective value = ${MDC(LogKeys.VALUE, settings.get(key))}, " +
+          log"user setting value = ${MDC(LogKeys.VALUE, value)}.")
+      return this
+    }
     settings.put(key, value)
     this
   }

core/src/main/scala/org/apache/spark/internal/config/package.scala

@@ -2838,4 +2838,12 @@ package object config {
       .checkValues(Set("connect", "classic"))
       .createWithDefault(
         if (sys.env.get("SPARK_CONNECT_MODE").contains("1")) "connect" else "classic")
+
+  private[spark] val SPARK_DRIVER_JVM_CONFIG_BLACKLIST =
+    ConfigBuilder("spark.driver.jvm.config.blacklist")
+      .doc("Spark driver jvm startup blacklist conf list.")
+      .version("3.2.1")
+      .stringConf.toSequence
+      .createWithDefault(
+        DRIVER_MEMORY.key :: DRIVER_MEMORY_OVERHEAD.key :: DRIVER_CORES.key :: Nil)
 }