[SPARK-32222][K8S][TESTS] Add K8s IT for conf propagation

### What changes were proposed in this pull request?

Added integration test - which tries to configure a log4j.properties and checks if, it is the one pickup by the driver.

### Why are the changes needed?

Improved test coverage.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

By running integration tests.

Closes #30388 from ScrapCodes/SPARK-32222/k8s-it-spark-conf-propagate.

Authored-by: Prashant Sharma <prashsh1@in.ibm.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>

resource-managers/kubernetes/integration-tests/src/test/resources/log-config-test-log4j.properties

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This log4j config file is for integration test SparkConfPropagateSuite.
+log4j.rootCategory=DEBUG, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c: %m%n

resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala

@@ -41,10 +41,10 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 
 class KubernetesSuite extends SparkFunSuite
-  with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SecretsTestsSuite
-  with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite with PVTestsSuite
-  with DepsTestsSuite with DecommissionSuite with RTestsSuite with Logging with Eventually
-  with Matchers {
+  with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SparkConfPropagateSuite
+  with SecretsTestsSuite with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite
+  with PVTestsSuite with DepsTestsSuite with DecommissionSuite with RTestsSuite with Logging
+  with Eventually with Matchers {
 
 
   import KubernetesSuite._

resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkConfPropagateSuite.scala

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.k8s.integrationtest
+
+import java.io.{BufferedWriter, File, FileWriter}
+import java.net.URL
+
+import scala.io.{BufferedSource, Source}
+
+import io.fabric8.kubernetes.api.model._
+
+import org.apache.spark.internal.config
+
+private[spark] trait SparkConfPropagateSuite { k8sSuite: KubernetesSuite =>
+  import KubernetesSuite.{k8sTestTag, SPARK_PI_MAIN_CLASS}
+
+  test("Verify logging configuration is picked from the provided SPARK_CONF_DIR/log4j.properties",
+    k8sTestTag) {
+    val loggingConfigFileName = "log-config-test-log4j.properties"
+    val loggingConfURL: URL = this.getClass.getClassLoader.getResource(loggingConfigFileName)
+    assert(loggingConfURL != null, "Logging configuration file not available.")
+
+    val content = Source.createBufferedSource(loggingConfURL.openStream()).getLines().mkString("\n")
+    val logConfFilePath = s"${sparkHomeDir.toFile}/conf/log4j.properties"
+
+    try {
+      val writer = new BufferedWriter(new FileWriter(logConfFilePath))
+      writer.write(content)
+      writer.close()
+
+      sparkAppConf.set("spark.driver.extraJavaOptions", "-Dlog4j.debug")
+
+      runSparkApplicationAndVerifyCompletion(
+        appResource = containerLocalSparkDistroExamplesJar,
+        mainClass = SPARK_PI_MAIN_CLASS,
+        expectedLogOnCompletion = (Seq("DEBUG",
+          s"log4j: Reading configuration from URL file:/opt/spark/conf/log4j.properties",
+          "Pi is roughly 3")),
+        appArgs = Array.empty[String],
+        driverPodChecker = doBasicDriverPodCheck,
+        executorPodChecker = doBasicExecutorPodCheck,
+        appLocator = appLocator,
+        isJVM = true)
+    } finally {
+      new File(logConfFilePath).delete()
+    }
+  }
+}