[MESOS][SPARK-23464] Fix Mesos Cluster Scheduler to allow configs con…

…taining spaces and quotes
mesosphere · Feb 24, 2018 · 4812ba3 · 4812ba3
1 parent 1a198ce
commit 4812ba3
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 1 deletion.
diff --git a/...mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/...mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -530,7 +530,7 @@ private[spark] class MesosClusterScheduler(
       .filter { case (key, _) => !replicatedOptionsBlacklist.contains(key) }
       .toMap
     (defaultConf ++ driverConf).foreach { case (key, value) =>
-      options ++= Seq("--conf", s""""$key=${shellEscape(value)}"""".stripMargin) }
+      options ++= Seq("--conf", s"$key=${shellEscape(value)}") }
 
     options
   }

diff --git a/.../src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala b/.../src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
@@ -199,6 +199,31 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
     })
   }
 
+  test("can pass config with spaces and quotes") {
+    setScheduler()
+
+    val mem = 1000
+    val cpu = 1
+
+    val response = scheduler.submitDriver(
+      new MesosDriverDescription("d1", "jar", mem, cpu, true,
+        command,
+        Map("spark.mesos.executor.home" -> "test",
+          "spark.app.name" -> "test",
+          "spark.driver.extraJavaOptions" ->
+            "-XX:+PrintGC -Dkey=value -Dnum=\"one two\""),
+        "s1",
+        new Date()))
+    assert(response.success)
+
+    val offer = Utils.createOffer("o1", "s1", mem, cpu)
+    scheduler.resourceOffers(driver, List(offer).asJava)
+    val tasks = Utils.verifyTaskLaunched(driver, "o1")
+    val driverCmd = tasks.head.getCommand.getValue
+    assert(driverCmd.contains(
+      "--conf spark.driver.extraJavaOptions=\"-XX:+PrintGC -Dkey=value -Dnum=\\\"one two\\\"\" "))
+  }
+
   test("supports spark.mesos.driverEnv.*") {
     setScheduler()