Bug fixes for passing the JAR URIs correctly

mesos · Apr 14, 2012 · b1b9a74 · b1b9a74
1 parent bb19ee7
commit b1b9a74
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 5 deletions.
diff --git a/core/src/main/scala/spark/Executor.scala b/core/src/main/scala/spark/Executor.scala
@@ -116,6 +116,7 @@ class Executor extends org.apache.mesos.Executor with Logging {
     // each URL has a unique file name so that no local filenames will clash
     // in this process. This is guaranteed by MesosScheduler.
     val uris = System.getProperty("spark.jar.uris", "")
+    logInfo("spark.jar.uris = " + uris)
     val localFiles = ArrayBuffer[String]()
     for (uri <- uris.split(",").filter(_.size > 0)) {
       val url = new URL(uri)
@@ -149,6 +150,7 @@ class Executor extends org.apache.mesos.Executor with Logging {
 
   // Download a file from a given URL to the local filesystem
   private def downloadFile(url: URL, localPath: String) {
+    logInfo("Downloading " + url + " to " + localPath)
     val in = url.openStream()
     val out = new FileOutputStream(localPath)
     Utils.copyStream(in, out, true)

diff --git a/core/src/main/scala/spark/MesosScheduler.scala b/core/src/main/scala/spark/MesosScheduler.scala
@@ -76,9 +76,6 @@ private class MesosScheduler(
   // URIs of JARs to pass to executor
   var jarUris: String = ""
 
-  // Create an ExecutorInfo for our tasks
-  val executorInfo = createExecutorInfo()
-
   // Sorts jobs in reverse order of run ID for use in our priority queue (so lower IDs run first)
   private val jobOrdering = new Ordering[Job] {
     override def compare(j1: Job, j2: Job): Int = {
@@ -108,7 +105,7 @@ private class MesosScheduler(
       setDaemon(true)
       override def run {
         val sched = MesosScheduler.this
-        driver = new MesosSchedulerDriver(sched, frameworkName, executorInfo, master)
+        driver = new MesosSchedulerDriver(sched, frameworkName, createExecutorInfo(), master)
         try {
           val ret = driver.run()
           logInfo("driver.run() returned with code " + ret)

diff --git a/core/src/main/scala/spark/SimpleJob.scala b/core/src/main/scala/spark/SimpleJob.scala
@@ -176,7 +176,6 @@ class SimpleJob(
           return Some(TaskDescription.newBuilder()
               .setTaskId(taskId)
               .setSlaveId(offer.getSlaveId)
-              .setExecutor(sched.executorInfo)
               .setName(taskName)
               .addResources(cpuRes)
               .setData(ByteString.copyFrom(serializedTask))