Optimize the schedule in Master

apache · Sep 5, 2014 · bbc7087 · bbc7087
1 parent 1904bac
commit bbc7087
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -481,12 +481,16 @@ private[spark] class Master(
     if (state != RecoveryState.ALIVE) { return }
 
     // First schedule drivers, they take strict precedence over applications
-    val shuffledWorkers = Random.shuffle(workers) // Randomization helps balance drivers
-    for (worker <- shuffledWorkers if worker.state == WorkerState.ALIVE) {
-      for (driver <- List(waitingDrivers: _*)) { // iterate over a copy of waitingDrivers
-        if (worker.memoryFree >= driver.desc.mem && worker.coresFree >= driver.desc.cores) {
+    for (driver <- List(waitingDrivers: _*)) {
+      val shuffledWorkers = Random.shuffle(workers) // Randomization helps balance drivers
+      val shuffledWorkersIter = shuffledWorkers.iterator
+      var launched = false
+      while(shuffledWorkersIter.hasNext && !launched) {
+        val worker = shuffledWorkersIter.next()
+        if (worker.state == WorkerState.ALIVE && worker.memoryFree >= driver.desc.mem && worker.coresFree >= driver.desc.cores) {
           launchDriver(worker, driver)
           waitingDrivers -= driver
+          launched = true
         }
       }
     }