Make jobFailed idempotent

apache · Jul 17, 2015 · 1a19268 · 1a19268
1 parent 7b22514
commit 1a19268
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala b/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala
@@ -83,6 +83,12 @@ private[spark] class JobWaiter[T](
   }
 
   override def jobFailed(exception: Exception): Unit = synchronized {
-    promise.failure(exception)
+    // There are certain situations where jobFailed can be called multiple times for the same
+    // job. We guard against this by making this method idempotent.
+    if (!isCompleted) {
+      promise.failure(exception)
+    } else {
+      assert(promiseFuture.value.get.isFailure)
+    }
   }
 }