From ecd1a142600daa31372d5604643c3757750bddb1 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 3 Jun 2020 15:45:38 -0700 Subject: [PATCH] Update core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala If we have a failure during block migration, log the exception. Co-authored-by: Attila Zsolt Piros <2017933+attilapiros@users.noreply.github.com> --- .../org/apache/spark/shuffle/IndexShuffleBlockResolver.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala index 79323be45845c..36ee3892017d3 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala @@ -226,6 +226,7 @@ private[spark] class IndexShuffleBlockResolver( override def onFailure(streamId: String, cause: Throwable): Unit = { // the framework handles the connection itself, we just need to do local cleanup + logWarning(s"Error while uploading $blockId", cause) channel.close() fileTmp.delete() }