From 930f619fe07768186b8ab9d455d87c7110e377a2 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Wed, 17 Sep 2014 01:18:24 +0900 Subject: [PATCH 1/2] Removed redundant putting failed results --- .../apache/spark/storage/ShuffleBlockFetcherIterator.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index c8e708aa6b1bc..e3241999298fe 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -125,9 +125,8 @@ final class ShuffleBlockFetcherIterator( // Note that there is a chance that some blocks have been fetched successfully, but we // still add them to the failed queue. This is fine because when the caller see a // FetchFailedException, it is going to fail the entire task anyway. - for ((blockId, size) <- req.blocks) { - results.put(new FetchResult(blockId, -1, null)) - } + val blocks = req.blocks + if (blocks.size > 0) results.put(new FetchResult(blocks(0)._1, -1, null)) } } ) From d4b604e8a689d8af358fddbca85903f0b894d707 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Thu, 18 Sep 2014 09:17:50 +0900 Subject: [PATCH 2/2] Modified ShuffleBlockFetcherIteratorSuite not to iterate after getting failed result --- .../spark/storage/ShuffleBlockFetcherIteratorSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala index 809bd70929656..01c967b15a131 100644 --- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala @@ -176,8 +176,9 @@ class ShuffleBlockFetcherIteratorSuite extends FunSuite { null, 48 * 1024 * 1024) - iterator.foreach { case (_, iterOption) => - assert(!iterOption.isDefined) - } + val result = iterator.next + assert(result._2 == None, "The 2nd element of the tuple should " + + "be None when the result is failed") + // When we meet failed result, we must not iterate any more. } }