From fcf9f1a381b751cea49e69a266bbf22692be2af0 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Fri, 13 Jun 2014 15:08:06 -0700 Subject: [PATCH] Remove BlockStatus if it is no longer cached In StorageStatusListener, a block that is no longer cached has its BlockStatus updated to reflect this in the StorageStatus blocks map. What we should really do, however, is to remove this from the map. Otherwise, the ExecutorsPage still thinks that this is block is cached. --- .../org/apache/spark/storage/StorageStatusListener.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala index a6e6627d54e01..c694fc8c347ec 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala @@ -37,7 +37,11 @@ class StorageStatusListener extends SparkListener { val filteredStatus = storageStatusList.find(_.blockManagerId.executorId == execId) filteredStatus.foreach { storageStatus => updatedBlocks.foreach { case (blockId, updatedStatus) => - storageStatus.blocks(blockId) = updatedStatus + if (updatedStatus.storageLevel == StorageLevel.NONE) { + storageStatus.blocks.remove(blockId) + } else { + storageStatus.blocks(blockId) = updatedStatus + } } } }