From 40cdcb2334dc1d66194a1516e7ec998908b7cc78 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Sat, 8 Mar 2014 01:08:16 +0800 Subject: [PATCH] Bug fix, and addressed PR comments from @mridulm Fixed bug: getRddId(blockId) returns an Option[Int], which never equals to rddId: Int. --- .../org/apache/spark/storage/MemoryStore.scala | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala index c104a1a47f02b..5a667a80e0f26 100644 --- a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala @@ -244,14 +244,9 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long) // accessed RDD, unless this is the same RDD as the one with the // new partition. In that case, we keep the old partition in memory // to prevent cycling partitions from the same RDD in and out. - // - // TODO implement LRU eviction - rddToAdd match { - case Some(rddId) if rddId == getRddId(blockId) => - // no-op - case _ => - selectedBlocks += blockId - selectedMemory += pair.getValue.size + if (rddToAdd.isEmpty || rddToAdd != getRddId(blockId)) { + selectedBlocks += blockId + selectedMemory += pair.getValue.size } } } @@ -274,6 +269,8 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long) } return true } else { + logInfo(s"Will not store $blockIdToAdd as it would require dropping another block " + + "from the same RDD") return false } }