Bug fix, and addressed PR comments from @mridulm

Fixed bug: getRddId(blockId) returns an Option[Int], which never equals to rddId: Int.
holdenk · Mar 7, 2014 · 40cdcb2 · 40cdcb2
1 parent 62c92ac
commit 40cdcb2
Showing 1 changed file with 5 additions and 8 deletions.
diff --git a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala
@@ -244,14 +244,9 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
           //    accessed RDD, unless this is the same RDD as the one with the
           //    new partition. In that case, we keep the old partition in memory
           //    to prevent cycling partitions from the same RDD in and out.
-          //
-          // TODO implement LRU eviction
-          rddToAdd match {
-            case Some(rddId) if rddId == getRddId(blockId) =>
-              // no-op
-            case _ =>
-              selectedBlocks += blockId
-              selectedMemory += pair.getValue.size
+          if (rddToAdd.isEmpty || rddToAdd != getRddId(blockId)) {
+            selectedBlocks += blockId
+            selectedMemory += pair.getValue.size
           }
         }
       }
@@ -274,6 +269,8 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
         }
         return true
       } else {
+        logInfo(s"Will not store $blockIdToAdd as it would require dropping another block " +
+          "from the same RDD")
         return false
       }
     }