Skip to content
Browse files

Fixed SPARK-1194 https://spark-project.atlassian.net/browse/SPARK-1194

  • Loading branch information...
1 parent 328c73d commit 62c92ac7b8e616529bdaa52b73eb70e50bc01b47 @liancheng liancheng committed Mar 7, 2014
Showing with 16 additions and 6 deletions.
  1. +16 −6 core/src/main/scala/org/apache/spark/storage/MemoryStore.scala
View
22 core/src/main/scala/org/apache/spark/storage/MemoryStore.scala
@@ -236,13 +236,23 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
while (maxMemory - (currentMemory - selectedMemory) < space && iterator.hasNext) {
val pair = iterator.next()
val blockId = pair.getKey
- if (rddToAdd.isDefined && rddToAdd == getRddId(blockId)) {
- logInfo("Will not store " + blockIdToAdd + " as it would require dropping another " +
- "block from the same RDD")
- return false
+ // Apply the same-RDD rule for cache replacement. Quoted from the
+ // original RDD paper:
+ //
+ // When a new RDD partition is computed but there is not enough
+ // space to store it, we evict a partition from the least recently
+ // accessed RDD, unless this is the same RDD as the one with the
+ // new partition. In that case, we keep the old partition in memory
+ // to prevent cycling partitions from the same RDD in and out.
+ //
+ // TODO implement LRU eviction
+ rddToAdd match {
+ case Some(rddId) if rddId == getRddId(blockId) =>
+ // no-op
+ case _ =>
+ selectedBlocks += blockId
+ selectedMemory += pair.getValue.size
}
- selectedBlocks += blockId
- selectedMemory += pair.getValue.size
}
}

0 comments on commit 62c92ac

Please sign in to comment.
Something went wrong with that request. Please try again.