From 1da81ab8cd6e7e45e1b2d03352ecbbb1635f644c Mon Sep 17 00:00:00 2001 From: wangxiaoyu8 Date: Thu, 27 Aug 2015 18:41:15 +0800 Subject: [PATCH] SPARK-10314 [CORE]RDD persist to OFF_HEAP tachyon got block rdd_x_x not found exception when parallelism is big than data split size --- .../org/apache/spark/storage/BlockManagerMasterEndpoint.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala index 68ed9096731c5..cfa76aefde06e 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala @@ -487,7 +487,7 @@ private[spark] class BlockManagerInfo( logInfo("Added %s on ExternalBlockStore on %s (size: %s)".format( blockId, blockManagerId.hostPort, Utils.bytesToString(externalBlockStoreSize))) } - if (!blockId.isBroadcast && blockStatus.isCached) { + if (!blockId.isBroadcast && (blockStatus.isCached || blockStatus.storageLevel.useOffHeap)) { _cachedBlocks += blockId } } else if (_blocks.containsKey(blockId)) {