Skip to content

Commit

Permalink
[SPARK-14243][CORE] update task metrics when removing blocks
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?

This PR try to use `incUpdatedBlockStatuses ` to update the `updatedBlockStatuses ` when removing blocks, making sure `BlockManager` correctly updates `updatedBlockStatuses`

## How was this patch tested?

test("updated block statuses") in BlockManagerSuite.scala

Author: jeanlyn <jeanlyn92@gmail.com>

Closes #12091 from jeanlyn/updateBlock.
  • Loading branch information
jeanlyn authored and Andrew Or committed Mar 31, 2016
1 parent 446c45b commit 8a333d2
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 2 deletions.
Expand Up @@ -1264,9 +1264,12 @@ private[spark] class BlockManager(
"the disk, memory, or external block store")
}
blockInfoManager.removeBlock(blockId)
val removeBlockStatus = getCurrentBlockStatus(blockId, info)
if (tellMaster && info.tellMaster) {
val status = getCurrentBlockStatus(blockId, info)
reportBlockStatus(blockId, info, status)
reportBlockStatus(blockId, info, removeBlockStatus)
}
Option(TaskContext.get()).foreach { c =>
c.taskMetrics().incUpdatedBlockStatuses(Seq((blockId, removeBlockStatus)))
}
}
}
Expand Down
Expand Up @@ -928,6 +928,16 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
assert(!store.diskStore.contains("list3"), "list3 was in disk store")
assert(!store.diskStore.contains("list4"), "list4 was in disk store")
assert(!store.diskStore.contains("list5"), "list5 was in disk store")

// remove block - list2 should be removed from disk
val updatedBlocks6 = getUpdatedBlocks {
store.removeBlock(
"list2", tellMaster = true)
}
assert(updatedBlocks6.size === 1)
assert(updatedBlocks6.head._1 === TestBlockId("list2"))
assert(updatedBlocks6.head._2.storageLevel == StorageLevel.NONE)
assert(!store.diskStore.contains("list2"), "list2 was in disk store")
}

test("query block statuses") {
Expand Down

0 comments on commit 8a333d2

Please sign in to comment.