Skip to content

Commit

Permalink
*UsedByRDD -> *UsedByRdd (minor)
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewor14
andrewor14 committed Aug 1, 2014
1 parent 6970bc8 commit af19bc0
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 44 deletions.
8 changes: 4 additions & 4 deletions core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,10 @@ class StorageStatus(val blockManagerId: BlockManagerId, val maxMem: Long) {
def offHeapUsed: Long = blocks.values.map(_.tachyonSize).reduceOption(_ + _).getOrElse(0L)

/** Return the memory used by the given RDD in this block manager in O(1) time. */
def memUsedByRDD(rddId: Int): Long = _rddStorageInfo.get(rddId).map(_._1).getOrElse(0L)
def memUsedByRdd(rddId: Int): Long = _rddStorageInfo.get(rddId).map(_._1).getOrElse(0L)

/** Return the disk space used by the given RDD in this block manager in O(1) time. */
def diskUsedByRDD(rddId: Int): Long = _rddStorageInfo.get(rddId).map(_._2).getOrElse(0L)
def diskUsedByRdd(rddId: Int): Long = _rddStorageInfo.get(rddId).map(_._2).getOrElse(0L)

/** Return the off-heap space used by the given RDD in this block manager in O(1) time. */
def offHeapUsedByRdd(rddId: Int): Long = _rddStorageInfo.get(rddId).map(_._3).getOrElse(0L)
Expand Down Expand Up @@ -253,8 +253,8 @@ private[spark] object StorageUtils {
.map(_.rddStorageLevel(rddId)).flatMap(s => s).headOption.getOrElse(StorageLevel.NONE)
val numCachedPartitions = statuses
.map(_.numRddBlocksById(rddId)).reduceOption(_ + _).getOrElse(0)
val memSize = statuses.map(_.memUsedByRDD(rddId)).reduceOption(_ + _).getOrElse(0L)
val diskSize = statuses.map(_.diskUsedByRDD(rddId)).reduceOption(_ + _).getOrElse(0L)
val memSize = statuses.map(_.memUsedByRdd(rddId)).reduceOption(_ + _).getOrElse(0L)
val diskSize = statuses.map(_.diskUsedByRdd(rddId)).reduceOption(_ + _).getOrElse(0L)
val tachyonSize = statuses.map(_.offHeapUsedByRdd(rddId)).reduceOption(_ + _).getOrElse(0L)

rddInfo.storageLevel = storageLevel
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,10 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
<tr>
<td>{status.blockManagerId.host + ":" + status.blockManagerId.port}</td>
<td>
{Utils.bytesToString(status.memUsedByRDD(rddId))}
{Utils.bytesToString(status.memUsedByRdd(rddId))}
({Utils.bytesToString(status.memRemaining)} Remaining)
</td>
<td>{Utils.bytesToString(status.diskUsedByRDD(rddId))}</td>
<td>{Utils.bytesToString(status.diskUsedByRdd(rddId))}</td>
</tr>
}

Expand Down
76 changes: 38 additions & 38 deletions core/src/test/scala/org/apache/spark/storage/StorageSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,12 @@ class StorageSuite extends FunSuite {
assert(status.rddBlocksById(2).contains(RDDBlockId(2, 2)))
assert(status.rddBlocksById(2).contains(RDDBlockId(2, 3)))
assert(status.rddBlocksById(2).contains(RDDBlockId(2, 4)))
assert(status.memUsedByRDD(0) === 10L)
assert(status.memUsedByRDD(1) === 100L)
assert(status.memUsedByRDD(2) === 30L)
assert(status.diskUsedByRDD(0) === 20L)
assert(status.diskUsedByRDD(1) === 200L)
assert(status.diskUsedByRDD(2) === 80L)
assert(status.memUsedByRdd(0) === 10L)
assert(status.memUsedByRdd(1) === 100L)
assert(status.memUsedByRdd(2) === 30L)
assert(status.diskUsedByRdd(0) === 20L)
assert(status.diskUsedByRdd(1) === 200L)
assert(status.diskUsedByRdd(2) === 80L)
assert(status.offHeapUsedByRdd(0) === 1L)
assert(status.offHeapUsedByRdd(1) === 1L)
assert(status.offHeapUsedByRdd(2) === 1L)
Expand All @@ -124,8 +124,8 @@ class StorageSuite extends FunSuite {

// Verify default values for RDDs that don't exist
assert(status.rddBlocksById(10).isEmpty)
assert(status.memUsedByRDD(10) === 0L)
assert(status.diskUsedByRDD(10) === 0L)
assert(status.memUsedByRdd(10) === 0L)
assert(status.diskUsedByRdd(10) === 0L)
assert(status.offHeapUsedByRdd(10) === 0L)
assert(status.rddStorageLevel(10) === None)
}
Expand All @@ -140,12 +140,12 @@ class StorageSuite extends FunSuite {
assert(status.rddBlocksById(0).size === 1)
assert(status.rddBlocksById(1).size === 1)
assert(status.rddBlocksById(2).size === 3)
assert(status.memUsedByRDD(0) === 0L)
assert(status.memUsedByRDD(1) === 100L)
assert(status.memUsedByRDD(2) === 20L)
assert(status.diskUsedByRDD(0) === 0L)
assert(status.diskUsedByRDD(1) === 200L)
assert(status.diskUsedByRDD(2) === 1060L)
assert(status.memUsedByRdd(0) === 0L)
assert(status.memUsedByRdd(1) === 100L)
assert(status.memUsedByRdd(2) === 20L)
assert(status.diskUsedByRdd(0) === 0L)
assert(status.diskUsedByRdd(1) === 200L)
assert(status.diskUsedByRdd(2) === 1060L)
assert(status.offHeapUsedByRdd(0) === 0L)
assert(status.offHeapUsedByRdd(1) === 1L)
assert(status.offHeapUsedByRdd(2) === 0L)
Expand All @@ -166,12 +166,12 @@ class StorageSuite extends FunSuite {
assert(status.rddBlocksById(1).size === 0)
assert(status.rddBlocksById(2).size === 1)
assert(status.rddBlocksById(2).contains(RDDBlockId(2, 3)))
assert(status.memUsedByRDD(0) === 10L)
assert(status.memUsedByRDD(1) === 0L)
assert(status.memUsedByRDD(2) === 10L)
assert(status.diskUsedByRDD(0) === 20L)
assert(status.diskUsedByRDD(1) === 0L)
assert(status.diskUsedByRDD(2) === 20L)
assert(status.memUsedByRdd(0) === 10L)
assert(status.memUsedByRdd(1) === 0L)
assert(status.memUsedByRdd(2) === 10L)
assert(status.diskUsedByRdd(0) === 20L)
assert(status.diskUsedByRdd(1) === 0L)
assert(status.diskUsedByRdd(2) === 20L)
assert(status.offHeapUsedByRdd(0) === 1L)
assert(status.offHeapUsedByRdd(1) === 0L)
assert(status.offHeapUsedByRdd(2) === 0L)
Expand Down Expand Up @@ -248,12 +248,12 @@ class StorageSuite extends FunSuite {
status.updateRddStorageInfo(0, 1000L, 1000L, 1000L, memOnly)
status.updateRddStorageInfo(1, 2000L, 2000L, 2000L, diskOnly)
status.updateRddStorageInfo(2, 3000L, 3000L, 3000L, memAndDisk)
assert(status.memUsedByRDD(0) === 1010L)
assert(status.memUsedByRDD(1) === 2100L)
assert(status.memUsedByRDD(2) === 3030L)
assert(status.diskUsedByRDD(0) === 1020L)
assert(status.diskUsedByRDD(1) === 2200L)
assert(status.diskUsedByRDD(2) === 3080L)
assert(status.memUsedByRdd(0) === 1010L)
assert(status.memUsedByRdd(1) === 2100L)
assert(status.memUsedByRdd(2) === 3030L)
assert(status.diskUsedByRdd(0) === 1020L)
assert(status.diskUsedByRdd(1) === 2200L)
assert(status.diskUsedByRdd(2) === 3080L)
assert(status.offHeapUsedByRdd(0) === 1001L)
assert(status.offHeapUsedByRdd(1) === 2001L)
assert(status.offHeapUsedByRdd(2) === 3001L)
Expand All @@ -265,12 +265,12 @@ class StorageSuite extends FunSuite {
status.updateRddStorageInfo(0, -100L, -100L, -100L, memOnly)
status.updateRddStorageInfo(1, -200L, -200L, -200L, diskOnly)
status.updateRddStorageInfo(2, -300L, -300L, -300L, memAndDisk)
assert(status.memUsedByRDD(0) === 910L)
assert(status.memUsedByRDD(1) === 1900L)
assert(status.memUsedByRDD(2) === 2730L)
assert(status.diskUsedByRDD(0) === 920L)
assert(status.diskUsedByRDD(1) === 2000L)
assert(status.diskUsedByRDD(2) === 2780L)
assert(status.memUsedByRdd(0) === 910L)
assert(status.memUsedByRdd(1) === 1900L)
assert(status.memUsedByRdd(2) === 2730L)
assert(status.diskUsedByRdd(0) === 920L)
assert(status.diskUsedByRdd(1) === 2000L)
assert(status.diskUsedByRdd(2) === 2780L)
assert(status.offHeapUsedByRdd(0) === 901L)
assert(status.offHeapUsedByRdd(1) === 1801L)
assert(status.offHeapUsedByRdd(2) === 2701L)
Expand All @@ -279,12 +279,12 @@ class StorageSuite extends FunSuite {
status.updateRddStorageInfo(0, -10000L, -10000L, -10000L, memOnly)
status.updateRddStorageInfo(1, -20000L, -20000L, -20000L, diskOnly)
status.updateRddStorageInfo(2, -30000L, -30000L, -30000L, memAndDisk)
assert(status.memUsedByRDD(0) === 0L)
assert(status.memUsedByRDD(1) === 0L)
assert(status.memUsedByRDD(2) === 0L)
assert(status.diskUsedByRDD(0) === 0L)
assert(status.diskUsedByRDD(1) === 0L)
assert(status.diskUsedByRDD(2) === 0L)
assert(status.memUsedByRdd(0) === 0L)
assert(status.memUsedByRdd(1) === 0L)
assert(status.memUsedByRdd(2) === 0L)
assert(status.diskUsedByRdd(0) === 0L)
assert(status.diskUsedByRdd(1) === 0L)
assert(status.diskUsedByRdd(2) === 0L)
assert(status.offHeapUsedByRdd(0) === 0L)
assert(status.offHeapUsedByRdd(1) === 0L)
assert(status.offHeapUsedByRdd(2) === 0L)
Expand Down

0 comments on commit af19bc0

Please sign in to comment.