Skip to content

Commit

Permalink
[SPARK-12716][WEB UI] Add a TOTALS row to the Executors Web UI
Browse files Browse the repository at this point in the history
Added a Totals table to the top of the page to display the totals of each applicable column in the executors table.

Old Description:
~~Created a TOTALS row containing the totals of each column in the executors UI. By default the TOTALS row appears at the top of the table. When a column is sorted the TOTALS row will always sort to either the top or bottom of the table.~~

Author: Alex Bozarth <ajbozart@us.ibm.com>

Closes #10668 from ajbozarth/spark12716.
  • Loading branch information
ajbozarth authored and Tom Graves committed Jan 15, 2016
1 parent 0bb7355 commit 61c4587
Showing 1 changed file with 64 additions and 10 deletions.
74 changes: 64 additions & 10 deletions core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,6 @@ private[ui] class ExecutorsPage(
}
(_storageStatusList, _execInfo)
}
val maxMem = storageStatusList.map(_.maxMem).sum
val memUsed = storageStatusList.map(_.memUsed).sum
val diskUsed = storageStatusList.map(_.diskUsed).sum
val execInfoSorted = execInfo.sortBy(_.id)
val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty

Expand Down Expand Up @@ -100,18 +97,15 @@ private[ui] class ExecutorsPage(
</table>

val content =
<div class="row-fluid">
<div class="row">
<div class="span12">
<ul class="unstyled">
<li><strong>Memory:</strong>
{Utils.bytesToString(memUsed)} Used
({Utils.bytesToString(maxMem)} Total) </li>
<li><strong>Disk:</strong> {Utils.bytesToString(diskUsed)} Used </li>
</ul>
<h4>Totals for {execInfo.size} Executors</h4>
{execSummary(execInfo)}
</div>
</div>
<div class = "row">
<div class="span12">
<h4>Active Executors</h4>
{execTable}
</div>
</div>;
Expand Down Expand Up @@ -179,6 +173,66 @@ private[ui] class ExecutorsPage(
</tr>
}

private def execSummary(execInfo: Seq[ExecutorSummary]): Seq[Node] = {
val maximumMemory = execInfo.map(_.maxMemory).sum
val memoryUsed = execInfo.map(_.memoryUsed).sum
val diskUsed = execInfo.map(_.diskUsed).sum
val totalDuration = execInfo.map(_.totalDuration).sum
val totalInputBytes = execInfo.map(_.totalInputBytes).sum
val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum
val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum

val sumContent =
<tr>
<td>{execInfo.map(_.rddBlocks).sum}</td>
<td sorttable_customkey={memoryUsed.toString}>
{Utils.bytesToString(memoryUsed)} /
{Utils.bytesToString(maximumMemory)}
</td>
<td sorttable_customkey={diskUsed.toString}>
{Utils.bytesToString(diskUsed)}
</td>
<td>{execInfo.map(_.activeTasks).sum}</td>
<td>{execInfo.map(_.failedTasks).sum}</td>
<td>{execInfo.map(_.completedTasks).sum}</td>
<td>{execInfo.map(_.totalTasks).sum}</td>
<td sorttable_customkey={totalDuration.toString}>
{Utils.msDurationToString(totalDuration)}
</td>
<td sorttable_customkey={totalInputBytes.toString}>
{Utils.bytesToString(totalInputBytes)}
</td>
<td sorttable_customkey={totalShuffleRead.toString}>
{Utils.bytesToString(totalShuffleRead)}
</td>
<td sorttable_customkey={totalShuffleWrite.toString}>
{Utils.bytesToString(totalShuffleWrite)}
</td>
</tr>;

<table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>RDD Blocks</th>
<th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th>
<th>Disk Used</th>
<th>Active Tasks</th>
<th>Failed Tasks</th>
<th>Complete Tasks</th>
<th>Total Tasks</th>
<th>Task Time</th>
<th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
<th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
<th>
<span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}>
Shuffle Write
</span>
</th>
</thead>
<tbody>
{sumContent}
</tbody>
</table>
}
}

private[spark] object ExecutorsPage {
Expand Down

0 comments on commit 61c4587

Please sign in to comment.