Skip to content

Commit

Permalink
[SPARK-46384][SPARK-46404][SS][UI] Fix Operation Duration Stack Chart…
Browse files Browse the repository at this point in the history 
… on Structured Streaming Page

### What changes were proposed in this pull request?

In this PR, we feed correct data to d3.stack() according to the API Change. https://d3js.org/d3-shape/stack

### Why are the changes needed?

Fix Operation Duration Stack Chart

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

#### UI UT tests Add

ui-test/tests/structured-streaming-page.test.js

#### Local tests

![image](https://github.com/apache/spark/assets/8326978/349de7af-aef6-4ed4-b1c1-2ebc54d6444e)

### Was this patch authored or co-authored using generative AI tooling?

no

Closes #44346 from yaooqinn/SPARK-46384.

Authored-by: Kent Yao <yao@apache.org>
Signed-off-by: Kent Yao <yao@apache.org>
  • Loading branch information
@yaooqinn
yaooqinn committed Dec 15, 2023
1 parent b1e8b50 commit 7d30bd7
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 72 deletions.
39 changes: 26 additions & 13 deletions core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
*/

/* global $, d3, timeFormat, timeTipStrings */
export { showBootstrapTooltip, hideBootstrapTooltip,
getMaxMarginLeftForTimeline, getOnClickTimelineFunction,
registerHistogram, drawHistogram,
registerTimeline, drawTimeline
}

// timeFormat: StreamingPage.scala will generate a global "timeFormat" dictionary to store the time
// and its formatted string. Because we cannot specify a timezone in JavaScript, to make sure the
Expand All @@ -37,14 +42,22 @@ var unitLabelYOffset = -10;
var onClickTimeline = function() {};

// Show a tooltip "text" for "node"
function showBootstrapTooltip(node, text) {
$(node).tooltip({title: text, trigger: "manual", container: "body"});
$(node).tooltip("show");
function showBootstrapTooltip(d3Selection, text) {
d3Selection.each(function() {
$(this).tooltip({title: text, trigger: "manual", container: "body"});
$(this).tooltip("show");
});
}

// Hide the tooltip for "node"
function hideBootstrapTooltip(node) {
$(node).tooltip("dispose");
function hideBootstrapTooltip(d3Selection) {
d3Selection.each(function() {
$(this).tooltip("dispose");
});
}

function getMaxMarginLeftForTimeline() {
return maxMarginLeftForTimeline;
}

/* eslint-disable no-unused-vars */
Expand Down Expand Up @@ -212,7 +225,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
.attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
.on('mouseover', function(d) {
var tip = yValueFormat(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
showBootstrapTooltip(d3.select(this).node(), tip);
showBootstrapTooltip(d3.select(this), tip);
// show the point
d3.select(this)
.attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
Expand All @@ -221,7 +234,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
.attr("r", "3");
})
.on('mouseout', function() {
hideBootstrapTooltip(d3.select(this).node());
hideBootstrapTooltip(d3.select(this));
// hide the point
d3.select(this)
.attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
Expand Down Expand Up @@ -296,10 +309,10 @@ function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
.on('mouseover', function(event, d) {
var percent = yValueFormat(d.length / values.length) + "%";
var tip = d.length + " batches (" + percent + ") between " + yValueFormat(d.x0) + " and " + yValueFormat(d.x1) + " " + unitY;
showBootstrapTooltip(d3.select(this).node(), tip);
showBootstrapTooltip(d3.select(this), tip);
})
.on('mouseout', function() {
hideBootstrapTooltip(d3.select(this).node());
hideBootstrapTooltip(d3.select(this));
});

if (batchInterval && batchInterval <= maxY) {
Expand All @@ -314,19 +327,19 @@ function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
.text("stable")
.on('mouseover', function(d) {
var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")";
showBootstrapTooltip(d3.select(this).node(), tip);
showBootstrapTooltip(d3.select(this), tip);
})
.on('mouseout', function() {
hideBootstrapTooltip(d3.select(this).node());
hideBootstrapTooltip(d3.select(this));
});
}
}
/* eslint-enable no-unused-vars */

$(function() {
$(function () {
var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true";

$("span.expand-input-rate").click(function() {
$("span.expand-input-rate").click(function () {
status = !status;
$("#inputs-table").toggle('collapsed');
// Toggle the class of the arrow between open and closed
Expand Down
82 changes: 38 additions & 44 deletions core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,22 @@
* limitations under the License.
*/

/* global d3, formattedTimeTipStrings, formattedTimeToValues, hideBootstrapTooltip, maxMarginLeftForTimeline, showBootstrapTooltip, unitLabelYOffset */
export { drawAreaStack };
import { getMaxMarginLeftForTimeline, hideBootstrapTooltip, showBootstrapTooltip } from './streaming-page.js';
/* global, d3 */
// pre-define some colors for legends.
var colorPool = ["#F8C471", "#F39C12", "#B9770E", "#73C6B6", "#16A085", "#117A65", "#B2BABB", "#7F8C8D", "#616A6B"];

/* eslint-disable no-unused-vars */
function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
const unitLabelYOffset = -10;

/* eslint-disable no-undef */
function drawAreaStack(id, labels, values) {
d3.select(d3.select(id).node().parentNode)
.style("padding", "8px 0 8px 8px")
.style("border-right", "0px solid white");

// Setup svg using Bostock's margin convention
var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline};
var margin = {top: 20, right: 40, bottom: 30, left: getMaxMarginLeftForTimeline()};
var width = 850 - margin.left - margin.right;
var height = 300 - margin.top - margin.bottom;

Expand All @@ -37,32 +41,32 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");

var data = values;

var parse = d3.timeParse("%H:%M:%S.%L");

// Transpose the data into layers
var dataset = d3.stack()(labels.map(function(fruit) {
return data.map(function(d) {
return {_x: d.x, x: parse(d.x), y: +d[fruit]};
var data = values.flatMap(function(d) {
return Object.keys(d).filter(k => k !== "x").map(function(key) {
return {x: d.x, label: key, duration: +d[key]};
});
}));
});

var dataset = d3.stack()
.keys(labels)
.value(([, group], label) => group.get(label).duration)(
d3.index(data, d => d.x, d => d.label));

// Set x, y and colors
var x = d3.scaleOrdinal()
.domain(dataset[0].map(function(d) { return d.x; }))
.rangeRoundBands([10, width-10], 0.02);
var x = d3.scaleBand()
.domain(dataset[0].map(d => d.data[0]))
.range([10, width-10], 0.02);

var y = d3.scaleLinear()
.domain([0, d3.max(dataset, function(d) { return d3.max(d, function(d) { return d.y0 + d.y; }); })])
.domain([0, d3.max(dataset, layer => d3.max(layer, d => d[1]))])
.range([height, 0]);

var colors = colorPool.slice(0, labels.length);

// Define and draw axes
var yAxis = d3.axisLeft(y).ticks(7).tickFormat( function(d) { return d } );

var xAxis = d3.axisBottom(x).tickFormat(d3.timeFormat("%H:%M:%S.%L"));
var xAxis = d3.axisBottom(x).tickFormat(function(d) { return d });

// Only show the first and last time in the graph
var xline = [];
Expand All @@ -89,33 +93,30 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
.attr("class", "cost")
.style("fill", function(d, i) { return colors[i]; });

var rect = groups.selectAll("rect")
groups.selectAll("rect")
.data(function(d) { return d; })
.enter()
.append("rect")
.attr("x", function(d) { return x(d.x); })
.attr("y", function(d) { return y(d.y0 + d.y); })
.attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); })
.attr("width", x.rangeBand())
.on('mouseover', function(d) {
.attr("x", d => x(d.data[0]))
.attr("y", d => y(d[1]))
.attr("height", d => y(d[0]) - y(d[1]))
.attr("width", x.bandwidth())
.on('mouseover', function(event, d) {
var tip = '';
var idx = 0;
var _values = formattedTimeToValues[d._x];
_values.forEach(function (k) {
tip += labels[idx] + ': ' + k + ' ';
idx += 1;
d.data[1].forEach(function (k) {
tip += k.label + ': ' + k.duration + ' ';
});
tip += " at " + formattedTimeTipStrings[d._x];
showBootstrapTooltip(d3.select(this).node(), tip);
tip += " at " + d.data[0];
showBootstrapTooltip(d3.select(this), tip);
})
.on('mouseout', function() {
hideBootstrapTooltip(d3.select(this).node());
hideBootstrapTooltip(d3.select(this));
})
.on("mousemove", (event, d) => {
var xPosition = d3.pointer(event)[0] - 15;
var yPosition = d3.pointer(event)[1] - 25;
tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
tooltip.select("text").text(d.y);
tooltip.select("text").text(d[1]);
});

// Draw legend
Expand All @@ -130,18 +131,11 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
.attr("width", 18)
.attr("height", 18)
.style("fill", function(d, i) {return colors.slice().reverse()[i];})
.on('mouseover', function(d, i) {
var len = labels.length;
showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]);
.on('mouseover', function(event, d) {
showBootstrapTooltip(d3.select(this), labels[labels.length - 1 - colors.indexOf(d)]);
})
.on('mouseout', function() {
hideBootstrapTooltip(d3.select(this).node());
})
.on("mousemove", (event, d) => {
var xPosition = d3.pointer(event)[0] - 15;
var yPosition = d3.pointer(event)[1] - 25;
tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
tooltip.select("text").text(d.y);
hideBootstrapTooltip(d3.select(this));
});

// Prep the tooltip bits, initial display is hidden
Expand All @@ -162,4 +156,4 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
.attr("font-size", "12px")
.attr("font-weight", "bold");
}
/* eslint-enable no-unused-vars */
/* eslint-enable no-undef */
26 changes: 16 additions & 10 deletions core/src/main/scala/org/apache/spark/ui/GraphUIData.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.ui
import java.{util => ju}
import java.lang.{Long => JLong}

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import scala.xml.{Node, Unparsed}
Expand Down Expand Up @@ -60,7 +61,9 @@ private[spark] class GraphUIData(
}

def generateTimelineHtml(jsCollector: JsCollector): Seq[Node] = {
jsCollector.addImports("import {registerTimeline} from '/static/streaming-page.js';")
jsCollector.addPreparedStatement(s"registerTimeline($minY, $maxY);")
jsCollector.addImports("import {drawTimeline} from '/static/streaming-page.js';")
if (batchInterval.isDefined) {
jsCollector.addStatement(
"drawTimeline(" +
Expand All @@ -77,7 +80,9 @@ private[spark] class GraphUIData(

def generateHistogramHtml(jsCollector: JsCollector): Seq[Node] = {
val histogramData = s"$dataJavaScriptName.map(function(d) { return d.y; })"
jsCollector.addImports("import {registerHistogram} from '/static/streaming-page.js';")
jsCollector.addPreparedStatement(s"registerHistogram($histogramData, $minY, $maxY);")
jsCollector.addImports("import {drawHistogram} from '/static/streaming-page.js';")
if (batchInterval.isDefined) {
jsCollector.addStatement(
"drawHistogram(" +
Expand All @@ -101,20 +106,13 @@ private[spark] class GraphUIData(
}.mkString("[", ",", "]")
val jsForLabels = operationLabels.toSeq.sorted.mkString("[\"", "\",\"", "\"]")

val (maxX, minX, maxY, minY) = if (values != null && values.length > 0) {
val xValues = values.map(_._1)
val yValues = values.map(_._2.asScala.toSeq.map(_._2.toLong).sum)
(xValues.max, xValues.min, yValues.max, yValues.min)
} else {
(0L, 0L, 0L, 0L)
}

dataJavaScriptName = jsCollector.nextVariableName
jsCollector.addPreparedStatement(s"var $dataJavaScriptName = $jsForData;")
val labels = jsCollector.nextVariableName
jsCollector.addPreparedStatement(s"var $labels = $jsForLabels;")
jsCollector.addImports("import {drawAreaStack} from '/static/structured-streaming-page.js';")
jsCollector.addStatement(
s"drawAreaStack('#$timelineDivId', $labels, $dataJavaScriptName, $minX, $maxX, $minY, $maxY)")
s"drawAreaStack('#$timelineDivId', $labels, $dataJavaScriptName)")
<div id={timelineDivId}></div>
}
}
Expand Down Expand Up @@ -145,6 +143,8 @@ private[spark] class JsCollector {
*/
private val statements = ArrayBuffer[String]()

private val imports = mutable.Set[String]()

def addPreparedStatement(js: String): Unit = {
preparedStatements += js
}
Expand All @@ -153,17 +153,23 @@ private[spark] class JsCollector {
statements += js
}

def addImports(js: String): Unit = {
imports.add(js)
}

/**
* Generate a html snippet that will execute all scripts when the DOM has finished loading.
*/
def toHtml: Seq[Node] = {
val js =
s"""
|${imports.mkString("\n")}
|
|$$(document).ready(function() {
| ${preparedStatements.mkString("\n")}
| ${statements.mkString("\n")}
|});""".stripMargin

<script>{Unparsed(js)}</script>
<script type="module">{Unparsed(js)}</script>
}
}
4 changes: 2 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
// scalastyle:off
<script src={SparkUIUtils.prependBaseUri(request, "/static/d3.min.js")}></script>
<link rel="stylesheet" href={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.css")} type="text/css"/>
<script src={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.js")}></script>
<script src={SparkUIUtils.prependBaseUri(request, "/static/structured-streaming-page.js")}></script>
<script type="module" src={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.js")}></script>
<script type="module" src={SparkUIUtils.prependBaseUri(request, "/static/structured-streaming-page.js")}></script>
// scalastyle:on
}

Expand Down
11 changes: 8 additions & 3 deletions streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,19 @@ private[ui] class StreamingPage(parent: StreamingTab)
// scalastyle:off
<script src={SparkUIUtils.prependBaseUri(request, "/static/d3.min.js")}></script>
<link rel="stylesheet" href={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.css")} type="text/css"/>
<script src={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.js")}></script>
<script type="module" src={SparkUIUtils.prependBaseUri(request, "/static/streaming-page.js")}></script>
// scalastyle:on
}

/** Generate html that will set onClickTimeline declared in streaming-page.js */
private def generateOnClickTimelineFunction(): Seq[Node] = {
val js = "onClickTimeline = getOnClickTimelineFunction();"
<script>{Unparsed(js)}</script>
val js =
s"""
|import {getOnClickTimelineFunction} from '/static/streaming-page.js';
|
|onClickTimeline = getOnClickTimelineFunction();
|""".stripMargin
<script type="module">{Unparsed(js)}</script>
}

/** Generate basic information of the streaming program */
Expand Down
Loading

0 comments on commit 7d30bd7

Please sign in to comment.