diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css new file mode 100644 index 0000000000000..b859d7d23bd5f --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#dag-viz-graph svg path { + stroke: #444444; + stroke-width: 1.5px; +} + +#dag-viz-graph svg g.cluster rect { + stroke-width: 4px; + stroke-opacity: 0.5; +} + +#dag-viz-graph svg g.node circle, +#dag-viz-graph svg g.node rect { + fill: #444444; +} + +#dag-viz-graph svg g.node.cached circle, +#dag-viz-graph svg g.node.cached rect { + fill: #FF0000; +} + +/* Job page specific styles */ + +#dag-viz-graph svg.job marker#marker-arrow path { + fill: #444444; + stroke-width: 0px; +} + +#dag-viz-graph svg.job g.cluster rect { + fill: #FFFFFF; + stroke: #AADFFF; +} + +#dag-viz-graph svg.job g.cluster[id*="stage"] rect { + stroke: #FFDDEE; + stroke-width: 6px; +} + +#dag-viz-graph svg.job g#cross-stage-edges path { + fill: none; +} + +#dag-viz-graph svg.job g.cluster text { + fill: #AAAAAA; + font-size: 11px; +} + +/* Stage page specific styles */ + +#dag-viz-graph svg.stage g.cluster rect { + fill: #F0F8FF; + stroke: #AADFFF; +} + +#dag-viz-graph svg.stage g.cluster[id*="stage"] rect { + fill: #FFFFFF; + stroke: #FFDDEE; + stroke-width: 6px; +} + +#dag-viz-graph svg.stage g.node g.label text tspan { + fill: #FFFFFF; +} + +#dag-viz-graph svg.stage g.cluster text { + fill: #444444; + font-size: 14px; + font-weight: bold; +} diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js index cfbc0a77cf7e6..733831eaeba42 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js +++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js @@ -52,13 +52,6 @@ */ var VizConstants = { - rddColor: "#444444", - rddCachedColor: "#FF0000", - rddOperationColor: "#AADFFF", - stageColor: "#FFDDEE", - clusterLabelColor: "#888888", - edgeColor: "#444444", - edgeWidth: "1.5px", svgMarginX: 20, svgMarginY: 20, stageSep: 50, @@ -113,17 +106,21 @@ function toggleDagViz(forJob) { function renderDagViz(forJob) { // If there is not a dot file to render, fail fast and report error + var jobOrStage = forJob ? "job" : "stage"; if (metadataContainer().empty()) { graphContainer().append("div").text( - "No visualization information available for this " + (forJob ? "job" : "stage")); + "No visualization information available for this " + jobOrStage); return; } - var svg = graphContainer().append("svg"); + var svg = graphContainer().append("svg").attr("class", jobOrStage); + if (forJob) { renderDagVizForJob(svg); + postProcessDagVizForJob(); } else { renderDagVizForStage(svg); + postProcessDagVizForStage(); } // Find cached RDDs @@ -137,9 +134,17 @@ function renderDagViz(forJob) { // resizeSvg(svg); +} - // - styleDagViz(forJob); +function postProcessDagVizForJob() { +} + +function postProcessDagVizForStage() { + // Round corners on RDDs on the stage page + graphContainer() + .selectAll("svg.stage g.node rect") + .attr("rx", "5") + .attr("ry", "5"); } /* @@ -216,6 +221,7 @@ function renderDagVizForJob(svgContainer) { var crossStageEdges = []; metadataContainer().selectAll(".stage-metadata").each(function(d, i) { + // Set up container var metadata = d3.select(this); var dot = metadata.select(".dot-file").text(); var stageId = metadata.attr("stageId"); @@ -226,11 +232,14 @@ function renderDagVizForJob(svgContainer) { var container = svgContainer .append("a").attr("xlink:href", stageLink) .append("g").attr("id", containerId); + // Now we need to shift the container for this stage so it doesn't overlap // with existing ones. We do not need to do this for the first stage. if (i > 0) { // Take into account the position and width of the last stage's container - var existingStages = stageClusters(); + var existingStages = graphContainer() + .selectAll("svg g.cluster") + .filter("[id*=\"" + VizConstants.stageClusterPrefix + "\"]"); if (!existingStages.empty()) { var lastStage = d3.select(existingStages[0].pop()); var lastStageId = lastStage.attr("id"); @@ -262,25 +271,6 @@ function renderDagVizForJob(svgContainer) { connectRDDs(fromRDDId, toRDDId, container); } } -} - -/* Render the dot file as an SVG in the given container. */ -function renderDot(dot, container) { - var escaped_dot = dot - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, "\""); - var g = graphlibDot.read(escaped_dot); - var renderer = new dagreD3.render(); - renderer(container, g); -} - -/* Style the visualization we just rendered. */ -function styleDagViz(forJob) { - graphContainer() - .selectAll("svg path") - .style("stroke", VizConstants.edgeColor) - .style("stroke-width", VizConstants.edgeWidth); // Put an arrow at the end of every edge // We need to do this because we manually render some edges ourselves @@ -289,77 +279,19 @@ function styleDagViz(forJob) { graphContainer().select("svg") .append(function() { return dagreD3Marker.cloneNode(true); }) .attr("id", "marker-arrow") - .select("path") - .attr("fill", VizConstants.edgeColor) - .attr("strokeWidth", "0px"); graphContainer().selectAll("svg g > path").attr("marker-end", "url(#marker-arrow)"); graphContainer().selectAll("svg g.edgePaths def").remove(); // We no longer need these - - // Apply any job or stage specific styles - if (forJob) { - styleDagVizForJob(); - } else { - styleDagVizForStage(); - } } -/* Apply job-page-specific style to the visualization. */ -function styleDagVizForJob() { - graphContainer() - .selectAll("svg g.cluster rect") - .style("fill", "white") - .style("stroke", VizConstants.rddOperationColor) - .style("stroke-width", "4px") - .style("stroke-opacity", "0.5"); - stageClusters() - .select("rect") - .style("stroke", VizConstants.stageColor) - .style("strokeWidth", "6px"); - graphContainer() - .selectAll("svg g.node circle") - .style("fill", VizConstants.rddColor); - // TODO: add a legend to explain what a highlighted dot means - graphContainer() - .selectAll("svg g.cached circle") - .style("fill", VizConstants.rddCachedColor); - graphContainer() - .selectAll("svg g#cross-stage-edges path") - .style("fill", "none"); - graphContainer() - .selectAll("svg g.cluster text") - .attr("fill", VizConstants.clusterLabelColor) - .attr("font-size", "11px"); -} - -/* Apply stage-page-specific style to the visualization. */ -function styleDagVizForStage() { - graphContainer() - .selectAll("svg g.cluster rect") - .style("fill", "#F0F8FF") - .style("stroke", VizConstants.rddOperationColor) - .style("stroke-width", "4px") - .style("stroke-opacity", "0.5"); - stageClusters() - .select("rect") - .style("fill", "white") - .style("stroke", VizConstants.stageColor) - .style("strokeWidth", "6px"); - graphContainer() - .selectAll("svg g.node rect") - .style("fill", "#444444") - .attr("rx", "5") // round corners - .attr("ry", "5"); - graphContainer() - .selectAll("svg g.cached rect") - .style("fill", VizConstants.rddCachedColor) - graphContainer() - .selectAll("svg g.node g.label text tspan") - .style("fill", "white"); - graphContainer() - .selectAll("svg g.cluster text") - .attr("fill", "#444444") - .attr("font-size", "14px") - .attr("font-weight", "bold") +/* Render the dot file as an SVG in the given container. */ +function renderDot(dot, container) { + var escaped_dot = dot + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, "\""); + var g = graphlibDot.read(escaped_dot); + var renderer = new dagreD3.render(); + renderer(container, g); } /* @@ -371,10 +303,8 @@ function getAbsolutePosition(d3selection) { throw "Attempted to get absolute position of an empty selection."; } var obj = d3selection; - var _x = obj.attr("x") || 0, - _y = obj.attr("y") || 0; - _x = toFloat(_x); - _y = toFloat(_y); + var _x = toFloat(obj.attr("x")) || 0; + var _y = toFloat(obj.attr("y")) || 0; while (!obj.empty()) { var transformText = obj.attr("transform"); if (transformText) { @@ -395,7 +325,7 @@ function getAbsolutePosition(d3selection) { /* (Job page only) Connect two RDD nodes with a curved edge. */ function connectRDDs(fromRDDId, toRDDId, container) { var fromNodeId = VizConstants.nodePrefix + fromRDDId; - var toNodeId = VizConstants.nodePrefix + toRDDId + var toNodeId = VizConstants.nodePrefix + toRDDId; var fromPos = getAbsolutePosition(graphContainer().select("#" + fromNodeId)); var toPos = getAbsolutePosition(graphContainer().select("#" + toNodeId)); @@ -445,15 +375,12 @@ function connectRDDs(fromRDDId, toRDDId, container) { container.append("path").datum(points).attr("d", line); } -/* Helper d3 accessor to clusters that represent stages. */ -function stageClusters() { - return graphContainer() - .selectAll("svg g.cluster") - .filter("[id*=\"" + VizConstants.stageClusterPrefix + "\"]"); -} - /* Helper method to convert attributes to numeric values. */ function toFloat(f) { - return parseFloat(f.toString().replace(/px$/, "")); + if (f) { + return parseFloat(f.toString().replace(/px$/, "")); + } else { + return f; + } } diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 2f3fb181e4026..d2a6736c590c9 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -156,13 +156,10 @@ private[spark] object UIUtils extends Logging { def commonHeaderNodes: Seq[Node] = { - - - - + + + + @@ -174,6 +171,7 @@ private[spark] object UIUtils extends Logging { } def vizHeaderNodes: Seq[Node] = { +