Not expose nodeId

apache · Apr 21, 2020 · da88978 · da88978
1 parent 7792741
commit da88978
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
@@ -83,13 +83,16 @@ private[v1] class SqlResource extends BaseAppResource {
     val metrics = groupedMap.mapValues[Seq[Metric]](sqlPlanMetrics =>
       sqlPlanMetrics.flatMap(m => getMetric(metricValues, m.accumulatorId, m.name.trim)))
 
-    val metricDetails = metrics.map {
+    val sortedMetrics = metrics.toSeq.sortBy
+    { case ((nodeId, nodeName), metrics) => nodeId }.reverse
+
+    val metricDetails = sortedMetrics.map {
       case ((nodeId: Long, nodeName: String), metrics: Seq[Metric]) =>
         val wholeStageCodegenId = nodeIdAndWSCGIdMap.get(nodeId).flatten
-        MetricDetails(nodeId = nodeId, nodeName = nodeName.trim, wholeStageCodegenId, metrics)
-    }.toSeq
+        MetricDetails(nodeName = nodeName.trim, wholeStageCodegenId, metrics)
+    }
 
-    metricDetails.sortBy(_.nodeId).reverse
+    metricDetails
   }
 
   private def prepareExecutionData(

diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
@@ -31,7 +31,6 @@ class ExecutionData private[spark] (
     val metricDetails: Seq[MetricDetails])
 
 case class MetricDetails private[spark] (
-    nodeId: Long,
     nodeName: String,
     wholeStageCodegenId: Option[Long] = None,
     metrics: Seq[Metric])

diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
@@ -75,11 +75,11 @@ object SqlResourceSuite {
   }
 
   private def getExpectedMetricDetails(): Seq[MetricDetails] = {
-    val metricDetails = MetricDetails(1, WHOLE_STAGE_CODEGEN_1,
+    val metricDetails = MetricDetails(WHOLE_STAGE_CODEGEN_1,
       wholeStageCodegenId = None, metrics = Seq(Metric(DURATION, "0 ms")))
-    val metricDetails2 = MetricDetails(2, FILTER,
+    val metricDetails2 = MetricDetails(FILTER,
       wholeStageCodegenId = Some(1), metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
-    val metricDetails3 = MetricDetails(3, SCAN_TEXT, wholeStageCodegenId = Some(1),
+    val metricDetails3 = MetricDetails(SCAN_TEXT, wholeStageCodegenId = Some(1),
       metrics = Seq(Metric(METADATA_TIME, "2 ms"),
         Metric(NUMBER_OF_FILES_READ, "1"),
         Metric(NUMBER_OF_OUTPUT_ROWS, "1"),
@@ -91,9 +91,9 @@ object SqlResourceSuite {
 
   private def getExpectedMetricDetailsWhenWholeStageCodegenIsOff(): Seq[MetricDetails] = {
     val metricDetails =
-      MetricDetails(1, WHOLE_STAGE_CODEGEN_1, metrics = Seq(Metric(DURATION, "0 ms")))
-    val metricDetails2 = MetricDetails(2, FILTER, metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
-    val metricDetails3 = MetricDetails(3, SCAN_TEXT,
+      MetricDetails(WHOLE_STAGE_CODEGEN_1, metrics = Seq(Metric(DURATION, "0 ms")))
+    val metricDetails2 = MetricDetails(FILTER, metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
+    val metricDetails3 = MetricDetails(SCAN_TEXT,
       metrics = Seq(Metric(METADATA_TIME, "2 ms"),
         Metric(NUMBER_OF_FILES_READ, "1"),
         Metric(NUMBER_OF_OUTPUT_ROWS, "1"),