From 0412c4b959b42cb0b41c669593e689430da59ec2 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 16:10:24 +0100 Subject: [PATCH 1/9] Fix for SPARK-23051 --- .../apache/spark/ui/jobs/AllJobsPage.scala | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 37e3b3b304a63..5998d77c6e801 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -429,20 +429,36 @@ private[ui] class JobDataSource( val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") val submissionTime = jobData.submissionTime val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") - val jobDescription = UIUtils.makeDescription(jobData.description.getOrElse(""), - basePath, plainText = false) + val lastStage = { + val stageAttempts = jobData.stageIds.map(store.stageData(_)).flatten + if (!stageAttempts.isEmpty) { + val lastId = stageAttempts.map(_.stageId).max + stageAttempts.find(_.stageId == lastId) + } else { + None + }} + val jobDescription = jobData.description.getOrElse { + lastStage.get.description.getOrElse(jobData.name)} val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) + val lastStageName = lastStage.map(_.name).getOrElse(jobData.name) + val lastStageDescription = lastStage.flatMap(_.description) + .getOrElse( + jobData.description + .getOrElse(jobData.name)) + + val formattedJobDescription = + UIUtils.makeDescription(jobDescription, basePath, plainText = false) new JobTableRowData( jobData, - jobData.name, - jobData.description.getOrElse(jobData.name), + lastStageName, + lastStageDescription, duration.getOrElse(-1), formattedDuration, submissionTime.map(_.getTime()).getOrElse(-1L), formattedSubmissionTime, - jobDescription, + formattedJobDescription, detailUrl ) } From 937e0fd579a7f133f860eb6f572f50ad7d7fdf89 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 16:25:44 +0100 Subject: [PATCH 2/9] fixed logic determining jobDescription --- .../src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 5998d77c6e801..3f839c561ec06 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -431,7 +431,7 @@ private[ui] class JobDataSource( val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") val lastStage = { - val stageAttempts = jobData.stageIds.map(store.stageData(_)).flatten + val stageAttempts = jobData.stageIds.flatMap(store.stageData(_)) if (!stageAttempts.isEmpty) { val lastId = stageAttempts.map(_.stageId).max stageAttempts.find(_.stageId == lastId) @@ -439,7 +439,7 @@ private[ui] class JobDataSource( None }} val jobDescription = jobData.description.getOrElse { - lastStage.get.description.getOrElse(jobData.name)} + lastStage.flatMap(_.description).getOrElse(jobData.name)} val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) val lastStageName = lastStage.map(_.name).getOrElse(jobData.name) val lastStageDescription = lastStage.flatMap(_.description) From df41000d99eef3b609ece4b072c5aee0650c21f1 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 16:51:06 +0100 Subject: [PATCH 3/9] Formatted code --- .../org/apache/spark/ui/jobs/AllJobsPage.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 3f839c561ec06..20a4ab6081c7d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -23,6 +23,8 @@ import javax.servlet.http.HttpServletRequest import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer +import scala.concurrent +import scala.concurrent.duration import scala.xml._ import org.apache.commons.lang3.StringEscapeUtils @@ -438,18 +440,22 @@ private[ui] class JobDataSource( } else { None }} - val jobDescription = jobData.description.getOrElse { - lastStage.flatMap(_.description).getOrElse(jobData.name)} - val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) + + val jobDescription = jobData.description + .getOrElse(lastStage.flatMap(_.description). + getOrElse(jobData.name)) + val lastStageName = lastStage.map(_.name).getOrElse(jobData.name) + val lastStageDescription = lastStage.flatMap(_.description) - .getOrElse( - jobData.description + .getOrElse(jobData.description .getOrElse(jobData.name)) val formattedJobDescription = UIUtils.makeDescription(jobDescription, basePath, plainText = false) + val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) + new JobTableRowData( jobData, lastStageName, From 3fa520a78098ea80148f45d1b4ccc80adee6becf Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 17:15:10 +0100 Subject: [PATCH 4/9] Removed unused import --- core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 20a4ab6081c7d..99c23549cb640 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -23,8 +23,6 @@ import javax.servlet.http.HttpServletRequest import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import scala.concurrent -import scala.concurrent.duration import scala.xml._ import org.apache.commons.lang3.StringEscapeUtils From 7c1f5fcf3455c7a12bf81b799e58b6eb463c44be Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 18:04:59 +0100 Subject: [PATCH 5/9] Minor changes requested by reviewers --- .../org/apache/spark/ui/jobs/AllJobsPage.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 99c23549cb640..54d6912243b22 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -432,16 +432,17 @@ private[ui] class JobDataSource( val lastStage = { val stageAttempts = jobData.stageIds.flatMap(store.stageData(_)) - if (!stageAttempts.isEmpty) { - val lastId = stageAttempts.map(_.stageId).max - stageAttempts.find(_.stageId == lastId) + if (stageAttempts.nonEmpty) { + val lastAttempt = stageAttempts.maxBy(_.stageId) + Some(lastAttempt) } else { None - }} + } + } val jobDescription = jobData.description - .getOrElse(lastStage.flatMap(_.description). - getOrElse(jobData.name)) + .getOrElse(lastStage.flatMap(_.description) + .getOrElse(jobData.name)) val lastStageName = lastStage.map(_.name).getOrElse(jobData.name) From 8b9b155a1d264098ba6769db3b2176b1831c942c Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 18:06:43 +0100 Subject: [PATCH 6/9] Renamed variable --- .../main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 54d6912243b22..9f01629b7d623 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -430,7 +430,7 @@ private[ui] class JobDataSource( val submissionTime = jobData.submissionTime val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") - val lastStage = { + val lastStageAttempt = { val stageAttempts = jobData.stageIds.flatMap(store.stageData(_)) if (stageAttempts.nonEmpty) { val lastAttempt = stageAttempts.maxBy(_.stageId) @@ -441,12 +441,12 @@ private[ui] class JobDataSource( } val jobDescription = jobData.description - .getOrElse(lastStage.flatMap(_.description) + .getOrElse(lastStageAttempt.flatMap(_.description) .getOrElse(jobData.name)) - val lastStageName = lastStage.map(_.name).getOrElse(jobData.name) + val lastStageName = lastStageAttempt.map(_.name).getOrElse(jobData.name) - val lastStageDescription = lastStage.flatMap(_.description) + val lastStageDescription = lastStageAttempt.flatMap(_.description) .getOrElse(jobData.description .getOrElse(jobData.name)) From 41fcdce67eb4f3f621a14df4833a7af70eab5f37 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 18:07:47 +0100 Subject: [PATCH 7/9] Minor code simplification --- core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 9f01629b7d623..84d73208bca21 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -433,8 +433,7 @@ private[ui] class JobDataSource( val lastStageAttempt = { val stageAttempts = jobData.stageIds.flatMap(store.stageData(_)) if (stageAttempts.nonEmpty) { - val lastAttempt = stageAttempts.maxBy(_.stageId) - Some(lastAttempt) + Some(stageAttempts.maxBy(_.stageId)) } else { None } From fb9ce507782a72d5e6ee6b373c3bda33fc819e13 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 21:43:46 +0100 Subject: [PATCH 8/9] Code was simplified and follows 2.2 logic better --- .../apache/spark/ui/jobs/AllJobsPage.scala | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 84d73208bca21..13e76bd66f08d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -65,12 +65,10 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We }.map { job => val jobId = job.jobId val status = job.status - val displayJobDescription = - if (job.description.isEmpty) { - job.name - } else { - UIUtils.makeDescription(job.description.get, "", plainText = true).text - } + val jobDescription = store.lastStageAttempt(job.stageIds.max).description + val displayJobDescription = jobDescription + .map(UIUtils.makeDescription(_, "", plainText = true).text) + .getOrElse("") val submissionTime = job.submissionTime.get.getTime() val completionTime = job.completionTime.map(_.getTime()).getOrElse(System.currentTimeMillis()) val classNameByStatus = status match { @@ -429,34 +427,17 @@ private[ui] class JobDataSource( val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") val submissionTime = jobData.submissionTime val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") - - val lastStageAttempt = { - val stageAttempts = jobData.stageIds.flatMap(store.stageData(_)) - if (stageAttempts.nonEmpty) { - Some(stageAttempts.maxBy(_.stageId)) - } else { - None - } - } - - val jobDescription = jobData.description - .getOrElse(lastStageAttempt.flatMap(_.description) - .getOrElse(jobData.name)) - - val lastStageName = lastStageAttempt.map(_.name).getOrElse(jobData.name) - - val lastStageDescription = lastStageAttempt.flatMap(_.description) - .getOrElse(jobData.description - .getOrElse(jobData.name)) + val lastStageAttempt = store.lastStageAttempt(jobData.stageIds.max) + val lastStageDescription = lastStageAttempt.description.getOrElse("") val formattedJobDescription = - UIUtils.makeDescription(jobDescription, basePath, plainText = false) + UIUtils.makeDescription(lastStageDescription, basePath, plainText = false) val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId) new JobTableRowData( jobData, - lastStageName, + lastStageAttempt.name, lastStageDescription, duration.getOrElse(-1), formattedDuration, From d9cdb07263f7a584cf217d30c55313283459ac92 Mon Sep 17 00:00:00 2001 From: Sandor Murakozi Date: Fri, 12 Jan 2018 23:55:49 +0100 Subject: [PATCH 9/9] Fixed indentation issue --- core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 13e76bd66f08d..ff916bb6a5759 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -67,7 +67,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We val status = job.status val jobDescription = store.lastStageAttempt(job.stageIds.max).description val displayJobDescription = jobDescription - .map(UIUtils.makeDescription(_, "", plainText = true).text) + .map(UIUtils.makeDescription(_, "", plainText = true).text) .getOrElse("") val submissionTime = job.submissionTime.get.getTime() val completionTime = job.completionTime.map(_.getTime()).getOrElse(System.currentTimeMillis())