From a41ac4bda10cd83018e651aaeae123ef4056a579 Mon Sep 17 00:00:00 2001 From: Joshi Date: Wed, 8 Jul 2015 12:05:42 -0700 Subject: [PATCH] History Server: updated order for multiple attempts --- .../deploy/history/FsHistoryProvider.scala | 7 +- .../spark/deploy/history/HistoryPage.scala | 187 ++++++++---------- 2 files changed, 86 insertions(+), 108 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index db383b9823d3c..f83f243ffec8f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -414,7 +414,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) /** * Comparison function that defines the sort order for application attempts within the same * application. Order is: running attempts before complete attempts, running attempts sorted - * by start time, completed attempts sorted by end time. + * by start time showing whichever started first, + * completed attempts sorted by end time showing whichever ended first. * * Normally applications should have a single running attempt; but failure to call sc.stop() * may cause multiple running attempts to show up. @@ -425,9 +426,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) a1: FsApplicationAttemptInfo, a2: FsApplicationAttemptInfo): Boolean = { if (a1.completed == a2.completed) { - if (a1.completed) a1.endTime >= a2.endTime else a1.startTime >= a2.startTime + if (a1.completed) a1.endTime <= a2.endTime else a1.startTime <= a2.startTime } else { - !a1.completed + a1.completed } } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index b42f92212692e..dd12898723348 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -35,6 +35,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") Option(request.getParameter("showIncomplete")).getOrElse("false").toBoolean val allApps = parent.getApplicationList() + .filter(_.attempts.head.completed != requestedIncomplete) val allAppsSize = allApps.size val actualFirst = if (requestedFirst < allAppsSize) requestedFirst else 0 @@ -50,15 +51,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") val hasMultipleAttempts = appsToShow.exists(_.attempts.size > 1) val appTable = if (hasMultipleAttempts) { - UIUtils.listingTable( - appWithAttemptHeader, - appWithAttemptRow(_, requestedIncomplete), - appsToShow) + UIUtils.listingTable(appWithAttemptHeader, appWithAttemptRow, appsToShow) } else { - UIUtils.listingTable( - appHeader, - appRow(_, requestedIncomplete), - appsToShow) + UIUtils.listingTable(appHeader, appRow, appsToShow) } val providerConfig = parent.getProviderConfig() @@ -69,61 +64,61 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {providerConfig.map { case (k, v) =>
  • {k}: {v}
    • }} { - // This displays the indices of pages that are within `plusOrMinus` pages of - // the current page. Regardless of where the current page is, this also links - // to the first and last page. If the current page +/- `plusOrMinus` is greater - // than the 2nd page from the first page or less than the 2nd page from the last - // page, `...` will be displayed. - if (allAppsSize > 0) { - val leftSideIndices = - rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete) - val rightSideIndices = - rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount, - requestedIncomplete) - -

    - Showing {actualFirst + 1}-{last + 1} of {allAppsSize} - {if (requestedIncomplete) "(Incomplete applications)"} - - { - if (actualPage > 1) { - < - 1 - } - } - {if (actualPage - plusOrMinus > secondPageFromLeft) " ... "} - {leftSideIndices} - {actualPage} - {rightSideIndices} - {if (actualPage + plusOrMinus < secondPageFromRight) " ... "} - { - if (actualPage < pageCount) { - {pageCount} - > - } - } - -

    ++ + // This displays the indices of pages that are within `plusOrMinus` pages of + // the current page. Regardless of where the current page is, this also links + // to the first and last page. If the current page +/- `plusOrMinus` is greater + // than the 2nd page from the first page or less than the 2nd page from the last + // page, `...` will be displayed. + if (allAppsSize > 0) { + val leftSideIndices = + rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete) + val rightSideIndices = + rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount, + requestedIncomplete) + +

    + Showing {actualFirst + 1}-{last + 1} of {allAppsSize} + {if (requestedIncomplete) "(Incomplete applications)"} + + { + if (actualPage > 1) { + < + 1 + } + } + {if (actualPage - plusOrMinus > secondPageFromLeft) " ... "} + {leftSideIndices} + {actualPage} + {rightSideIndices} + {if (actualPage + plusOrMinus < secondPageFromRight) " ... "} + { + if (actualPage < pageCount) { + {pageCount} + > + } + } + +

    ++ appTable - } else if (requestedIncomplete) { -

    No incomplete applications found!

    - } else { -

    No completed applications found!

    ++ + } else if (requestedIncomplete) { +

    No incomplete applications found!

    + } else { +

    No completed applications found!

    ++

    Did you specify the correct logging directory? Please verify your setting of spark.history.fs.logDirectory and whether you have the permissions to access it.
    It is also possible that your application did not run to completion or did not stop the SparkContext.

    - } + } } { - if (requestedIncomplete) { - "Back to completed applications" - } else { - "Show incomplete applications" - } + if (requestedIncomplete) { + "Back to completed applications" + } else { + "Show incomplete applications" + } } @@ -151,19 +146,18 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") "Last Updated") private def rangeIndices( - range: Seq[Int], - condition: Int => Boolean, - showIncomplete: Boolean): Seq[Node] = { + range: Seq[Int], + condition: Int => Boolean, + showIncomplete: Boolean): Seq[Node] = { range.filter(condition).map(nextPage => {nextPage} ) } private def attemptRow( - renderAttemptIdColumn: Boolean, - info: ApplicationHistoryInfo, - attempt: ApplicationAttemptInfo, - isFirst: Boolean, - requestedIncomplete: Boolean): Seq[Node] = { + renderAttemptIdColumn: Boolean, + info: ApplicationHistoryInfo, + attempt: ApplicationAttemptInfo, + isFirst: Boolean): Seq[Node] = { val uiAddress = HistoryServer.getAttemptURI(info.id, attempt.attemptId) val startTime = UIUtils.formatDate(attempt.startTime) val endTime = if (attempt.endTime > 0) UIUtils.formatDate(attempt.endTime) else "-" @@ -174,67 +168,50 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") "-" } val lastUpdated = UIUtils.formatDate(attempt.lastUpdated) - var someAttemptCompleted = false - info.attempts.foreach{ attempt => - if (attempt.completed) someAttemptCompleted = true - } { - if (isFirst) { - if (info.attempts.size > 1 || renderAttemptIdColumn) { - - {info.id} + if (isFirst) { + if (info.attempts.size > 1 || renderAttemptIdColumn) { + + {info.id} {info.name} - } else { - {info.id} - {info.name} - } } else { - Nil + {info.id} + {info.name} } + } else { + Nil } - { - if (renderAttemptIdColumn && - (requestedIncomplete || (!requestedIncomplete && someAttemptCompleted))) { - if (info.attempts.size > 1 && attempt.attemptId.isDefined) { - - {attempt.attemptId.get} - } else { -   - } - } else { - Nil - } } { - if (requestedIncomplete || (!requestedIncomplete && someAttemptCompleted)) { - {startTime} - {endTime} - - {duration} - {attempt.sparkUser} - {lastUpdated} + if (renderAttemptIdColumn) { + if (info.attempts.size > 1 && attempt.attemptId.isDefined) { + + {attempt.attemptId.get} } else { - Nil +   } + } else { + Nil + } } + {startTime} + {endTime} + + {duration} + {attempt.sparkUser} + {lastUpdated} } - - - private def appRow( - info: ApplicationHistoryInfo, - requestedIncomplete: Boolean): Seq[Node] = { - attemptRow(false, info, info.attempts.head, true, requestedIncomplete) + private def appRow(info: ApplicationHistoryInfo): Seq[Node] = { + attemptRow(false, info, info.attempts.head, true) } - private def appWithAttemptRow( - info: ApplicationHistoryInfo, - requestedIncomplete: Boolean): Seq[Node] = { - attemptRow(true, info, info.attempts.head, true, requestedIncomplete) ++ - info.attempts.drop(1).flatMap(attemptRow(true, info, _, false, requestedIncomplete)) + private def appWithAttemptRow(info: ApplicationHistoryInfo): Seq[Node] = { + attemptRow(true, info, info.attempts.head, true) ++ + info.attempts.drop(1).flatMap(attemptRow(true, info, _, false)) } private def makePageLink(linkPage: Int, showIncomplete: Boolean): String = {