From acf685508aa766bc3b9f310ade67249076e834fd Mon Sep 17 00:00:00 2001 From: luogankun Date: Thu, 14 May 2015 05:19:08 -0700 Subject: [PATCH 1/4] [SPARK-7634][SQL] thrift server UI optimization --- .../thriftserver/ui/ThriftServerPage.scala | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala index 6a2be4a58e5cb..5aa7483c0b1a2 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala @@ -45,7 +45,13 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage("" {listener.onlineSessionNum} session(s) are online, running {listener.totalRunning} SQL statement(s) ++ - generateSessionStatsTable() ++ + generateSessionStatsTable(true) ++ +
++ +

+ {listener.sessionList.size - listener.onlineSessionNum} session(s) are offline +

++ + generateSessionStatsTable(false) ++ +
++ generateSQLStatsTable() UIUtils.headerSparkPage("ThriftServer", content, parent, Some(5000)) } @@ -135,11 +141,15 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage("" } /** Generate stats of batch sessions of the thrift server program */ - private def generateSessionStatsTable(): Seq[Node] = { - val numBatches = listener.sessionList.size + private def generateSessionStatsTable(online: Boolean): Seq[Node] = { + val numBatches = if(online) listener.onlineSessionNum else (listener.sessionList.size-listener.onlineSessionNum) val table = if (numBatches > 0) { - val dataRows = - listener.sessionList.values + val dataRowsTmp = listener.sessionList.values.toSeq + val dataRows = if(online) { + dataRowsTmp.filter(_.finishTimestamp == 0).sortBy(_.startTimestamp).reverse + } else { + dataRowsTmp.filter(_.finishTimestamp != 0).sortBy(_.startTimestamp).reverse + } val headerRow = Seq("User", "IP", "Session ID", "Start Time", "Finish Time", "Duration", "Total Execute") def generateDataRow(session: SessionInfo): Seq[Node] = { From a1c10600497e402e256064671eed0b0c78e90a87 Mon Sep 17 00:00:00 2001 From: luogankun Date: Sun, 31 May 2015 19:34:33 -0700 Subject: [PATCH 2/4] [SPARK-7634][SQL] thrift server UI optimization --- .../sql/hive/thriftserver/ui/ThriftServerPage.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala index 5aa7483c0b1a2..9af7a93e387d5 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala @@ -75,7 +75,7 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage("" val table = if (numStatement > 0) { val headerRow = Seq("User", "JobID", "GroupID", "Start Time", "Finish Time", "Duration", "Statement", "State", "Detail") - val dataRows = listener.executionList.values + val dataRows = listener.executionList.values.toSeq.reverse def generateDataRow(info: ExecutionInfo): Seq[Node] = { val jobLink = info.jobId.map { id: String => @@ -142,13 +142,15 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage("" /** Generate stats of batch sessions of the thrift server program */ private def generateSessionStatsTable(online: Boolean): Seq[Node] = { - val numBatches = if(online) listener.onlineSessionNum else (listener.sessionList.size-listener.onlineSessionNum) + val numBatches = if(online) listener.onlineSessionNum + else (listener.sessionList.size - listener.onlineSessionNum) + val table = if (numBatches > 0) { val dataRowsTmp = listener.sessionList.values.toSeq val dataRows = if(online) { - dataRowsTmp.filter(_.finishTimestamp == 0).sortBy(_.startTimestamp).reverse + dataRowsTmp.filter(_.finishTimestamp == 0).reverse } else { - dataRowsTmp.filter(_.finishTimestamp != 0).sortBy(_.startTimestamp).reverse + dataRowsTmp.filter(_.finishTimestamp != 0).reverse } val headerRow = Seq("User", "IP", "Session ID", "Start Time", "Finish Time", "Duration", "Total Execute") From 43129ef7c575772d83b0b7e1492833bea20da597 Mon Sep 17 00:00:00 2001 From: luogankun Date: Sun, 31 May 2015 19:38:53 -0700 Subject: [PATCH 3/4] [SPARK-7634][SQL] thrift server UI optimization --- .../spark/sql/hive/thriftserver/ui/ThriftServerPage.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala index 9af7a93e387d5..f85a540f6d3d3 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala @@ -143,7 +143,7 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage("" /** Generate stats of batch sessions of the thrift server program */ private def generateSessionStatsTable(online: Boolean): Seq[Node] = { val numBatches = if(online) listener.onlineSessionNum - else (listener.sessionList.size - listener.onlineSessionNum) + else (listener.sessionList.size - listener.onlineSessionNum) val table = if (numBatches > 0) { val dataRowsTmp = listener.sessionList.values.toSeq From 642148fd09d0292f05f28952eaf617255d3050ff Mon Sep 17 00:00:00 2001 From: luogankun Date: Wed, 10 Jun 2015 23:56:06 -0700 Subject: [PATCH 4/4] [SPARK-7634][SQL]The offline session table is fold by default --- .../thriftserver/ui/ThriftServerPage.scala | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala index f85a540f6d3d3..315ed092a866a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala @@ -44,13 +44,23 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""

{listener.onlineSessionNum} session(s) are online, running {listener.totalRunning} SQL statement(s) -

++ - generateSessionStatsTable(true) ++ -
++ -

- {listener.sessionList.size - listener.onlineSessionNum} session(s) are offline -

++ - generateSessionStatsTable(false) ++ + +
+ {generateSessionStatsTable(true)} +
+
+
+ + + offline session(s) + + +

++ generateSQLStatsTable() UIUtils.headerSparkPage("ThriftServer", content, parent, Some(5000))