From eb823b822043ced54b82de39571c36d0c8b42f96 Mon Sep 17 00:00:00 2001 From: Joshi Date: Tue, 23 Jun 2015 20:36:30 -0700 Subject: [PATCH 1/3] SPARK-5768: Fix for incorrect memory in Spark UI --- core/src/main/scala/org/apache/spark/ui/ToolTips.scala | 3 +++ .../main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala index 063e2a1f8b18e..cee49aa7b2e66 100644 --- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala +++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala @@ -35,6 +35,9 @@ private[spark] object ToolTips { val OUTPUT = "Bytes and records written to Hadoop." + val STORAGE_MEMORY = + "Total memory used for caching of the total available memory for caching.This is the storage memory used on the executor." + val SHUFFLE_WRITE = "Bytes and records written to disk in order to be read by a shuffle in a future stage." diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index b247e4cdc3bd4..01cddda4c62cd 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -67,7 +67,7 @@ private[ui] class ExecutorsPage( Executor ID Address RDD Blocks - Memory Used + Storage Memory Disk Used Active Tasks Failed Tasks From 2fe53d98d6f22e2f7d249720315e805e54dfc6f8 Mon Sep 17 00:00:00 2001 From: Joshi Date: Wed, 24 Jun 2015 20:51:02 -0700 Subject: [PATCH 2/3] Fix for incorrect memory in Spark UI --- core/src/main/scala/org/apache/spark/ui/ToolTips.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala index cee49aa7b2e66..0e69949b84d03 100644 --- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala +++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala @@ -36,7 +36,8 @@ private[spark] object ToolTips { val OUTPUT = "Bytes and records written to Hadoop." val STORAGE_MEMORY = - "Total memory used for caching of the total available memory for caching.This is the storage memory used on the executor." + """Memory used / total available memory for storage of data like RDD partitions cached in + memory. """ val SHUFFLE_WRITE = "Bytes and records written to disk in order to be read by a shuffle in a future stage." From b678a91ce73007b3d8a656e204e2f9ec9bd709a1 Mon Sep 17 00:00:00 2001 From: Joshi Date: Wed, 24 Jun 2015 21:17:08 -0700 Subject: [PATCH 3/3] Fix for incorrect memory in Spark UI --- core/src/main/scala/org/apache/spark/ui/ToolTips.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala index 0e69949b84d03..e2d25e36365fa 100644 --- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala +++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala @@ -36,8 +36,8 @@ private[spark] object ToolTips { val OUTPUT = "Bytes and records written to Hadoop." val STORAGE_MEMORY = - """Memory used / total available memory for storage of data like RDD partitions cached in - memory. """ + "Memory used / total available memory for storage of data " + + "like RDD partitions cached in memory. " val SHUFFLE_WRITE = "Bytes and records written to disk in order to be read by a shuffle in a future stage."