From 3911c46892304a5eec7d603dd8a015c9cf1fe0af Mon Sep 17 00:00:00 2001 From: "jianran.tfh" Date: Fri, 17 Mar 2017 17:30:28 +0800 Subject: [PATCH 1/5] rdd id info --- core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala index d47b75544fdba..cd484fa10334e 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala @@ -47,7 +47,8 @@ class BlockRDD[T: ClassTag](sc: SparkContext, @transient val blockIds: Array[Blo blockManager.get[T](blockId) match { case Some(block) => block.data.asInstanceOf[Iterator[T]] case None => - throw new Exception("Could not compute split, block " + blockId + " not found") + throw new Exception("Could not compute split, block " + blockId + + " of RDD " + id + " not found") } } From e426dc5243fca185ff0ef8dafcbfcba14e031b83 Mon Sep 17 00:00:00 2001 From: jianran Date: Fri, 17 Mar 2017 23:03:05 +0800 Subject: [PATCH 2/5] [SPARK-19998] add RDD id info --- core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala index cd484fa10334e..cbbf13b7aba49 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala @@ -47,8 +47,9 @@ class BlockRDD[T: ClassTag](sc: SparkContext, @transient val blockIds: Array[Blo blockManager.get[T](blockId) match { case Some(block) => block.data.asInstanceOf[Iterator[T]] case None => - throw new Exception("Could not compute split, block " + blockId + - " of RDD " + id + " not found") + throw new Exception( + s"Could not compute split, block $blockId not found of RDD $id not found") + } } From 98393f8e346be0cd1cbe73b6861668e17446990d Mon Sep 17 00:00:00 2001 From: "jianran.tfh" Date: Sat, 18 Mar 2017 21:55:08 +0800 Subject: [PATCH 3/5] remove redundant 'not found' --- core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala index cbbf13b7aba49..e8df1acc85c2e 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala @@ -47,8 +47,7 @@ class BlockRDD[T: ClassTag](sc: SparkContext, @transient val blockIds: Array[Blo blockManager.get[T](blockId) match { case Some(block) => block.data.asInstanceOf[Iterator[T]] case None => - throw new Exception( - s"Could not compute split, block $blockId not found of RDD $id not found") + throw new Exception(s"Could not compute split, block $blockId of RDD $id not found") } } From 4bd4b8d573e1bd93176e67e8f3487e0292acd6f6 Mon Sep 17 00:00:00 2001 From: "jianran.tfh" Date: Mon, 20 Mar 2017 10:24:15 +0800 Subject: [PATCH 4/5] remove empty line --- core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala index e8df1acc85c2e..4e036c2ed49b5 100644 --- a/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/BlockRDD.scala @@ -48,7 +48,6 @@ class BlockRDD[T: ClassTag](sc: SparkContext, @transient val blockIds: Array[Blo case Some(block) => block.data.asInstanceOf[Iterator[T]] case None => throw new Exception(s"Could not compute split, block $blockId of RDD $id not found") - } } From ad5ef3cca2956ef689fd3cd33a36eec72cef4ba7 Mon Sep 17 00:00:00 2001 From: "jianran.tfh" Date: Tue, 28 Mar 2017 10:12:30 +0800 Subject: [PATCH 5/5] TaskSetManager checkSpeculatableTasks variables immutability and Use string interpolation --- .../scala/org/apache/spark/scheduler/TaskSetManager.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index a177aab5f95de..86ac1839718c0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -936,15 +936,15 @@ private[spark] class TaskSetManager( } var foundTasks = false val minFinishedForSpeculation = (SPECULATION_QUANTILE * numTasks).floor.toInt - logDebug("Checking for speculative tasks: minFinished = " + minFinishedForSpeculation) + logDebug(s"Checking for speculative tasks: minFinished = $minFinishedForSpeculation") if (tasksSuccessful >= minFinishedForSpeculation && tasksSuccessful > 0) { val time = clock.getTimeMillis() - var medianDuration = successfulTaskDurations.median + val medianDuration = successfulTaskDurations.median val threshold = max(SPECULATION_MULTIPLIER * medianDuration, minTimeToSpeculation) // TODO: Threshold should also look at standard deviation of task durations and have a lower // bound based on that. - logDebug("Task length threshold for speculation: " + threshold) + logDebug(s"Task length threshold for speculation: $threshold") for (tid <- runningTasksSet) { val info = taskInfos(tid) val index = info.index