From d39b028a30f4cf5a63156b18fccce00c8165ac9a Mon Sep 17 00:00:00 2001 From: Zheng RuiFeng Date: Wed, 20 Sep 2017 13:54:48 +0800 Subject: [PATCH 1/3] create pr --- .../org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala index ce2bd7b430f43..e32447a79abb8 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala @@ -360,7 +360,9 @@ private[spark] object GradientBoostedTrees extends Logging { logInfo("Internal timing for DecisionTree:") logInfo(s"$timer") + predErrorCheckpointer.unpersistDataSet() predErrorCheckpointer.deleteAllCheckpoints() + validatePredErrorCheckpointer.unpersistDataSet() validatePredErrorCheckpointer.deleteAllCheckpoints() if (persistedInput) input.unpersist() From 723c44965bc502333aef4f1c5108d487c0facc50 Mon Sep 17 00:00:00 2001 From: Zheng RuiFeng Date: Wed, 20 Sep 2017 15:35:23 +0800 Subject: [PATCH 2/3] fix pregel --- graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala index 755c6febc48e6..08c7fb126c187 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala @@ -170,6 +170,7 @@ object Pregel extends Logging { i += 1 } messageCheckpointer.unpersistDataSet() + graphCheckpointer.unpersistDataSet() graphCheckpointer.deleteAllCheckpoints() messageCheckpointer.deleteAllCheckpoints() g From 0f84ab9ee440dec1fe0fa36438af45f7b21c3e63 Mon Sep 17 00:00:00 2001 From: Zheng RuiFeng Date: Wed, 20 Sep 2017 17:44:25 +0800 Subject: [PATCH 3/3] revert pregel --- graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala index 08c7fb126c187..755c6febc48e6 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala @@ -170,7 +170,6 @@ object Pregel extends Logging { i += 1 } messageCheckpointer.unpersistDataSet() - graphCheckpointer.unpersistDataSet() graphCheckpointer.deleteAllCheckpoints() messageCheckpointer.deleteAllCheckpoints() g