From 929d784c28de5ea46f3727ba3ffc8eaa524e2a32 Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 2 May 2014 10:55:32 +0800 Subject: [PATCH 1/4] get sparkhome from sparkconf Preferentially --- .../src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index dd0a1360abe14..35c6062f78264 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -83,7 +83,8 @@ private[spark] class Worker( @volatile var registered = false @volatile var connected = false val workerId = generateWorkerId() - val sparkHome = new File(Option(System.getenv("SPARK_HOME")).getOrElse(".")) + val sparkHome = new File(conf.getOption("spark.home").orElse( + Option(System.getenv("SPARK_HOME")).getOrElse("."))) var workDir: File = null val executors = new HashMap[String, ExecutorRunner] val finishedExecutors = new HashMap[String, ExecutorRunner] From f2fccb3e938626bea31cbc0d099aa27e45697afb Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 2 May 2014 11:17:32 +0800 Subject: [PATCH 2/4] update --- .../main/scala/org/apache/spark/deploy/worker/Worker.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 35c6062f78264..095a6d117f937 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -73,6 +73,7 @@ private[spark] class Worker( // Index into masterUrls that we're currently trying to register with. var masterIndex = 0 + var wf = "wf" val masterLock: Object = new Object() var master: ActorSelection = null @@ -83,8 +84,8 @@ private[spark] class Worker( @volatile var registered = false @volatile var connected = false val workerId = generateWorkerId() - val sparkHome = new File(conf.getOption("spark.home").orElse( - Option(System.getenv("SPARK_HOME")).getOrElse("."))) + val sparkHome = new File(conf.getOption("spark.home").orElse(Option( + Option(System.getenv("SPARK_HOME")).getOrElse(".")))) var workDir: File = null val executors = new HashMap[String, ExecutorRunner] val finishedExecutors = new HashMap[String, ExecutorRunner] From c00678d96000ca93cc7e18addff76567fa2ae177 Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 2 May 2014 11:20:16 +0800 Subject: [PATCH 3/4] update --- core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 095a6d117f937..56383000d2b13 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -73,7 +73,6 @@ private[spark] class Worker( // Index into masterUrls that we're currently trying to register with. var masterIndex = 0 - var wf = "wf" val masterLock: Object = new Object() var master: ActorSelection = null From a4aef1ccd37fc72cfa73deb0744e77ba8a4259cb Mon Sep 17 00:00:00 2001 From: wangfei Date: Fri, 2 May 2014 11:49:35 +0800 Subject: [PATCH 4/4] fix error --- core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 56383000d2b13..bf8d0e5328934 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -84,7 +84,7 @@ private[spark] class Worker( @volatile var connected = false val workerId = generateWorkerId() val sparkHome = new File(conf.getOption("spark.home").orElse(Option( - Option(System.getenv("SPARK_HOME")).getOrElse(".")))) + Option(System.getenv("SPARK_HOME")).getOrElse("."))).get) var workDir: File = null val executors = new HashMap[String, ExecutorRunner] val finishedExecutors = new HashMap[String, ExecutorRunner]