Pass PYTHONPATH to executor

apache · Apr 12, 2015 · 413fa25 · 413fa25
1 parent 5d8f7b9
commit 413fa25
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -371,6 +371,11 @@ private[spark] class Client(
       env(ENV_DIST_CLASSPATH) = dcp
     }
 
+    sys.env.get("PYTHONPATH") match {
+      case Some(pythonPath) => env("PYTHONPATH") = pythonPath
+      case None => // do nothing
+    }
+
     env
   }
 

diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -299,6 +299,12 @@ class ExecutorRunnable(
     }
 
     System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k, v) => env(k) = v }
+
+    sys.env.get("PYTHONPATH") match {
+      case Some(pythonPath) => env("PYTHONPATH") = pythonPath
+      case None => // do nothing
+    }
+
     env
   }
 }