From eedbbcc3d24a7ab94e5ae2fc89d9f08f8b69459c Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 5 May 2014 19:08:25 +0000 Subject: [PATCH 1/2] Include the sbin/spark-config.sh in spark-executor This is needed because broadcast values are broken on pyspark on Mesos, it tries to import pyspark but can't. --- sbin/spark-executor | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/spark-executor b/sbin/spark-executor index de5bfab563125..6bb500c9f4bbd 100755 --- a/sbin/spark-executor +++ b/sbin/spark-executor @@ -19,5 +19,10 @@ FWDIR="$(cd `dirname $0`/..; pwd)" +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` + +. "$sbin/spark-config.sh" + echo "Running spark-executor with framework dir = $FWDIR" exec $FWDIR/bin/spark-class org.apache.spark.executor.MesosExecutorBackend From b2f12954c695fb3e24512a1b2fa9e9ac9cea8cbc Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Thu, 8 May 2014 17:18:04 -0400 Subject: [PATCH 2/2] Inline PYTHONPATH in spark-executor --- sbin/spark-executor | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sbin/spark-executor b/sbin/spark-executor index 6bb500c9f4bbd..336549f29c9ce 100755 --- a/sbin/spark-executor +++ b/sbin/spark-executor @@ -19,10 +19,8 @@ FWDIR="$(cd `dirname $0`/..; pwd)" -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` - -. "$sbin/spark-config.sh" +export PYTHONPATH=$FWDIR/python:$PYTHONPATH +export PYTHONPATH=$FWDIR/python/lib/py4j-0.8.1-src.zip:$PYTHONPATH echo "Running spark-executor with framework dir = $FWDIR" exec $FWDIR/bin/spark-class org.apache.spark.executor.MesosExecutorBackend