Spark new cluster misc changes

metabrainz · Apr 19, 2021 · f4e1227 · f4e1227
1 parent 9d0be21
commit f4e1227
Showing 1 changed file with 19 additions and 20 deletions.
diff --git a/docker/spark-new-cluster/start-request-consumer-container.sh b/docker/spark-new-cluster/start-request-consumer-container.sh
@@ -6,26 +6,25 @@ source spark_config.sh
 
 CONTAINER_NAME="spark-request-consumer"
 
-docker pull metabrainz/listenbrainz-spark-new-cluster:latest
+python -m venv pyspark_venv
+source pyspark_venv/bin/activate
+pip install -r requirements_spark.txt
+pip uninstall pyspark py4j
+venv-pack -o pyspark_venv.tar.gz
+
+export PYSPARK_DRIVER_PYTHON=python
+export PYSPARK_PYTHON=./environment/bin/python
 
 zip -rq listenbrainz_spark_request_consumer.zip listenbrainz_spark/
-docker run \
-    -d \
-    -v /spark:/spark \
-    -v /hadoop:/hadoop \
-    -v /usr/lib/jvm/adoptopenjdk-11-hotspot-amd64:/usr/lib/jvm/adoptopenjdk-11-hotspot-amd64 \
-    -v `pwd`:/rec \
-    --network host \
-    --name $CONTAINER_NAME \
-    metabrainz/listenbrainz-spark-new-cluster:latest \
-    /spark/bin/spark-submit \
-        --packages org.apache.spark:spark-avro_2.11:2.4.1 \
-        --master yarn \
-        --deploy-mode cluster \
-        --conf "spark.scheduler.listenerbus.eventqueue.capacity"=$LISTENERBUS_CAPACITY \
-        --conf "spark.cores.max"=$MAX_CORES \
-        --conf "spark.executor.cores"=$EXECUTOR_CORES \
-        --conf "spark.executor.memory"=$EXECUTOR_MEMORY \
-        --conf "spark.driver.memory"=$DRIVER_MEMORY \
-        --py-files listenbrainz_spark_request_consumer.zip \
+
+/spark/bin/spark-submit \
+    --packages org.apache.spark:spark-avro_2.12:3.1.1 \
+    --master yarn \
+    --conf "spark.yarn.dist.archives"=pyspark_venv.tar.gz#environment \
+    --conf "spark.scheduler.listenerbus.eventqueue.capacity"=$LISTENERBUS_CAPACITY \
+    --conf "spark.cores.max"=$MAX_CORES \
+    --conf "spark.executor.cores"=$EXECUTOR_CORES \
+    --conf "spark.executor.memory"=$EXECUTOR_MEMORY \
+    --conf "spark.driver.memory"=$DRIVER_MEMORY \
+    --py-files listenbrainz_spark_request_consumer.zip \
 	spark_manage.py request_consumer