Merge pull request #200 from jbouffard/serializers

Added Kyro Registrator to SparkConf
locationtech-labs · May 15, 2017 · e3f0b5f · e3f0b5f
2 parents 2370c60 + 44d70b5
commit e3f0b5f
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -24,7 +24,9 @@ ${WHEEL}: ${DIST-ASSEMBLY} $(call rwildcard, geopyspark, *.py) setup.py
 wheel: ${WHEEL}
 
 pyspark: ${DIST-ASSEMBLY}
-	pyspark --jars ${DIST-ASSEMBLY}
+	pyspark --jars ${DIST-ASSEMBLY} \
+		--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+		--conf spark.kyro.registrator=geotrellis.spark.io.kyro.KryoRegistrator
 
 docker/archives/${ASSEMBLYNAME}: ${DIST-ASSEMBLY}
 	cp -f ${DIST-ASSEMBLY} docker/archives/${ASSEMBLYNAME}

diff --git a/docker/kernels/local/kernel.json b/docker/kernels/local/kernel.json
@@ -10,7 +10,7 @@
       "PYSPARK_PYTHON": "/usr/bin/python3.4",
       "SPARK_HOME": "/usr/local/spark-2.1.0-bin-hadoop2.7",
       "PYTHONPATH": "/usr/local/spark-2.1.0-bin-hadoop2.7/python/lib/pyspark.zip:/usr/local/spark-2.1.0-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip",
-      "PYSPARK_SUBMIT_ARGS": "--master local --archives /blobs/gdal-and-friends.tar.gz,/blobs/geopyspark-and-friends.tar.gz --jars /blobs/geotrellis-backend-assembly-0.1.0.jar --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.executorEnv.LD_LIBRARY_PATH=gdal-and-friends.tar.gz/lib/ --conf spark.executorEnv.PYTHONPATH=geopyspark-and-friends.tar.gz/ pyspark-shell"
+      "PYSPARK_SUBMIT_ARGS": "--master local --archives /blobs/gdal-and-friends.tar.gz,/blobs/geopyspark-and-friends.tar.gz --jars /blobs/geotrellis-backend-assembly-0.1.0.jar --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.kyro.registrator=geotrellis.spark.io.kyro.KryoRegistrator --conf spark.executorEnv.LD_LIBRARY_PATH=gdal-and-friends.tar.gz/lib/ --conf spark.executorEnv.PYTHONPATH=geopyspark-and-friends.tar.gz/ pyspark-shell"
   },
   "language": "python",
   "display_name": "PySpark (local)"

diff --git a/geopyspark/geopyspark_utils.py b/geopyspark/geopyspark_utils.py
@@ -72,13 +72,15 @@ def setup_environment():
         os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars {} \
             --conf spark.ui.enabled=false \
             --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+            --conf spark.kyro.registrator=geotrellis.spark.io.kyro.KryoRegistrator \
             --driver-memory 2G \
             --executor-memory 2G \
             pyspark-shell".format(jar_string)
     else:
         os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars {} \
             --conf spark.ui.enabled=false \
             --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+            --conf spark.kyro.registrator=geotrellis.spark.io.kyro.KryoRegistrator \
             --driver-memory 8G \
             --executor-memory 8G \
             pyspark-shell".format(jar_string)