Merge pull request #153 from jamesmcclain/fix/travis

Improve Travis Reliability
locationtech-labs · May 2, 2017 · 0448b81 · 0448b81
2 parents 50c416d + df8fd2c
commit 0448b81
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 3 deletions.
diff --git a/Makefile b/Makefile
@@ -34,10 +34,13 @@ docker-build: docker/archives/${ASSEMBLYNAME} docker/archives/${WHEELNAME}
 	(cd docker && make stage1)
 
 clean:
-	(cd geopyspark-backend && ./sbt "project geotrellis-backend" clean)
 	rm -f ${WHEEL} ${DIST-ASSEMBLY}
+	(cd geopyspark-backend && ./sbt "project geotrellis-backend" clean)
+	(cd docker && make clean)
 
 cleaner: clean
 	rm -f `find ./build ./geopyspark | grep "\.pyc"`
+	(cd docker && make cleaner)
 
 cleanest: cleaner
+	(cd docker && make cleanest)
diff --git a/docker/Makefile b/docker/Makefile
@@ -55,6 +55,6 @@ clean:
 	rm -f $(WHL) $(JAR)
 
 cleaner: clean
-	rm -f $(GDAL-BLOB) $(PYTHON-BLOB)
+	rm -f $(GDAL-BLOB) $(PYTHON-BLOB) blobs/*
 
 cleanest: cleaner
diff --git a/geopyspark/geopyspark_utils.py b/geopyspark/geopyspark_utils.py
@@ -70,7 +70,15 @@ def setup_environment():
     os.environ['JARS'] = jar_string
     os.environ["PYSPARK_PYTHON"] = "python3"
     os.environ["PYSPARK_DRIVER_PYTHON"] = "python3"
-    os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars {} \
+    if 'TRAVIS' in os.environ:
+        os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars {} \
+            --conf spark.ui.enabled=false \
+            --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+            --driver-memory 2G \
+            --executor-memory 2G \
+            pyspark-shell".format(jar_string)
+    else:
+        os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars {} \
             --conf spark.ui.enabled=false \
             --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
             --driver-memory 8G \