Fix ClassNotFoundException errors in PySpark streaming tests.

JoshRosen · Mar 14, 2016 · 267aaf9 · 267aaf9
1 parent 2c10193
commit 267aaf9
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py
@@ -1636,7 +1636,13 @@ def search_kinesis_asl_assembly_jar():
         jars = "%s,%s,%s,%s,%s" % (kafka_assembly_jar, flume_assembly_jar, mqtt_assembly_jar,
                                    mqtt_test_jar, kinesis_asl_assembly_jar)
 
-    os.environ["PYSPARK_SUBMIT_ARGS"] = "--jars %s pyspark-shell" % jars
+    # We need to set userClassPathFirst here because the streaming data source classes are also
+    # loadable from the root classloader (because of SPARK_PREPEND_CLASSES) but their dependencies
+    # are only present in the data source assembly JARs.
+    os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([
+        "--conf spark.driver.userClassPathFirst=true",
+        "--jars %s pyspark-shell" % jars,
+    ])
     testcases = [BasicOperationTests, WindowFunctionTests, StreamingContextTests, CheckpointTests,
                  KafkaStreamTests, FlumeStreamTests, FlumePollingStreamTests, MQTTStreamTests,
                  StreamingListenerTests]