diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 3effbca1bbcd5..18a5366986113 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -332,8 +332,9 @@ object SparkSubmit { // that can be distributed with the job if (args.isPython && clusterManager == YARN) { var pyArchives: String = null - if (sys.env.contains("PYSPARK_ARCHIVES_PATH")) { - pyArchives = sys.env.get("PYSPARK_ARCHIVES_PATH").get + val pyArchivesEnvOpt = sys.env.get("PYSPARK_ARCHIVES_PATH") + if (pyArchivesEnvOpt.isDefined) { + pyArchives = pyArchivesEnvOpt.get } else { if (!sys.env.contains("SPARK_HOME")) { printErrorAndExit("SPARK_HOME does not exist for python application in yarn mode.")