fix(python): Add insecure env variable

ValueError: You are trying to pass an insecure Py4j gateway to Spark. This presents a security risk. If you are sure you understand and accept this risk, you can set the environment variable 'PYSPARK_ALLOW_INSECURE_GATEWAY=1', but note this option will be removed in Spark 3.0 Change-Id: Ifea54c64c282d2bfe3fa7be6b05743166aedafc9
spark-jobserver · Feb 25, 2020 · 1e451a0 · 1e451a0
1 parent da9dfd4
commit 1e451a0
Showing 1 changed file with 20 additions and 10 deletions.
diff --git a/job-server-python/src/test/scala/spark/jobserver/python/SubprocessSpec.scala b/job-server-python/src/test/scala/spark/jobserver/python/SubprocessSpec.scala
@@ -162,7 +162,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (0)
       endpoint.result should matchPattern {
@@ -181,7 +182,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (1)
       endpoint.validationProblems should be (Some(Seq("config input.strings not found")))
@@ -205,7 +207,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (0)
       endpoint.result should matchPattern {
@@ -235,7 +238,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (0)
       endpoint.result should matchPattern {
@@ -268,7 +272,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (0)
       endpoint.result should be ("done")
@@ -283,7 +288,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw2.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode2 = process2.!
       pythonExitCode2 should be (0)
       endpoint2.result should matchPattern {
@@ -308,7 +314,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (2)
       stopGateway(gw)
@@ -323,7 +330,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (4)
       stopGateway(gw)
@@ -338,7 +346,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (3)
       stopGateway(gw)
@@ -356,7 +365,8 @@ class SubprocessSpec extends FunSpec with Matchers with BeforeAndAfterAll {
         Process(
           Seq("python", "-m", "sparkjobserver.subprocess", gw.getListeningPort.toString),
           None,
-          "PYTHONPATH" -> pythonPath)
+          "PYTHONPATH" -> pythonPath,
+          "PYSPARK_ALLOW_INSECURE_GATEWAY" -> "1")
       val pythonExitCode = process.!
       pythonExitCode should be (0)
       endpoint.result should be ("Hello World 3")