From ce4391b6916ee043ca3218b09f8e66a3a61f019f Mon Sep 17 00:00:00 2001 From: Muhammad Raihan Muhaimin Date: Mon, 4 Apr 2022 16:25:41 -0400 Subject: [PATCH] fix: build spark step load right deploy type Fix #1253 --- awswrangler/emr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awswrangler/emr.py b/awswrangler/emr.py index 18d81139c..6998d8774 100644 --- a/awswrangler/emr.py +++ b/awswrangler/emr.py @@ -1022,7 +1022,7 @@ def build_spark_step( else: config: str = "hdfs:///user/hadoop/config.json" cmd = ( - f"spark-submit --deploy-mode cluster " + f"spark-submit --deploy-mode {deploy_mode} " f"--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_TYPE=docker " f"--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE={docker_image} " f"--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG={config} "