Add default conf parameter to Spark JDBC Hook (#8787)

apache · May 9, 2020 · 7506c73 · 7506c73
1 parent bc19778
commit 7506c73
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 3 deletions.
diff --git a/airflow/providers/apache/spark/example_dags/example_spark_dag.py b/airflow/providers/apache/spark/example_dags/example_spark_dag.py
@@ -48,7 +48,6 @@
     jdbc_to_spark_job = SparkJDBCOperator(
         cmd_type='jdbc_to_spark',
         jdbc_table="foo",
-        spark_conf={},
         spark_jars="${SPARK_HOME}/jars/postgresql-42.2.12.jar",
         jdbc_driver="org.postgresql.Driver",
         metastore_table="bar",
@@ -60,7 +59,6 @@
     spark_to_jdbc_job = SparkJDBCOperator(
         cmd_type='spark_to_jdbc',
         jdbc_table="foo",
-        spark_conf={},
         spark_jars="${SPARK_HOME}/jars/postgresql-42.2.12.jar",
         jdbc_driver="org.postgresql.Driver",
         metastore_table="bar",

diff --git a/airflow/providers/apache/spark/hooks/spark_jdbc.py b/airflow/providers/apache/spark/hooks/spark_jdbc.py
@@ -147,7 +147,7 @@ def __init__(self,
         super().__init__(*args, **kwargs)
         self._name = spark_app_name
         self._conn_id = spark_conn_id
-        self._conf = spark_conf
+        self._conf = spark_conf or {}
         self._py_files = spark_py_files
         self._files = spark_files
         self._jars = spark_jars