Fix invariant check in databricks job/pipeline path (#6718)

* Fix invariant check in databricks job/pipeline path * Modify test-case to test for job path Co-authored-by: Iswariya Manivannan <iswariya.manivannan@thoughtworks.com>
dagster-io · Feb 22, 2022 · fe88147 · fe88147
1 parent ef56abe
commit fe88147
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/...dules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py b/...dules/libraries/dagster-databricks/dagster_databricks/databricks_pyspark_step_launcher.py
@@ -142,7 +142,7 @@ def __init__(
             "Missing config: need to provide either 'local_dagster_job_package_path' or 'local_pipeline_package_path' config entry",
         )
         check.invariant(
-            local_dagster_job_package_path is None or local_pipeline_package_path,
+            local_dagster_job_package_path is None or local_pipeline_package_path is None,
             "Error in config: Provided both 'local_dagster_job_package_path' and 'local_pipeline_package_path' entries. Need to specify one or the other.",
         )
         self.local_dagster_job_package_path = check.str_param(

diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py b/python_modules/libraries/dagster-databricks/dagster_databricks_tests/test_pyspark.py
@@ -182,16 +182,24 @@ def test_pyspark_databricks(
             for record in instance.get_event_records()
             if record.event_log_entry.step_key == "do_nothing_solid"
         ]
-
+    config = BASE_DATABRICKS_PYSPARK_STEP_LAUNCHER_CONFIG.copy()
+    config.pop("local_pipeline_package_path")
     result = execute_pipeline(
         pipeline=reconstructable(define_do_nothing_pipe),
         mode="test",
         run_config={
             "resources": {
                 "pyspark_step_launcher": {
                     "config": deep_merge_dicts(
-                        BASE_DATABRICKS_PYSPARK_STEP_LAUNCHER_CONFIG,
-                        {"databricks_host": "", "databricks_token": "", "poll_interval_sec": 0.1},
+                        config,
+                        {
+                            "databricks_host": "",
+                            "databricks_token": "",
+                            "poll_interval_sec": 0.1,
+                            "local_dagster_job_package_path": os.path.abspath(
+                                os.path.dirname(__file__)
+                            ),
+                        },
                     ),
                 },
             },