Merge pull request #48 from fivetran/MagicBot/databricks-compatibility

Feature: Databricks compatibility
fivetran · Jun 27, 2023 · 1e03e64 · 1e03e64
2 parents 7d6d3b1 + ab08996
commit 1e03e64
Show file tree

Hide file tree

Showing 12 changed files with 38 additions and 15 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -56,4 +56,18 @@ steps:
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      bash .buildkite/scripts/run_models.sh redshift
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# dbt_quickbooks_source v0.9.0
+## 🎉 Feature Update 🎉
+- Databricks compatibility! ([#48](https://github.com/fivetran/dbt_quickbooks_source/pull/48))
+
 # dbt_quickbooks_source v0.8.0
 
 ## 🎉 Feature Updates 🎉

diff --git a/README.md b/README.md
@@ -43,7 +43,7 @@ from [Fivetran's connector](https://fivetran.com/docs/applications/quickbooks) f
 To use this dbt package, you must have the following:
 
 - At least one Fivetran QuickBooks connector syncing data into your destination.
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
 ## Step 2: Install the package (skip if also using the `quickbooks` transformation package)
 If you are **not** using the [QuickBooks transformation package](https://github.com/fivetran/dbt_quickbooks), include the following `quickbooks_source` package version in your `packages.yml` file.
@@ -52,7 +52,7 @@ If you are **not** using the [QuickBooks transformation package](https://github.
 ```yaml
 packages:
   - package: fivetran/quickbooks_source
-    version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.9.0", "<0.10.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ## Step 3: Define database and schema variables

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 name: 'quickbooks_source'
 
-version: '0.8.0' 
+version: '0.9.0'
 
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -45,10 +45,10 @@ integration_tests:
       schema: quickbooks_source_integrations_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: quickbooks_source_integrations_tests
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'quickbooks_source_integration_tests'
-version: '0.8.0'
+version: '0.9.0'
 
 
 profile: 'integration_tests'
@@ -66,3 +66,7 @@ clean-targets:
 - target
 - dbt_modules
 - dbt_packages
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
diff --git a/models/src_quickbooks.yml b/models/src_quickbooks.yml
@@ -3,7 +3,7 @@ version: 2
 sources:
   - name: quickbooks
     schema: "{{ var('quickbooks_schema', 'quickbooks') }}"
-    database: "{{ var('quickbooks_database', target.database) }}"
+    database: "{% if target.type != 'spark' %}{{ var('quickbooks_database', target.database)}}{% endif %}"
     loader: fivetran
     loaded_at_field: _fivetran_synced
     freshness: