fivetran · fivetran-catfritz · May 16, 2023 · Apr 21, 2023 · May 5, 2023 · May 5, 2023
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -57,3 +57,18 @@ steps:
             - "CI_REDSHIFT_DBT_USER"
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -21,3 +21,5 @@ dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{mailchimp_using_automations: false, mailchimp_using_segments: false}' --target "$db" --full-refresh
 dbt test --target "$db"
+
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
@@ -0,0 +1,51 @@
+## PR Overview
+**This PR will address the following Issue/Feature:**
+
+**This PR will result in the following new package version:**
+<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+
+**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+
+## PR Checklist
+### Basic Validation
+Please acknowledge that you have successfully performed the following commands locally:
+- [ ] dbt compile
+- [ ] dbt run –full-refresh
+- [ ] dbt run
+- [ ] dbt test
+- [ ] dbt run –vars (if applicable)
+
+Before marking this PR as "ready for review" the following have been applied:
+- [ ] The appropriate issue has been linked and tagged
+- [ ] You are assigned to the corresponding issue and this PR
+- [ ] BuildKite integration tests are passing
+
+### Detailed Validation
+Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
+- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
+- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
+- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+<!--- Provide the steps you took to validate your changes below. -->
+
+### Standard Updates
+Please acknowledge that your PR contains the following standard updates:
+- Package versioning has been appropriately indexed in the following locations:
+    - [ ] indexed within dbt_project.yml
+    - [ ] indexed within integration_tests/dbt_project.yml
+- [ ] CHANGELOG has individual entries for each respective change in this PR
+    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
+- [ ] README updates have been applied (if applicable)
+    <!--- Remember to check the following README locations for common updates. →
+        <!--- Suggested install range (needed for breaking changes) →
+        <!--- Dependency matrix is appropriately updated (if applicable) →
+        <!--- New variable documentation (if applicable) -->
+- [ ] DECISIONLOG updates have been updated (if applicable)
+- [ ] Appropriate yml documentation has been added (if applicable)
+
+### dbt Docs
+Please acknowledge that after the above were all completed the below were applied to your branch:
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,52 +1,33 @@
-**Are you a current Fivetran customer?** 
-<!--- Please tell us your name, title and company -->
+**Please provide your name and company**
 
-**What change(s) does this PR introduce?** 
-<!--- Describe what changes your PR introduces to the package and how to leverage this new feature. -->
+**Link the issue/feature request which this PR is meant to address**
+<!--- If an issue was not created, please create one first so we may discuss the PR prior to opening one. -->
 
-**Did you update the CHANGELOG?** 
+**Detail what changes this PR introduces and how this addresses the issue/feature request linked above.**
+
+**How did you validate the changes introduced within this PR?**
+
+**Which warehouse did you use to develop these changes?**
+
+**Did you update the CHANGELOG?**
 <!--- Please update the new package version’s CHANGELOG entry detailing the changes included in this PR. -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Does this PR introduce a breaking change?**
-<!--- Does this PR introduce changes that will cause current package users' jobs to fail or require a `--full-refresh`? -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes (please provide breaking change details below.)
-- [ ] No  (please provide an explanation as to how the change is non-breaking below.)
-
-**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)** 
+**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)**
 <!--- The dbt_project.yml and the integration_tests/dbt_project.yml files contain the version number. Be sure to upgrade it accordingly -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Is this PR in response to a previously created Bug or Feature Request**
-<!--- If an Issue was created it is helpful to track the progress by linking it in the PR. -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes, Issue/Feature [link bug/feature number here]
-- [ ] No 
-
-**How did you test the PR changes?** 
-<!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
-- [ ] Local (please provide additional testing details below)
-
-**Select which warehouse(s) were used to test the PR**
-<!--- To check a warehouse remove the space and insert an x in the box (eg. [x] Bigquery). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] BigQuery
-- [ ] Redshift
-- [ ] Snowflake
-- [ ] Postgres
-- [ ] Databricks
-- [ ] Other (provide details below)
-
 **Provide an emoji that best describes your current mood**
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
 :dancer:
 
 **Feedback**
 
 We are so excited you decided to contribute to the Fivetran community dbt package! We continue to work to improve the packages and would greatly appreciate your [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next.
+
+**PR Template** 
+- [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)
+
+- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,12 @@
+# dbt_mailchimp v0.8.0
+## 🎉 Feature Update
+- Databricks compatibility! ([#37](https://github.com/fivetran/dbt_mailchimp/pull/37))
+- Small updates to documentation. ([#37](https://github.com/fivetran/dbt_mailchimp/pull/37))
+
+ ## Under the Hood:
+- Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job. ([#34](https://github.com/fivetran/dbt_mailchimp/pull/34))
+- Updated the pull request [templates](/.github). ([#34](https://github.com/fivetran/dbt_mailchimp/pull/34))
+
 # dbt_mailchimp v0.7.0
 [PR #30](https://github.com/fivetran/dbt_mailchimp/pull/30) includes the following breaking changes:
 ## 🚨 Breaking Changes 🚨:

diff --git a/README.md b/README.md
@@ -37,16 +37,24 @@ The following table provides a detailed list of all models materialized within t
 To use this dbt package, you must have the following:
 
 - At least one Fivetran Mailchimp connector syncing data into your destination.
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
 ## Step 2: Install the package
 Include the following mailchimp package version in your `packages.yml` file:
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 ```yaml
 packages:
   - package: fivetran/mailchimp
-    version: [">=0.7.0", "<0.8.0"]
+    version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
+Do **NOT** include the `mailchimp_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well. 
+
+### Databricks dispatch configuration
+If you are using a Databricks destination with this package, you must add the following (or a variation of the following) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
 
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `mailchimp` schema. If this is not where your Mailchimp data is (for example, if your Mailchimp schema is named `mailchimp_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -113,7 +121,7 @@ packages:
       version: [">=1.0.0", "<2.0.0"]
 
     - package: fivetran/mailchip_source
-      version: [">=0.4.0", "<0.5.0"]
+      version: [">=0.5.0", "<0.6.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'mailchimp'
-version: '0.7.0'
+version: '0.8.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   mailchimp:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -45,7 +45,7 @@ integration_tests:
       schema: mailchimp_integration_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: mailchimp_integration_tests

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'mailchimp_integration_tests'
-version: '0.7.0'
+version: '0.8.0'
 profile: 'integration_tests'
 vars:
   mailchimp_source:
@@ -76,3 +76,7 @@ seeds:
     unsubscribe:
       +column_types:
         timestamp: timestamp
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
diff --git a/models/mailchimp.yml b/models/mailchimp.yml
@@ -378,6 +378,7 @@ models:
     description: Each record represents the send of a campaign email, enriched with click, open and unsubscribe metrics.
     columns:
       - name: email_id
+        description: The ID of the email sent. Surrogate key generated in dbt from 'campaign_id' and 'member_id'.
         tests:
           - unique
           - not_null
@@ -415,6 +416,9 @@ models:
       - name: was_clicked
         description: Whether the email sent was clicked.
 
+      - name: send_timestamp
+        description: The date and time a campaign was sent.
+
       - name: first_open_timestamp
         description: The timestamp of the first open for an email.
 
@@ -469,7 +473,7 @@ models:
         description: The ID of the combination the member was part of for a campaign. 
 
       - name: email_id
-        description: The ID of the email sent. (Created in dbt, not Mailchimp.)
+        description: The ID of the email sent. Surrogate key generated in dbt from 'campaign_id' and 'member_id'.
 
       - name: send_timestamp
         description: The time the campaign was sent.
@@ -499,6 +503,7 @@ models:
         description: The ID of the Segment an automation was sent to.
 
       - name: segment_text
+        description: Description of the segment. Formatted as a string marked up with HTML.
 
       - name: started_timestamp
         description: The date and time the Automation was started.

diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,4 @@
 packages:
-- package: fivetran/mailchimp_source
-  version: [">=0.4.0", "<0.5.0"]
+ - package: fivetran/mailchimp_source
+   version: [">=0.5.0", "<0.6.0"]
+