From 532edbb9c8343c1643fe390e4f4bdc280f7a197b Mon Sep 17 00:00:00 2001 From: Gemma Down <52132406+glsdown@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:02:50 +0100 Subject: [PATCH] Add variable to allow users to exclude all results columns (#382) * Add ability to exclude all results column * Add documentation about the variable --- README.md | 54 ++++++++++++++---------- integration_test_project/dbt_project.yml | 1 + macros/upload_exposures.sql | 12 +++++- macros/upload_models.sql | 12 +++++- macros/upload_seeds.sql | 12 +++++- macros/upload_snapshots.sql | 12 +++++- macros/upload_sources.sql | 12 +++++- macros/upload_tests.sql | 12 +++++- 8 files changed, 92 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b546e98c..a63bba3c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,37 @@ vars: ] ``` +## Creating custom marts tables + +Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +models: + dbt_artifacts: + +enabled: false + sources: + +enabled: true +``` + +In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis: + +- exposures +- models +- seeds +- snapshots +- sources +- tests + +This column can cause queries to become too long - particularly in BigQuery. Therefore, if you want to disable this column, you can make use of the `dbt_artifacts_exclude_all_results` variable, and set this to `true` in your `dbt_project.yml` file. + +``` +# dbt_project.yml +vars: + dbt_artifacts_exclude_all_results: true +``` + ## Upgrading from 1.x to >=2.0.0 If you were using the following variables: @@ -189,29 +220,6 @@ An example operation is as follows: dbt run-operation migrate_from_v0_to_v1 --args '{old_database: analytics, old_schema: dbt_artifacts, new_database: analytics, new_schema: artifact_sources}' ``` -## Creating custom marts tables - -Multiple modelled `dim` and `fct` models have been provided for ease of use, but we recognise that some use cases may require custom ones. To this end, you can disable all but the raw sources tables using the following in your `dbt_project.yml` file: - -```yml -# dbt_project.yml - -models: - dbt_artifacts: - +enabled: false - sources: - +enabled: true -``` - -In these sources tables, you will find a JSON column `all_results` which contains a JSON blob of the results object used, which you can use in your own analysis: - -- exposures -- models -- seeds -- snapshots -- sources -- tests - ## Acknowledgements Thank you to [Tails.com](https://tails.com/gb/careers/) for initial development and maintenance of this package. On 2021/12/20, the repository was transferred from the Tails.com GitHub organization to Brooklyn Data Co. diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index a6550009..dba89a50 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -21,6 +21,7 @@ vars: env_vars: ["TEST_ENV_VAR_NUMBER", "TEST_ENV_VAR_EMPTY", "TEST_ENV_VAR_WITH_QUOTE"] dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"] + dbt_artifacts_exclude_all_results: true models: +persist_docs: diff --git a/macros/upload_exposures.sql b/macros/upload_exposures.sql index d8653154..8ef51ce2 100644 --- a/macros/upload_exposures.sql +++ b/macros/upload_exposures.sql @@ -41,7 +41,11 @@ '{{ exposure.package_name }}', {# package_name #} '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} '{{ tojson(exposure.tags) }}', {# tags #} - '{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -70,7 +74,11 @@ '{{ exposure.package_name }}', {# package_name #} {{ tojson(exposure.depends_on.nodes) }}, {# depends_on_nodes #} {{ tojson(exposure.tags) }}, {# tags #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(exposure) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_models.sql b/macros/upload_models.sql index cb396a05..f0748876 100644 --- a/macros/upload_models.sql +++ b/macros/upload_models.sql @@ -40,7 +40,11 @@ '{{ tojson(model.tags) }}', {# tags #} '{{ tojson(model.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ model.alias }}', {# alias #} - '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -71,7 +75,11 @@ {{ tojson(model.tags) }}, {# tags #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model.config.meta)) }}, {# meta #} '{{ model.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(model) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_seeds.sql b/macros/upload_seeds.sql index 0a766bcb..ea288e21 100644 --- a/macros/upload_seeds.sql +++ b/macros/upload_seeds.sql @@ -37,7 +37,11 @@ '{{ seed.checksum.checksum | replace('\\', '\\\\') }}', {# checksum #} '{{ tojson(seed.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ seed.alias }}', {# alias #} - '{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -64,7 +68,11 @@ '{{ seed.checksum.checksum | replace('\\', '\\\\')}}', {# checksum #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed.config.meta)) }}, {# meta #} '{{ seed.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(seed) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_snapshots.sql b/macros/upload_snapshots.sql index 05dad1be..3c7a7cbf 100644 --- a/macros/upload_snapshots.sql +++ b/macros/upload_snapshots.sql @@ -42,7 +42,11 @@ '{{ snapshot.config.strategy }}', {# strategy #} '{{ tojson(snapshot.config.meta) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}', {# meta #} '{{ snapshot.alias }}', {# alias #} - '{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -71,7 +75,11 @@ '{{ snapshot.config.strategy }}', {# strategy #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot.config.meta)) }}, {# meta #} '{{ snapshot.alias }}', {# alias #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(snapshot) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index 38ab6ec5..23208d6d 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -33,7 +33,11 @@ '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} '{{ tojson(source.freshness) | replace("'","\\'") }}', {# freshness #} - '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -60,7 +64,11 @@ '{{ source.identifier }}', {# identifier #} '{{ source.loaded_at_field | replace("'","\\'") }}', {# loaded_at_field #} {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source.freshness) | replace("'","\\'")) }}, {# freshness #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(source) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"')) }} {# all_results #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index 9d6a02a6..d29341ff 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -27,7 +27,11 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} '{{ tojson(test.tags) }}', {# tags #} - '{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_fields #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"') }}' {# all_fields #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %} @@ -51,7 +55,11 @@ '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} {{ tojson(test.tags) }}, {# tags #} - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(test) | replace("\\", "\\\\") | replace("'","\\'") | replace('"', '\\"')) }} {# all_fields #} + {% endif %} ) {%- if not loop.last %},{%- endif %} {%- endfor %}