diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index f583c684a..dcb074371 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -14,6 +14,7 @@ on: - databricks - databricks_catalog - spark + - athena elementary-ref: type: string required: false diff --git a/elementary/monitor/dbt_project/macros/create_temp_table.sql b/elementary/monitor/dbt_project/macros/create_temp_table.sql deleted file mode 100644 index c00a31604..000000000 --- a/elementary/monitor/dbt_project/macros/create_temp_table.sql +++ /dev/null @@ -1,26 +0,0 @@ -{% macro create_temp_table(database_name, schema_name, table_name, sql_query) %} - {% do return(adapter.dispatch('create_temp_table','elementary_cli')(database_name, schema_name, table_name, sql_query)) %} -{%- endmacro %} - -{% macro default__create_temp_table(database_name, schema_name, table_name, sql_query) %} - {% do return(elementary.create_temp_table(database_name, schema_name, table_name, sql_query)) %} -{% endmacro %} - -{% macro snowflake__create_temp_table(database_name, schema_name, table_name, sql_query) %} - {% set temp_table_exists, temp_table_relation = dbt.get_or_create_relation(database=database_name, - schema=schema_name, - identifier=table_name, - type='table') -%} - {% set temp_table_relation = elementary.edr_make_temp_relation(temp_table_relation) %} - {% set create_query %} - create or replace temporary table {{ temp_table_relation }} - as ( - {{ sql_query }} - ); - - {% endset %} - - {% do elementary.run_query(create_query) %} - - {{ return(temp_table_relation) }} -{% endmacro %} diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index dad6bebee..492f95153 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -11,7 +11,7 @@ *, {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('detected_at'), elementary.edr_current_timestamp(), 'day') }} as days_diff, {# When we split test into multiple test results, we want to have the same invocation order for the test results from the same run so we use rank. #} - rank() over (partition by elementary_unique_id order by detected_at desc) as invocations_rank_index + rank() over (partition by elementary_unique_id order by {{elementary.edr_cast_as_timestamp('detected_at')}} desc) as invocations_rank_index from test_results ) @@ -22,7 +22,7 @@ test_results.model_unique_id, test_results.test_unique_id, test_results.elementary_unique_id, - test_results.detected_at, + {{elementary.edr_cast_as_timestamp('test_results.detected_at')}} as detected_at, test_results.database_name, test_results.schema_name, test_results.table_name, @@ -58,7 +58,7 @@ {% set test_results = [] %} {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} - {% set ordered_test_results_relation = elementary_cli.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %} + {% set ordered_test_results_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %} {% set test_results_agate_sql %} select * from {{ ordered_test_results_relation }} @@ -72,6 +72,9 @@ {% set test_results_agate = elementary.run_query(test_results_agate_sql) %} {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %} + {% if not elementary.has_temp_table_support() %} + {% do elementary.fully_drop_relation(ordered_test_results_relation) %} + {% endif %} {% set tests = elementary.agate_to_dicts(test_results_agate) %} {% set filtered_tests = [] %} diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml index 5947f6715..5e2b22285 100644 --- a/elementary/monitor/dbt_project/package-lock.yml +++ b/elementary/monitor/dbt_project/package-lock.yml @@ -1,6 +1,6 @@ packages: - package: dbt-labs/dbt_utils version: 0.8.6 - - package: elementary-data/elementary - version: 0.18.0 -sha1_hash: 24666e9f3179ea742d9fb6adce8005b9af3fc466 + - git: https://github.com/elementary-data/dbt-data-reliability.git + revision: cc226bf865f922d6c4c5c6d65b2a142540e222f8 +sha1_hash: a838112140dab39e15226f415d0b0bcbfbe311ff diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml index 7f5e68a7a..dec2ea19f 100644 --- a/elementary/monitor/dbt_project/packages.yml +++ b/elementary/monitor/dbt_project/packages.yml @@ -1,10 +1,10 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<0.9.0"] - - package: elementary-data/elementary - version: 0.18.0 + # - package: elementary-data/elementary + # version: 0.18.0 # NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this # commented, so it will be easy to access) - # - git: https://github.com/elementary-data/dbt-data-reliability.git - # revision: 0f82a3f6c76706237172184b8bde3058040f5a8e + - git: https://github.com/elementary-data/dbt-data-reliability.git + revision: cc226bf865f922d6c4c5c6d65b2a142540e222f8 diff --git a/tests/tests_with_db/test_dbt_runner.py b/tests/tests_with_db/test_dbt_runner.py index b3ebbdd01..e039e1d31 100644 --- a/tests/tests_with_db/test_dbt_runner.py +++ b/tests/tests_with_db/test_dbt_runner.py @@ -29,10 +29,7 @@ def test_run(self, custom_dbt_runner: BaseDbtRunner): result = custom_dbt_runner.run(select="one", vars={"test_marker": test_marker}) assert result is True - invocations = self._run_query( - custom_dbt_runner, - f"""select * from {{{{ ref("dbt_invocations") }}}} where vars ilike '%{test_marker}%'""", - ) + invocations = self._get_invocations_by_marker(custom_dbt_runner, test_marker) assert len(invocations) == 1 invocation = invocations[0] @@ -47,16 +44,21 @@ def test_test(self, custom_dbt_runner: BaseDbtRunner): result = custom_dbt_runner.test(select="one", vars={"test_marker": test_marker}) assert result is True - invocations = self._run_query( - custom_dbt_runner, - f"""select * from {{{{ ref("dbt_invocations") }}}} where vars ilike '%{test_marker}%'""", - ) + invocations = self._get_invocations_by_marker(custom_dbt_runner, test_marker) assert len(invocations) == 1 invocation = invocations[0] assert invocation["command"] == "test" assert invocation["selected"] == json.dumps(["one"]) + def _get_invocations_by_marker( + self, custom_dbt_runner: BaseDbtRunner, test_marker: str + ): + return self._run_query( + custom_dbt_runner, + f"""select * from {{{{ ref("dbt_invocations") }}}} where LOWER(vars) LIKE LOWER('%{test_marker}%')""", + ) + @staticmethod def _run_query(dbt_runner: SubprocessDbtRunner, query: str): return json.loads(