From f130aed5dc8197ec786e1e37319f46bf013aafd2 Mon Sep 17 00:00:00 2001 From: Thomas Schmidt Date: Fri, 29 Dec 2023 08:37:52 +0100 Subject: [PATCH] Add dbt support --- docsource/index.rst | 1 + docsource/usage/dbt.md | 130 +++++ examples/dbt/test_example.py | 47 ++ src/sql_mock/config.py | 12 + src/sql_mock/dbt.py | 179 ++++++ .../my_first_dbt_model.sql | 26 + .../my_second_dbt_model.sql | 7 + tests/resources/dbt/dbt_manifest.json | 536 ++++++++++++++++++ tests/sql_mock/test_dbt.py | 241 ++++++++ 9 files changed, 1179 insertions(+) create mode 100644 docsource/usage/dbt.md create mode 100644 examples/dbt/test_example.py create mode 100644 src/sql_mock/config.py create mode 100644 src/sql_mock/dbt.py create mode 100644 tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql create mode 100644 tests/resources/dbt/compiled_example_models/my_second_dbt_model.sql create mode 100644 tests/resources/dbt/dbt_manifest.json create mode 100644 tests/sql_mock/test_dbt.py diff --git a/docsource/index.rst b/docsource/index.rst index 98da015..2f84192 100644 --- a/docsource/index.rst +++ b/docsource/index.rst @@ -30,6 +30,7 @@ It provides a consistent and convenient way to test the execution of your query :caption: Basic Usage usage/defining_table_mocks + usage/dbt usage/your_sql_query_to_test usage/result_assertion usage/default_values diff --git a/docsource/usage/dbt.md b/docsource/usage/dbt.md new file mode 100644 index 0000000..6b11434 --- /dev/null +++ b/docsource/usage/dbt.md @@ -0,0 +1,130 @@ +# Enhanced SQLMock with dbt Integration Guide + +## Introduction + +This guide will provide a quick start on how to use SQLMock with dbt (data build tool). You can use it to mock dbt models, sources, and seed models. We'll cover how to use these features effectively in your unit tests. + +## Prerequisites + +- A working dbt project with a `manifest.json` file **that has the latest compiled run.** (make sure to run `dbt compile`). +- The SQLMock library installed in your Python environment. + +## Configuration + +### Setting the dbt Manifest Path + +Initialize your testing environment by setting the global path to your dbt manifest file: + +```python +from sql_mock.config import SQLMockConfig + +SQLMockConfig.set_dbt_manifest_path('/path/to/your/dbt/manifest.json') +``` + +## Creating Mock Tables + +SQLMock offers specialized decorators for different dbt entities: models, sources, and seeds. + +### dbt Model Mock Table + +For dbt models, use the `dbt_model_meta` decorator from `sql_mock.dbt`. This decorator is suited for mocking the transformed data produced by dbt models. + +```python +from sql_mock.dbt import dbt_model_meta +from sql_mock.bigquery.table_mocks import BigQueryMockTable + +@dbt_model_meta(model_name="your_dbt_model_name") +class YourDBTModelTable(BigQueryMockTable): + # Define your table columns and other necessary attributes here + ... +``` + +### dbt Source Mock Table + +For dbt sources, use the `dbt_source_meta` decorator from `sql_mock.dbt`. This is ideal for mocking the raw data sources that dbt models consume. + +```python +from sql_mock.dbt import dbt_source_meta +from sql_mock.bigquery.table_mocks import BigQueryMockTable + +@dbt_source_meta(source_name="your_source_name", table_name="your_source_table") +class YourDBTSourceTable(BigQueryMockTable): + # Define your table columns and other necessary attributes here + ... +``` + +### dbt Seed Mock Table + +For dbt seeds, which are static data sets loaded into the database, use the `dbt_seed_meta` decorator from `sql_mock.dbt`. + +```python +from sql_mock.dbt import dbt_seed_meta +from sql_mock.bigquery.table_mocks import BigQueryMockTable + +@dbt_seed_meta(seed_name="your_dbt_seed_name") +class YourDBTSeedTable(BigQueryMockTable): + # Define your table columns and other necessary attributes here + ... +``` + +## Example: Testing a dbt Model with Upstream Source and Seed Data + +Let’s consider a dbt model named `monthly_user_spend` that aggregates data from a source `user_transactions` and a seed `user_categories`. + +### Step 1: Define Your Source and Seed Mock Tables + +```python +@dbt_source_meta(source_name="transactions", table_name="user_transactions") +class UserTransactionsTable(BigQueryMockTable): + transaction_id = col.Int(default=1) + user_id = col.Int(default=1) + amount = col.Float(default=1.0) + transaction_date = col.Date(default='2023-12-24') + +@dbt_seed_meta(seed_name="user_categories") +class UserCategoriesTable(BigQueryMockTable): + user_id = col.Int(default=1) + category = col.String(default='foo') +``` + +### Step 2: Define Your Model Mock Table + +```python +@dbt_model_meta(model_name="monthly_user_spend") +class MonthlyUserSpendTable(BigQueryMockTable): + user_id = col.Int(default=1) + month = col.String(default='foo') + total_spend = col.Float(default=1.0) + category = col.String(default='foo') +``` + +### Step 3: Write Your Test Case + +```python +import datetime + +def test_monthly_user_spend_model(): + # Mock input data for UserTransactionsTable and UserCategoriesTable + transactions_data = [ + {"transaction_id": 1, "user_id": 1, "amount": 120.0, "transaction_date": datetime.date(2023, 1, 10)}, + {"transaction_id": 2, "user_id": 2, "amount": 150.0, "transaction_date": datetime.date(2023, 1, 20)}, + ] + + categories_data = [ + {"user_id": 1, "category": "Premium"}, + {"user_id": 2, "category": "Standard"} + ] + + transactions_table = UserTransactionsTable.from_dicts(transactions_data) + categories_table = UserCategoriesTable.from_dicts(categories_data) + + # Expected result + expected_output = [ + {"user_id": 1, "month": "2023-01", "total_spend": 120.0, "category": "Premium"}, + {"user_id": 2, "month": "2023-01", "total_spend": 150.0, "category": "Standard"}, + ] + + monthly_spend_table = MonthlyUserSpendTable.from_mocks(input_data=[transactions_table, categories_table]) + + monthly_spend_table.assert_equal(expected_output) +``` diff --git a/examples/dbt/test_example.py b/examples/dbt/test_example.py new file mode 100644 index 0000000..451293e --- /dev/null +++ b/examples/dbt/test_example.py @@ -0,0 +1,47 @@ +from sql_mock.bigquery import column_mocks as col +from sql_mock.bigquery.table_mocks import BigQueryMockTable +from sql_mock.config import SQLMockConfig +from sql_mock.dbt import dbt_model_meta, dbt_seed_meta, dbt_source_meta + +SQLMockConfig.set_dbt_manifest_path("./tests/resources/dbt/dbt_manifest.json") + + +# NOTE: The Source and Seed classes will not be used in the example test. They are only here for demonstration purpose. +@dbt_source_meta(source_name="source_data", table_name="opportunity_events") +class OpportunityEventsSource(BigQueryMockTable): + event_id = col.Int(default=1) + event_type = col.String(default="foo") + event_date = col.Date(default="2023-12-24") + + +@dbt_seed_meta(seed_name="country_codes") +class CountryCodesSeed(BigQueryMockTable): + country_code = col.String(default="foo") + country_name = col.String(default="foo") + + +@dbt_model_meta(model_name="my_first_dbt_model") +class MyFirstDBTModel(BigQueryMockTable): + id = col.Int(default=1) + + +@dbt_model_meta(model_name="my_second_dbt_model") +class MySecondDBTModel(BigQueryMockTable): + id = col.Int(default=1) + + +def test_my_second_dbt_model(): + # Mock data for the first model + first_model_data = [{"id": 1}, {"id": 2}, {"id": 3}] + + # Create a mock table instance with the data + first_model_table = MyFirstDBTModel.from_dicts(first_model_data) + + # Expected result for the second model + expected_output = [{"id": 1}] # Assuming the second model filters for entries with id 1 only + + # Instantiate the second dbt model mock table with the first model as input + second_model_table = MySecondDBTModel.from_mocks(input_data=[first_model_table]) + + # Assert that the dbt model's output matches the expected output + second_model_table.assert_equal(expected_output) diff --git a/src/sql_mock/config.py b/src/sql_mock/config.py new file mode 100644 index 0000000..fa4a525 --- /dev/null +++ b/src/sql_mock/config.py @@ -0,0 +1,12 @@ +class SQLMockConfig: + _dbt_manifest_path = None + + @classmethod + def set_dbt_manifest_path(cls, path: str): + cls._dbt_manifest_path = path + + @classmethod + def get_dbt_manifest_path(cls): + if cls._dbt_manifest_path is None: + raise ValueError("DBT manifest path is not set. Please set it using set_dbt_manifest_path()") + return cls._dbt_manifest_path diff --git a/src/sql_mock/dbt.py b/src/sql_mock/dbt.py new file mode 100644 index 0000000..41ea82a --- /dev/null +++ b/src/sql_mock/dbt.py @@ -0,0 +1,179 @@ +import json +from typing import TYPE_CHECKING + +from sql_mock.config import SQLMockConfig +from sql_mock.helpers import parse_table_refs, validate_input_mocks +from sql_mock.table_mocks import MockTableMeta + +# Needed to avoid circular imports on type check +if TYPE_CHECKING: + from sql_mock.table_mocks import BaseMockTable + + +def _get_model_metadata_from_dbt_manifest(manifest_path: str, model_name: str) -> dict: + """ + Extracts the rendered SQL query for a specified model from the dbt manifest file. + + Args: + manifest_path (str): Path to the dbt manifest.json file. + model_name (str): Name of the dbt model. + + Returns: + dict: Dictionary of metadata from dbt (path to compiled sql query and table ref) + """ + with open(manifest_path, "r") as file: + manifest = json.load(file) + + for node in manifest["nodes"].values(): + if node["resource_type"] == "model" and node["name"] == model_name: + return { + "query_path": node["compiled_path"], + "table_ref": node["relation_name"], + } + + raise ValueError(f"Model '{model_name}' not found in dbt manifest.") + + +def _get_source_metadata_from_dbt_manifest(manifest_path: str, source_name: str, table_name: str) -> dict: + """ + Extracts the table metadata for dbt source from the manifest file. + + Args: + manifest_path (str): Path to the dbt manifest.json file. + source_name (str): Name of the dbt source. + table_name (str): Name of the table in the dbt source. + + Returns: + dict: Dictionary of metadata from dbt + """ + with open(manifest_path, "r") as file: + manifest = json.load(file) + + for node in manifest["sources"].values(): + if ( + node["resource_type"] == "source" + and node["source_name"] == source_name + and node["identifier"] == table_name + ): + return { + "table_ref": node["relation_name"], + } + + raise ValueError(f"Source '{source_name}' not found in dbt manifest.") + + +def _get_seed_metadata_from_dbt_manifest(manifest_path: str, seed_name: str) -> dict: + """ + Extracts the table metadata for dbt seed from the manifest file. + + Args: + manifest_path (str): Path to the dbt manifest.json file. + seed_name (str): Name of the dbt seed. + + Returns: + dict: Dictionary of metadata from dbt + """ + with open(manifest_path, "r") as file: + manifest = json.load(file) + + for node in manifest["nodes"].values(): + if node["resource_type"] == "seed" and node["name"] == seed_name: + return { + "table_ref": node["relation_name"], + } + + raise ValueError(f"Seed '{seed_name}' not found in dbt manifest.") + + +def dbt_model_meta(model_name: str, manifest_path: str = None, default_inputs: ["BaseMockTable"] = None): + """ + Decorator that is used to define MockTable metadata for dbt models. + + Args: + model_name (string) : Name of the dbt model + manifest_path (string): Path to the dbt manifest file + default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided. + """ + + def decorator(cls): + path = manifest_path or SQLMockConfig.get_dbt_manifest_path() + + dbt_meta = _get_model_metadata_from_dbt_manifest(manifest_path=path, model_name=model_name) + + parsed_query = "" + with open(dbt_meta["query_path"]) as f: + parsed_query = f.read() + + if default_inputs: + validate_input_mocks(default_inputs) + + cls._sql_mock_meta = MockTableMeta( + table_ref=parse_table_refs(dbt_meta["table_ref"], dialect=cls._sql_dialect), + query=parsed_query, + default_inputs=default_inputs or [], + ) + return cls + + return decorator + + +def dbt_source_meta( + source_name: str, table_name: str, manifest_path: str = None, default_inputs: ["BaseMockTable"] = None +): + """ + Decorator that is used to define MockTable metadata for dbt sources. + + Args: + source_name (string) : Name of source + table_name (string): Name of the table in the source + manifest_path (string): Path to the dbt manifest file + default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided. + """ + + def decorator(cls): + path = manifest_path or SQLMockConfig.get_dbt_manifest_path() + + dbt_meta = _get_source_metadata_from_dbt_manifest( + manifest_path=path, source_name=source_name, table_name=table_name + ) + + if default_inputs: + validate_input_mocks(default_inputs) + + cls._sql_mock_meta = MockTableMeta( + table_ref=parse_table_refs(dbt_meta["table_ref"], dialect=cls._sql_dialect), + default_inputs=default_inputs or [], + ) + return cls + + return decorator + + +def dbt_seed_meta(seed_name: str, manifest_path: str = None, default_inputs: ["BaseMockTable"] = None): + """ + Decorator that is used to define MockTable metadata for dbt sources. + + Args: + seed_name (string) : Name of the dbt seed + manifest_path (string): Path to the dbt manifest file + default_inputs: List of default input mock instances that serve as default input if no other instance of that class is provided. + """ + + def decorator(cls): + path = manifest_path or SQLMockConfig.get_dbt_manifest_path() + + dbt_meta = _get_seed_metadata_from_dbt_manifest( + manifest_path=path, + seed_name=seed_name, + ) + + if default_inputs: + validate_input_mocks(default_inputs) + + cls._sql_mock_meta = MockTableMeta( + table_ref=parse_table_refs(dbt_meta["table_ref"], dialect=cls._sql_dialect), + default_inputs=default_inputs or [], + ) + return cls + + return decorator diff --git a/tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql b/tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql new file mode 100644 index 0000000..2f45e2a --- /dev/null +++ b/tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql @@ -0,0 +1,26 @@ +-- [hv_1.1|Classification: CONFIDENTIAL, DeepL SE] + +/* + Welcome to your first dbt model! + Did you know that you can also configure models directly within SQL files? + This will override configurations stated in dbt_project.yml + + Try changing "table" to "view" below +*/ + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +/* + Uncomment the line below to remove records with null `id` values +*/ + +-- where id is not null \ No newline at end of file diff --git a/tests/resources/dbt/compiled_example_models/my_second_dbt_model.sql b/tests/resources/dbt/compiled_example_models/my_second_dbt_model.sql new file mode 100644 index 0000000..6ad009f --- /dev/null +++ b/tests/resources/dbt/compiled_example_models/my_second_dbt_model.sql @@ -0,0 +1,7 @@ +-- [hv_1.1|Classification: CONFIDENTIAL, DeepL SE] + +-- Use the `ref` function to select from other models + +select * +from `sql_mock_db`.`my_first_dbt_model` +where id = 1 diff --git a/tests/resources/dbt/dbt_manifest.json b/tests/resources/dbt/dbt_manifest.json new file mode 100644 index 0000000..6082617 --- /dev/null +++ b/tests/resources/dbt/dbt_manifest.json @@ -0,0 +1,536 @@ +{ + "metadata":{ + "dbt_schema_version":"https://schemas.getdbt.com/dbt/manifest/v11.json", + "dbt_version":"1.7.4", + "generated_at":"2023-12-28T12:23:14.131356Z", + "invocation_id":"33a2953c-7739-4451-b106-5005b3fbb7d4", + "env":{ + + }, + "project_name":"sql_mock", + "project_id":"7bdc11bc0f5dd7bc0117761f9502d6b0", + "user_id":"74953650-5c59-4b93-b2b3-a5e358c374c0", + "send_anonymous_usage_stats":true, + "adapter_type":"clickhouse" + }, + "nodes":{ + "model.sql_mock.my_first_dbt_model":{ + "database":"", + "schema":"sql_mock_db", + "name":"my_first_dbt_model", + "resource_type":"model", + "package_name":"sql_mock", + "path":"example/my_first_dbt_model.sql", + "original_file_path":"models/example/my_first_dbt_model.sql", + "unique_id":"model.sql_mock.my_first_dbt_model", + "fqn":[ + "sql_mock", + "example", + "my_first_dbt_model" + ], + "alias":"my_first_dbt_model", + "checksum":{ + "name":"sha256", + "checksum":"8f5a6188069c550051ab3d45e044450f86a9ca6e1bcfd334cce9c8ef79740d48" + }, + "config":{ + "enabled":true, + "alias":null, + "schema":"example_schema", + "database":null, + "tags":[ + + ], + "meta":{ + + }, + "group":null, + "materialized":"distributed_table", + "incremental_strategy":null, + "persist_docs":{ + + }, + "post-hook":[ + + ], + "pre-hook":[ + + ], + "quoting":{ + + }, + "column_types":{ + + }, + "full_refresh":null, + "unique_key":null, + "on_schema_change":"ignore", + "on_configuration_change":"apply", + "grants":{ + + }, + "packages":[ + + ], + "docs":{ + "show":true, + "node_color":null + }, + "contract":{ + "enforced":false, + "alias_types":true + }, + "access":"protected", + "engine":"ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}/{uuid}','{replica}')", + "sharding_key":"rand()" + }, + "tags":[ + + ], + "description":"A starter dbt model", + "columns":{ + "id":{ + "name":"id", + "description":"The primary key for this table", + "meta":{ + + }, + "data_type":null, + "constraints":[ + + ], + "quote":null, + "tags":[ + + ] + } + }, + "meta":{ + + }, + "group":null, + "docs":{ + "show":true, + "node_color":null + }, + "patch_path":"sql_mock://models/example/schema.yml", + "build_path":null, + "deferred":false, + "unrendered_config":{ + "schema":"example_schema", + "materialized":"distributed_table", + "engine":"ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}/{uuid}','{replica}')", + "sharding_key":"rand()" + }, + "created_at":1703751270.463069, + "relation_name":"`sql_mock_db`.`my_first_dbt_model`", + "raw_code":"/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n select 1 as id\n union all\n select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", + "language":"sql", + "refs":[ + + ], + "sources":[ + + ], + "metrics":[ + + ], + "depends_on":{ + "macros":[ + + ], + "nodes":[ + + ] + }, + "compiled_path":"tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql", + "compiled":true, + "compiled_code":"\n\n/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\nwith source_data as (\n\n select 1 as id\n union all\n select null as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", + "extra_ctes_injected":true, + "extra_ctes":[ + + ], + "contract":{ + "enforced":false, + "alias_types":true, + "checksum":null + }, + "access":"protected", + "constraints":[ + + ], + "version":null, + "latest_version":null, + "deprecation_date":null + }, + "model.sql_mock.my_second_dbt_model":{ + "database":"", + "schema":"sql_mock_db", + "name":"my_second_dbt_model", + "resource_type":"model", + "package_name":"sql_mock", + "path":"example/my_second_dbt_model.sql", + "original_file_path":"models/example/my_second_dbt_model.sql", + "unique_id":"model.sql_mock.my_second_dbt_model", + "fqn":[ + "sql_mock", + "example", + "my_second_dbt_model" + ], + "alias":"my_second_dbt_model", + "checksum":{ + "name":"sha256", + "checksum":"4bb5c04f6ea2e92b0939316ebf7f4ad7ad142a45cc9348e74c0f56e5261f3bbc" + }, + "config":{ + "enabled":true, + "alias":null, + "schema":"example_schema", + "database":null, + "tags":[ + + ], + "meta":{ + + }, + "group":null, + "materialized":"distributed_table", + "incremental_strategy":null, + "persist_docs":{ + + }, + "post-hook":[ + + ], + "pre-hook":[ + + ], + "quoting":{ + + }, + "column_types":{ + + }, + "full_refresh":null, + "unique_key":null, + "on_schema_change":"ignore", + "on_configuration_change":"apply", + "grants":{ + + }, + "packages":[ + + ], + "docs":{ + "show":true, + "node_color":null + }, + "contract":{ + "enforced":false, + "alias_types":true + }, + "access":"protected", + "engine":"ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}/{uuid}','{replica}')", + "sharding_key":"rand()" + }, + "tags":[ + + ], + "description":"A starter dbt model", + "columns":{ + "id":{ + "name":"id", + "description":"The primary key for this table", + "meta":{ + + }, + "data_type":null, + "constraints":[ + + ], + "quote":null, + "tags":[ + + ] + } + }, + "meta":{ + + }, + "group":null, + "docs":{ + "show":true, + "node_color":null + }, + "patch_path":"sql_mock://models/example/schema.yml", + "build_path":null, + "deferred":false, + "unrendered_config":{ + "schema":"example_schema", + "materialized":"distributed_table", + "engine":"ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}/{uuid}','{replica}')", + "sharding_key":"rand()" + }, + "created_at":1703751270.46336, + "relation_name":"`sql_mock_db`.`my_second_dbt_model`", + "raw_code":"\n\n-- Use the `ref` function to select from other models\n\nselect *\nfrom {{ ref('my_first_dbt_model') }}\nwhere id = 1", + "language":"sql", + "refs":[ + { + "name":"my_first_dbt_model", + "package":null, + "version":null + } + ], + "sources":[ + + ], + "metrics":[ + + ], + "depends_on":{ + "macros":[ + + ], + "nodes":[ + "model.sql_mock.my_first_dbt_model" + ] + }, + "compiled_path":"tests/resources/dbt/compiled_example_models/my_second_dbt_model.sql", + "compiled":true, + "compiled_code":"\n\n-- Use the `ref` function to select from other models\n\nselect *\nfrom `sql_mock_db`.`my_first_dbt_model`\nwhere id = 1", + "extra_ctes_injected":true, + "extra_ctes":[ + + ], + "contract":{ + "enforced":false, + "alias_types":true, + "checksum":null + }, + "access":"protected", + "constraints":[ + + ], + "version":null, + "latest_version":null, + "deprecation_date":null + }, + "seed.sql_mock.country_codes":{ + "database":"", + "schema":"sql_mock_db", + "name":"country_codes", + "resource_type":"seed", + "package_name":"sql_mock", + "path":"country_codes.csv", + "original_file_path":"seeds/country_codes.csv", + "unique_id":"seed.sql_mock.country_codes", + "fqn":[ + "sql_mock", + "country_codes" + ], + "alias":"country_codes", + "checksum":{ + "name":"sha256", + "checksum":"b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c" + }, + "config":{ + "enabled":true, + "alias":null, + "schema":null, + "database":null, + "tags":[ + + ], + "meta":{ + + }, + "group":null, + "materialized":"seed", + "incremental_strategy":null, + "persist_docs":{ + + }, + "post-hook":[ + + ], + "pre-hook":[ + + ], + "quoting":{ + + }, + "column_types":{ + + }, + "full_refresh":null, + "unique_key":null, + "on_schema_change":"ignore", + "on_configuration_change":"apply", + "grants":{ + + }, + "packages":[ + + ], + "docs":{ + "show":true, + "node_color":null + }, + "contract":{ + "enforced":false, + "alias_types":true + }, + "delimiter":",", + "quote_columns":null + }, + "tags":[ + + ], + "description":"", + "columns":{ + + }, + "meta":{ + + }, + "group":null, + "docs":{ + "show":true, + "node_color":null + }, + "patch_path":null, + "build_path":null, + "deferred":false, + "unrendered_config":{ + + }, + "created_at":1703766194.209625, + "relation_name":"`sql_mock_db`.`country_codes`", + "raw_code":"", + "root_path":"/Users/foo/dbt", + "depends_on":{ + "macros":[ + + ] + } + } + }, + "sources":{ + "source.sql_mock.salesforce.opportunity_events":{ + "database":"", + "schema":"source_data", + "name":"opportunity_events", + "resource_type":"source", + "package_name":"sql_mock", + "path":"models/sources/source_data.yml", + "original_file_path":"models/sources/source_data.yml", + "unique_id":"source.sql_mock.source_data.opportunity_events", + "fqn":[ + "sql_mock", + "sources", + "source_data", + "opportunity_events" + ], + "source_name":"source_data", + "source_description":"", + "loader":"", + "identifier":"opportunity_events", + "quoting":{ + "database":null, + "schema":null, + "identifier":null, + "column":null + }, + "loaded_at_field":"datetime", + "freshness":{}, + "external":null, + "description":"Opportunity change events", + "columns":{ + + }, + "meta":{ + + }, + "source_meta":{ + + }, + "tags":[ + + ], + "config":{ + "enabled":true + }, + "patch_path":null, + "unrendered_config":{ + + }, + "relation_name":"`source_data`.`opportunity_events`", + "created_at":1703751270.504808 + } + }, + "macros":{}, + "docs":{ + "doc.dbt.__overview__":{ + "name":"__overview__", + "resource_type":"doc", + "package_name":"dbt", + "path":"overview.md", + "original_file_path":"docs/overview.md", + "unique_id":"doc.dbt.__overview__", + "block_contents":"### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion" + } + }, + "exposures":{ + + }, + "metrics":{ + + }, + "groups":{ + + }, + "selectors":{ + + }, + "disabled":{ + + }, + "parent_map":{ + "model.sql_mock.my_first_dbt_model":[ + + ], + "model.sql_mock.my_second_dbt_model":[ + "model.sql_mock.my_first_dbt_model" + ], + "seed.sql_mock.country_codes":[ + + ], + "source.sql_mock.source_data.opportunity_events":[ + + ] + }, + "child_map":{ + "model.sql_mock.my_first_dbt_model":[ + "model.sql_mock.my_second_dbt_model" + ], + "model.sql_mock.my_second_dbt_model":[ + + ], + "seed.sql_mock.country_codes":[ + + ], + "source.sql_mock.source_data.opportunity_events":[ + + ] + }, + "group_map":{ + + }, + "saved_queries":{ + + }, + "semantic_models":{ + + } + } diff --git a/tests/sql_mock/test_dbt.py b/tests/sql_mock/test_dbt.py new file mode 100644 index 0000000..f2b3567 --- /dev/null +++ b/tests/sql_mock/test_dbt.py @@ -0,0 +1,241 @@ +import pytest + +from sql_mock.config import SQLMockConfig +from sql_mock.dbt import ( + _get_model_metadata_from_dbt_manifest, + _get_seed_metadata_from_dbt_manifest, + _get_source_metadata_from_dbt_manifest, + dbt_model_meta, + dbt_seed_meta, + dbt_source_meta, +) +from sql_mock.table_mocks import BaseMockTable + + +class TestDbtModelMeta: + def test_manifest_path_provided(self, mocker): + """...then metadata should be extracted from that manifest path""" + manifest_path = "path/to/my/manifest" + + # We set another path in the config but it should be overwritten + SQLMockConfig.set_dbt_manifest_path("some/other/path") + + model_name = "my_model" + returned_query_path = "some/path/to/query.sql" + returned_table_ref = "db.my_model" + + mocked_get_model_metadata_from_dbt_manifest = mocker.patch( + "sql_mock.dbt._get_model_metadata_from_dbt_manifest" + ) + mocked_get_model_metadata_from_dbt_manifest.return_value = { + "query_path": returned_query_path, + "table_ref": returned_table_ref, + } + + query = "SELECT bar FROM foo" + mock_open = mocker.patch("builtins.open") + # Configure the mock to return the file content + mock_open.return_value.__enter__.return_value.read.return_value = query + + @dbt_model_meta(model_name=model_name, manifest_path=manifest_path) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query == query + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mock_open.assert_called_once_with(returned_query_path) + mocked_get_model_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, model_name=model_name + ) + + def test_manifest_path_not_provided_but_set_in_config(self, mocker): + """...then metadata should be extracted from the manifest path provided in the config""" + manifest_path = "path/to/my/manifest" + SQLMockConfig.set_dbt_manifest_path(manifest_path) + + model_name = "my_model" + returned_query_path = "some/path/to/query.sql" + returned_table_ref = "db.my_model" + + mocked_get_model_metadata_from_dbt_manifest = mocker.patch( + "sql_mock.dbt._get_model_metadata_from_dbt_manifest" + ) + mocked_get_model_metadata_from_dbt_manifest.return_value = { + "query_path": returned_query_path, + "table_ref": returned_table_ref, + } + + query = "SELECT bar FROM foo" + mock_open = mocker.patch("builtins.open") + # Configure the mock to return the file content + mock_open.return_value.__enter__.return_value.read.return_value = query + + @dbt_model_meta(model_name=model_name) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query == query + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mock_open.assert_called_once_with(returned_query_path) + mocked_get_model_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, model_name=model_name + ) + + +class TestDbtSourceMeta: + def test_manifest_path_provided(self, mocker): + """...then metadata should be extracted from that manifest path""" + manifest_path = "path/to/my/manifest" + + # We set another path in the config but it should be overwritten + SQLMockConfig.set_dbt_manifest_path("some/other/path") + + source_name = "my_source" + table_name = "my_table" + returned_table_ref = "db.my_model" + + mocked_get_source_metadata_from_dbt_manifest = mocker.patch( + "sql_mock.dbt._get_source_metadata_from_dbt_manifest" + ) + mocked_get_source_metadata_from_dbt_manifest.return_value = {"table_ref": returned_table_ref} + + @dbt_source_meta(source_name=source_name, table_name=table_name, manifest_path=manifest_path) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query is None + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mocked_get_source_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, source_name=source_name, table_name=table_name + ) + + def test_manifest_path_not_provided_but_set_in_config(self, mocker): + """...then metadata should be extracted from the manifest path provided in the config""" + manifest_path = "path/to/my/manifest" + SQLMockConfig.set_dbt_manifest_path(manifest_path) + + source_name = "my_source" + table_name = "my_table" + returned_table_ref = "db.my_model" + + mocked_get_source_metadata_from_dbt_manifest = mocker.patch( + "sql_mock.dbt._get_source_metadata_from_dbt_manifest" + ) + mocked_get_source_metadata_from_dbt_manifest.return_value = {"table_ref": returned_table_ref} + + @dbt_source_meta(source_name=source_name, table_name=table_name, manifest_path=manifest_path) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query is None + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mocked_get_source_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, source_name=source_name, table_name=table_name + ) + + +class TestDbtSeedMeta: + def test_manifest_path_provided(self, mocker): + """...then metadata should be extracted from that manifest path""" + manifest_path = "path/to/my/manifest" + + # We set another path in the config but it should be overwritten + SQLMockConfig.set_dbt_manifest_path("some/other/path") + + seed_name = "my_model" + returned_query_path = "some/path/to/query.sql" + returned_table_ref = "db.my_model" + + mocked_get_seed_metadata_from_dbt_manifest = mocker.patch("sql_mock.dbt._get_seed_metadata_from_dbt_manifest") + mocked_get_seed_metadata_from_dbt_manifest.return_value = { + "query_path": returned_query_path, + "table_ref": returned_table_ref, + } + + @dbt_seed_meta(seed_name=seed_name, manifest_path=manifest_path) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query is None + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mocked_get_seed_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, seed_name=seed_name + ) + + def test_manifest_path_not_provided_but_set_in_config(self, mocker): + """...then metadata should be extracted from the manifest path provided in the config""" + manifest_path = "path/to/my/manifest" + SQLMockConfig.set_dbt_manifest_path(manifest_path) + + seed_name = "my_model" + returned_query_path = "some/path/to/query.sql" + returned_table_ref = "db.my_model" + + mocked_get_seed_metadata_from_dbt_manifest = mocker.patch("sql_mock.dbt._get_seed_metadata_from_dbt_manifest") + mocked_get_seed_metadata_from_dbt_manifest.return_value = { + "query_path": returned_query_path, + "table_ref": returned_table_ref, + } + + @dbt_seed_meta(seed_name=seed_name) + class TestMock(BaseMockTable): + pass + + assert TestMock._sql_mock_meta.query is None + assert TestMock._sql_mock_meta.table_ref == returned_table_ref + mocked_get_seed_metadata_from_dbt_manifest.assert_called_once_with( + manifest_path=manifest_path, seed_name=seed_name + ) + + +MANIFEST_FILE = "./tests/resources/dbt/dbt_manifest.json" + + +class TestGetModelMetadataDbtFromManifest: + def test_model_does_not_exist_in_file(self): + """...then the method should raise a ValueError""" + with pytest.raises(ValueError): + _get_model_metadata_from_dbt_manifest(manifest_path=MANIFEST_FILE, model_name="I don not exist") + + def test_model_does_exist_in_file(self): + """...then the method should return the correct values""" + data = _get_model_metadata_from_dbt_manifest(manifest_path=MANIFEST_FILE, model_name="my_first_dbt_model") + + assert data["query_path"] == "tests/resources/dbt/compiled_example_models/my_first_dbt_model.sql" + assert data["table_ref"] == "`sql_mock_db`.`my_first_dbt_model`" + + +class TestGetSourceMetadataDbtFromManifest: + def test_source_does_not_exist_in_file(self): + """...then the method should raise a ValueError""" + with pytest.raises(ValueError): + _get_source_metadata_from_dbt_manifest( + manifest_path=MANIFEST_FILE, source_name="I don not exist", table_name="I don not exist either" + ) + + def test_source_does_exist_in_file(self): + """...then the method should return the correct values""" + data = _get_source_metadata_from_dbt_manifest( + manifest_path=MANIFEST_FILE, source_name="source_data", table_name="opportunity_events" + ) + + assert data["table_ref"] == "`source_data`.`opportunity_events`" + + +class TestGetSeedMetadataDbtFromManifest: + def test_seed_does_not_exist_in_file(self): + """...then the method should raise a ValueError""" + with pytest.raises(ValueError): + _get_seed_metadata_from_dbt_manifest( + manifest_path=MANIFEST_FILE, + seed_name="I don not exist", + ) + + def test_seed_does_exist_in_file(self): + """...then the method should return the correct values""" + data = _get_seed_metadata_from_dbt_manifest( + manifest_path=MANIFEST_FILE, + seed_name="country_codes", + ) + + assert data["table_ref"] == "`sql_mock_db`.`country_codes`"