From c094ca58b896d0019f82f82118fabe1dd36400f6 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 8 Aug 2022 15:16:10 -0700 Subject: [PATCH 1/2] Upgrade to support dbt-core==1.2.0 - Addresses issue https://github.com/oracle/dbt-oracle/issues/32 - Added 2 new basic tests BaseDocsGenerate and BaseDocsGenReferences - Upgraded dbt-core dependency to v1.2.0 - Upgraded dbt-adapter-tests to v1.2.0 - Do not change case of object names when fetching metadata either in adapter.sql or catalog.sql --- .gitignore | 2 +- Makefile | 2 +- dbt/adapters/oracle/__version__.py | 2 +- dbt/include/oracle/macros/adapters.sql | 20 +- dbt/include/oracle/macros/catalog.sql | 6 +- requirements.txt | 2 +- requirements_dev.txt | 2 +- setup.cfg | 6 +- setup.py | 6 +- .../functional/adapter/test_docs_generate.py | 187 +++++++++++++++ .../adapter/test_docs_genreferences.py | 222 ++++++++++++++++++ 11 files changed, 433 insertions(+), 24 deletions(-) create mode 100644 tests/functional/adapter/test_docs_generate.py create mode 100644 tests/functional/adapter/test_docs_genreferences.py diff --git a/.gitignore b/.gitignore index 9e6f1e1..b342ead 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,4 @@ doc/build.gitbak .venv1.1.0 .venv .bldenv - +.venv1.2/ diff --git a/Makefile b/Makefile index fb6c880..19f8782 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.1.2 +VERSION=1.2.0 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index 08147bd..a79a1d1 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.1.2" +version = "1.2.0" diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index 4eb25ad..07efda9 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -58,7 +58,7 @@ 'SYNONYM', 'MATERIALIZED VIEW' ) - AND owner = '{{ schema | upper }}') + AND upper(owner) = '{{ schema | upper }}') LOOP BEGIN IF cur_rec.object_type = 'TABLE' @@ -130,8 +130,8 @@ {% call statement('get_columns_in_relation', fetch_result=True) %} with columns as ( select - UPPER(SYS_CONTEXT('userenv', 'DB_NAME')) table_catalog, - UPPER(owner) table_schema, + SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, + owner table_schema, table_name, column_name, data_type, @@ -178,9 +178,9 @@ numeric_precision as "numeric_precision", numeric_scale as "numeric_scale" from columns - where table_name = upper('{{ relation.identifier }}') + where upper(table_name) = upper('{{ relation.identifier }}') {% if relation.schema %} - and table_schema = upper('{{ relation.schema }}') + and upper(table_schema) = upper('{{ relation.schema }}') {% endif %} order by ordinal_position @@ -314,8 +314,8 @@ {% macro oracle__list_relations_without_caching(schema_relation) %} {% call statement('list_relations_without_caching', fetch_result=True) -%} with tables as - (select UPPER(SYS_CONTEXT('userenv', 'DB_NAME')) table_catalog, - UPPER(owner) table_schema, + (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, + owner table_schema, table_name, case when iot_type = 'Y' @@ -326,8 +326,8 @@ end table_type from sys.all_tables union all - select UPPER(SYS_CONTEXT('userenv', 'DB_NAME')), - UPPER(owner), + select SYS_CONTEXT('userenv', 'DB_NAME'), + owner, view_name, 'VIEW' from sys.all_views @@ -341,7 +341,7 @@ end as "kind" from tables where table_type in ('BASE TABLE', 'VIEW') - and table_schema = upper('{{ schema_relation.schema }}') + and upper(table_schema) = upper('{{ schema_relation.schema }}') {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} diff --git a/dbt/include/oracle/macros/catalog.sql b/dbt/include/oracle/macros/catalog.sql index c1b2673..d8eac12 100644 --- a/dbt/include/oracle/macros/catalog.sql +++ b/dbt/include/oracle/macros/catalog.sql @@ -71,8 +71,8 @@ from sys.all_tab_columns ), tables as - (select UPPER(SYS_CONTEXT('userenv', 'DB_NAME')) table_catalog, - UPPER(owner) table_schema, + (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, + owner table_schema, table_name, case when iot_type = 'Y' @@ -117,7 +117,7 @@ and all_col_comments.column_name = columns.column_name where ( {%- for schema in schemas -%} - tables.table_schema = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + upper(tables.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} {%- endfor -%} ) order by diff --git a/requirements.txt b/requirements.txt index c656129..644a856 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -dbt-core==1.1.2 +dbt-core==1.2.0 cx_Oracle==8.3.0 oracledb==1.0.3 diff --git a/requirements_dev.txt b/requirements_dev.txt index cefeb06..3817949 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,4 +6,4 @@ tox coverage twine pytest -dbt-tests-adapter==1.1.2 \ No newline at end of file +dbt-tests-adapter==1.2.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 211a346..5f94579 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.1.2 +version = 1.2.0 description = dbt (data build tool) adapter for the Oracle database long_description = file: README.md long_description_content_type = text/markdown @@ -32,12 +32,12 @@ zip_safe = False packages = find: include_package_data = True install_requires = - dbt-core==1.1.2 + dbt-core==1.2.0 cx_Oracle==8.3.0 oracledb==1.0.3 test_suite=tests test_requires = - dbt-tests-adapter==1.1.2 + dbt-tests-adapter==1.2.0 pytest scripts = bin/create-pem-from-p12 diff --git a/setup.py b/setup.py index 6d2a21c..a7a9356 100644 --- a/setup.py +++ b/setup.py @@ -32,13 +32,13 @@ requirements = [ - "dbt-core==1.1.2", + "dbt-core==1.2.0", "cx_Oracle==8.3.0", "oracledb==1.0.3" ] test_requirements = [ - "dbt-tests-adapter==1.1.2", + "dbt-tests-adapter==1.2.0", "pytest" ] @@ -52,7 +52,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.1.2' +VERSION = '1.2.0' setup( author="Oracle", python_requires='>=3.7.2', diff --git a/tests/functional/adapter/test_docs_generate.py b/tests/functional/adapter/test_docs_generate.py new file mode 100644 index 0000000..381c2fa --- /dev/null +++ b/tests/functional/adapter/test_docs_generate.py @@ -0,0 +1,187 @@ +import pytest + +from dbt.tests.adapter.basic.test_docs_generate import (models__schema_yml, + models__readme_md, + models__model_sql, + BaseDocsGenerate) + + +from dbt.tests.adapter.basic.expected_catalog import no_stats +from dbt.tests.util import AnyInteger + +models__second_model_sql = """ +{{ + config(materialized='view') +}} + +select * from {{ ref('seed') }} +""" + + +class TextType: + + def __eq__(self, other): + return other.upper().startswith('VARCHAR') + + +def base_expected_catalog( + project, + role, + id_type, + text_type, + time_type, + view_type, + table_type, + model_stats, + seed_stats=None, + case=None, + case_columns=False, +): + + if case is None: + + def case(x): + return x + + col_case = case if case_columns else lambda x: x + + if seed_stats is None: + seed_stats = model_stats + + model_database = case(project.database) + my_schema_name = case(project.test_schema) + alternate_schema = case(project.test_schema) + + expected_cols = { + col_case("id"): { + "name": col_case("id"), + "index": AnyInteger(), + "type": id_type, + "comment": None, + }, + col_case("first_name"): { + "name": col_case("first_name"), + "index": AnyInteger(), + "type": TextType(), + "comment": None, + }, + col_case("email"): { + "name": col_case("email"), + "index": AnyInteger(), + "type": TextType(), + "comment": None, + }, + col_case("ip_address"): { + "name": col_case("ip_address"), + "index": AnyInteger(), + "type": TextType(), + "comment": None, + }, + col_case("updated_at"): { + "name": col_case("updated_at"), + "index": AnyInteger(), + "type": time_type, + "comment": None, + }, + } + + return { + "nodes": { + "model.test.model": { + "unique_id": "model.test.model", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": case("model"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": expected_cols, + }, + "model.test.second_model": { + "unique_id": "model.test.second_model", + "metadata": { + "schema": alternate_schema, + "database": project.database.upper(), + "name": case("second_model"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": expected_cols, + }, + "seed.test.seed": { + "unique_id": "seed.test.seed", + "metadata": { + "schema": my_schema_name, + "database": project.database.upper(), + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": expected_cols, + }, + }, + "sources": { + "source.test.my_source.my_table": { + "unique_id": "source.test.my_source.my_table", + "metadata": { + "schema": my_schema_name, + "database": project.database.upper(), + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": expected_cols, + }, + }, + } + + +class TestOracleDocsGenerate(BaseDocsGenerate): + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "second_model.sql": models__second_model_sql, + "readme.md": models__readme_md, + "model.sql": models__model_sql, + } + + @pytest.fixture(scope="class") + def expected_catalog(self, project, profile_user): + return base_expected_catalog( + project, + role=profile_user.upper(), + id_type="NUMBER", + text_type="VARCHAR2(16)", + time_type="TIMESTAMP(6)", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + case=lambda x: x.upper(), + case_columns=False + ) + + @pytest.fixture(scope="class") + def project_config_update(self, unique_schema): + alternate_schema = unique_schema + return { + "asset-paths": ["assets", "invalid-asset-paths"], + "vars": { + "test_schema": unique_schema, + "alternate_schema": alternate_schema, + }, + "seeds": { + "quote_columns": True, + }, + "quoting": {"identifier": False}, + } diff --git a/tests/functional/adapter/test_docs_genreferences.py b/tests/functional/adapter/test_docs_genreferences.py new file mode 100644 index 0000000..fcf649e --- /dev/null +++ b/tests/functional/adapter/test_docs_genreferences.py @@ -0,0 +1,222 @@ +import pytest + +from dbt.tests.adapter.basic.test_docs_generate import (BaseDocsGenReferences, + ref_models__schema_yml, + ref_sources__schema_yml, + ref_models__ephemeral_copy_sql, + ref_models__docs_md) + +from dbt.tests.adapter.basic.expected_catalog import no_stats + +ref_models__ephemeral_summary_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select "first_name", count(*) as ct from {{ref('ephemeral_copy')}} +group by "first_name" +order by "first_name" asc + +""" + +ref_models__view_summary_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select "first_name", ct from {{ref('ephemeral_summary')}} +order by ct asc + +""" + + +class TextType: + + def __eq__(self, other): + return other.upper().startswith('VARCHAR') + + +def expected_references_catalog( + project, + role, + id_type, + text_type, + time_type, + view_type, + table_type, + model_stats, + bigint_type=None, + seed_stats=None, + case=None, + case_columns=False, + view_summary_stats=None, +): + if case is None: + + def case(x): + return x + + col_case = case if case_columns else lambda x: x + + if seed_stats is None: + seed_stats = model_stats + + if view_summary_stats is None: + view_summary_stats = model_stats + + model_database = project.database.upper() + my_schema_name = project.test_schema.upper() + + summary_columns = { + "first_name": { + "name": "first_name", + "index": 1, + "type": TextType() , + "comment": None, + }, + "CT": { + "name": "CT", + "index": 2, + "type": bigint_type, + "comment": None, + }, + } + + seed_columns = { + "id": { + "name": col_case("id"), + "index": 1, + "type": id_type, + "comment": None, + }, + "first_name": { + "name": col_case("first_name"), + "index": 2, + "type": TextType(), + "comment": None, + }, + "email": { + "name": col_case("email"), + "index": 3, + "type": TextType(), + "comment": None, + }, + "ip_address": { + "name": col_case("ip_address"), + "index": 4, + "type": TextType(), + "comment": None, + }, + "updated_at": { + "name": col_case("updated_at"), + "index": 5, + "type": time_type, + "comment": None, + }, + } + return { + "nodes": { + "seed.test.seed": { + "unique_id": "seed.test.seed", + "metadata": { + "schema": my_schema_name, + "database": project.database.upper(), + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": seed_columns, + }, + "model.test.ephemeral_summary": { + "unique_id": "model.test.ephemeral_summary", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": case("ephemeral_summary"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": summary_columns, + }, + "model.test.view_summary": { + "unique_id": "model.test.view_summary", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": case("view_summary"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": view_summary_stats, + "columns": summary_columns, + }, + }, + "sources": { + "source.test.my_source.my_table": { + "unique_id": "source.test.my_source.my_table", + "metadata": { + "schema": my_schema_name, + "database": project.database.upper(), + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": seed_columns, + }, + }, + } + + +class TestOracleDocsGenReferences(BaseDocsGenReferences): + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": ref_models__schema_yml, + "sources.yml": ref_sources__schema_yml, + "view_summary.sql": ref_models__view_summary_sql, + "ephemeral_summary.sql": ref_models__ephemeral_summary_sql, + "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, + "docs.md": ref_models__docs_md, + } + + @pytest.fixture(scope="class") + def project_config_update(self, unique_schema): + alternate_schema = unique_schema + return { + "asset-paths": ["assets", "invalid-asset-paths"], + "vars": { + "test_schema": unique_schema, + "alternate_schema": alternate_schema, + }, + "seeds": { + "quote_columns": True, + }, + "quoting": {"identifier": True}, + } + + @pytest.fixture(scope="class") + def expected_catalog(self, project, profile_user): + return expected_references_catalog( + project, + role=profile_user.upper(), + id_type="NUMBER", + text_type="VARCHAR2(16)", + time_type="TIMESTAMP(6)", + bigint_type="NUMBER", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + case_columns=False + ) From 217ae59c55a454c66cb08da7dbaa992a13e8d372 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 8 Aug 2022 15:31:23 -0700 Subject: [PATCH 2/2] Fix: Upgraded dbt-tests-adapter version to 1.2.0 in Github actions --- .github/workflows/oracle-xe-adapter-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/oracle-xe-adapter-tests.yml b/.github/workflows/oracle-xe-adapter-tests.yml index c22ea61..56ee3f7 100644 --- a/.github/workflows/oracle-xe-adapter-tests.yml +++ b/.github/workflows/oracle-xe-adapter-tests.yml @@ -45,7 +45,7 @@ jobs: - name: Install dbt-oracle with core dependencies run: | python -m pip install --upgrade pip - pip install pytest dbt-tests-adapter==1.1.2 + pip install pytest dbt-tests-adapter==1.2.0 pip install -r requirements.txt pip install -e .