From 68f1650638bcf96a65039cc39cc69000d9d4d3fd Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 19 Jul 2022 17:37:58 -0700 Subject: [PATCH 1/5] Feature enhancement to support quoting for column names - This addresses github issue https://github.com/oracle/dbt-oracle/issues/8 - Oracle Adapter class exposes a new method called `should_identifier_be_quoted(identifier)` which returns true or false if the identifier is either in list of Oracle keywords or does not start with alphabet - Macro {{ column.name }} will return quoted name if column name is either in list of Oracle keywords or does not start with alphabet - List of Oracle keywords is defined in dbt/adapters/oracle/keyword_catalog.py from https://docs.oracle.com/en/database/oracle/oracle-database/21/zzpre/Oracle-reserved-words-keywords-namespaces.html#GUID-25FE5FB4-5B17-4AFA-9B59-77B6036EF579 - Handled column quoting in incremental materialization (MERGE, INSERT, UPDATE) --- dbt/adapters/oracle/column.py | 14 + dbt/adapters/oracle/impl.py | 14 +- dbt/adapters/oracle/keyword_catalog.py | 292 ++++++++++++++++++ .../materializations/incremental/helpers.sql | 57 +++- setup.cfg | 2 +- setup.py | 2 +- .../functional/adapter/test_quoted_columns.py | 120 +++++++ 7 files changed, 487 insertions(+), 14 deletions(-) create mode 100644 dbt/adapters/oracle/keyword_catalog.py create mode 100644 tests/functional/adapter/test_quoted_columns.py diff --git a/dbt/adapters/oracle/column.py b/dbt/adapters/oracle/column.py index caef028..ab4572c 100644 --- a/dbt/adapters/oracle/column.py +++ b/dbt/adapters/oracle/column.py @@ -19,6 +19,7 @@ from dbt.adapters.base.column import Column +from dbt.adapters.oracle.keyword_catalog import KEYWORDS @dataclass @@ -66,3 +67,16 @@ def is_string(self) -> bool: if self.dtype.lower() in self.STRING_DATATYPES: return True return super().is_string() + + def should_be_quoted(self) -> bool: + if self.column.upper() in KEYWORDS: + return True + if not self.column[0].isalpha(): + return True + return False + + @property + def name(self) -> str: + if self.should_be_quoted(): + return self.quoted + return self.column diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 65fd9e3..d7b92af 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -19,6 +19,8 @@ ) from itertools import chain +import agate + import dbt.exceptions from dbt.adapters.base.relation import BaseRelation, InformationSchema from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME @@ -32,8 +34,8 @@ from dbt.exceptions import raise_compiler_error from dbt.utils import filter_null_values +from dbt.adapters.oracle.keyword_catalog import KEYWORDS -import agate COLUMNS_EQUAL_SQL = ''' with diff_count as ( @@ -232,6 +234,14 @@ def list_relations_without_caching( )) return relations + @available + def should_identifier_be_quoted(self, identifier) -> bool: + if identifier.upper() in KEYWORDS: + return True + if not identifier[0].isalpha(): + return True + return False + @available def quote_seed_column( self, column: str, quote_config: Optional[bool] @@ -239,6 +249,8 @@ def quote_seed_column( quote_columns: bool = False if isinstance(quote_config, bool): quote_columns = quote_config + elif self.should_identifier_be_quoted(column): + quote_columns = True elif quote_config is None: pass else: diff --git a/dbt/adapters/oracle/keyword_catalog.py b/dbt/adapters/oracle/keyword_catalog.py new file mode 100644 index 0000000..533fd97 --- /dev/null +++ b/dbt/adapters/oracle/keyword_catalog.py @@ -0,0 +1,292 @@ + +# https://docs.oracle.com/en/database/oracle/oracle-database/21/zzpre/Oracle-reserved-words-keywords-namespaces.html#GUID-25FE5FB4-5B17-4AFA-9B59-77B6036EF579 +RESERVED_KEYWORDS = { + "ACCESS" + "ELSE", + "MODIFY", + "START", + "ADD", + "EXCLUSIVE", + "NOAUDIT", + "SELECT", + "ALL", + "EXISTS", + "NOCOMPRESS", + "SESSION", + "ALTER", + "FILE", + "NOT", + "SET", + "AND", + "FLOAT", + "NOTFOUND", + "SHARE", + "ANY", + "FOR", + "NOWAIT", + "SIZE", + "ARRAYLEN", + "FROM", + "NULL", + "SMALLINT", + "AS", + "GRANT", + "NUMBER", + "SQLBUF", + "ASC", + "GROUP", + "OF", + "SUCCESSFUL", + "AUDIT", + "HAVING", + "OFFLINE", + "SYNONYM", + "BETWEEN", + "IDENTIFIED", + "ON", + "SYSDATE", + "BY", + "IMMEDIATE", + "ONLINE", + "TABLE", + "CHAR", + "IN", + "OPTION", + "THEN", + "CHECK", + "INCREMENT", + "OR", + "TO", + "CLUSTER", + "INDEX", + "ORDER", + "TRIGGER", + "COLUMN", + "INITIAL", + "PCTFREE", + "UID", + "COMMENT", + "INSERT", + "PRIOR", + "UNION", + "COMPRESS", + "INTEGER", + "PRIVILEGES", + "UNIQUE", + "CONNECT", + "INTERSECT", + "PUBLIC", + "UPDATE", + "CREATE", + "INTO", + "RAW", + "USER", + "CURRENT", + "IS", + "RENAME", + "VALIDATE", + "DATE", + "LEVEL", + "RESOURCE", + "VALUES", + "DECIMAL", + "LIKE", + "REVOKE", + "VARCHAR", + "DEFAULT", + "LOCK", + "ROW", + "VARCHAR2", + "DELETE", + "LONG", + "ROWID", + "VIEW", + "DESC", + "MAXEXTENTS", + "ROWLABEL", + "WHENEVER", + "DISTINCT", + "MINUS", + "ROWNUM", + "WHERE", + "DROP", + "MODE", + "ROWS", + "WITH" +} + +NON_RESERVED_KEYWORDS = { + "ADMIN", + "CURSOR", + "FOUND", + "MOUNT", + "AFTER", + "CYCLE", + "FUNCTION", + "NEXT", + "ALLOCATE", + "DATABASE", + "GO", + "NEW", + "ANALYZE", + "DATAFILE", + "GOTO", + "NOARCHIVELOG", + "ARCHIVE", + "DBA", + "GROUPS", + "NOCACHE", + "ARCHIVELOG", + "DEC", + "INCLUDING", + "NOCYCLE", + "AUTHORIZATION", + "DECLARE", + "INDICATOR", + "NOMAXVALUE", + "AVG", + "DISABLE", + "INITRANS", + "NOMINVALUE", + "BACKUP", + "DISMOUNT", + "INSTANCE", + "NONE", + "BEGIN", + "DOUBLE", + "INT", + "NOORDER", + "BECOME", + "DUMP", + "KEY", + "NORESETLOGS", + "BEFORE", + "EACH", + "LANGUAGE", + "NORMAL", + "BLOCK", + "ENABLE", + "LAYER", + "NOSORT", + "BODY", + "END", + "LINK", + "NUMERIC", + "CACHE", + "ESCAPE", + "LISTS", + "OFF", + "CANCEL", + "EVENTS", + "LOGFILE", + "OLD", + "CASCADE", + "EXCEPT", + "MANAGE", + "ONLY", + "CHANGE", + "EXCEPTIONS", + "MANUAL", + "OPEN", + "CHARACTER", + "EXEC", + "MAX", + "OPTIMAL", + "CHECKPOINT", + "EXPLAIN", + "MAXDATAFILES", + "OWN", + "CLOSE", + "EXECUTE", + "MAXINSTANCES", + "PACKAGE", + "COBOL", + "EXTENT", + "MAXLOGFILES", + "PARALLEL", + "COMMIT", + "EXTERNALLY", + "MAXLOGHISTORY", + "PCTINCREASE", + "COMPILE", + "FETCH", + "MAXLOGMEMBERS", + "PCTUSED", + "CONSTRAINT", + "FLUSH", + "MAXTRANS", + "PLAN", + "CONSTRAINTS", + "FREELIST", + "MAXVALUE", + "PLI", + "CONTENTS", + "FREELISTS", + "MIN", + "PRECISION", + "CONTINUE", + "FORCE", + "MINEXTENTS", + "PRIMARY", + "CONTROLFILE", + "FOREIGN", + "MINVALUE", + "PRIVATE", + "COUNT", + "FORTRAN", + "MODULE", + "PROCEDURE", + "PROFILE", + "SAVEPOINT", + "SQLSTATE", + "TRACING", + "QUOTA", + "SCHEMA", + "STATEMENT_ID", + "TRANSACTION", + "READ", + "SCN", + "STATISTICS", + "TRIGGERS", + "REAL", + "SECTION", + "STOP", + "TRUNCATE", + "RECOVER", + "SEGMENT", + "STORAGE", + "UNDER", + "REFERENCES", + "SEQUENCE", + "SUM", + "UNLIMITED", + "REFERENCING", + "SHARED", + "SWITCH", + "UNTIL", + "RESETLOGS", + "SNAPSHOT", + "SYSTEM", + "USE", + "RESTRICTED", + "SOME", + "TABLES", + "USING", + "REUSE", + "SORT", + "TABLESPACE", + "WHEN", + "ROLE", + "SQL", + "TEMPORARY", + "WRITE", + "ROLES", + "SQLCODE", + "THREAD", + "WORK", + "ROLLBACK", + "SQLERROR", + "TIME" +} + +KEYWORDS = RESERVED_KEYWORDS | NON_RESERVED_KEYWORDS diff --git a/dbt/include/oracle/macros/materializations/incremental/helpers.sql b/dbt/include/oracle/macros/materializations/incremental/helpers.sql index 0988c1d..fdd794c 100644 --- a/dbt/include/oracle/macros/materializations/incremental/helpers.sql +++ b/dbt/include/oracle/macros/materializations/incremental/helpers.sql @@ -34,29 +34,64 @@ ) {%- endmacro %} -{% macro oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=none, statement_name="main") %} - {%- set on_predicates = [] -%} - {%- set unique_key_list = [] -%} - {%- set dest_cols_csv = dest_columns | map(attribute='name') | join(', ') -%} - {%- set update_columns = config.get('merge_update_columns', default=dest_columns | map(attribute='name'))-%} +{% macro oracle_conditional_quote_column_names_for_incremental_merge(dest_columns) %} + {%- set quote = "\"" -%} + {%- set final_update_columns = [] -%} + {%- set user_defined_merge_update_columns = config.get("merge_update_columns") -%} + {% if user_defined_merge_update_columns is none %} + {%- set final_update_columns = dest_columns | map(attribute='name') -%} + {% else %} + {% for col in user_defined_merge_update_columns %} + {% if adapter.should_identifier_be_quoted(col) == true %} + {% do final_update_columns.append(quote ~ col ~ quote) %} + {% else %} + {% do final_update_columns.append(col) %} + {% endif %} + {% endfor %} + {% endif %} + {{ return(final_update_columns)}} +{% endmacro %} - {%- if unique_key -%} - {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %} +{% macro oracle_conditional_quote_unique_key_for_incremental_merge(unique_key) %} + {%- set quote = "\"" -%} + {%- set unique_key_list = [] -%} + {%- set unique_key_merge_predicates = [] -%} + {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %} {% for key in unique_key | unique %} - {% do unique_key_list.append(key.upper()) %} + {% if adapter.should_identifier_be_quoted(key) == true %} + {% do unique_key_list.append(quote ~ key ~ quote) %} + {% else %} + {% do unique_key_list.append(key.upper()) %} + {% endif %} {% endfor %} {% else %} - {% do unique_key_list.append(unique_key.upper()) %} + {% if adapter.should_identifier_be_quoted(unique_key) == true %} + {% do unique_key_list.append(quote ~ unique_key ~ quote) %} + {% else %} + {% do unique_key_list.append(unique_key.upper()) %} + {% endif %} {% endif %} {% for key in unique_key_list %} {% set this_key_match %} temp.{{ key }} = target.{{ key }} {% endset %} - {% do on_predicates.append(this_key_match) %} + {% do unique_key_merge_predicates.append(this_key_match) %} {% endfor %} + {%- set unique_key_result = {'unique_key_list': unique_key_list, 'unique_key_merge_predicates': unique_key_merge_predicates} -%} + {{ return(unique_key_result)}} +{% endmacro %} + + +{% macro oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=none, statement_name="main") %} + {%- set dest_cols_csv = dest_columns | map(attribute='name') | join(', ') -%} + {%- set update_columns = oracle_conditional_quote_column_names_for_incremental_merge(dest_columns) -%} + {%- if unique_key -%} + {%- set unique_key_result = oracle_conditional_quote_unique_key_for_incremental_merge(unique_key) -%} + {%- set unique_key_list = unique_key_result['unique_key_list'] -%} + {%- set unique_key_merge_predicates = unique_key_result['unique_key_merge_predicates'] -%} merge into {{ target_relation }} target using {{ tmp_relation }} temp - on ({{ on_predicates | join(' AND ') }}) + on ({{ unique_key_merge_predicates | join(' AND ') }}) when matched then update set {% for col in update_columns if col.upper() not in unique_key_list -%} diff --git a/setup.cfg b/setup.cfg index a896571..7b01a43 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.1.1 +version = 1.1.2rc1 description = dbt (data build tool) adapter for the Oracle database long_description = file: README.md long_description_content_type = text/markdown diff --git a/setup.py b/setup.py index 9313ba1..239fcd7 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION='1.1.1' +VERSION='1.1.2rc1' setup( author="Oracle", python_requires='>=3.7.2', diff --git a/tests/functional/adapter/test_quoted_columns.py b/tests/functional/adapter/test_quoted_columns.py new file mode 100644 index 0000000..189f775 --- /dev/null +++ b/tests/functional/adapter/test_quoted_columns.py @@ -0,0 +1,120 @@ +import datetime +from pathlib import Path + +import pytest +from dbt.tests.util import run_dbt + +# seeds/my_seed.csv +seed_csv = """ +_user_id,user_name,birth_date,income,last_login_date,desc +1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login +""".lstrip() + +# models/my_incr_model.sql +my_incr_model_sql = """ +{{config(materialized='incremental', + unique_key='_user_id', + merge_update_columns=['user_name', 'income', 'last_login_date', 'desc'])}} +SELECT * FROM {{ ref('seed') }} +{% if is_incremental() %} + WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) +{% endif %} + +""" + +# seeds/add_new_rows.sql +seeds__add_new_rows_sql = """ +-- insert two new rows, both of which should be in incremental model +INSERT ALL + INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES + (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') + INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES + (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') +SELECT * FROM dual +""" + + +class TestIncrementalMergeQuotedColumns: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_incr_model.sql": my_incr_model_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run incremental model + - add new rows + - run incremental model + + The following SQL is expected to run. + + merge into dbt_test.my_incr_model target + using o$pt_my_incr_model150332 temp + on ( + temp.USER_ID = target.USER_ID + ) + when matched then + update set + target.user_name = temp.user_name, + target.income = temp.income, + target.last_login_date = temp.last_login_date + when not matched then + insert(USER_ID, USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE) + values( + temp.USER_ID, + temp.USER_NAME, + temp.BIRTH_DATE, + temp.INCOME, + temp.LAST_LOGIN_DATE + ) + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) + + results = run_dbt(['run']) + assert len(results) == 1 + + user_id_2_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 2) + expected_result = [(2, 'Lillian Sr.', + datetime.datetime(1978, 9, 3, 0, 0), + 200000, + datetime.datetime(2022, 5, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(user_id_2_query, fetch="all") + assert result == expected_result + + used_id_5_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 5) + expected_result = [(5, 'John Doe', + datetime.datetime(1992, 10, 1, 0, 0), + 300000, + datetime.datetime(2022, 6, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(used_id_5_query, fetch="all") + assert result == expected_result + + From 12c068a0938c1929ccbbaaf5d10d640eff9ad03a Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Fri, 22 Jul 2022 17:05:48 -0700 Subject: [PATCH 2/5] Support Quoting in a better way and handle all cases - adapter.should_identifier_be_quoted(identifier) handles 3 cases when an identifier should be quoted. - adapter.check_and_quote_identifier(identifier) is exposed to quote an identifier - A new macro is get_quoted_column_csv(model, column_names) is added to quote a list of column names - The implementation of ``{{column.name}}`` is reverted in exchange for a single quoting api - Incremental materialization macros are fixed to use the adapter.check_and_quote_identifier() - 4 new unit test cases to test quoting for different scenarios --- Makefile | 2 +- dbt/adapters/oracle/column.py | 13 -- dbt/adapters/oracle/impl.py | 55 ++++++- dbt/adapters/oracle/keyword_catalog.py | 16 ++ dbt/include/oracle/macros/columns.sql | 13 +- .../materializations/incremental/helpers.sql | 40 ++--- .../incremental_materialization/__init__.py | 0 .../quotes/__init__.py | 0 ...t_quote_special_characters_and_keywords.py | 123 +++++++++++++++ .../test_quoted_columns_incremental_insert.py | 116 ++++++++++++++ .../quotes/test_quotes_enabled_in_model.py | 129 ++++++++++++++++ .../sync_schema/__init__.py | 0 ...t_quote_special_characters_and_keywords.py | 143 ++++++++++++++++++ .../test_merge_update_columns.py} | 0 .../test_quotes_with_merge_update_columns.py} | 19 ++- .../test_unique_id.py} | 0 16 files changed, 619 insertions(+), 50 deletions(-) create mode 100644 tests/functional/adapter/incremental_materialization/__init__.py create mode 100644 tests/functional/adapter/incremental_materialization/quotes/__init__.py create mode 100644 tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py create mode 100644 tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py create mode 100644 tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py create mode 100644 tests/functional/adapter/incremental_materialization/sync_schema/__init__.py create mode 100644 tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py rename tests/functional/adapter/{test_incr_merge_update_columns.py => incremental_materialization/test_merge_update_columns.py} (100%) rename tests/functional/adapter/{test_quoted_columns.py => incremental_materialization/test_quotes_with_merge_update_columns.py} (89%) rename tests/functional/adapter/{test_incremental_unique_id.py => incremental_materialization/test_unique_id.py} (100%) diff --git a/Makefile b/Makefile index e36b4fd..2a8271f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.1.1 +VERSION=1.1.2rc1 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/column.py b/dbt/adapters/oracle/column.py index ab4572c..82a9b4f 100644 --- a/dbt/adapters/oracle/column.py +++ b/dbt/adapters/oracle/column.py @@ -67,16 +67,3 @@ def is_string(self) -> bool: if self.dtype.lower() in self.STRING_DATATYPES: return True return super().is_string() - - def should_be_quoted(self) -> bool: - if self.column.upper() in KEYWORDS: - return True - if not self.column[0].isalpha(): - return True - return False - - @property - def name(self) -> str: - if self.should_be_quoted(): - return self.quoted - return self.column diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index d7b92af..882bc41 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -30,12 +30,15 @@ from dbt.adapters.oracle.column import OracleColumn from dbt.adapters.oracle.relation import OracleRelation from dbt.contracts.graph.manifest import Manifest +from dbt.events import AdapterLogger from dbt.exceptions import raise_compiler_error from dbt.utils import filter_null_values from dbt.adapters.oracle.keyword_catalog import KEYWORDS +logger = AdapterLogger("oracle") + COLUMNS_EQUAL_SQL = ''' with diff_count as ( @@ -234,14 +237,62 @@ def list_relations_without_caching( )) return relations + @staticmethod + def is_valid_identifier(identifier) -> bool: + """Returns True if an identifier is valid + + An identifier is considered valid if the following conditions are True + + 1. First character is alphabetic + 2. Rest of the characters is either alphanumeric or any one of the literals '#', '$', '_' + + """ + # The first character should be alphabetic + if not identifier[0].isalpha(): + return False + # Rest of the characters is either alphanumeric or any one of the literals '#', '$', '_' + idx = 1 + while idx < len(identifier): + identifier_chr = identifier[idx] + if not identifier_chr.isalnum() and identifier_chr not in ('#', '$', '_'): + return False + idx += 1 + return True + @available - def should_identifier_be_quoted(self, identifier) -> bool: + def should_identifier_be_quoted(self, + identifier, + models_column_dict=None) -> bool: + """Returns True if identifier should be quoted else False + + An identifier should be quoted in the following 3 cases: + + - 1. Identifier is an Oracle keyword + + - 2. Identifier is not valid according to the following rules + - First character is alphabetic + - Rest of the characters is either alphanumeric or any one of the literals '#', '$', '_' + + - 3. User has enabled quoting for the column in the model configuration + + """ if identifier.upper() in KEYWORDS: return True - if not identifier[0].isalpha(): + elif not self.is_valid_identifier(identifier): return True + elif models_column_dict and identifier in models_column_dict: + return models_column_dict[identifier].get('quote', False) + elif models_column_dict and self.quote(identifier) in models_column_dict: + return models_column_dict[self.quote(identifier)].get('quote', False) return False + @available + def check_and_quote_identifier(self, identifier, models_column_dict=None) -> str: + if self.should_identifier_be_quoted(identifier, models_column_dict): + return self.quote(identifier) + else: + return identifier + @available def quote_seed_column( self, column: str, quote_config: Optional[bool] diff --git a/dbt/adapters/oracle/keyword_catalog.py b/dbt/adapters/oracle/keyword_catalog.py index 533fd97..27fcd5f 100644 --- a/dbt/adapters/oracle/keyword_catalog.py +++ b/dbt/adapters/oracle/keyword_catalog.py @@ -1,3 +1,19 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # https://docs.oracle.com/en/database/oracle/oracle-database/21/zzpre/Oracle-reserved-words-keywords-namespaces.html#GUID-25FE5FB4-5B17-4AFA-9B59-77B6036EF579 RESERVED_KEYWORDS = { diff --git a/dbt/include/oracle/macros/columns.sql b/dbt/include/oracle/macros/columns.sql index 1ba6b17..cfa0867 100644 --- a/dbt/include/oracle/macros/columns.sql +++ b/dbt/include/oracle/macros/columns.sql @@ -29,7 +29,7 @@ ALTER {{ relation.type }} {{ relation }} ADD ( {% for column in add_columns %} - {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {{ adapter.check_and_quote_identifier(column.name, model.columns) }} {{ column.data_type }}{{ ',' if not loop.last }} {% endfor %} ) {% endset %} @@ -41,10 +41,19 @@ ALTER {{ relation.type }} {{ relation }} DROP ( {% for column in remove_columns %} - {{ column.name }}{{ ',' if not loop.last }} + {{ adapter.check_and_quote_identifier(column.name, model.columns) }}{{ ',' if not loop.last }} {% endfor %} ) CASCADE CONSTRAINTS {% endset %} {% do run_query(remove_sql)%} {% endif %} {% endmacro %} + +{% macro get_quoted_column_csv(model, column_names) %} + {%- set quoted = [] -%} + {% for col in column_names %} + {%- do quoted.append(adapter.check_and_quote_identifier(col, model.columns)) -%} + {% endfor %} + {%- set cols_csv = quoted | join(', ') -%} + {{ return(cols_csv) }} +{% endmacro %} \ No newline at end of file diff --git a/dbt/include/oracle/macros/materializations/incremental/helpers.sql b/dbt/include/oracle/macros/materializations/incremental/helpers.sql index fdd794c..82e9b8a 100644 --- a/dbt/include/oracle/macros/materializations/incremental/helpers.sql +++ b/dbt/include/oracle/macros/materializations/incremental/helpers.sql @@ -34,38 +34,29 @@ ) {%- endmacro %} -{% macro oracle_conditional_quote_column_names_for_incremental_merge(dest_columns) %} - {%- set quote = "\"" -%} - {%- set final_update_columns = [] -%} - {%- set user_defined_merge_update_columns = config.get("merge_update_columns") -%} - {% if user_defined_merge_update_columns is none %} - {%- set final_update_columns = dest_columns | map(attribute='name') -%} - {% else %} - {% for col in user_defined_merge_update_columns %} - {% if adapter.should_identifier_be_quoted(col) == true %} - {% do final_update_columns.append(quote ~ col ~ quote) %} - {% else %} - {% do final_update_columns.append(col) %} - {% endif %} - {% endfor %} - {% endif %} - {{ return(final_update_columns)}} +{% macro oracle_check_and_quote_column_names_for_incremental_merge(dest_column_names) %} + {%- set quoted_update_columns = [] -%} + {%- set update_columns = config.get("merge_update_columns", default=dest_column_names) -%} + {% for col in update_columns %} + {% do quoted_update_columns.append(adapter.check_and_quote_identifier(col, model.columns)) %} + {% endfor %} + {{ return(quoted_update_columns)}} {% endmacro %} -{% macro oracle_conditional_quote_unique_key_for_incremental_merge(unique_key) %} +{% macro oracle_check_and_quote_unique_key_for_incremental_merge(unique_key) %} {%- set quote = "\"" -%} {%- set unique_key_list = [] -%} {%- set unique_key_merge_predicates = [] -%} {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %} {% for key in unique_key | unique %} - {% if adapter.should_identifier_be_quoted(key) == true %} + {% if adapter.should_identifier_be_quoted(key, model.columns) == true %} {% do unique_key_list.append(quote ~ key ~ quote) %} {% else %} {% do unique_key_list.append(key.upper()) %} {% endif %} {% endfor %} {% else %} - {% if adapter.should_identifier_be_quoted(unique_key) == true %} + {% if adapter.should_identifier_be_quoted(unique_key, model.columns) == true %} {% do unique_key_list.append(quote ~ unique_key ~ quote) %} {% else %} {% do unique_key_list.append(unique_key.upper()) %} @@ -83,10 +74,11 @@ {% macro oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=none, statement_name="main") %} - {%- set dest_cols_csv = dest_columns | map(attribute='name') | join(', ') -%} - {%- set update_columns = oracle_conditional_quote_column_names_for_incremental_merge(dest_columns) -%} + {%- set dest_column_names = dest_columns | map(attribute='name') | list -%} + {%- set dest_cols_csv = get_quoted_column_csv(model, dest_column_names) -%} + {%- set update_columns = oracle_check_and_quote_column_names_for_incremental_merge(dest_column_names) -%} {%- if unique_key -%} - {%- set unique_key_result = oracle_conditional_quote_unique_key_for_incremental_merge(unique_key) -%} + {%- set unique_key_result = oracle_check_and_quote_unique_key_for_incremental_merge(unique_key) -%} {%- set unique_key_list = unique_key_result['unique_key_list'] -%} {%- set unique_key_merge_predicates = unique_key_result['unique_key_merge_predicates'] -%} merge into {{ target_relation }} target @@ -94,14 +86,14 @@ on ({{ unique_key_merge_predicates | join(' AND ') }}) when matched then update set - {% for col in update_columns if col.upper() not in unique_key_list -%} + {% for col in update_columns if (col.upper() not in unique_key_list and col not in unique_key_list) -%} target.{{ col }} = temp.{{ col }}{% if not loop.last %}, {% endif %} {% endfor -%} when not matched then insert({{ dest_cols_csv }}) values( {% for col in dest_columns -%} - temp.{{ col.name }}{% if not loop.last %}, {% endif %} + temp.{{ adapter.check_and_quote_identifier(col.name, model.columns) }}{% if not loop.last %}, {% endif %} {% endfor -%} ) {%- else -%} diff --git a/tests/functional/adapter/incremental_materialization/__init__.py b/tests/functional/adapter/incremental_materialization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/adapter/incremental_materialization/quotes/__init__.py b/tests/functional/adapter/incremental_materialization/quotes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py b/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py new file mode 100644 index 0000000..c79cfaa --- /dev/null +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py @@ -0,0 +1,123 @@ +import datetime +from pathlib import Path + +import pytest +from dbt.tests.util import run_dbt + +# seeds/my_seed.csv +seed_csv = """ +_user_id,user name,birth_ date in yyyy-mm-dd,income,last_login_date,desc +1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login +""".lstrip() + +# models/my_incr_model.sql +my_incr_model_sql = """ +{{config(materialized='incremental', + unique_key='_user_id')}} +SELECT * FROM {{ ref('seed') }} +{% if is_incremental() %} + WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) +{% endif %} + +""" + +# seeds/add_new_rows.sql +seeds__add_new_rows_sql = """ +-- insert two new rows, both of which should be in incremental model +INSERT ALL + INTO {schema}.seed ("_user_id", "user name", "birth_ date in yyyy-mm-dd", income, last_login_date, "desc") VALUES + (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') + INTO {schema}.seed ("_user_id", "user name", "birth_ date in yyyy-mm-dd", income, last_login_date, "desc") VALUES + (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') +SELECT * FROM dual +""" + + +class TestIncrementalMergeQuoteWithKeywordsandSpecialChars: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_incr_model.sql": my_incr_model_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run incremental model + - add new rows + - run incremental model + + The following SQL is expected to run. + + merge into dbt_test.my_incr_model target + using o$pt_my_incr_model175348 temp + on ( + temp."_user_id" = target."_user_id" + ) + when matched then + update set + target.USER_NAME = temp.USER_NAME, + target.BIRTH_DATE = temp.BIRTH_DATE, + target.INCOME = temp.INCOME, + target.LAST_LOGIN_DATE = temp.LAST_LOGIN_DATE, + target."desc" = temp."desc" + when not matched then + insert("_user_id", USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE, "desc") + values( + temp."_user_id", + temp.USER_NAME, + temp.BIRTH_DATE, + temp.INCOME, + temp.LAST_LOGIN_DATE, + temp."desc" + ) + + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) + + results = run_dbt(['run']) + assert len(results) == 1 + + user_id_2_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 2) + expected_result = [(2, 'Lillian Sr.', + datetime.datetime(1982, 2, 3, 0, 0), + 200000, + datetime.datetime(2022, 5, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(user_id_2_query, fetch="all") + assert result == expected_result + + used_id_5_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 5) + expected_result = [(5, 'John Doe', + datetime.datetime(1992, 10, 1, 0, 0), + 300000, + datetime.datetime(2022, 6, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(used_id_5_query, fetch="all") + assert result == expected_result + + diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py b/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py new file mode 100644 index 0000000..7e58eaf --- /dev/null +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py @@ -0,0 +1,116 @@ +import datetime +from pathlib import Path + +import pytest +from dbt.tests.util import run_dbt + +# seeds/my_seed.csv +seed_csv = """ +_user_id,user_name,birth_date,income,last_login_date,desc +1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login +""".lstrip() + +# models/my_incr_model.sql +my_incr_model_sql = """ +{{config(materialized='incremental')}} +SELECT * FROM {{ ref('seed') }} +{% if is_incremental() %} + WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) +{% endif %} + +""" + +# seeds/add_new_rows.sql +seeds__add_new_rows_sql = """ +-- insert two new rows, both of which should be in incremental model +INSERT ALL + INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES + (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') + INTO {schema}.seed ("_user_id", user_name, birth_date, income, last_login_date, "desc") VALUES + (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') +SELECT * FROM dual +""" + + +class TestIncrementalInsertQuotedColumnsAllCols: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_incr_model.sql": my_incr_model_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run incremental model + - add new rows + - run incremental model + + The following SQL is expected to run. + + merge into dbt_test.my_incr_model target + using o$pt_my_incr_model175348 temp + on ( + temp."_user_id" = target."_user_id" + ) + when matched then + update set + target.USER_NAME = temp.USER_NAME, + target.BIRTH_DATE = temp.BIRTH_DATE, + target.INCOME = temp.INCOME, + target.LAST_LOGIN_DATE = temp.LAST_LOGIN_DATE, + target."desc" = temp."desc" + when not matched then + insert("_user_id", USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE, "desc") + values( + temp."_user_id", + temp.USER_NAME, + temp.BIRTH_DATE, + temp.INCOME, + temp.LAST_LOGIN_DATE, + temp."desc" + ) + + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) + + results = run_dbt(['run']) + assert len(results) == 1 + + user_id_2_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 2) + result = project.run_sql(user_id_2_query, fetch="all") + assert len(result) == 2 + + used_id_5_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 5) + expected_result = [(5, 'John Doe', + datetime.datetime(1992, 10, 1, 0, 0), + 300000, + datetime.datetime(2022, 6, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(used_id_5_query, fetch="all") + assert result == expected_result + + diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py b/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py new file mode 100644 index 0000000..340aede --- /dev/null +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py @@ -0,0 +1,129 @@ +import datetime +from pathlib import Path + +import pytest +from dbt.tests.util import run_dbt + +# seeds/my_seed.csv +seed_csv = """ +user_id,user_name,birth_date,income,last_login_date,description +1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login +""".lstrip() + +model_yml = """ +version: 2 +models: + - name: my_incr_model + columns: + - name: user_name + quote: true + - name: user_id + quote: true + - name: birth_date + quote: true + - name: income + quote: true + - name: last_login_date + quote: true + - name: description + quote: true +""" + + +# models/my_incr_model.sql +my_incr_model_sql = """ +{{config(materialized='incremental', + merge_update_columns=["user_name", "birth_date", "income", "last_login_date", "description"], + unique_key='user_id')}} +SELECT * FROM {{ ref('seed') }} +{% if is_incremental() %} + WHERE "last_login_date" > (SELECT max("last_login_date") FROM {{ this }}) +{% endif %} + +""" + +# seeds/add_new_rows.sql +seeds__add_new_rows_sql = """ +-- insert two new rows, both of which should be in incremental model +INSERT ALL + INTO {schema}.seed ("user_id", "user_name", "birth_date", "income", "last_login_date", "description") VALUES + (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') + INTO {schema}.seed ("user_id", "user_name", "birth_date", "income", "last_login_date", "description") VALUES + (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') +SELECT * FROM dual +""" + + +class TestIncrementalMergeQuotedColumnsConfigYml: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_incr_model.sql": my_incr_model_sql, + "schema.yml": model_yml + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": True + }, + } + + def test_run_dbt(self, project): + """ + - run seed + - run incremental model + - add new rows + - run incremental model + + The following SQL is expected to run. + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) + + results = run_dbt(['run']) + assert len(results) == 1 + + user_id_2_query = 'SELECT * FROM {}.{} WHERE "user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 2) + expected_result = [(2, 'Lillian Sr.', + datetime.datetime(1982, 2, 3, 0, 0), + 200000, + datetime.datetime(2022, 5, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(user_id_2_query, fetch="all") + assert result == expected_result + + used_id_5_query = 'SELECT * FROM {}.{} WHERE "user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 5) + expected_result = [(5, 'John Doe', + datetime.datetime(1992, 10, 1, 0, 0), + 300000, + datetime.datetime(2022, 6, 1, 6, 1, 31), + 'Login')] + + result = project.run_sql(used_id_5_query, fetch="all") + assert result == expected_result + + diff --git a/tests/functional/adapter/incremental_materialization/sync_schema/__init__.py b/tests/functional/adapter/incremental_materialization/sync_schema/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py b/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py new file mode 100644 index 0000000..af781a0 --- /dev/null +++ b/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py @@ -0,0 +1,143 @@ +import datetime +from pathlib import Path + +import pytest +from dbt.tests.util import run_dbt + +# seeds/my_seed.csv +seed_csv = """ +_user_id,user name,birth_ date in yyyy-mm-dd,income,last_login_date,desc +1,Easton,1981-05-20,40000,2022-04-25T08:57:59,login +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59,login +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59,login +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59,login +""".lstrip() + +# models/my_incr_model.sql +my_incr_model_sql = """ +{{config(materialized='incremental', + on_schema_change='sync_all_columns', + unique_key='_user_id')}} +SELECT * FROM {{ ref('seed') }} +{% if is_incremental() %} + WHERE last_login_date > (SELECT max(last_login_date) FROM {{ this }}) +{% endif %} + +""" + +# seeds/add_new_rows.sql +seeds__add_new_rows_sql = """ +-- insert two new rows, both of which should be in incremental model +INSERT ALL + INTO {schema}.seed ("_user_id", "user name", "birth date in yyyy-mm-dd", income, last_login_date, "desc") VALUES + (2,'Lillian Sr.', TO_DATE('1982-02-03', 'YYYY-MM-DD'), 200000, TO_DATE('2022-05-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') + INTO {schema}.seed ("_user_id", "user name", "birth date in yyyy-mm-dd", income, last_login_date, "desc") VALUES + (5,'John Doe',TO_DATE('1992-10-01', 'YYYY-MM-DD'), 300000, TO_DATE('2022-06-01 06:01:31', 'YYYY-MM-DD HH:MI:SS'), 'Login') +SELECT * FROM dual +""" + +# seeds/add_column.sql +add_column_sql = """ +-- Add a new column +ALTER TABLE {schema}.seed ADD ("birth date in yyyy-mm-dd" DATE ) + +""" + +# seeds/drop_column.sql +drop_column_sql = """ +-- Drop a column +ALTER TABLE {schema}.seed DROP ("birth_ date in yyyy-mm-dd") CASCADE CONSTRAINTS +""" + + +class TestSyncSchemaIncrementalMergeQuotedColumns: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql, + "add_column.sql": add_column_sql, + "drop_column.sql": drop_column_sql + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_incr_model.sql": my_incr_model_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run incremental model + - add new rows + - run incremental model + + The following SQL is expected to run. + + merge into dbt_test.my_incr_model target + using o$pt_my_incr_model175348 temp + on ( + temp."_user_id" = target."_user_id" + ) + when matched then + update set + target.USER_NAME = temp.USER_NAME, + target.BIRTH_DATE = temp.BIRTH_DATE, + target.INCOME = temp.INCOME, + target.LAST_LOGIN_DATE = temp.LAST_LOGIN_DATE, + target."desc" = temp."desc" + when not matched then + insert("_user_id", USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE, "desc") + values( + temp."_user_id", + temp.USER_NAME, + temp.BIRTH_DATE, + temp.INCOME, + temp.LAST_LOGIN_DATE, + temp."desc" + ) + + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + project.run_sql_file(Path("seeds") / Path("add_column.sql")) + project.run_sql_file(Path("seeds") / Path("drop_column.sql")) + project.run_sql_file(Path("seeds") / Path("add_new_rows.sql")) + + results = run_dbt(['run']) + assert len(results) == 1 + + + + user_id_2_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 2) + expected_result = [(2, 'Lillian Sr.', + 200000, + datetime.datetime(2022, 5, 1, 6, 1, 31), + 'Login', + datetime.datetime(1982, 2, 3, 0, 0))] + + result = project.run_sql(user_id_2_query, fetch="all") + assert result == expected_result + + used_id_5_query = 'SELECT * FROM {}.{} WHERE "_user_id" = {}'.format(project.test_schema, + 'my_incr_model', + 5) + expected_result = [(5, 'John Doe', + 300000, + datetime.datetime(2022, 6, 1, 6, 1, 31), + 'Login', + datetime.datetime(1992, 10, 1, 0, 0))] + + result = project.run_sql(used_id_5_query, fetch="all") + assert result == expected_result + + diff --git a/tests/functional/adapter/test_incr_merge_update_columns.py b/tests/functional/adapter/incremental_materialization/test_merge_update_columns.py similarity index 100% rename from tests/functional/adapter/test_incr_merge_update_columns.py rename to tests/functional/adapter/incremental_materialization/test_merge_update_columns.py diff --git a/tests/functional/adapter/test_quoted_columns.py b/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py similarity index 89% rename from tests/functional/adapter/test_quoted_columns.py rename to tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py index 189f775..597d110 100644 --- a/tests/functional/adapter/test_quoted_columns.py +++ b/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py @@ -62,25 +62,28 @@ def test_run_dbt(self, project): The following SQL is expected to run. merge into dbt_test.my_incr_model target - using o$pt_my_incr_model150332 temp + using o$pt_my_incr_model180849 temp on ( - temp.USER_ID = target.USER_ID + temp."_user_id" = target."_user_id" ) when matched then update set - target.user_name = temp.user_name, - target.income = temp.income, - target.last_login_date = temp.last_login_date + target.USER_NAME = temp.USER_NAME, + target.INCOME = temp.INCOME, + target.LAST_LOGIN_DATE = temp.LAST_LOGIN_DATE, + target."desc" = temp."desc" when not matched then - insert(USER_ID, USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE) + insert("_user_id", USER_NAME, BIRTH_DATE, INCOME, LAST_LOGIN_DATE, "desc") values( - temp.USER_ID, + temp."_user_id", temp.USER_NAME, temp.BIRTH_DATE, temp.INCOME, - temp.LAST_LOGIN_DATE + temp.LAST_LOGIN_DATE, + temp."desc" ) + """ results = run_dbt(['seed']) assert len(results) == 1 diff --git a/tests/functional/adapter/test_incremental_unique_id.py b/tests/functional/adapter/incremental_materialization/test_unique_id.py similarity index 100% rename from tests/functional/adapter/test_incremental_unique_id.py rename to tests/functional/adapter/incremental_materialization/test_unique_id.py From c5595af245b203d373669cb326ae06a04322fffc Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Fri, 22 Jul 2022 17:37:32 -0700 Subject: [PATCH 3/5] Added copyright notice --- ...est_quote_special_characters_and_keywords.py | 17 +++++++++++++++++ .../test_quoted_columns_incremental_insert.py | 17 +++++++++++++++++ .../quotes/test_quotes_enabled_in_model.py | 17 +++++++++++++++++ ...est_quote_special_characters_and_keywords.py | 17 +++++++++++++++++ .../test_merge_update_columns.py | 17 +++++++++++++++++ .../test_quotes_with_merge_update_columns.py | 17 +++++++++++++++++ .../adapter/macros/test_alter_column_type.py | 17 +++++++++++++++++ 7 files changed, 119 insertions(+) diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py b/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py index c79cfaa..2686446 100644 --- a/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quote_special_characters_and_keywords.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py b/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py index 7e58eaf..55c7508 100644 --- a/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quoted_columns_incremental_insert.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py b/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py index 340aede..0df3383 100644 --- a/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py +++ b/tests/functional/adapter/incremental_materialization/quotes/test_quotes_enabled_in_model.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py b/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py index af781a0..e2a014c 100644 --- a/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py +++ b/tests/functional/adapter/incremental_materialization/sync_schema/test_quote_special_characters_and_keywords.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/incremental_materialization/test_merge_update_columns.py b/tests/functional/adapter/incremental_materialization/test_merge_update_columns.py index 558c2f8..e99bf81 100644 --- a/tests/functional/adapter/incremental_materialization/test_merge_update_columns.py +++ b/tests/functional/adapter/incremental_materialization/test_merge_update_columns.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py b/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py index 597d110..e719fae 100644 --- a/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py +++ b/tests/functional/adapter/incremental_materialization/test_quotes_with_merge_update_columns.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import datetime from pathlib import Path diff --git a/tests/functional/adapter/macros/test_alter_column_type.py b/tests/functional/adapter/macros/test_alter_column_type.py index f705f1f..b9ea967 100644 --- a/tests/functional/adapter/macros/test_alter_column_type.py +++ b/tests/functional/adapter/macros/test_alter_column_type.py @@ -1,3 +1,20 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + import pytest from dbt.tests.util import run_dbt, get_relation_columns From 9caa59118a0846fac88b781e64ff3a90d3c83d92 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Sun, 24 Jul 2022 20:21:58 -0700 Subject: [PATCH 4/5] Better quoting for relation names - Removed hardcoding of quoting configurations from all macros. Config should be picked from dbt project config - Added 4 new test cases for relation names --- dbt/include/oracle/macros/adapters.sql | 25 ++- .../materializations/snapshot/snapshot.sql | 4 +- .../snapshot/snapshot_merge.sql | 4 +- .../materializations/snapshot/strategies.sql | 2 +- .../macros/materializations/table/table.sql | 2 + .../adapter/test_quoted_relations.py | 204 ++++++++++++++++++ 6 files changed, 223 insertions(+), 18 deletions(-) create mode 100644 tests/functional/adapter/test_quoted_relations.py diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index 04bc09f..4eb25ad 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -94,10 +94,9 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create {% if temporary -%} global temporary - {%- endif %} table {{ relation.include(schema=(not temporary)).quote(schema=False, identifier=False) }} + {%- endif %} table {{ relation.include(schema=(not temporary)) }} {% if temporary -%} on commit preserve rows {%- endif %} as {{ sql }} @@ -112,7 +111,7 @@ create {% if temporary -%} global temporary - {%- endif %} table {{ relation.include(schema=(not temporary)).quote(schema=False, identifier=False) }} + {%- endif %} table {{ relation.include(schema=(not temporary)) }} {% if temporary -%} on commit preserve rows {%- endif %} as {{ sql }} @@ -122,7 +121,7 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create or replace view {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} as + create or replace view {{ relation }} as {{ sql }} {% endmacro %} @@ -205,7 +204,7 @@ {% macro oracle__alter_relation_comment(relation, comment) %} {% set escaped_comment = oracle_escape_comment(comment) %} {# "comment on table" even for views #} - comment on table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} is {{ escaped_comment }} + comment on table {{ relation }} is {{ escaped_comment }} {% endmacro %} {% macro oracle__persist_docs(relation, model, for_relation, for_columns) -%} @@ -218,7 +217,7 @@ {% set comment = column_dict[column_name]['description'] %} {% set escaped_comment = oracle_escape_comment(comment) %} {% call statement('alter _column comment', fetch_result=False) -%} - comment on column {{ relation.include(False, True, True).quote(schema=False, identifier=False) }}.{{ column_name }} is {{ escaped_comment }} + comment on column {{ relation }}.{{ column_name }} is {{ escaped_comment }} {%- endcall %} {% endfor %} {% endif %} @@ -234,16 +233,16 @@ {%- set tmp_column = column_name + "__dbt_alter" -%} {% call statement('alter_column_type 1', fetch_result=False) %} - alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} add {{ tmp_column }} {{ new_column_type }} + alter table {{ relation }} add {{ tmp_column }} {{ new_column_type }} {% endcall %} {% call statement('alter_column_type 2', fetch_result=False) %} - update {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} set {{ tmp_column }} = {{ column_name }} + update {{ relation }} set {{ tmp_column }} = {{ column_name }} {% endcall %} {% call statement('alter_column_type 3', fetch_result=False) %} - alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} drop column {{ column_name }} cascade constraints + alter table {{ relation }} drop column {{ column_name }} cascade constraints {% endcall %} {% call statement('alter_column_type 4', fetch_result=False) %} - alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} rename column {{ tmp_column }} to {{ column_name }} + alter table {{ relation }} rename column {{ tmp_column }} to {{ column_name }} {% endcall %} {% endmacro %} @@ -257,7 +256,7 @@ pragma EXCEPTION_INIT(attempted_ddl_on_in_use_GTT, -14452); BEGIN SAVEPOINT start_transaction; - EXECUTE IMMEDIATE 'DROP {{ relation.type }} {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} cascade constraint'; + EXECUTE IMMEDIATE 'DROP {{ relation.type }} {{ relation }} cascade constraint'; COMMIT; EXCEPTION WHEN attempted_ddl_on_in_use_GTT THEN @@ -272,14 +271,14 @@ {#-- To avoid `ORA-01702: a view is not appropriate here` we check that the relation to be truncated is a table #} {% if relation.is_table %} {% call statement('truncate_relation') -%} - truncate table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} + truncate table {{ relation }} {%- endcall %} {% endif %} {% endmacro %} {% macro oracle__rename_relation(from_relation, to_relation) -%} {% call statement('rename_relation') -%} - ALTER {{ from_relation.type }} {{ from_relation.include(False, True, True).quote(schema=False, identifier=False) }} rename to {{ to_relation.include(False, False, True).quote(schema=False, identifier=False) }} + ALTER {{ from_relation.type }} {{ from_relation }} rename to {{ to_relation.include(schema=False) }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index c53bfc3..07bf2b2 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -51,10 +51,10 @@ snapshotted_data as ( - select {{ target_relation.quote(schema=False, identifier=False) }}.*, + select {{ target_relation }}.*, {{ strategy.unique_key }} as dbt_unique_key - from {{ target_relation.quote(schema=False, identifier=False) }} + from {{ target_relation }} where dbt_valid_to is null ), diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot_merge.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot_merge.sql index 0393cd1..ad355fe 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot_merge.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot_merge.sql @@ -27,8 +27,8 @@ {% do dest_cols_csv.append("d." + column) %} {% endfor %} - merge into {{ target.quote(schema=False, identifier=False) }} d - using {{ source.quote(schema=False, identifier=False) }} s + merge into {{ target }} d + using {{ source }} s on (s.dbt_scd_id = d.dbt_scd_id) when matched diff --git a/dbt/include/oracle/macros/materializations/snapshot/strategies.sql b/dbt/include/oracle/macros/materializations/snapshot/strategies.sql index ce5ebaf..ea501f3 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/strategies.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/strategies.sql @@ -84,7 +84,7 @@ {%- set target_table = node.get('alias', node.get('name')) -%} {%- set target_relation = adapter.get_relation(database=node_database, schema=node.schema, identifier=target_table) -%} - {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation.quote(schema=False, identifier=False)) -%} + {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%} {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #} {%- set ns.column_added = false -%} diff --git a/dbt/include/oracle/macros/materializations/table/table.sql b/dbt/include/oracle/macros/materializations/table/table.sql index 48541a0..f08320d 100644 --- a/dbt/include/oracle/macros/materializations/table/table.sql +++ b/dbt/include/oracle/macros/materializations/table/table.sql @@ -18,8 +18,10 @@ {% set identifier = model['alias'] %} {% set tmp_identifier = model['name'] + '__dbt_tmp' %} {% set backup_identifier = model['name'] + '__dbt_backup' %} + {{ log("SCHEMA __<>>>><<>>>>>>>>>>" ~ schema, info=True) }} {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + {{ log("FOUND OLD RELATION __<>>>><<>>>>>>>>>>" ~ old_relation, info=True) }} {% set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, diff --git a/tests/functional/adapter/test_quoted_relations.py b/tests/functional/adapter/test_quoted_relations.py new file mode 100644 index 0000000..531fe08 --- /dev/null +++ b/tests/functional/adapter/test_quoted_relations.py @@ -0,0 +1,204 @@ +""" +Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import pytest + +import datetime + +from dbt.tests.util import run_dbt, relation_from_name + + +# seeds/my_seed.csv +seed_csv = """ +user_id,user_name,birth_date,income,last_login_date +1,Easton,1981-05-20,40000,2022-04-25T08:57:59 +2,Lillian,1978-09-03,54000,2022-04-25T08:58:59 +3,Jeremiah,1982-03-11,10000,2022-04-25T09:57:59 +4,Nolan,1976-05-06,900000,2022-04-25T09:58:59 +""".lstrip() + +# models/foo.sql +my_model_foo_sql = """ +{{config(materialized='table', + alias="foo")}} +SELECT * FROM {{ ref('seed') }} +""" + +# models/foo2.sql +my_model_foo2_sql = """ +{{config(materialized='table', + alias="Foo")}} +SELECT * FROM {{ ref('seed') }} +""" + + +class TestQuotedLowerCaseTableName: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_foo.sql": my_model_foo_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "quoting": { + "database": False, + "schema": False, + "identifier": True + }, + } + + def test_run_dbt(self, project): + """ + - run seed + - run model + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + sql = 'SELECT COUNT(*) from "foo"' + result = project.run_sql(sql, fetch="all") + assert result == [(4,)] + + +class TestQuotedTitleCaseTableName: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_foo.sql": my_model_foo2_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "quoting": { + "database": False, + "schema": False, + "identifier": True + }, + } + + def test_run_dbt(self, project): + """ + - run seed + - run model + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + sql = 'SELECT COUNT(*) from "Foo"' + result = project.run_sql(sql, fetch="all") + assert result == [(4,)] + + +class TestUnQuotedLowerCaseTableName: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_foo.sql": my_model_foo_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run model + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + sql = 'SELECT COUNT(*) from FOO' + result = project.run_sql(sql, fetch="all") + assert result == [(4,)] + + +class TestUnQuotedTitleCaseTableName: + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_foo.sql": my_model_foo2_sql, + } + + def test_run_dbt(self, project): + """ + - run seed + - run model + + """ + results = run_dbt(['seed']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + results = run_dbt(['run']) + assert len(results) == 1 + + sql = 'SELECT COUNT(*) from FOO' + result = project.run_sql(sql, fetch="all") + assert result == [(4,)] + + From c87d91278775c05b1bc0bbdd62cdaf43baa1c7bc Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Sun, 24 Jul 2022 20:41:57 -0700 Subject: [PATCH 5/5] Removed debug log statement --- dbt/include/oracle/macros/materializations/table/table.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbt/include/oracle/macros/materializations/table/table.sql b/dbt/include/oracle/macros/materializations/table/table.sql index f08320d..de04c53 100644 --- a/dbt/include/oracle/macros/materializations/table/table.sql +++ b/dbt/include/oracle/macros/materializations/table/table.sql @@ -18,10 +18,7 @@ {% set identifier = model['alias'] %} {% set tmp_identifier = model['name'] + '__dbt_tmp' %} {% set backup_identifier = model['name'] + '__dbt_backup' %} - {{ log("SCHEMA __<>>>><<>>>>>>>>>>" ~ schema, info=True) }} - {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} - {{ log("FOUND OLD RELATION __<>>>><<>>>>>>>>>>" ~ old_relation, info=True) }} {% set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,