From ad580d7ebd23a29949a02f152641009d66f197db Mon Sep 17 00:00:00 2001 From: shiyuhang0 <52435083+shiyuhang0@users.noreply.github.com> Date: Mon, 22 Aug 2022 17:11:51 +0800 Subject: [PATCH] support dbt v1.2 (#6) Support dbt v1.2 1. Lift + shift for cross-db macros 2. Support connection retry 3. Support grant 4. Doc Update Co-authored-by: Shi Yuhang <1136742008.com> --- .github/workflows/main.yml | 16 ++ README.md | 32 ++- dbt/adapters/tidb/connections.py | 40 ++-- dbt/include/tidb/macros/apply_grants.sql | 29 +++ .../incremental/incremental.sql | 5 + .../materializations/snapshot/snapshot.sql | 5 + .../materializations/snapshot/strategies.sql | 41 ++-- dbt/include/tidb/macros/utils/bool_or.sql | 7 + .../tidb/macros/utils/cast_bool_to_text.sql | 8 + dbt/include/tidb/macros/utils/date_trunc.sql | 31 +++ dbt/include/tidb/macros/utils/dateadd.sql | 9 + dbt/include/tidb/macros/utils/datediff.sql | 14 ++ dbt/include/tidb/macros/utils/hash.sql | 5 + dbt/include/tidb/macros/utils/split_part.sql | 16 ++ requirements_dev.txt | 4 +- setup.py | 9 +- tests/README.md | 17 +- .../adapter/tidb/basic/test_tidb.py | 22 ++ .../adapter/tidb/grant/create_user.sql | 5 + .../adapter/tidb/grant/fixture_snapshots.py | 27 +++ .../adapter/tidb/grant/test_grant.py | 41 ++++ .../adapter/tidb/utils/fixture_bool_or.py | 34 +++ .../adapter/tidb/utils/fixture_dateadd.py | 31 +++ .../adapter/tidb/utils/fixture_datediff.py | 67 ++++++ .../adapter/tidb/utils/fixture_safe_cast.py | 27 +++ .../adapter/tidb/utils/test_util.py | 156 ++++++++++++++ .../adapter/tidb4_0/basic/test_tidb.py | 22 ++ .../adapter/tidb4_0/grant/create_user.sql | 5 + .../adapter/tidb4_0/grant/test_grant.py | 38 ++++ .../adapter/tidb4_0/utils/fixture_bool_or.py | 34 +++ .../adapter/tidb4_0/utils/fixture_dateadd.py | 31 +++ .../adapter/tidb4_0/utils/fixture_datediff.py | 68 ++++++ .../tidb4_0/utils/fixture_datetrunc.py | 38 ++++ .../tidb4_0/utils/fixture_safe_cast.py | 27 +++ .../tidb4_0/utils/fixture_split_part.py | 52 +++++ .../adapter/tidb4_0/utils/test_util.py | 195 ++++++++++++++++++ .../adapter/tidb5_1/basic/test_tidb.py | 22 ++ .../adapter/tidb5_1/grant/create_user.sql | 5 + .../adapter/tidb5_1/grant/test_grant.py | 42 ++++ .../adapter/tidb5_1/utils/fixture_bool_or.py | 34 +++ .../adapter/tidb5_1/utils/fixture_dateadd.py | 31 +++ .../adapter/tidb5_1/utils/fixture_datediff.py | 68 ++++++ .../tidb5_1/utils/fixture_datetrunc.py | 38 ++++ .../tidb5_1/utils/fixture_safe_cast.py | 27 +++ .../tidb5_1/utils/fixture_split_part.py | 52 +++++ .../adapter/tidb5_1/utils/test_util.py | 178 ++++++++++++++++ 46 files changed, 1657 insertions(+), 48 deletions(-) create mode 100644 dbt/include/tidb/macros/apply_grants.sql create mode 100644 dbt/include/tidb/macros/utils/bool_or.sql create mode 100644 dbt/include/tidb/macros/utils/cast_bool_to_text.sql create mode 100644 dbt/include/tidb/macros/utils/date_trunc.sql create mode 100644 dbt/include/tidb/macros/utils/dateadd.sql create mode 100644 dbt/include/tidb/macros/utils/datediff.sql create mode 100644 dbt/include/tidb/macros/utils/hash.sql create mode 100644 dbt/include/tidb/macros/utils/split_part.sql create mode 100644 tests/functional/adapter/tidb/grant/create_user.sql create mode 100644 tests/functional/adapter/tidb/grant/fixture_snapshots.py create mode 100644 tests/functional/adapter/tidb/grant/test_grant.py create mode 100644 tests/functional/adapter/tidb/utils/fixture_bool_or.py create mode 100644 tests/functional/adapter/tidb/utils/fixture_dateadd.py create mode 100644 tests/functional/adapter/tidb/utils/fixture_datediff.py create mode 100644 tests/functional/adapter/tidb/utils/fixture_safe_cast.py create mode 100644 tests/functional/adapter/tidb/utils/test_util.py create mode 100644 tests/functional/adapter/tidb4_0/grant/create_user.sql create mode 100644 tests/functional/adapter/tidb4_0/grant/test_grant.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_bool_or.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_dateadd.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_datediff.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_datetrunc.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_safe_cast.py create mode 100644 tests/functional/adapter/tidb4_0/utils/fixture_split_part.py create mode 100644 tests/functional/adapter/tidb4_0/utils/test_util.py create mode 100644 tests/functional/adapter/tidb5_1/grant/create_user.sql create mode 100644 tests/functional/adapter/tidb5_1/grant/test_grant.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_dateadd.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_datediff.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_datetrunc.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_safe_cast.py create mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_split_part.py create mode 100644 tests/functional/adapter/tidb5_1/utils/test_util.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32ec332..0528b1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,10 @@ jobs: - name: Run tests run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb tidb_5_3: @@ -66,6 +70,10 @@ jobs: - name: Run tests run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb tidb_5_1: @@ -97,6 +105,10 @@ jobs: - name: Run tests run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb5_1/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb5_1 tidb_4_0: @@ -128,4 +140,8 @@ jobs: - name: Run tests run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb4_0/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb4_0 \ No newline at end of file diff --git a/README.md b/README.md index a4059f8..4af78c6 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Thanks to them for their excellent work. ## Table of Contents * [Installation](#installation) * [Supported features](#supported-features) + * [Supported functions](#supported-functions) * [Profile Configuration](#profile-configuration) * [Database User Privileges](#database-user-privileges) * [Running Tests](#running-tests) @@ -46,6 +47,8 @@ $ pip install dbt-tidb | ✅ | ✅ | ✅ | Custom data tests | | ✅ | ✅ | ✅ | Docs generate | | ❌ | ❌ | ✅ | Snapshots | +| ✅ | ✅ | ✅ | Connection retry | +| ✅ | ✅ | ✅ | Grant | Note: @@ -55,6 +58,31 @@ Note: * TiDB 4.X does not support using SQL func in `CREATE VIEW`, avoid it in your SQL code. You can find more detail [here](https://github.com/pingcap/tidb/pull/27252). +## Supported functions + +cross-db macros are moved from dbt-utils into dbt-core, so you can use the following functions directly, see [dbt-util](https://github.com/dbt-labs/dbt-utils) on how to use them. +- bool_or +- cast_bool_to_text +- dateadd +- datediff +- date_trunc +- hash +- safe_cast +- split_part +- last_day +- cast_bool_to_text +- concat +- escape_single_quotes +- except +- intersect +- length +- position +- replace +- right +- listagg (not support yet) + +> pay attention that datediff is a little different from dbt-util that it will round down rather than round up. + ## Profile Configuration TiDB targets should be set up using the following configuration in your `profiles.yml` file. @@ -72,6 +100,7 @@ your_profile_name: schema: database_name username: tidb_username password: tidb_password + retries: 2 ``` | Option | Description | Required? | Example | @@ -82,6 +111,7 @@ your_profile_name: | schema | Specify the schema (database) to build models into | Required | `analytics` | | username | The username to use to connect to the server | Required | `dbt_admin` | | password | The password to use for authenticating to the server | Required | `correct-horse-battery-staple` | +| retries | The retry times for connection to TiDB (1 in default) | Optional | `2` | ## Database User Privileges @@ -102,7 +132,7 @@ You can find some help [here](https://docs.pingcap.com/tidb/v4.0/privilege-manag ## Running Tests -See [test/README.md](test/README.md) for details on running the integration tests. +See [tests/README.md](tests/README.md) for details on running the integration tests. ## Example diff --git a/dbt/adapters/tidb/connections.py b/dbt/adapters/tidb/connections.py index 22b0733..e767430 100644 --- a/dbt/adapters/tidb/connections.py +++ b/dbt/adapters/tidb/connections.py @@ -23,6 +23,7 @@ class TiDBCredentials(Credentials): username: Optional[str] = None password: Optional[str] = None charset: Optional[str] = None + retries: int = 1 _ALIASES = { "UID": "username", @@ -87,35 +88,20 @@ def open(cls, connection): if credentials.port: kwargs["port"] = credentials.port - try: - connection.handle = mysql.connector.connect(**kwargs) - connection.state = "open" - except mysql.connector.Error: - - try: - logger.debug( - "Failed connection without supplying the `database`. " - "Trying again with `database` included." - ) - - # Try again with the database included - kwargs["database"] = credentials.schema + def connect(): + handle = mysql.connector.connect(**kwargs) + return handle - connection.handle = mysql.connector.connect(**kwargs) - connection.state = "open" - except mysql.connector.Error as e: + # we just retry for any error now + retryable_exceptions = [mysql.connector.Error] - logger.debug( - "Got an error when attempting to open a tidb " - "connection: '{}'".format(e) - ) - - connection.handle = None - connection.state = "fail" - - raise dbt.exceptions.FailedToConnectException(str(e)) - - return connection + return cls.retry_connection( + connection, + connect=connect, + logger=logger, + retry_limit=credentials.retries, + retryable_exceptions=retryable_exceptions, + ) @classmethod def get_credentials(cls, credentials): diff --git a/dbt/include/tidb/macros/apply_grants.sql b/dbt/include/tidb/macros/apply_grants.sql new file mode 100644 index 0000000..b55e501 --- /dev/null +++ b/dbt/include/tidb/macros/apply_grants.sql @@ -0,0 +1,29 @@ +-- support Select/Insert/Delete/Update now +{% macro default__get_show_grant_sql(relation) %} + + select case(Table_priv) when null then null else 'select' end as privilege_type, `User` as grantee from mysql.tables_priv where `DB` = '{{relation.schema}}' and `Table_name` = '{{relation.identifier}}' and Table_priv like '%Select%' + union ALL + select case(Table_priv) when null then null else 'insert' end as privilege_type, `User` as grantee from mysql.tables_priv where `DB` = '{{relation.schema}}' and `Table_name` = '{{relation.identifier}}' and Table_priv like '%Insert%' + union ALL + select case(Table_priv) when null then null else 'update' end as privilege_type, `User` as grantee from mysql.tables_priv where `DB` = '{{relation.schema}}' and `Table_name` = '{{relation.identifier}}' and Table_priv like '%Update%' + union ALL + select case(Table_priv) when null then null else 'delete' end as privilege_type, `User` as grantee from mysql.tables_priv where `DB` = '{{relation.schema}}' and `Table_name` = '{{relation.identifier}}' and Table_priv like '%Delete%' + +{% endmacro %} + +{%- macro tidb__get_grant_sql(relation, privilege, grantees) -%} + grant {{ privilege }} on {{ relation }} to {{ '\"' + grantees|join('\", \"') + '\"' }} +{%- endmacro -%} + + {%- macro tidb__get_revoke_sql(relation, privilege, grantees) -%} + revoke {{ privilege }} on {{ relation }} from {{ '\"' + grantees|join('\", \"') + '\"' }} +{%- endmacro -%} + +-- tidb-dbt does not support multi=true now, so we need to split every grant/revoke statement +{% macro tidb__call_dcl_statements(dcl_statement_list) %} + {% for dcl_statement in dcl_statement_list %} + {% call statement('grant_or_revoke') %} + {{ dcl_statement }} + {% endcall %} + {% endfor %} +{% endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/materializations/incremental/incremental.sql b/dbt/include/tidb/macros/materializations/incremental/incremental.sql index fc3088b..4d62c61 100644 --- a/dbt/include/tidb/macros/materializations/incremental/incremental.sql +++ b/dbt/include/tidb/macros/materializations/incremental/incremental.sql @@ -7,6 +7,9 @@ {% set existing_relation = load_relation(this) %} {% set tmp_relation = make_temp_relation(this) %} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {{ run_hooks(pre_hooks, inside_transaction=False) }} -- `BEGIN` happens here: @@ -44,6 +47,8 @@ {{ build_sql }} {% endcall %} + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} {% do persist_docs(target_relation, model) %} {{ run_hooks(post_hooks, inside_transaction=True) }} diff --git a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql index d8ec576..34c586e 100644 --- a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql @@ -11,6 +11,8 @@ {%- set strategy_name = config.get('strategy') -%} {%- set unique_key = config.get('unique_key') %} + -- grab current tables grants config for comparision later on + {%- set grant_config = config.get('grants') -%} {% if not adapter.check_schema_exists(model.database, model.schema) %} {% do create_schema(model.database, model.schema) %} @@ -98,6 +100,9 @@ {% endif %} + {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + {% do persist_docs(target_relation, model) %} {{ run_hooks(post_hooks, inside_transaction=True) }} diff --git a/dbt/include/tidb/macros/materializations/snapshot/strategies.sql b/dbt/include/tidb/macros/materializations/snapshot/strategies.sql index 92ceed6..b1485d2 100644 --- a/dbt/include/tidb/macros/materializations/snapshot/strategies.sql +++ b/dbt/include/tidb/macros/materializations/snapshot/strategies.sql @@ -6,26 +6,41 @@ {%- endfor -%})) {%- endmacro %} -{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%} - {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%} +-- copy from dbt-core v1.2, just alter database=None in adapter.get_relation +{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%} {%- if not target_exists -%} - {# no table yet -> return whatever the query does #} - {{ return([false, query_columns]) }} + {#-- no table yet -> return whatever the query does --#} + {{ return((false, query_columns)) }} {%- endif -%} - {# handle any schema changes #} - {%- set target_table = node.get('alias', node.get('name')) -%} - {%- set target_relation = adapter.get_relation(database=None, schema=node.schema, identifier=target_table) -%} - {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%} - {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #} + + {#-- handle any schema changes --#} + {%- set target_relation = adapter.get_relation(database=None, schema=node.schema, identifier=node.alias) -%} + + {% if check_cols_config == 'all' %} + {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%} + + {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %} + {#-- query for proper casing/quoting, to support comparison below --#} + {%- set select_check_cols_from_target -%} + select {{ check_cols_config | join(', ') }} from ({{ node['compiled_sql'] }}) subq + {%- endset -%} + {% set query_columns = get_columns_in_query(select_check_cols_from_target) %} + + {% else %} + {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} + {% endif %} + + {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%} + {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#} {%- set ns.column_added = false -%} {%- set intersection = [] -%} {%- for col in query_columns -%} {%- if col in existing_cols -%} - {%- do intersection.append(col) -%} + {%- do intersection.append(adapter.quote(col)) -%} {%- else -%} {% set ns.column_added = true %} - {%- endif -%} + {%- endif -%} {%- endfor -%} - {{ return([ns.column_added, intersection]) }} -{%- endmacro %} \ No newline at end of file + {{ return((ns.column_added, intersection)) }} +{%- endmacro %} diff --git a/dbt/include/tidb/macros/utils/bool_or.sql b/dbt/include/tidb/macros/utils/bool_or.sql new file mode 100644 index 0000000..90a4f29 --- /dev/null +++ b/dbt/include/tidb/macros/utils/bool_or.sql @@ -0,0 +1,7 @@ +-- bool_or is an agg function which will return true once any expression is true +-- use max to replace it +{% macro tidb__bool_or(expression) -%} + + max({{ expression }}) + +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/cast_bool_to_text.sql b/dbt/include/tidb/macros/utils/cast_bool_to_text.sql new file mode 100644 index 0000000..92d4023 --- /dev/null +++ b/dbt/include/tidb/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,8 @@ +{% macro tidb__cast_bool_to_text(field) %} + + case {{ field }} + when true then 'true' + when false then 'false' + end + +{% endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/date_trunc.sql b/dbt/include/tidb/macros/utils/date_trunc.sql new file mode 100644 index 0000000..f1e3618 --- /dev/null +++ b/dbt/include/tidb/macros/utils/date_trunc.sql @@ -0,0 +1,31 @@ +-- date_trunc can truncate date with given datepart(year,month,quarter,day is supported) +{% macro tidb__date_trunc(datepart, date) -%} + + {%- if datepart =='day' -%} + + DATE_FORMAT({{date}}, '%Y-%m-%d') + + {%- elif datepart == 'month' -%} + + DATE_FORMAT({{date}}, '%Y-%m-01') + + {%- elif datepart == 'quarter' -%} + + case QUARTER({{date}}) + when 1 then DATE_FORMAT({{date}}, '%Y-01-01') + when 2 then DATE_FORMAT({{date}}, '%Y-04-01') + when 2 then DATE_FORMAT({{date}}, '%Y-07-01') + when 2 then DATE_FORMAT({{date}}, '%Y-10-01') + end + + {%- elif datepart == 'year' -%} + + DATE_FORMAT({{date}}, '%Y-01-01') + + {%- else -%} + + {{ exceptions.raise_compiler_error("macro date_trunc not implemented for datepart ~ '" ~ datepart ~ "' ~ on TiDB") }} + + {%- endif -%} + +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/dateadd.sql b/dbt/include/tidb/macros/utils/dateadd.sql new file mode 100644 index 0000000..31e3506 --- /dev/null +++ b/dbt/include/tidb/macros/utils/dateadd.sql @@ -0,0 +1,9 @@ +-- add interval to given from_date_or_timestamp with datepart(day,month,hour...) +{% macro tidb__dateadd(datepart, interval, from_date_or_timestamp) %} + + DATE_ADD( + {{ from_date_or_timestamp }}, + interval {{ interval }} {{ datepart }} + ) + +{% endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/datediff.sql b/dbt/include/tidb/macros/utils/datediff.sql new file mode 100644 index 0000000..000f889 --- /dev/null +++ b/dbt/include/tidb/macros/utils/datediff.sql @@ -0,0 +1,14 @@ +-- the behavior is a little different from default_datediff that it will round down rather than round up +-- and millisecond is not supported +{% macro tidb__datediff(first_date, second_date, datepart) -%} + {%- if datepart =='millisecond' -%} + + {{ exceptions.raise_compiler_error("macro datediff not implemented for datepart ~ '" ~ datepart ~ "' ~ on TiDB") }} + + {%- else -%} + + TIMESTAMPDIFF({{datepart}},{{first_date}},{{second_date}}) + + {%- endif -%} + +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/hash.sql b/dbt/include/tidb/macros/utils/hash.sql new file mode 100644 index 0000000..0335a5c --- /dev/null +++ b/dbt/include/tidb/macros/utils/hash.sql @@ -0,0 +1,5 @@ +{% macro tidb__hash(field) -%} + + md5(cast({{ field }} as CHAR)) + +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/tidb/macros/utils/split_part.sql b/dbt/include/tidb/macros/utils/split_part.sql new file mode 100644 index 0000000..41d192c --- /dev/null +++ b/dbt/include/tidb/macros/utils/split_part.sql @@ -0,0 +1,16 @@ +-- split_part will split string with given delimiter_text, and it will return the element at element +{% macro tidb__split_part(string_text, delimiter_text, part_number) %} + + + {% if part_number >= 0 %} + + SUBSTRING_INDEX(SUBSTRING_INDEX({{ string_text }}, {{ delimiter_text }}, {{ part_number }}), {{ delimiter_text }}, -1) + + {% else %} + + SUBSTRING_INDEX(SUBSTRING_INDEX({{ string_text }}, {{ delimiter_text }}, {{ part_number }}), {{ delimiter_text }}, 1) + + {% endif %} + + +{% endmacro %} \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index b6ae115..9e87633 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ -dbt-core~=1.1.0 +dbt-core~=1.2.0 mysql-connector-python>=8.0.0,<8.1 pytest~=7.0 markupsafe==2.0.1 -dbt-tests-adapter~=1.1.0 +dbt-tests-adapter~=1.2.0 pytest-dotenv \ No newline at end of file diff --git a/setup.py b/setup.py index fcd8e0d..01e4a30 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,10 @@ import os import sys -if sys.version_info < (3, 8) or sys.version_info >= (3, 11): +if sys.version_info < (3, 8) or sys.version_info > (3, 10): print("Error: dbt-tidb does not support this version of Python.") - print("Please install Python 3.8 or higher but less than 3.11.") + print("Please install Python 3.8 or higher but less than 3.10.") + sys.exit(1) # require version of setuptools that supports find_namespace_packages from setuptools import setup @@ -26,8 +27,8 @@ long_description = f.read() package_name = "dbt-tidb" -package_version = "1.1.0" -dbt_core_version = "1.1.0" +package_version = "1.2.0" +dbt_core_version = "1.2.0" description = """The TiDB adapter plugin for dbt""" setup( diff --git a/tests/README.md b/tests/README.md index 4d0e4aa..052752a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -55,7 +55,22 @@ If you specify a package, all Python files under the package will be tested. Don ``` # basic PYTHONPATH=. pytest tests/functional/adapter/tidb/basic -`` +# utils +PYTHONPATH=. pytest tests/functional/adapter/tidb/utils +``` + +## Test grant + +When you test grant, you need to [create three users](https://docs.pingcap.com/tidb/stable/basic-sql-operations#create-authorize-and-delete-a-user) in TiDB and set environment variables like: +``` +export DBT_TEST_USER_1=user1 +export DBT_TEST_USER_2=user2 +export DBT_TEST_USER_3=user3 +``` +Then test grant: +``` +PYTHONPATH=. pytest tests/functional/adapter/tidb/grant +``` ## Test other versions of TiDB diff --git a/tests/functional/adapter/tidb/basic/test_tidb.py b/tests/functional/adapter/tidb/basic/test_tidb.py index 0e69727..8938749 100644 --- a/tests/functional/adapter/tidb/basic/test_tidb.py +++ b/tests/functional/adapter/tidb/basic/test_tidb.py @@ -12,6 +12,9 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog from dbt.tests.util import run_dbt, check_relations_equal from dbt.tests.adapter.incremental.test_incremental_unique_id import ( BaseIncrementalUniqueKey, @@ -61,5 +64,24 @@ def test_adapter_methods(self, project, equal_tables): check_relations_equal(project.adapter, equal_tables) +class TestValidateConnection(BaseValidateConnection): + pass + + +class TestDocsGenerate(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=None, + id_type="int(11)", + text_type="text", + time_type="timestamp", + view_type="view", + table_type="table", + model_stats=no_stats(), + ) + + class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): pass diff --git a/tests/functional/adapter/tidb/grant/create_user.sql b/tests/functional/adapter/tidb/grant/create_user.sql new file mode 100644 index 0000000..f6701cb --- /dev/null +++ b/tests/functional/adapter/tidb/grant/create_user.sql @@ -0,0 +1,5 @@ +CREATE USER 'user1'@'%' IDENTIFIED BY ''; + +CREATE USER 'user2'@'%' IDENTIFIED BY ''; + +CREATE USER 'user3'@'%' IDENTIFIED BY ''; \ No newline at end of file diff --git a/tests/functional/adapter/tidb/grant/fixture_snapshots.py b/tests/functional/adapter/tidb/grant/fixture_snapshots.py new file mode 100644 index 0000000..3731cc7 --- /dev/null +++ b/tests/functional/adapter/tidb/grant/fixture_snapshots.py @@ -0,0 +1,27 @@ +my_snapshot_sql = """ +{% snapshot my_snapshot %} + {{ config( + check_cols='all', unique_key='id', strategy='check', + target_database=database, target_schema=schema + ) }} + select 1 as id, cast('blue' as char) as color +{% endsnapshot %} +""".strip() + +snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_1') }}"] +""" + +user2_snapshot_schema_yml = """ +version: 2 +snapshots: + - name: my_snapshot + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" diff --git a/tests/functional/adapter/tidb/grant/test_grant.py b/tests/functional/adapter/tidb/grant/test_grant.py new file mode 100644 index 0000000..8cf0898 --- /dev/null +++ b/tests/functional/adapter/tidb/grant/test_grant.py @@ -0,0 +1,41 @@ +import pytest + +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants +from tests.functional.adapter.tidb.grant.fixture_snapshots import ( + my_snapshot_sql, + snapshot_schema_yml, +) + + +# need to export DBT_TEST_USER_1,DBT_TEST_USER_2,DBT_TEST_USER_3 +class TestModelGrantsTiDB(BaseModelGrants): + pass + + +class TestIncrementalGrantsTiDB(BaseIncrementalGrants): + pass + + +class TestSeedGrantsTiDB(BaseSeedGrants): + pass + + +class TestSnapshotGrantsTiDB(BaseSnapshotGrants): + @pytest.fixture(scope="class") + def snapshots(self): + return { + "my_snapshot.sql": my_snapshot_sql, + "schema.yml": self.interpolate_name_overrides(snapshot_schema_yml), + } + + +class TestInvalidGrantsTiDB(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "You are not allowed to create a user with GRANT" + + def privilege_does_not_exist_error(self): + return "Illegal privilege level specified for" diff --git a/tests/functional/adapter/tidb/utils/fixture_bool_or.py b/tests/functional/adapter/tidb/utils/fixture_bool_or.py new file mode 100644 index 0000000..c14848c --- /dev/null +++ b/tests/functional/adapter/tidb/utils/fixture_bool_or.py @@ -0,0 +1,34 @@ +# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql + +models__test_bool_or_sql = """ +with data as ( + select * from {{ ref('data_bool_or') }} +), +data_output as ( + select * from {{ ref('data_bool_or_expected') }} +), +calculate as ( + select + `key`, + {{ bool_or('val1 = val2') }} as value + from data + group by `key` +) +select + calculate.value as actual, + data_output.value as expected +from calculate +left join data_output +on calculate.key = data_output.key +""" + + +models__test_bool_or_yml = """ +version: 2 +models: + - name: test_bool_or + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb/utils/fixture_dateadd.py b/tests/functional/adapter/tidb/utils/fixture_dateadd.py new file mode 100644 index 0000000..d2a4e44 --- /dev/null +++ b/tests/functional/adapter/tidb/utils/fixture_dateadd.py @@ -0,0 +1,31 @@ +# tidb does not support cast to timestamp, so cast to datatime + +models__test_dateadd_sql = """ +with data as ( + + select * from {{ ref('data_dateadd') }} + +) + +select + case + when datepart = 'hour' then cast({{ dateadd('hour', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'day' then cast({{ dateadd('day', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'month' then cast({{ dateadd('month', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'year' then cast({{ dateadd('year', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + else null + end as actual, + result as expected + +from data +""" + +models__test_dateadd_yml = """ +version: 2 +models: + - name: test_dateadd + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb/utils/fixture_datediff.py b/tests/functional/adapter/tidb/utils/fixture_datediff.py new file mode 100644 index 0000000..c640d95 --- /dev/null +++ b/tests/functional/adapter/tidb/utils/fixture_datediff.py @@ -0,0 +1,67 @@ +# datediff in dbt-tidb will round down rather than round up + +# change 2019-12-31 00:00:00,2020-01-06 02:00:00,week,1 to 2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +# change 2019-12-31 00:00:00,2019-12-27 00:00:00,week,-1 to 2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 +seeds__data_datediff_csv = """first_date,second_date,datepart,result +2018-01-01 01:00:00,2018-01-02 01:00:00,day,1 +2018-01-01 01:00:00,2018-02-01 01:00:00,month,1 +2018-01-01 01:00:00,2019-01-01 01:00:00,year,1 +2018-01-01 01:00:00,2018-01-01 02:00:00,hour,1 +2018-01-01 01:00:00,2018-01-01 02:01:00,minute,61 +2018-01-01 01:00:00,2018-01-01 02:00:01,second,3601 +2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 +2019-12-31 00:00:00,2019-12-30 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-02 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +,2018-01-01 02:00:00,hour, +2018-01-01 02:00:00,,hour, +""" + + +models__test_datediff_sql = """ +with data as ( + + select * from {{ ref('data_datediff') }} + +) + +select + + case + when datepart = 'second' then {{ datediff('first_date', 'second_date', 'second') }} + when datepart = 'minute' then {{ datediff('first_date', 'second_date', 'minute') }} + when datepart = 'hour' then {{ datediff('first_date', 'second_date', 'hour') }} + when datepart = 'day' then {{ datediff('first_date', 'second_date', 'day') }} + when datepart = 'week' then {{ datediff('first_date', 'second_date', 'week') }} + when datepart = 'month' then {{ datediff('first_date', 'second_date', 'month') }} + when datepart = 'year' then {{ datediff('first_date', 'second_date', 'year') }} + else null + end as actual, + result as expected + +from data + +-- Also test correct casting of literal values. +-- all the expected value except microsecond are changed from 1 to 0, and the test for millisecond is excluded +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 0 as expected + +""" + + +models__test_datediff_yml = """ +version: 2 +models: + - name: test_datediff + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb/utils/fixture_safe_cast.py b/tests/functional/adapter/tidb/utils/fixture_safe_cast.py new file mode 100644 index 0000000..0c8af33 --- /dev/null +++ b/tests/functional/adapter/tidb/utils/fixture_safe_cast.py @@ -0,0 +1,27 @@ +seeds__data_safe_cast_csv = """field,output +abc,abc +123,123 +, +""" + +# tidb does not support cast to text, so cast to char +models__test_safe_cast_sql = """ +with data as ( + select * from {{ ref('data_safe_cast') }} +) +select + {{ safe_cast('field', api.Column.translate_type('char')) }} as actual, + output as expected +from data +""" + + +models__test_safe_cast_yml = """ +version: 2 +models: + - name: test_safe_cast + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb/utils/test_util.py b/tests/functional/adapter/tidb/utils/test_util.py new file mode 100644 index 0000000..fa1e450 --- /dev/null +++ b/tests/functional/adapter/tidb/utils/test_util.py @@ -0,0 +1,156 @@ +import pytest +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesQuote, +) +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from tests.functional.adapter.tidb.utils.fixture_bool_or import ( + models__test_bool_or_sql, + models__test_bool_or_yml, +) +from tests.functional.adapter.tidb.utils.fixture_dateadd import ( + models__test_dateadd_yml, + models__test_dateadd_sql, +) +from tests.functional.adapter.tidb.utils.fixture_datediff import ( + seeds__data_datediff_csv, + models__test_datediff_sql, + models__test_datediff_yml, +) +from tests.functional.adapter.tidb.utils.fixture_safe_cast import ( + models__test_safe_cast_yml, + models__test_safe_cast_sql, +) + + +class TestAnyValue(BaseAnyValue): + pass + + +class TestBoolOr(BaseBoolOr): + @pytest.fixture(scope="class") + def models(self): + return { + "test_bool_or.yml": models__test_bool_or_yml, + "test_bool_or.sql": self.interpolate_macro_namespace( + models__test_bool_or_sql, "bool_or" + ), + } + + +class TestCastBoolToText(BaseCastBoolToText): + pass + + +class TestConcat(BaseConcat): + pass + + +class TestDateAdd(BaseDateAdd): + @pytest.fixture(scope="class") + def models(self): + return { + "test_dateadd.yml": models__test_dateadd_yml, + "test_dateadd.sql": self.interpolate_macro_namespace( + models__test_dateadd_sql, "dateadd" + ), + } + + +class TestDateDiff(BaseDateDiff): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_datediff.csv": seeds__data_datediff_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_datediff.yml": models__test_datediff_yml, + "test_datediff.sql": self.interpolate_macro_namespace( + models__test_datediff_sql, "datediff" + ), + } + + +class TestDateTrunc(BaseDateTrunc): + pass + + +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesQuote): + pass + + +class TestExcept(BaseExcept): + pass + + +class TestHash(BaseHash): + pass + + +class TestIntersect(BaseIntersect): + pass + + +class TestLastDay(BaseLastDay): + pass + + +class TestLength(BaseLength): + pass + + +@pytest.mark.skip(reason="unsupport") +class TestListagg(BaseListagg): + pass + + +class TestPosition(BasePosition): + pass + + +class TestReplace(BaseReplace): + pass + + +class TestRight(BaseRight): + pass + + +class TestSafeCast(BaseSafeCast): + @pytest.fixture(scope="class") + def models(self): + return { + "test_safe_cast.yml": models__test_safe_cast_yml, + "test_safe_cast.sql": self.interpolate_macro_namespace( + self.interpolate_macro_namespace( + models__test_safe_cast_sql, "safe_cast" + ), + "type_string", + ), + } + + +class TestSplitPart(BaseSplitPart): + pass + + +class TestStringLiteral(BaseStringLiteral): + pass diff --git a/tests/functional/adapter/tidb4_0/basic/test_tidb.py b/tests/functional/adapter/tidb4_0/basic/test_tidb.py index 575d5c0..be070f9 100644 --- a/tests/functional/adapter/tidb4_0/basic/test_tidb.py +++ b/tests/functional/adapter/tidb4_0/basic/test_tidb.py @@ -15,6 +15,9 @@ from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection from dbt.tests.adapter.incremental.test_incremental_unique_id import ( BaseIncrementalUniqueKey, ) @@ -74,6 +77,25 @@ def test_adapter_methods(self, project, equal_tables): check_relations_equal(project.adapter, equal_tables) +class TestValidateConnection(BaseValidateConnection): + pass + + +class TestDocsGenerate(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=None, + id_type="int(11)", + text_type="text", + time_type="timestamp", + view_type="view", + table_type="table", + model_stats=no_stats(), + ) + + @pytest.mark.skip( reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." ) diff --git a/tests/functional/adapter/tidb4_0/grant/create_user.sql b/tests/functional/adapter/tidb4_0/grant/create_user.sql new file mode 100644 index 0000000..f6701cb --- /dev/null +++ b/tests/functional/adapter/tidb4_0/grant/create_user.sql @@ -0,0 +1,5 @@ +CREATE USER 'user1'@'%' IDENTIFIED BY ''; + +CREATE USER 'user2'@'%' IDENTIFIED BY ''; + +CREATE USER 'user3'@'%' IDENTIFIED BY ''; \ No newline at end of file diff --git a/tests/functional/adapter/tidb4_0/grant/test_grant.py b/tests/functional/adapter/tidb4_0/grant/test_grant.py new file mode 100644 index 0000000..036dc31 --- /dev/null +++ b/tests/functional/adapter/tidb4_0/grant/test_grant.py @@ -0,0 +1,38 @@ +import pytest + +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +# need to export DBT_TEST_USER_1,DBT_TEST_USER_2,DBT_TEST_USER_3 +class TestModelGrantsTiDB(BaseModelGrants): + pass + + +@pytest.mark.skip( + reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." +) +class TestIncrementalGrantsTiDB(BaseIncrementalGrants): + pass + + +class TestSeedGrantsTiDB(BaseSeedGrants): + pass + + +@pytest.mark.skip( + reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." +) +class TestSnapshotGrantsTiDB(BaseSnapshotGrants): + pass + + +class TestInvalidGrantsTiDB(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "You are not allowed to create a user with GRANT" + + def privilege_does_not_exist_error(self): + return "You have an error in your SQL syntax; check the manual that corresponds to your TiDB version for the right syntax to use" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_bool_or.py b/tests/functional/adapter/tidb4_0/utils/fixture_bool_or.py new file mode 100644 index 0000000..c14848c --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_bool_or.py @@ -0,0 +1,34 @@ +# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql + +models__test_bool_or_sql = """ +with data as ( + select * from {{ ref('data_bool_or') }} +), +data_output as ( + select * from {{ ref('data_bool_or_expected') }} +), +calculate as ( + select + `key`, + {{ bool_or('val1 = val2') }} as value + from data + group by `key` +) +select + calculate.value as actual, + data_output.value as expected +from calculate +left join data_output +on calculate.key = data_output.key +""" + + +models__test_bool_or_yml = """ +version: 2 +models: + - name: test_bool_or + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_dateadd.py b/tests/functional/adapter/tidb4_0/utils/fixture_dateadd.py new file mode 100644 index 0000000..d2a4e44 --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_dateadd.py @@ -0,0 +1,31 @@ +# tidb does not support cast to timestamp, so cast to datatime + +models__test_dateadd_sql = """ +with data as ( + + select * from {{ ref('data_dateadd') }} + +) + +select + case + when datepart = 'hour' then cast({{ dateadd('hour', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'day' then cast({{ dateadd('day', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'month' then cast({{ dateadd('month', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'year' then cast({{ dateadd('year', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + else null + end as actual, + result as expected + +from data +""" + +models__test_dateadd_yml = """ +version: 2 +models: + - name: test_dateadd + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_datediff.py b/tests/functional/adapter/tidb4_0/utils/fixture_datediff.py new file mode 100644 index 0000000..5257c88 --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_datediff.py @@ -0,0 +1,68 @@ +# datediff in dbt-tidb will round down rather than round up + +# change 2019-12-31 00:00:00,2020-01-06 02:00:00,week,1 to 2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +# change 2019-12-31 00:00:00,2019-12-27 00:00:00,week,-1 to 2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 + +# change models__test_datediff_sql: +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); +seeds__data_datediff_csv = """first_date,second_date,datepart,result +2018-01-01 01:00:00,2018-01-02 01:00:00,day,1 +2018-01-01 01:00:00,2018-02-01 01:00:00,month,1 +2018-01-01 01:00:00,2019-01-01 01:00:00,year,1 +2018-01-01 01:00:00,2018-01-01 02:00:00,hour,1 +2018-01-01 01:00:00,2018-01-01 02:01:00,minute,61 +2018-01-01 01:00:00,2018-01-01 02:00:01,second,3601 +2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 +2019-12-31 00:00:00,2019-12-30 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-02 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +,2018-01-01 02:00:00,hour, +2018-01-01 02:00:00,,hour, +""" + + +models__test_datediff_sql = """ +with data as ( + +select + + case + when datepart = 'second' then {{ datediff('first_date', 'second_date', 'second') }} + when datepart = 'minute' then {{ datediff('first_date', 'second_date', 'minute') }} + when datepart = 'hour' then {{ datediff('first_date', 'second_date', 'hour') }} + when datepart = 'day' then {{ datediff('first_date', 'second_date', 'day') }} + when datepart = 'week' then {{ datediff('first_date', 'second_date', 'week') }} + when datepart = 'month' then {{ datediff('first_date', 'second_date', 'month') }} + when datepart = 'year' then {{ datediff('first_date', 'second_date', 'year') }} + else null + end as actual, + result as expected + +from {{ ref('data_datediff') }} + +-- Also test correct casting of literal values. +-- all the expected value except microsecond are changed from 1 to 0, and the test for millisecond is excluded +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 0 as expected +) +select * from data +""" + + +models__test_datediff_yml = """ +version: 2 +models: + - name: test_datediff + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_datetrunc.py b/tests/functional/adapter/tidb4_0/utils/fixture_datetrunc.py new file mode 100644 index 0000000..09521bb --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_datetrunc.py @@ -0,0 +1,38 @@ +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); + +seeds__data_date_trunc_csv = """updated_at,day,month +2018-01-05 12:00:00,2018-01-05,2018-01-01 +,, +""" + +models__test_date_trunc_sql = """ +with data as ( + +select + cast({{date_trunc('day', 'updated_at') }} as date) as actual, + day as expected + +from {{ ref('data_date_trunc') }} + +union all + +select + cast({{ date_trunc('month', 'updated_at') }} as date) as actual, + month as expected + +from {{ ref('data_date_trunc') }} + +) +select * from data +""" + +models__test_date_trunc_yml = """ +version: 2 +models: + - name: test_date_trunc + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_safe_cast.py b/tests/functional/adapter/tidb4_0/utils/fixture_safe_cast.py new file mode 100644 index 0000000..0c8af33 --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_safe_cast.py @@ -0,0 +1,27 @@ +seeds__data_safe_cast_csv = """field,output +abc,abc +123,123 +, +""" + +# tidb does not support cast to text, so cast to char +models__test_safe_cast_sql = """ +with data as ( + select * from {{ ref('data_safe_cast') }} +) +select + {{ safe_cast('field', api.Column.translate_type('char')) }} as actual, + output as expected +from data +""" + + +models__test_safe_cast_yml = """ +version: 2 +models: + - name: test_safe_cast + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/fixture_split_part.py b/tests/functional/adapter/tidb4_0/utils/fixture_split_part.py new file mode 100644 index 0000000..ae4cc0f --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/fixture_split_part.py @@ -0,0 +1,52 @@ +# split_part +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); + +seeds__data_split_part_csv = """parts,split_on,result_1,result_2,result_3 +a|b|c,|,a,b,c +1|2|3,|,1,2,3 +,|,,, +""" + + +models__test_split_part_sql = """ +with data as ( + +select + {{ split_part('parts', 'split_on', 1) }} as actual, + result_1 as expected + +from {{ ref('data_split_part') }} + +union all + +select + {{ split_part('parts', 'split_on', 2) }} as actual, + result_2 as expected + +from {{ ref('data_split_part') }} + +union all + +select + {{ split_part('parts', 'split_on', 3) }} as actual, + result_3 as expected + +from {{ ref('data_split_part') }} + +) + +select * from data + +""" + + +models__test_split_part_yml = """ +version: 2 +models: + - name: test_split_part + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb4_0/utils/test_util.py b/tests/functional/adapter/tidb4_0/utils/test_util.py new file mode 100644 index 0000000..49103bf --- /dev/null +++ b/tests/functional/adapter/tidb4_0/utils/test_util.py @@ -0,0 +1,195 @@ +import pytest +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesQuote, +) +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from tests.functional.adapter.tidb4_0.utils.fixture_bool_or import ( + models__test_bool_or_sql, + models__test_bool_or_yml, +) +from tests.functional.adapter.tidb4_0.utils.fixture_dateadd import ( + models__test_dateadd_yml, + models__test_dateadd_sql, +) +from tests.functional.adapter.tidb4_0.utils.fixture_datediff import ( + seeds__data_datediff_csv, + models__test_datediff_sql, + models__test_datediff_yml, +) +from tests.functional.adapter.tidb4_0.utils.fixture_datetrunc import ( + models__test_date_trunc_yml, + models__test_date_trunc_sql, +) +from tests.functional.adapter.tidb4_0.utils.fixture_safe_cast import ( + models__test_safe_cast_yml, + models__test_safe_cast_sql, +) +from tests.functional.adapter.tidb4_0.utils.fixture_split_part import ( + models__test_split_part_yml, + models__test_split_part_sql, +) + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestAnyValue(BaseAnyValue): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestBoolOr(BaseBoolOr): + @pytest.fixture(scope="class") + def models(self): + return { + "test_bool_or.yml": models__test_bool_or_yml, + "test_bool_or.sql": self.interpolate_macro_namespace( + models__test_bool_or_sql, "bool_or" + ), + } + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestCastBoolToText(BaseCastBoolToText): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestConcat(BaseConcat): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestDateAdd(BaseDateAdd): + @pytest.fixture(scope="class") + def models(self): + return { + "test_dateadd.yml": models__test_dateadd_yml, + "test_dateadd.sql": self.interpolate_macro_namespace( + models__test_dateadd_sql, "dateadd" + ), + } + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestDateDiff(BaseDateDiff): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_datediff.csv": seeds__data_datediff_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_datediff.yml": models__test_datediff_yml, + "test_datediff.sql": self.interpolate_macro_namespace( + models__test_datediff_sql, "datediff" + ), + } + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestDateTrunc(BaseDateTrunc): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date_trunc.yml": models__test_date_trunc_yml, + "test_date_trunc.sql": self.interpolate_macro_namespace( + models__test_date_trunc_sql, "date_trunc" + ), + } + + +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesQuote): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestExcept(BaseExcept): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestHash(BaseHash): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestIntersect(BaseIntersect): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestLastDay(BaseLastDay): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestLength(BaseLength): + pass + + +@pytest.mark.skip(reason="unsupport") +class TestListagg(BaseListagg): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestPosition(BasePosition): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestReplace(BaseReplace): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestRight(BaseRight): + pass + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestSafeCast(BaseSafeCast): + @pytest.fixture(scope="class") + def models(self): + return { + "test_safe_cast.yml": models__test_safe_cast_yml, + "test_safe_cast.sql": self.interpolate_macro_namespace( + self.interpolate_macro_namespace( + models__test_safe_cast_sql, "safe_cast" + ), + "type_string", + ), + } + + +@pytest.mark.skip(reason="need to rewrite test for TiDB4.x does not support CTE") +class TestSplitPart(BaseSplitPart): + @pytest.fixture(scope="class") + def models(self): + return { + "test_split_part.yml": models__test_split_part_yml, + "test_split_part.sql": self.interpolate_macro_namespace( + models__test_split_part_sql, "split_part" + ), + } + + +class TestStringLiteral(BaseStringLiteral): + pass diff --git a/tests/functional/adapter/tidb5_1/basic/test_tidb.py b/tests/functional/adapter/tidb5_1/basic/test_tidb.py index bff9146..5d71ed0 100644 --- a/tests/functional/adapter/tidb5_1/basic/test_tidb.py +++ b/tests/functional/adapter/tidb5_1/basic/test_tidb.py @@ -15,6 +15,9 @@ from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog +from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate +from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection from dbt.tests.adapter.incremental.test_incremental_unique_id import ( BaseIncrementalUniqueKey, ) @@ -72,6 +75,25 @@ def test_adapter_methods(self, project, equal_tables): check_relations_equal(project.adapter, equal_tables) +class TestValidateConnection(BaseValidateConnection): + pass + + +class TestDocsGenerate(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role=None, + id_type="int(11)", + text_type="text", + time_type="timestamp", + view_type="view", + table_type="table", + model_stats=no_stats(), + ) + + @pytest.mark.skip( reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." ) diff --git a/tests/functional/adapter/tidb5_1/grant/create_user.sql b/tests/functional/adapter/tidb5_1/grant/create_user.sql new file mode 100644 index 0000000..f6701cb --- /dev/null +++ b/tests/functional/adapter/tidb5_1/grant/create_user.sql @@ -0,0 +1,5 @@ +CREATE USER 'user1'@'%' IDENTIFIED BY ''; + +CREATE USER 'user2'@'%' IDENTIFIED BY ''; + +CREATE USER 'user3'@'%' IDENTIFIED BY ''; \ No newline at end of file diff --git a/tests/functional/adapter/tidb5_1/grant/test_grant.py b/tests/functional/adapter/tidb5_1/grant/test_grant.py new file mode 100644 index 0000000..5e74a65 --- /dev/null +++ b/tests/functional/adapter/tidb5_1/grant/test_grant.py @@ -0,0 +1,42 @@ +import pytest + +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + +# need to export DBT_TEST_USER_1,DBT_TEST_USER_2,DBT_TEST_USER_3 +class TestModelGrantsTiDB(BaseModelGrants): + pass + + +@pytest.mark.skip( + reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." +) +class TestIncrementalGrantsTiDB(BaseIncrementalGrants): + pass + + +class TestSeedGrantsTiDB(BaseSeedGrants): + pass + + +@pytest.mark.skip( + reason="TiDB 4.0 ~ 5.2 does not support creating a temporary table or view." +) +class TestSnapshotGrantsTiDB(BaseSnapshotGrants): + @pytest.fixture(scope="class") + def snapshots(self): + return { + "my_snapshot.sql": my_snapshot_sql, + "schema.yml": self.interpolate_name_overrides(snapshot_schema_yml), + } + + +class TestInvalidGrantsTiDB(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "You are not allowed to create a user with GRANT" + + def privilege_does_not_exist_error(self): + return "Illegal privilege level specified for" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py b/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py new file mode 100644 index 0000000..c14848c --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py @@ -0,0 +1,34 @@ +# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql + +models__test_bool_or_sql = """ +with data as ( + select * from {{ ref('data_bool_or') }} +), +data_output as ( + select * from {{ ref('data_bool_or_expected') }} +), +calculate as ( + select + `key`, + {{ bool_or('val1 = val2') }} as value + from data + group by `key` +) +select + calculate.value as actual, + data_output.value as expected +from calculate +left join data_output +on calculate.key = data_output.key +""" + + +models__test_bool_or_yml = """ +version: 2 +models: + - name: test_bool_or + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_dateadd.py b/tests/functional/adapter/tidb5_1/utils/fixture_dateadd.py new file mode 100644 index 0000000..d2a4e44 --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_dateadd.py @@ -0,0 +1,31 @@ +# tidb does not support cast to timestamp, so cast to datatime + +models__test_dateadd_sql = """ +with data as ( + + select * from {{ ref('data_dateadd') }} + +) + +select + case + when datepart = 'hour' then cast({{ dateadd('hour', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'day' then cast({{ dateadd('day', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'month' then cast({{ dateadd('month', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + when datepart = 'year' then cast({{ dateadd('year', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('DATETIME') }}) + else null + end as actual, + result as expected + +from data +""" + +models__test_dateadd_yml = """ +version: 2 +models: + - name: test_dateadd + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_datediff.py b/tests/functional/adapter/tidb5_1/utils/fixture_datediff.py new file mode 100644 index 0000000..5257c88 --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_datediff.py @@ -0,0 +1,68 @@ +# datediff in dbt-tidb will round down rather than round up + +# change 2019-12-31 00:00:00,2020-01-06 02:00:00,week,1 to 2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +# change 2019-12-31 00:00:00,2019-12-27 00:00:00,week,-1 to 2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 + +# change models__test_datediff_sql: +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); +seeds__data_datediff_csv = """first_date,second_date,datepart,result +2018-01-01 01:00:00,2018-01-02 01:00:00,day,1 +2018-01-01 01:00:00,2018-02-01 01:00:00,month,1 +2018-01-01 01:00:00,2019-01-01 01:00:00,year,1 +2018-01-01 01:00:00,2018-01-01 02:00:00,hour,1 +2018-01-01 01:00:00,2018-01-01 02:01:00,minute,61 +2018-01-01 01:00:00,2018-01-01 02:00:01,second,3601 +2019-12-31 00:00:00,2019-12-24 00:00:00,week,-1 +2019-12-31 00:00:00,2019-12-30 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-02 00:00:00,week,0 +2019-12-31 00:00:00,2020-01-07 02:00:00,week,1 +,2018-01-01 02:00:00,hour, +2018-01-01 02:00:00,,hour, +""" + + +models__test_datediff_sql = """ +with data as ( + +select + + case + when datepart = 'second' then {{ datediff('first_date', 'second_date', 'second') }} + when datepart = 'minute' then {{ datediff('first_date', 'second_date', 'minute') }} + when datepart = 'hour' then {{ datediff('first_date', 'second_date', 'hour') }} + when datepart = 'day' then {{ datediff('first_date', 'second_date', 'day') }} + when datepart = 'week' then {{ datediff('first_date', 'second_date', 'week') }} + when datepart = 'month' then {{ datediff('first_date', 'second_date', 'month') }} + when datepart = 'year' then {{ datediff('first_date', 'second_date', 'year') }} + else null + end as actual, + result as expected + +from {{ ref('data_datediff') }} + +-- Also test correct casting of literal values. +-- all the expected value except microsecond are changed from 1 to 0, and the test for millisecond is excluded +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 0 as expected +union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 0 as expected +) +select * from data +""" + + +models__test_datediff_yml = """ +version: 2 +models: + - name: test_datediff + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_datetrunc.py b/tests/functional/adapter/tidb5_1/utils/fixture_datetrunc.py new file mode 100644 index 0000000..09521bb --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_datetrunc.py @@ -0,0 +1,38 @@ +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); + +seeds__data_date_trunc_csv = """updated_at,day,month +2018-01-05 12:00:00,2018-01-05,2018-01-01 +,, +""" + +models__test_date_trunc_sql = """ +with data as ( + +select + cast({{date_trunc('day', 'updated_at') }} as date) as actual, + day as expected + +from {{ ref('data_date_trunc') }} + +union all + +select + cast({{ date_trunc('month', 'updated_at') }} as date) as actual, + month as expected + +from {{ ref('data_date_trunc') }} + +) +select * from data +""" + +models__test_date_trunc_yml = """ +version: 2 +models: + - name: test_date_trunc + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_safe_cast.py b/tests/functional/adapter/tidb5_1/utils/fixture_safe_cast.py new file mode 100644 index 0000000..0c8af33 --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_safe_cast.py @@ -0,0 +1,27 @@ +seeds__data_safe_cast_csv = """field,output +abc,abc +123,123 +, +""" + +# tidb does not support cast to text, so cast to char +models__test_safe_cast_sql = """ +with data as ( + select * from {{ ref('data_safe_cast') }} +) +select + {{ safe_cast('field', api.Column.translate_type('char')) }} as actual, + output as expected +from data +""" + + +models__test_safe_cast_yml = """ +version: 2 +models: + - name: test_safe_cast + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_split_part.py b/tests/functional/adapter/tidb5_1/utils/fixture_split_part.py new file mode 100644 index 0000000..ae4cc0f --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/fixture_split_part.py @@ -0,0 +1,52 @@ +# split_part +# source: create view `test_view` as (with data as ( select * from `test`) select * from data union all select 1 as id ); +# target: create view `test_view` as (with data as ( select * from `test` union all select 1 as id) select * from data ); + +seeds__data_split_part_csv = """parts,split_on,result_1,result_2,result_3 +a|b|c,|,a,b,c +1|2|3,|,1,2,3 +,|,,, +""" + + +models__test_split_part_sql = """ +with data as ( + +select + {{ split_part('parts', 'split_on', 1) }} as actual, + result_1 as expected + +from {{ ref('data_split_part') }} + +union all + +select + {{ split_part('parts', 'split_on', 2) }} as actual, + result_2 as expected + +from {{ ref('data_split_part') }} + +union all + +select + {{ split_part('parts', 'split_on', 3) }} as actual, + result_3 as expected + +from {{ ref('data_split_part') }} + +) + +select * from data + +""" + + +models__test_split_part_yml = """ +version: 2 +models: + - name: test_split_part + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/functional/adapter/tidb5_1/utils/test_util.py b/tests/functional/adapter/tidb5_1/utils/test_util.py new file mode 100644 index 0000000..bc1ae33 --- /dev/null +++ b/tests/functional/adapter/tidb5_1/utils/test_util.py @@ -0,0 +1,178 @@ +import pytest +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesQuote, +) +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from tests.functional.adapter.tidb5_1.utils.fixture_bool_or import ( + models__test_bool_or_sql, + models__test_bool_or_yml, +) +from tests.functional.adapter.tidb5_1.utils.fixture_dateadd import ( + models__test_dateadd_yml, + models__test_dateadd_sql, +) +from tests.functional.adapter.tidb5_1.utils.fixture_datediff import ( + seeds__data_datediff_csv, + models__test_datediff_sql, + models__test_datediff_yml, +) +from tests.functional.adapter.tidb5_1.utils.fixture_datetrunc import ( + models__test_date_trunc_sql, + models__test_date_trunc_yml, +) +from tests.functional.adapter.tidb5_1.utils.fixture_safe_cast import ( + models__test_safe_cast_yml, + models__test_safe_cast_sql, +) +from tests.functional.adapter.tidb5_1.utils.fixture_split_part import ( + models__test_split_part_yml, + models__test_split_part_sql, +) + + +class TestAnyValue(BaseAnyValue): + pass + + +class TestBoolOr(BaseBoolOr): + @pytest.fixture(scope="class") + def models(self): + return { + "test_bool_or.yml": models__test_bool_or_yml, + "test_bool_or.sql": self.interpolate_macro_namespace( + models__test_bool_or_sql, "bool_or" + ), + } + + +class TestCastBoolToText(BaseCastBoolToText): + pass + + +class TestConcat(BaseConcat): + pass + + +class TestDateAdd(BaseDateAdd): + @pytest.fixture(scope="class") + def models(self): + return { + "test_dateadd.yml": models__test_dateadd_yml, + "test_dateadd.sql": self.interpolate_macro_namespace( + models__test_dateadd_sql, "dateadd" + ), + } + + +class TestDateDiff(BaseDateDiff): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_datediff.csv": seeds__data_datediff_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_datediff.yml": models__test_datediff_yml, + "test_datediff.sql": self.interpolate_macro_namespace( + models__test_datediff_sql, "datediff" + ), + } + + +class TestDateTrunc(BaseDateTrunc): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date_trunc.yml": models__test_date_trunc_yml, + "test_date_trunc.sql": self.interpolate_macro_namespace( + models__test_date_trunc_sql, "date_trunc" + ), + } + + +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesQuote): + pass + + +class TestExcept(BaseExcept): + pass + + +class TestHash(BaseHash): + pass + + +class TestIntersect(BaseIntersect): + pass + + +class TestLastDay(BaseLastDay): + pass + + +class TestLength(BaseLength): + pass + + +@pytest.mark.skip(reason="unsupport") +class TestListagg(BaseListagg): + pass + + +class TestPosition(BasePosition): + pass + + +class TestReplace(BaseReplace): + pass + + +class TestRight(BaseRight): + pass + + +class TestSafeCast(BaseSafeCast): + @pytest.fixture(scope="class") + def models(self): + return { + "test_safe_cast.yml": models__test_safe_cast_yml, + "test_safe_cast.sql": self.interpolate_macro_namespace( + self.interpolate_macro_namespace( + models__test_safe_cast_sql, "safe_cast" + ), + "type_string", + ), + } + + +class TestSplitPart(BaseSplitPart): + @pytest.fixture(scope="class") + def models(self): + return { + "test_split_part.yml": models__test_split_part_yml, + "test_split_part.sql": self.interpolate_macro_namespace( + models__test_split_part_sql, "split_part" + ), + } + + +class TestStringLiteral(BaseStringLiteral): + pass