From 4e5d58a562ec02651fc147c9c7877825c18ba1c5 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Fri, 25 Nov 2022 10:55:10 +0800 Subject: [PATCH 1/6] Support dbt v1.3 --- .github/workflows/main.yml | 1 + dbt/adapters/tidb/__version__.py | 2 +- dbt/adapters/tidb/impl.py | 41 +++++++++++++++++- dbt/include/tidb/macros/adapters.sql | 4 -- .../materializations/incremental/helpers.sql | 30 +++++++++++++ .../incremental/incremental.sql | 14 +++--- .../materializations/snapshot/snapshot.sql | 5 --- dbt/include/tidb/macros/utils/timestamps.sql | 12 ++++++ requirements_dev.txt | 4 +- setup.py | 4 +- tests/README.md | 16 +++++++ .../adapter/tidb/basic/test_tidb.py | 7 ++- .../tidb/concurrency/test_concurrency.py | 25 +++++++++++ .../adapter/tidb/ephemeral/test_ephemeral.py | 15 +++++++ .../tidb/incremental/test_incremental.py | 7 +++ .../tidb/utils/data_types/test_data_types.py | 43 +++++++++++++++++++ .../adapter/tidb/utils/fixture_bool_or.py | 34 --------------- .../adapter/tidb/utils/set_timezone.sql | 1 + .../adapter/tidb/utils/test_util.py | 28 ++++++------ 19 files changed, 218 insertions(+), 75 deletions(-) create mode 100644 dbt/include/tidb/macros/utils/timestamps.sql create mode 100644 tests/functional/adapter/tidb/concurrency/test_concurrency.py create mode 100644 tests/functional/adapter/tidb/ephemeral/test_ephemeral.py create mode 100644 tests/functional/adapter/tidb/incremental/test_incremental.py create mode 100644 tests/functional/adapter/tidb/utils/data_types/test_data_types.py delete mode 100644 tests/functional/adapter/tidb/utils/fixture_bool_or.py create mode 100644 tests/functional/adapter/tidb/utils/set_timezone.sql diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0528b1a..9340d24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,6 +36,7 @@ jobs: - name: Run tests run: | mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/utils/set_timezone.sql export DBT_TEST_USER_1=user1 export DBT_TEST_USER_2=user2 export DBT_TEST_USER_3=user3 diff --git a/dbt/adapters/tidb/__version__.py b/dbt/adapters/tidb/__version__.py index 11a716e..d28b3dd 100644 --- a/dbt/adapters/tidb/__version__.py +++ b/dbt/adapters/tidb/__version__.py @@ -1 +1 @@ -version = "1.0.0" +version = "1.3.0" diff --git a/dbt/adapters/tidb/impl.py b/dbt/adapters/tidb/impl.py index 3f8cb52..6104f02 100644 --- a/dbt/adapters/tidb/impl.py +++ b/dbt/adapters/tidb/impl.py @@ -11,7 +11,7 @@ from dbt.adapters.tidb import TiDBConnectionManager from dbt.adapters.tidb import TiDBRelation from dbt.adapters.tidb import TiDBColumn -from dbt.adapters.base import BaseRelation +from dbt.adapters.base import BaseRelation, available from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -35,6 +35,11 @@ def date_function(cls): def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" + @classmethod + def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] + return "float" if decimals else "integer" + def quote(self, identifier): return "`{}`".format(identifier) @@ -277,3 +282,37 @@ def get_rows_different_sql( ) return sql + + def valid_incremental_strategies(self): + """The set of standard builtin strategies which this adapter supports out-of-the-box. + Not used to validate custom strategies defined by end users. + """ + return ["delete+insert", "append"] + + @available.parse_none + def get_incremental_strategy_macro(self, model_context, strategy: str): + # Construct macro_name from strategy name + if strategy is None: + strategy = "default" + + # validate strategies for this adapter + valid_strategies = self.valid_incremental_strategies() + valid_strategies.append("default") + builtin_strategies = self.builtin_incremental_strategies() + if strategy in builtin_strategies and strategy not in valid_strategies: + raise dbt.RuntimeException( + f"The incremental strategy '{strategy}' is not valid for this adapter" + ) + + strategy = strategy.replace("+", "_") + macro_name = f"get_incremental_{strategy}_sql" + # The model_context should have MacroGenerator callable objects for all macros + if macro_name not in model_context: + raise dbt.RuntimeException( + 'dbt could not find an incremental strategy macro with the name "{}" in {}'.format( + macro_name, self.config.project_name + ) + ) + + # This returns a callable macro + return model_context[macro_name] diff --git a/dbt/include/tidb/macros/adapters.sql b/dbt/include/tidb/macros/adapters.sql index d30e632..a823516 100644 --- a/dbt/include/tidb/macros/adapters.sql +++ b/dbt/include/tidb/macros/adapters.sql @@ -78,10 +78,6 @@ {% endmacro %} -{% macro tidb__current_timestamp() -%} - current_timestamp() -{%- endmacro %} - {% macro tidb__rename_relation(from_relation, to_relation) -%} {# tidb rename fails when the relation already exists, so a 2-step process is needed: diff --git a/dbt/include/tidb/macros/materializations/incremental/helpers.sql b/dbt/include/tidb/macros/materializations/incremental/helpers.sql index ad82e9a..b399dd9 100644 --- a/dbt/include/tidb/macros/materializations/incremental/helpers.sql +++ b/dbt/include/tidb/macros/materializations/incremental/helpers.sql @@ -1,3 +1,33 @@ +{% macro tidb__get_incremental_default_sql(arg_dict) %} + {% if arg_dict["unique_key"] %} + {% do return(get_incremental_delete_insert_sql(arg_dict)) %} + {% else %} + {% do return(get_incremental_append_sql(arg_dict)) %} + {% endif %} +{% endmacro %} + +{% macro get_incremental_delete_insert_sql(arg_dict) %} + {% set target_relation = arg_dict["target_relation"] %} + {% set tmp_relation = arg_dict["temp_relation"] %} + {% set unique_key = arg_dict["unique_key"] %} + {% set dest_columns = arg_dict["dest_columns"] %} + -- use get_delete_insert_merge_sql after support multi sql + -- we will delete then insert now + {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %} + {% call statement("pre_main") %} + {{ build_sql }} + {% endcall %} + {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %} +{% endmacro %} + +{% macro get_incremental_append_sql(arg_dict) %} + {% set target_relation = arg_dict["target_relation"] %} + {% set tmp_relation = arg_dict["temp_relation"] %} + {% set unique_key = arg_dict["unique_key"] %} + {% set dest_columns = arg_dict["dest_columns"] %} + {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %} +{% endmacro %} + -- need to support unique_key is sequence {% macro incremental_delete(target, source, unique_key, dest_columns) %} diff --git a/dbt/include/tidb/macros/materializations/incremental/incremental.sql b/dbt/include/tidb/macros/materializations/incremental/incremental.sql index 4d62c61..eb148db 100644 --- a/dbt/include/tidb/macros/materializations/incremental/incremental.sql +++ b/dbt/include/tidb/macros/materializations/incremental/incremental.sql @@ -1,4 +1,3 @@ - {% materialization incremental, adapter='tidb' %} {% set unique_key = config.get('unique_key') %} @@ -34,13 +33,12 @@ from_relation=tmp_relation, to_relation=target_relation) %} {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} - -- use get_delete_insert_merge_sql after support multi sql - -- we will delete then insert now - {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %} - {% call statement("pre_main") %} - {{ build_sql }} - {% endcall %} - {% set build_sql = incremental_insert(target_relation, tmp_relation, unique_key, dest_columns) %} + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set incremental_predicates = config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %} + {% set build_sql = get_incremental_delete_insert_sql(strategy_arg_dict) %} {% endif %} {% call statement("main") %} diff --git a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql index 34c586e..0f9d5e8 100644 --- a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql @@ -1,9 +1,4 @@ -{% macro tidb__snapshot_string_as_time(timestamp) -%} - {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} - {{ return(result) }} -{%- endmacro %} - {% materialization snapshot, adapter='tidb' %} {%- set config = model['config'] -%} diff --git a/dbt/include/tidb/macros/utils/timestamps.sql b/dbt/include/tidb/macros/utils/timestamps.sql new file mode 100644 index 0000000..2d8a0f3 --- /dev/null +++ b/dbt/include/tidb/macros/utils/timestamps.sql @@ -0,0 +1,12 @@ +{% macro tidb__current_timestamp() -%} + current_timestamp() +{%- endmacro %} + +{% macro tidb__snapshot_string_as_time(timestamp) -%} + {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} + {{ return(result) }} +{%- endmacro %} + +{% macro tidb__current_timestamp_backcompat() -%} + current_timestamp() +{%- endmacro %} \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 9e87633..edafd16 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ -dbt-core~=1.2.0 +dbt-core~=1.3.1 mysql-connector-python>=8.0.0,<8.1 pytest~=7.0 markupsafe==2.0.1 -dbt-tests-adapter~=1.2.0 +dbt-tests-adapter~=1.3.1 pytest-dotenv \ No newline at end of file diff --git a/setup.py b/setup.py index 01e4a30..6e1083c 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,8 @@ long_description = f.read() package_name = "dbt-tidb" -package_version = "1.2.0" -dbt_core_version = "1.2.0" +package_version = "1.3.0" +dbt_core_version = "1.3.1" description = """The TiDB adapter plugin for dbt""" setup( diff --git a/tests/README.md b/tests/README.md index 052752a..2657261 100644 --- a/tests/README.md +++ b/tests/README.md @@ -49,6 +49,16 @@ You can also install tidb with [TiUP playground](https://docs.pingcap.com/tidb/s tiup playground ${version} ``` +## Set TiDB variables +TiDB use `SYSTEM` as the default value of `time_zone`. For more information, see [TiDB Time Zone Support](https://docs.pingcap.com/tidb/stable/configure-time-zone). + +To pass the `CurrentTimestamp` tests, you should set `time_zone` to `UTC` first. + +```sql +mysql -u -h -P -p +mysql> set @@global.time_zone='UTC'; +``` + ## Use pytest to test If you specify a package, all Python files under the package will be tested. Don't forget to configure PYTHONPATH: @@ -57,6 +67,12 @@ If you specify a package, all Python files under the package will be tested. Don PYTHONPATH=. pytest tests/functional/adapter/tidb/basic # utils PYTHONPATH=. pytest tests/functional/adapter/tidb/utils +# concurrentcy +PYTHONPATH=. pytest tests/functional/adapter/tidb/concurrency +# ephemeral +PYTHONPATH=. pytest tests/functional/adapter/tidb/ephemeral +# incremental +PYTHONPATH=. pytest tests/functional/adapter/tidb/incremental ``` ## Test grant diff --git a/tests/functional/adapter/tidb/basic/test_tidb.py b/tests/functional/adapter/tidb/basic/test_tidb.py index 8938749..c316c69 100644 --- a/tests/functional/adapter/tidb/basic/test_tidb.py +++ b/tests/functional/adapter/tidb/basic/test_tidb.py @@ -15,10 +15,9 @@ from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog +from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange + from dbt.tests.util import run_dbt, check_relations_equal -from dbt.tests.adapter.incremental.test_incremental_unique_id import ( - BaseIncrementalUniqueKey, -) class TestEmptyMyAdapter(BaseEmpty): @@ -83,5 +82,5 @@ def expected_catalog(self, project): ) -class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): +class TestIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): pass diff --git a/tests/functional/adapter/tidb/concurrency/test_concurrency.py b/tests/functional/adapter/tidb/concurrency/test_concurrency.py new file mode 100644 index 0000000..175b92b --- /dev/null +++ b/tests/functional/adapter/tidb/concurrency/test_concurrency.py @@ -0,0 +1,25 @@ +import pytest + +from dbt.tests.util import ( + run_dbt, + check_relations_equal, + rm_file, + write_file + ) +from dbt.tests.adapter.concurrency.test_concurrency import ( + BaseConcurrency, + seeds__update_csv + ) + +class TestConcurrencyTiDB(BaseConcurrency): + def test_conncurrency_tidb(self, project): + run_dbt(["seed", "--select", "seed"]) + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) + + rm_file(project.project_root, "seeds", "seed.csv") + write_file(seeds__update_csv, project.project_root + '/seeds', "seed.csv") + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) diff --git a/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py b/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py new file mode 100644 index 0000000..994c65f --- /dev/null +++ b/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py @@ -0,0 +1,15 @@ +import pytest + +from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti + + +class TestEphemeralMultiTiDB(BaseEphemeralMulti): + def test_ephemeral_multi_snowflake(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + check_relations_equal(project.adapter, ["seed", "dependent"]) + # TiDB does not support double dependent + # check_relations_equal(project.adapter, ["seed", "double_dependent"]) + check_relations_equal(project.adapter, ["seed", "super_dependent"]) diff --git a/tests/functional/adapter/tidb/incremental/test_incremental.py b/tests/functional/adapter/tidb/incremental/test_incremental.py new file mode 100644 index 0000000..4aca118 --- /dev/null +++ b/tests/functional/adapter/tidb/incremental/test_incremental.py @@ -0,0 +1,7 @@ +import pytest + +from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey + + +class TestIncrementalUniqueKeyTiDB(BaseIncrementalUniqueKey): + pass diff --git a/tests/functional/adapter/tidb/utils/data_types/test_data_types.py b/tests/functional/adapter/tidb/utils/data_types/test_data_types.py new file mode 100644 index 0000000..8070cac --- /dev/null +++ b/tests/functional/adapter/tidb/utils/data_types/test_data_types.py @@ -0,0 +1,43 @@ +import pytest + +from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp + + +@pytest.mark.skip(reason="TiDB does not support cast as int") +class TestTypeIntTiDB(BaseTypeInt): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as bigint") +class TestTypeBigIntTiDB(BaseTypeBigInt): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as boolean") +class TestTypeBooleanTiDB(BaseTypeBoolean): + pass + + +@pytest.mark.skip(reason="TiDB does not support numeric type") +class TestTypeNumericTiDb(BaseTypeNumeric): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as text") +class TestTypeStringTiDB(BaseTypeString): + pass + + +class TestTypeFloatTiDB(BaseTypeFloat): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as timestamp") +class TestTypeTimestampTiDB(BaseTypeTimestamp): + pass diff --git a/tests/functional/adapter/tidb/utils/fixture_bool_or.py b/tests/functional/adapter/tidb/utils/fixture_bool_or.py deleted file mode 100644 index c14848c..0000000 --- a/tests/functional/adapter/tidb/utils/fixture_bool_or.py +++ /dev/null @@ -1,34 +0,0 @@ -# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql - -models__test_bool_or_sql = """ -with data as ( - select * from {{ ref('data_bool_or') }} -), -data_output as ( - select * from {{ ref('data_bool_or_expected') }} -), -calculate as ( - select - `key`, - {{ bool_or('val1 = val2') }} as value - from data - group by `key` -) -select - calculate.value as actual, - data_output.value as expected -from calculate -left join data_output -on calculate.key = data_output.key -""" - - -models__test_bool_or_yml = """ -version: 2 -models: - - name: test_bool_or - tests: - - assert_equal: - actual: actual - expected: expected -""" diff --git a/tests/functional/adapter/tidb/utils/set_timezone.sql b/tests/functional/adapter/tidb/utils/set_timezone.sql new file mode 100644 index 0000000..a28490f --- /dev/null +++ b/tests/functional/adapter/tidb/utils/set_timezone.sql @@ -0,0 +1 @@ +set @@global.time_zone='UTC'; \ No newline at end of file diff --git a/tests/functional/adapter/tidb/utils/test_util.py b/tests/functional/adapter/tidb/utils/test_util.py index fa1e450..107a598 100644 --- a/tests/functional/adapter/tidb/utils/test_util.py +++ b/tests/functional/adapter/tidb/utils/test_util.py @@ -1,4 +1,5 @@ import pytest + from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText @@ -6,9 +7,7 @@ from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc -from dbt.tests.adapter.utils.test_escape_single_quotes import ( - BaseEscapeSingleQuotesQuote, -) +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote from dbt.tests.adapter.utils.test_except import BaseExcept from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect @@ -21,9 +20,9 @@ from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral from dbt.tests.adapter.utils.test_listagg import BaseListagg -from tests.functional.adapter.tidb.utils.fixture_bool_or import ( - models__test_bool_or_sql, - models__test_bool_or_yml, +from dbt.tests.adapter.utils.test_current_timestamp import ( + BaseCurrentTimestamp, + BaseCurrentTimestampNaive, ) from tests.functional.adapter.tidb.utils.fixture_dateadd import ( models__test_dateadd_yml, @@ -45,14 +44,7 @@ class TestAnyValue(BaseAnyValue): class TestBoolOr(BaseBoolOr): - @pytest.fixture(scope="class") - def models(self): - return { - "test_bool_or.yml": models__test_bool_or_yml, - "test_bool_or.sql": self.interpolate_macro_namespace( - models__test_bool_or_sql, "bool_or" - ), - } + pass class TestCastBoolToText(BaseCastBoolToText): @@ -154,3 +146,11 @@ class TestSplitPart(BaseSplitPart): class TestStringLiteral(BaseStringLiteral): pass + + +class TestCurrentTimestampTiDB(BaseCurrentTimestamp): + pass + + +class TestCurrentTimestampNaiveTiDB(BaseCurrentTimestampNaive): + pass From 309411c157d296665493ce4fdc870712fa78b295 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Fri, 25 Nov 2022 11:15:54 +0800 Subject: [PATCH 2/6] fmt --- .../tidb/concurrency/test_concurrency.py | 22 +++++++++---------- .../tidb/incremental/test_incremental.py | 4 +++- .../adapter/tidb/utils/test_util.py | 4 +++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/functional/adapter/tidb/concurrency/test_concurrency.py b/tests/functional/adapter/tidb/concurrency/test_concurrency.py index 175b92b..5f67958 100644 --- a/tests/functional/adapter/tidb/concurrency/test_concurrency.py +++ b/tests/functional/adapter/tidb/concurrency/test_concurrency.py @@ -1,25 +1,25 @@ import pytest -from dbt.tests.util import ( - run_dbt, - check_relations_equal, - rm_file, - write_file - ) +from dbt.tests.util import run_dbt, check_relations_equal, rm_file, write_file from dbt.tests.adapter.concurrency.test_concurrency import ( BaseConcurrency, - seeds__update_csv - ) + seeds__update_csv, +) + class TestConcurrencyTiDB(BaseConcurrency): def test_conncurrency_tidb(self, project): run_dbt(["seed", "--select", "seed"]) results = run_dbt(["run"], expect_pass=False) assert len(results) == 7 - check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) + check_relations_equal( + project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"] + ) rm_file(project.project_root, "seeds", "seed.csv") - write_file(seeds__update_csv, project.project_root + '/seeds', "seed.csv") + write_file(seeds__update_csv, project.project_root + "/seeds", "seed.csv") results = run_dbt(["run"], expect_pass=False) assert len(results) == 7 - check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]) + check_relations_equal( + project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"] + ) diff --git a/tests/functional/adapter/tidb/incremental/test_incremental.py b/tests/functional/adapter/tidb/incremental/test_incremental.py index 4aca118..69b0107 100644 --- a/tests/functional/adapter/tidb/incremental/test_incremental.py +++ b/tests/functional/adapter/tidb/incremental/test_incremental.py @@ -1,6 +1,8 @@ import pytest -from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +from dbt.tests.adapter.incremental.test_incremental_unique_id import ( + BaseIncrementalUniqueKey, +) class TestIncrementalUniqueKeyTiDB(BaseIncrementalUniqueKey): diff --git a/tests/functional/adapter/tidb/utils/test_util.py b/tests/functional/adapter/tidb/utils/test_util.py index 107a598..28f75b4 100644 --- a/tests/functional/adapter/tidb/utils/test_util.py +++ b/tests/functional/adapter/tidb/utils/test_util.py @@ -7,7 +7,9 @@ from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc -from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesQuote, +) from dbt.tests.adapter.utils.test_except import BaseExcept from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect From b03dead1e27eedd738df6b3f7dfe3cf18ca765a4 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Fri, 25 Nov 2022 11:24:02 +0800 Subject: [PATCH 3/6] fix test --- .github/workflows/main.yml | 37 ++++++++++++++++++- .../adapter/tidb5_1/utils/fixture_bool_or.py | 34 ----------------- .../adapter/tidb5_1/utils/test_util.py | 13 +------ 3 files changed, 37 insertions(+), 47 deletions(-) delete mode 100644 tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9340d24..0c67e2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,12 +36,47 @@ jobs: - name: Run tests run: | mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql - mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/utils/set_timezone.sql export DBT_TEST_USER_1=user1 export DBT_TEST_USER_2=user2 export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb + tidb_6_1: + name: Python ${{ matrix.python-version }} | TiDB 6.1 + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ '3.8', '3.9', '3.10' ] + + services: + tidb_nightly: + image: pingcap/tidb:v6.1.0 + ports: + - 4000:4000 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install requirements + run: | + pip install -r requirements_dev.txt + + - name: Run tests + run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 + PYTHONPATH=. pytest tests/functional/adapter/tidb + + tidb_5_3: name: Python ${{ matrix.python-version }} | TiDB 5.3 runs-on: ubuntu-latest diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py b/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py deleted file mode 100644 index c14848c..0000000 --- a/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py +++ /dev/null @@ -1,34 +0,0 @@ -# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql - -models__test_bool_or_sql = """ -with data as ( - select * from {{ ref('data_bool_or') }} -), -data_output as ( - select * from {{ ref('data_bool_or_expected') }} -), -calculate as ( - select - `key`, - {{ bool_or('val1 = val2') }} as value - from data - group by `key` -) -select - calculate.value as actual, - data_output.value as expected -from calculate -left join data_output -on calculate.key = data_output.key -""" - - -models__test_bool_or_yml = """ -version: 2 -models: - - name: test_bool_or - tests: - - assert_equal: - actual: actual - expected: expected -""" diff --git a/tests/functional/adapter/tidb5_1/utils/test_util.py b/tests/functional/adapter/tidb5_1/utils/test_util.py index bc1ae33..f68b6f5 100644 --- a/tests/functional/adapter/tidb5_1/utils/test_util.py +++ b/tests/functional/adapter/tidb5_1/utils/test_util.py @@ -21,10 +21,6 @@ from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral from dbt.tests.adapter.utils.test_listagg import BaseListagg -from tests.functional.adapter.tidb5_1.utils.fixture_bool_or import ( - models__test_bool_or_sql, - models__test_bool_or_yml, -) from tests.functional.adapter.tidb5_1.utils.fixture_dateadd import ( models__test_dateadd_yml, models__test_dateadd_sql, @@ -53,14 +49,7 @@ class TestAnyValue(BaseAnyValue): class TestBoolOr(BaseBoolOr): - @pytest.fixture(scope="class") - def models(self): - return { - "test_bool_or.yml": models__test_bool_or_yml, - "test_bool_or.sql": self.interpolate_macro_namespace( - models__test_bool_or_sql, "bool_or" - ), - } + pass class TestCastBoolToText(BaseCastBoolToText): From 437466dab8c6788a33f642e3ba7e551ad5b9ee37 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Fri, 25 Nov 2022 11:35:09 +0800 Subject: [PATCH 4/6] Use tidb v6.1.2 image --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c67e2b..45b10ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,7 +51,7 @@ jobs: services: tidb_nightly: - image: pingcap/tidb:v6.1.0 + image: pingcap/tidb:v6.1.2 ports: - 4000:4000 From 9d0c983788b561c97e816797692117c138cb2e60 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Tue, 29 Nov 2022 20:53:45 +0800 Subject: [PATCH 5/6] Update impl.py --- dbt/adapters/tidb/impl.py | 54 +++++++++---------- .../incremental/incremental.sql | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/dbt/adapters/tidb/impl.py b/dbt/adapters/tidb/impl.py index 6104f02..1452159 100644 --- a/dbt/adapters/tidb/impl.py +++ b/dbt/adapters/tidb/impl.py @@ -289,30 +289,30 @@ def valid_incremental_strategies(self): """ return ["delete+insert", "append"] - @available.parse_none - def get_incremental_strategy_macro(self, model_context, strategy: str): - # Construct macro_name from strategy name - if strategy is None: - strategy = "default" - - # validate strategies for this adapter - valid_strategies = self.valid_incremental_strategies() - valid_strategies.append("default") - builtin_strategies = self.builtin_incremental_strategies() - if strategy in builtin_strategies and strategy not in valid_strategies: - raise dbt.RuntimeException( - f"The incremental strategy '{strategy}' is not valid for this adapter" - ) - - strategy = strategy.replace("+", "_") - macro_name = f"get_incremental_{strategy}_sql" - # The model_context should have MacroGenerator callable objects for all macros - if macro_name not in model_context: - raise dbt.RuntimeException( - 'dbt could not find an incremental strategy macro with the name "{}" in {}'.format( - macro_name, self.config.project_name - ) - ) - - # This returns a callable macro - return model_context[macro_name] + # @available.parse_none + # def get_incremental_strategy_macro(self, model_context, strategy: str): + # # Construct macro_name from strategy name + # if strategy is None: + # strategy = "default" + # + # # validate strategies for this adapter + # valid_strategies = self.valid_incremental_strategies() + # valid_strategies.append("default") + # builtin_strategies = self.builtin_incremental_strategies() + # if strategy in builtin_strategies and strategy not in valid_strategies: + # raise dbt.RuntimeException( + # f"The incremental strategy '{strategy}' is not valid for this adapter" + # ) + # + # strategy = strategy.replace("+", "_") + # macro_name = f"get_incremental_{strategy}_sql" + # # The model_context should have MacroGenerator callable objects for all macros + # if macro_name not in model_context: + # raise dbt.RuntimeException( + # 'dbt could not find an incremental strategy macro with the name "{}" in {}'.format( + # macro_name, self.config.project_name + # ) + # ) + # + # # This returns a callable macro + # return model_context[macro_name] diff --git a/dbt/include/tidb/macros/materializations/incremental/incremental.sql b/dbt/include/tidb/macros/materializations/incremental/incremental.sql index eb148db..f0a6caa 100644 --- a/dbt/include/tidb/macros/materializations/incremental/incremental.sql +++ b/dbt/include/tidb/macros/materializations/incremental/incremental.sql @@ -38,7 +38,7 @@ {% set incremental_predicates = config.get('incremental_predicates', none) %} {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %} - {% set build_sql = get_incremental_delete_insert_sql(strategy_arg_dict) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} {% endif %} {% call statement("main") %} From 92f9676bde9d7ee961211ff6caecfbf1cb174c16 Mon Sep 17 00:00:00 2001 From: Daemonxiao <735462752@qq.com> Date: Thu, 1 Dec 2022 14:03:56 +0800 Subject: [PATCH 6/6] Update impl.py --- dbt/adapters/tidb/impl.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/dbt/adapters/tidb/impl.py b/dbt/adapters/tidb/impl.py index 1452159..afe2eeb 100644 --- a/dbt/adapters/tidb/impl.py +++ b/dbt/adapters/tidb/impl.py @@ -287,32 +287,4 @@ def valid_incremental_strategies(self): """The set of standard builtin strategies which this adapter supports out-of-the-box. Not used to validate custom strategies defined by end users. """ - return ["delete+insert", "append"] - - # @available.parse_none - # def get_incremental_strategy_macro(self, model_context, strategy: str): - # # Construct macro_name from strategy name - # if strategy is None: - # strategy = "default" - # - # # validate strategies for this adapter - # valid_strategies = self.valid_incremental_strategies() - # valid_strategies.append("default") - # builtin_strategies = self.builtin_incremental_strategies() - # if strategy in builtin_strategies and strategy not in valid_strategies: - # raise dbt.RuntimeException( - # f"The incremental strategy '{strategy}' is not valid for this adapter" - # ) - # - # strategy = strategy.replace("+", "_") - # macro_name = f"get_incremental_{strategy}_sql" - # # The model_context should have MacroGenerator callable objects for all macros - # if macro_name not in model_context: - # raise dbt.RuntimeException( - # 'dbt could not find an incremental strategy macro with the name "{}" in {}'.format( - # macro_name, self.config.project_name - # ) - # ) - # - # # This returns a callable macro - # return model_context[macro_name] + return ["delete+insert"]