diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0528b1a..45b10ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,6 +41,42 @@ jobs: export DBT_TEST_USER_3=user3 PYTHONPATH=. pytest tests/functional/adapter/tidb + tidb_6_1: + name: Python ${{ matrix.python-version }} | TiDB 6.1 + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ '3.8', '3.9', '3.10' ] + + services: + tidb_nightly: + image: pingcap/tidb:v6.1.2 + ports: + - 4000:4000 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install requirements + run: | + pip install -r requirements_dev.txt + + - name: Run tests + run: | + mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql + export DBT_TEST_USER_1=user1 + export DBT_TEST_USER_2=user2 + export DBT_TEST_USER_3=user3 + PYTHONPATH=. pytest tests/functional/adapter/tidb + + tidb_5_3: name: Python ${{ matrix.python-version }} | TiDB 5.3 runs-on: ubuntu-latest diff --git a/dbt/adapters/tidb/__version__.py b/dbt/adapters/tidb/__version__.py index 11a716e..d28b3dd 100644 --- a/dbt/adapters/tidb/__version__.py +++ b/dbt/adapters/tidb/__version__.py @@ -1 +1 @@ -version = "1.0.0" +version = "1.3.0" diff --git a/dbt/adapters/tidb/impl.py b/dbt/adapters/tidb/impl.py index 3f8cb52..afe2eeb 100644 --- a/dbt/adapters/tidb/impl.py +++ b/dbt/adapters/tidb/impl.py @@ -11,7 +11,7 @@ from dbt.adapters.tidb import TiDBConnectionManager from dbt.adapters.tidb import TiDBRelation from dbt.adapters.tidb import TiDBColumn -from dbt.adapters.base import BaseRelation +from dbt.adapters.base import BaseRelation, available from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER from dbt.events import AdapterLogger from dbt.utils import executor @@ -35,6 +35,11 @@ def date_function(cls): def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "timestamp" + @classmethod + def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] + return "float" if decimals else "integer" + def quote(self, identifier): return "`{}`".format(identifier) @@ -277,3 +282,9 @@ def get_rows_different_sql( ) return sql + + def valid_incremental_strategies(self): + """The set of standard builtin strategies which this adapter supports out-of-the-box. + Not used to validate custom strategies defined by end users. + """ + return ["delete+insert"] diff --git a/dbt/include/tidb/macros/adapters.sql b/dbt/include/tidb/macros/adapters.sql index d30e632..a823516 100644 --- a/dbt/include/tidb/macros/adapters.sql +++ b/dbt/include/tidb/macros/adapters.sql @@ -78,10 +78,6 @@ {% endmacro %} -{% macro tidb__current_timestamp() -%} - current_timestamp() -{%- endmacro %} - {% macro tidb__rename_relation(from_relation, to_relation) -%} {# tidb rename fails when the relation already exists, so a 2-step process is needed: diff --git a/dbt/include/tidb/macros/materializations/incremental/helpers.sql b/dbt/include/tidb/macros/materializations/incremental/helpers.sql index ad82e9a..b399dd9 100644 --- a/dbt/include/tidb/macros/materializations/incremental/helpers.sql +++ b/dbt/include/tidb/macros/materializations/incremental/helpers.sql @@ -1,3 +1,33 @@ +{% macro tidb__get_incremental_default_sql(arg_dict) %} + {% if arg_dict["unique_key"] %} + {% do return(get_incremental_delete_insert_sql(arg_dict)) %} + {% else %} + {% do return(get_incremental_append_sql(arg_dict)) %} + {% endif %} +{% endmacro %} + +{% macro get_incremental_delete_insert_sql(arg_dict) %} + {% set target_relation = arg_dict["target_relation"] %} + {% set tmp_relation = arg_dict["temp_relation"] %} + {% set unique_key = arg_dict["unique_key"] %} + {% set dest_columns = arg_dict["dest_columns"] %} + -- use get_delete_insert_merge_sql after support multi sql + -- we will delete then insert now + {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %} + {% call statement("pre_main") %} + {{ build_sql }} + {% endcall %} + {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %} +{% endmacro %} + +{% macro get_incremental_append_sql(arg_dict) %} + {% set target_relation = arg_dict["target_relation"] %} + {% set tmp_relation = arg_dict["temp_relation"] %} + {% set unique_key = arg_dict["unique_key"] %} + {% set dest_columns = arg_dict["dest_columns"] %} + {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %} +{% endmacro %} + -- need to support unique_key is sequence {% macro incremental_delete(target, source, unique_key, dest_columns) %} diff --git a/dbt/include/tidb/macros/materializations/incremental/incremental.sql b/dbt/include/tidb/macros/materializations/incremental/incremental.sql index 4d62c61..f0a6caa 100644 --- a/dbt/include/tidb/macros/materializations/incremental/incremental.sql +++ b/dbt/include/tidb/macros/materializations/incremental/incremental.sql @@ -1,4 +1,3 @@ - {% materialization incremental, adapter='tidb' %} {% set unique_key = config.get('unique_key') %} @@ -34,13 +33,12 @@ from_relation=tmp_relation, to_relation=target_relation) %} {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} - -- use get_delete_insert_merge_sql after support multi sql - -- we will delete then insert now - {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %} - {% call statement("pre_main") %} - {{ build_sql }} - {% endcall %} - {% set build_sql = incremental_insert(target_relation, tmp_relation, unique_key, dest_columns) %} + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set incremental_predicates = config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} {% endif %} {% call statement("main") %} diff --git a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql index 34c586e..0f9d5e8 100644 --- a/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/tidb/macros/materializations/snapshot/snapshot.sql @@ -1,9 +1,4 @@ -{% macro tidb__snapshot_string_as_time(timestamp) -%} - {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} - {{ return(result) }} -{%- endmacro %} - {% materialization snapshot, adapter='tidb' %} {%- set config = model['config'] -%} diff --git a/dbt/include/tidb/macros/utils/timestamps.sql b/dbt/include/tidb/macros/utils/timestamps.sql new file mode 100644 index 0000000..2d8a0f3 --- /dev/null +++ b/dbt/include/tidb/macros/utils/timestamps.sql @@ -0,0 +1,12 @@ +{% macro tidb__current_timestamp() -%} + current_timestamp() +{%- endmacro %} + +{% macro tidb__snapshot_string_as_time(timestamp) -%} + {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%} + {{ return(result) }} +{%- endmacro %} + +{% macro tidb__current_timestamp_backcompat() -%} + current_timestamp() +{%- endmacro %} \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 9e87633..edafd16 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ -dbt-core~=1.2.0 +dbt-core~=1.3.1 mysql-connector-python>=8.0.0,<8.1 pytest~=7.0 markupsafe==2.0.1 -dbt-tests-adapter~=1.2.0 +dbt-tests-adapter~=1.3.1 pytest-dotenv \ No newline at end of file diff --git a/setup.py b/setup.py index 01e4a30..6e1083c 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,8 @@ long_description = f.read() package_name = "dbt-tidb" -package_version = "1.2.0" -dbt_core_version = "1.2.0" +package_version = "1.3.0" +dbt_core_version = "1.3.1" description = """The TiDB adapter plugin for dbt""" setup( diff --git a/tests/README.md b/tests/README.md index 052752a..2657261 100644 --- a/tests/README.md +++ b/tests/README.md @@ -49,6 +49,16 @@ You can also install tidb with [TiUP playground](https://docs.pingcap.com/tidb/s tiup playground ${version} ``` +## Set TiDB variables +TiDB use `SYSTEM` as the default value of `time_zone`. For more information, see [TiDB Time Zone Support](https://docs.pingcap.com/tidb/stable/configure-time-zone). + +To pass the `CurrentTimestamp` tests, you should set `time_zone` to `UTC` first. + +```sql +mysql -u -h -P -p +mysql> set @@global.time_zone='UTC'; +``` + ## Use pytest to test If you specify a package, all Python files under the package will be tested. Don't forget to configure PYTHONPATH: @@ -57,6 +67,12 @@ If you specify a package, all Python files under the package will be tested. Don PYTHONPATH=. pytest tests/functional/adapter/tidb/basic # utils PYTHONPATH=. pytest tests/functional/adapter/tidb/utils +# concurrentcy +PYTHONPATH=. pytest tests/functional/adapter/tidb/concurrency +# ephemeral +PYTHONPATH=. pytest tests/functional/adapter/tidb/ephemeral +# incremental +PYTHONPATH=. pytest tests/functional/adapter/tidb/incremental ``` ## Test grant diff --git a/tests/functional/adapter/tidb/basic/test_tidb.py b/tests/functional/adapter/tidb/basic/test_tidb.py index 8938749..c316c69 100644 --- a/tests/functional/adapter/tidb/basic/test_tidb.py +++ b/tests/functional/adapter/tidb/basic/test_tidb.py @@ -15,10 +15,9 @@ from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog +from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange + from dbt.tests.util import run_dbt, check_relations_equal -from dbt.tests.adapter.incremental.test_incremental_unique_id import ( - BaseIncrementalUniqueKey, -) class TestEmptyMyAdapter(BaseEmpty): @@ -83,5 +82,5 @@ def expected_catalog(self, project): ) -class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): +class TestIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): pass diff --git a/tests/functional/adapter/tidb/concurrency/test_concurrency.py b/tests/functional/adapter/tidb/concurrency/test_concurrency.py new file mode 100644 index 0000000..5f67958 --- /dev/null +++ b/tests/functional/adapter/tidb/concurrency/test_concurrency.py @@ -0,0 +1,25 @@ +import pytest + +from dbt.tests.util import run_dbt, check_relations_equal, rm_file, write_file +from dbt.tests.adapter.concurrency.test_concurrency import ( + BaseConcurrency, + seeds__update_csv, +) + + +class TestConcurrencyTiDB(BaseConcurrency): + def test_conncurrency_tidb(self, project): + run_dbt(["seed", "--select", "seed"]) + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal( + project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"] + ) + + rm_file(project.project_root, "seeds", "seed.csv") + write_file(seeds__update_csv, project.project_root + "/seeds", "seed.csv") + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal( + project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"] + ) diff --git a/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py b/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py new file mode 100644 index 0000000..994c65f --- /dev/null +++ b/tests/functional/adapter/tidb/ephemeral/test_ephemeral.py @@ -0,0 +1,15 @@ +import pytest + +from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti + + +class TestEphemeralMultiTiDB(BaseEphemeralMulti): + def test_ephemeral_multi_snowflake(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + check_relations_equal(project.adapter, ["seed", "dependent"]) + # TiDB does not support double dependent + # check_relations_equal(project.adapter, ["seed", "double_dependent"]) + check_relations_equal(project.adapter, ["seed", "super_dependent"]) diff --git a/tests/functional/adapter/tidb/incremental/test_incremental.py b/tests/functional/adapter/tidb/incremental/test_incremental.py new file mode 100644 index 0000000..69b0107 --- /dev/null +++ b/tests/functional/adapter/tidb/incremental/test_incremental.py @@ -0,0 +1,9 @@ +import pytest + +from dbt.tests.adapter.incremental.test_incremental_unique_id import ( + BaseIncrementalUniqueKey, +) + + +class TestIncrementalUniqueKeyTiDB(BaseIncrementalUniqueKey): + pass diff --git a/tests/functional/adapter/tidb/utils/data_types/test_data_types.py b/tests/functional/adapter/tidb/utils/data_types/test_data_types.py new file mode 100644 index 0000000..8070cac --- /dev/null +++ b/tests/functional/adapter/tidb/utils/data_types/test_data_types.py @@ -0,0 +1,43 @@ +import pytest + +from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp + + +@pytest.mark.skip(reason="TiDB does not support cast as int") +class TestTypeIntTiDB(BaseTypeInt): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as bigint") +class TestTypeBigIntTiDB(BaseTypeBigInt): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as boolean") +class TestTypeBooleanTiDB(BaseTypeBoolean): + pass + + +@pytest.mark.skip(reason="TiDB does not support numeric type") +class TestTypeNumericTiDb(BaseTypeNumeric): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as text") +class TestTypeStringTiDB(BaseTypeString): + pass + + +class TestTypeFloatTiDB(BaseTypeFloat): + pass + + +@pytest.mark.skip(reason="TiDB does not support cast as timestamp") +class TestTypeTimestampTiDB(BaseTypeTimestamp): + pass diff --git a/tests/functional/adapter/tidb/utils/fixture_bool_or.py b/tests/functional/adapter/tidb/utils/fixture_bool_or.py deleted file mode 100644 index c14848c..0000000 --- a/tests/functional/adapter/tidb/utils/fixture_bool_or.py +++ /dev/null @@ -1,34 +0,0 @@ -# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql - -models__test_bool_or_sql = """ -with data as ( - select * from {{ ref('data_bool_or') }} -), -data_output as ( - select * from {{ ref('data_bool_or_expected') }} -), -calculate as ( - select - `key`, - {{ bool_or('val1 = val2') }} as value - from data - group by `key` -) -select - calculate.value as actual, - data_output.value as expected -from calculate -left join data_output -on calculate.key = data_output.key -""" - - -models__test_bool_or_yml = """ -version: 2 -models: - - name: test_bool_or - tests: - - assert_equal: - actual: actual - expected: expected -""" diff --git a/tests/functional/adapter/tidb/utils/set_timezone.sql b/tests/functional/adapter/tidb/utils/set_timezone.sql new file mode 100644 index 0000000..a28490f --- /dev/null +++ b/tests/functional/adapter/tidb/utils/set_timezone.sql @@ -0,0 +1 @@ +set @@global.time_zone='UTC'; \ No newline at end of file diff --git a/tests/functional/adapter/tidb/utils/test_util.py b/tests/functional/adapter/tidb/utils/test_util.py index fa1e450..28f75b4 100644 --- a/tests/functional/adapter/tidb/utils/test_util.py +++ b/tests/functional/adapter/tidb/utils/test_util.py @@ -1,4 +1,5 @@ import pytest + from dbt.tests.adapter.utils.test_any_value import BaseAnyValue from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText @@ -21,9 +22,9 @@ from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral from dbt.tests.adapter.utils.test_listagg import BaseListagg -from tests.functional.adapter.tidb.utils.fixture_bool_or import ( - models__test_bool_or_sql, - models__test_bool_or_yml, +from dbt.tests.adapter.utils.test_current_timestamp import ( + BaseCurrentTimestamp, + BaseCurrentTimestampNaive, ) from tests.functional.adapter.tidb.utils.fixture_dateadd import ( models__test_dateadd_yml, @@ -45,14 +46,7 @@ class TestAnyValue(BaseAnyValue): class TestBoolOr(BaseBoolOr): - @pytest.fixture(scope="class") - def models(self): - return { - "test_bool_or.yml": models__test_bool_or_yml, - "test_bool_or.sql": self.interpolate_macro_namespace( - models__test_bool_or_sql, "bool_or" - ), - } + pass class TestCastBoolToText(BaseCastBoolToText): @@ -154,3 +148,11 @@ class TestSplitPart(BaseSplitPart): class TestStringLiteral(BaseStringLiteral): pass + + +class TestCurrentTimestampTiDB(BaseCurrentTimestamp): + pass + + +class TestCurrentTimestampNaiveTiDB(BaseCurrentTimestampNaive): + pass diff --git a/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py b/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py deleted file mode 100644 index c14848c..0000000 --- a/tests/functional/adapter/tidb5_1/utils/fixture_bool_or.py +++ /dev/null @@ -1,34 +0,0 @@ -# key is the keyword in tidb,so use `key` rather than key in models__test_bool_or_sql - -models__test_bool_or_sql = """ -with data as ( - select * from {{ ref('data_bool_or') }} -), -data_output as ( - select * from {{ ref('data_bool_or_expected') }} -), -calculate as ( - select - `key`, - {{ bool_or('val1 = val2') }} as value - from data - group by `key` -) -select - calculate.value as actual, - data_output.value as expected -from calculate -left join data_output -on calculate.key = data_output.key -""" - - -models__test_bool_or_yml = """ -version: 2 -models: - - name: test_bool_or - tests: - - assert_equal: - actual: actual - expected: expected -""" diff --git a/tests/functional/adapter/tidb5_1/utils/test_util.py b/tests/functional/adapter/tidb5_1/utils/test_util.py index bc1ae33..f68b6f5 100644 --- a/tests/functional/adapter/tidb5_1/utils/test_util.py +++ b/tests/functional/adapter/tidb5_1/utils/test_util.py @@ -21,10 +21,6 @@ from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral from dbt.tests.adapter.utils.test_listagg import BaseListagg -from tests.functional.adapter.tidb5_1.utils.fixture_bool_or import ( - models__test_bool_or_sql, - models__test_bool_or_yml, -) from tests.functional.adapter.tidb5_1.utils.fixture_dateadd import ( models__test_dateadd_yml, models__test_dateadd_sql, @@ -53,14 +49,7 @@ class TestAnyValue(BaseAnyValue): class TestBoolOr(BaseBoolOr): - @pytest.fixture(scope="class") - def models(self): - return { - "test_bool_or.yml": models__test_bool_or_yml, - "test_bool_or.sql": self.interpolate_macro_namespace( - models__test_bool_or_sql, "bool_or" - ), - } + pass class TestCastBoolToText(BaseCastBoolToText):