Support dbt v1.3

.github/workflows/main.yml

@@ -41,6 +41,42 @@ jobs:
           export DBT_TEST_USER_3=user3
           PYTHONPATH=. pytest tests/functional/adapter/tidb  
 
+  tidb_6_1:
+    name: Python ${{ matrix.python-version }} | TiDB 6.1
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+
+    services:
+      tidb_nightly:
+        image: pingcap/tidb:v6.1.2
+        ports:
+          - 4000:4000
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements_dev.txt
+
+      - name: Run tests
+        run: |
+          mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql
+          export DBT_TEST_USER_1=user1
+          export DBT_TEST_USER_2=user2
+          export DBT_TEST_USER_3=user3
+          PYTHONPATH=. pytest tests/functional/adapter/tidb  
+
+
   tidb_5_3:
     name: Python ${{ matrix.python-version }} | TiDB 5.3
     runs-on: ubuntu-latest

dbt/adapters/tidb/__version__.py

@@ -1 +1 @@
-version = "1.0.0"
+version = "1.3.0"

dbt/adapters/tidb/impl.py

@@ -11,7 +11,7 @@
 from dbt.adapters.tidb import TiDBConnectionManager
 from dbt.adapters.tidb import TiDBRelation
 from dbt.adapters.tidb import TiDBColumn
-from dbt.adapters.base import BaseRelation
+from dbt.adapters.base import BaseRelation, available
 from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER
 from dbt.events import AdapterLogger
 from dbt.utils import executor
@@ -35,6 +35,11 @@ def date_function(cls):
     def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
         return "timestamp"
 
+    @classmethod
+    def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))  # type: ignore[attr-defined]
+        return "float" if decimals else "integer"
+
     def quote(self, identifier):
         return "`{}`".format(identifier)
 
@@ -277,3 +282,9 @@ def get_rows_different_sql(
         )
 
         return sql
+
+    def valid_incremental_strategies(self):
+        """The set of standard builtin strategies which this adapter supports out-of-the-box.
+        Not used to validate custom strategies defined by end users.
+        """
+        return ["delete+insert"]

dbt/include/tidb/macros/adapters.sql

@@ -78,10 +78,6 @@
 
 {% endmacro %}
 
-{% macro tidb__current_timestamp() -%}
-  current_timestamp()
-{%- endmacro %}
-
 {% macro tidb__rename_relation(from_relation, to_relation) -%}
   {#
     tidb rename fails when the relation already exists, so a 2-step process is needed:

dbt/include/tidb/macros/materializations/incremental/helpers.sql

@@ -1,3 +1,33 @@
+{% macro tidb__get_incremental_default_sql(arg_dict) %}
+    {% if arg_dict["unique_key"] %}
+        {% do return(get_incremental_delete_insert_sql(arg_dict)) %}
+    {% else %}
+        {% do return(get_incremental_append_sql(arg_dict)) %}
+    {% endif %}
+{% endmacro %}
+
+{% macro get_incremental_delete_insert_sql(arg_dict) %}
+    {% set target_relation = arg_dict["target_relation"] %}
+    {% set tmp_relation = arg_dict["temp_relation"] %}
+    {% set unique_key = arg_dict["unique_key"] %}
+    {% set dest_columns = arg_dict["dest_columns"] %}
+    --  use get_delete_insert_merge_sql after support multi sql
+    --  we will delete then insert now
+    {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %}
+    {% call statement("pre_main") %}
+        {{ build_sql }}
+    {% endcall %}
+    {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %}
+{% endmacro %}
+
+{% macro get_incremental_append_sql(arg_dict) %}
+    {% set target_relation = arg_dict["target_relation"] %}
+    {% set tmp_relation = arg_dict["temp_relation"] %}
+    {% set unique_key = arg_dict["unique_key"] %}
+    {% set dest_columns = arg_dict["dest_columns"] %}
+    {% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %}
+{% endmacro %}
+
 -- need to support unique_key is sequence
 {% macro incremental_delete(target, source, unique_key, dest_columns) %}
 

dbt/include/tidb/macros/materializations/incremental/incremental.sql

@@ -1,4 +1,3 @@
-
 {% materialization incremental, adapter='tidb' %}
 
   {% set unique_key = config.get('unique_key') %}
@@ -34,13 +33,12 @@
              from_relation=tmp_relation,
              to_relation=target_relation) %}
      {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
-     --  use get_delete_insert_merge_sql after support multi sql
-     --  we will delete then insert now
-     {% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %}
-      {% call statement("pre_main") %}
-          {{ build_sql }}
-      {% endcall %}
-     {% set build_sql = incremental_insert(target_relation, tmp_relation, unique_key, dest_columns) %}
+     {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
+     {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
+     {% set incremental_predicates = config.get('incremental_predicates', none) %}
+     {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
+     {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %}
+      {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
   {% endif %}
 
   {% call statement("main") %}

dbt/include/tidb/macros/materializations/snapshot/snapshot.sql

@@ -1,9 +1,4 @@
 
-{% macro tidb__snapshot_string_as_time(timestamp) -%}
-    {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%}
-    {{ return(result) }}
-{%- endmacro %}
-
 {% materialization snapshot, adapter='tidb' %}
   {%- set config = model['config'] -%}
 

dbt/include/tidb/macros/utils/timestamps.sql

@@ -0,0 +1,12 @@
+{% macro tidb__current_timestamp() -%}
+  current_timestamp()
+{%- endmacro %}
+
+{% macro tidb__snapshot_string_as_time(timestamp) -%}
+    {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%}
+    {{ return(result) }}
+{%- endmacro %}
+
+{% macro tidb__current_timestamp_backcompat() -%}
+  current_timestamp()
+{%- endmacro %}

requirements_dev.txt

@@ -1,6 +1,6 @@
-dbt-core~=1.2.0
+dbt-core~=1.3.1
 mysql-connector-python>=8.0.0,<8.1
 pytest~=7.0
 markupsafe==2.0.1
-dbt-tests-adapter~=1.2.0
+dbt-tests-adapter~=1.3.1
 pytest-dotenv

setup.py

@@ -27,8 +27,8 @@
     long_description = f.read()
 
 package_name = "dbt-tidb"
-package_version = "1.2.0"
-dbt_core_version = "1.2.0"
+package_version = "1.3.0"
+dbt_core_version = "1.3.1"
 description = """The TiDB adapter plugin for dbt"""
 
 setup(

tests/README.md

@@ -49,6 +49,16 @@ You can also install tidb with [TiUP playground](https://docs.pingcap.com/tidb/s
 tiup playground ${version}
 ```
 
+## Set TiDB variables
+TiDB use `SYSTEM` as the default value of `time_zone`. For more information, see [TiDB Time Zone Support](https://docs.pingcap.com/tidb/stable/configure-time-zone). 
+
+To pass the `CurrentTimestamp` tests, you should set `time_zone` to `UTC` first.
+
+```sql
+mysql -u <user_name> -h <host> -P <port> -p
+mysql> set @@global.time_zone='UTC';
+```
+
 ## Use pytest to test
 
 If you specify a package, all Python files under the package will be tested. Don't forget to configure PYTHONPATH:
@@ -57,6 +67,12 @@ If you specify a package, all Python files under the package will be tested. Don
 PYTHONPATH=. pytest tests/functional/adapter/tidb/basic
 # utils
 PYTHONPATH=. pytest tests/functional/adapter/tidb/utils
+# concurrentcy
+PYTHONPATH=. pytest tests/functional/adapter/tidb/concurrency
+# ephemeral
+PYTHONPATH=. pytest tests/functional/adapter/tidb/ephemeral
+# incremental
+PYTHONPATH=. pytest tests/functional/adapter/tidb/incremental
 ```
 
 ## Test grant

tests/functional/adapter/tidb/basic/test_tidb.py

@@ -15,10 +15,9 @@
 from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection
 from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate
 from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog
+from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange
+
 from dbt.tests.util import run_dbt, check_relations_equal
-from dbt.tests.adapter.incremental.test_incremental_unique_id import (
-    BaseIncrementalUniqueKey,
-)
 
 
 class TestEmptyMyAdapter(BaseEmpty):
@@ -83,5 +82,5 @@ def expected_catalog(self, project):
         )
 
 
-class TestIncrementalUniqueKey(BaseIncrementalUniqueKey):
+class TestIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange):
     pass

tests/functional/adapter/tidb/concurrency/test_concurrency.py

@@ -0,0 +1,25 @@
+import pytest
+
+from dbt.tests.util import run_dbt, check_relations_equal, rm_file, write_file
+from dbt.tests.adapter.concurrency.test_concurrency import (
+    BaseConcurrency,
+    seeds__update_csv,
+)
+
+
+class TestConcurrencyTiDB(BaseConcurrency):
+    def test_conncurrency_tidb(self, project):
+        run_dbt(["seed", "--select", "seed"])
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results) == 7
+        check_relations_equal(
+            project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]
+        )
+
+        rm_file(project.project_root, "seeds", "seed.csv")
+        write_file(seeds__update_csv, project.project_root + "/seeds", "seed.csv")
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results) == 7
+        check_relations_equal(
+            project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]
+        )

tests/functional/adapter/tidb/ephemeral/test_ephemeral.py

@@ -0,0 +1,15 @@
+import pytest
+
+from dbt.tests.util import run_dbt, check_relations_equal
+from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti
+
+
+class TestEphemeralMultiTiDB(BaseEphemeralMulti):
+    def test_ephemeral_multi_snowflake(self, project):
+        run_dbt(["seed"])
+        results = run_dbt(["run"])
+        assert len(results) == 3
+        check_relations_equal(project.adapter, ["seed", "dependent"])
+        # TiDB does not support double dependent
+        # check_relations_equal(project.adapter, ["seed", "double_dependent"])
+        check_relations_equal(project.adapter, ["seed", "super_dependent"])

tests/functional/adapter/tidb/incremental/test_incremental.py

@@ -0,0 +1,9 @@
+import pytest
+
+from dbt.tests.adapter.incremental.test_incremental_unique_id import (
+    BaseIncrementalUniqueKey,
+)
+
+
+class TestIncrementalUniqueKeyTiDB(BaseIncrementalUniqueKey):
+    pass

tests/functional/adapter/tidb/utils/data_types/test_data_types.py

@@ -0,0 +1,43 @@
+import pytest
+
+from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt
+from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
+from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean
+from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric
+from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString
+from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
+from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp
+
+
+@pytest.mark.skip(reason="TiDB does not support cast as int")
+class TestTypeIntTiDB(BaseTypeInt):
+    pass
+
+
+@pytest.mark.skip(reason="TiDB does not support cast as bigint")
+class TestTypeBigIntTiDB(BaseTypeBigInt):
+    pass
+
+
+@pytest.mark.skip(reason="TiDB does not support cast as boolean")
+class TestTypeBooleanTiDB(BaseTypeBoolean):
+    pass
+
+
+@pytest.mark.skip(reason="TiDB does not support numeric type")
+class TestTypeNumericTiDb(BaseTypeNumeric):
+    pass
+
+
+@pytest.mark.skip(reason="TiDB does not support cast as text")
+class TestTypeStringTiDB(BaseTypeString):
+    pass
+
+
+class TestTypeFloatTiDB(BaseTypeFloat):
+    pass
+
+
+@pytest.mark.skip(reason="TiDB does not support cast as timestamp")
+class TestTypeTimestampTiDB(BaseTypeTimestamp):
+    pass

tests/functional/adapter/tidb/utils/set_timezone.sql

@@ -0,0 +1 @@
+set @@global.time_zone='UTC';