Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support dbt v1.3 #29

Merged
merged 6 commits into from
Dec 5, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,42 @@ jobs:
export DBT_TEST_USER_3=user3
PYTHONPATH=. pytest tests/functional/adapter/tidb

tidb_6_1:
name: Python ${{ matrix.python-version }} | TiDB 6.1
runs-on: ubuntu-latest

strategy:
matrix:
python-version: [ '3.8', '3.9', '3.10' ]

services:
tidb_nightly:
image: pingcap/tidb:v6.1.2
ports:
- 4000:4000

steps:
- name: Checkout
uses: actions/checkout@v2

- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

- name: Install requirements
run: |
pip install -r requirements_dev.txt

- name: Run tests
run: |
mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql
export DBT_TEST_USER_1=user1
export DBT_TEST_USER_2=user2
export DBT_TEST_USER_3=user3
PYTHONPATH=. pytest tests/functional/adapter/tidb


tidb_5_3:
name: Python ${{ matrix.python-version }} | TiDB 5.3
runs-on: ubuntu-latest
Expand Down
2 changes: 1 addition & 1 deletion dbt/adapters/tidb/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version = "1.0.0"
version = "1.3.0"
13 changes: 12 additions & 1 deletion dbt/adapters/tidb/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from dbt.adapters.tidb import TiDBConnectionManager
from dbt.adapters.tidb import TiDBRelation
from dbt.adapters.tidb import TiDBColumn
from dbt.adapters.base import BaseRelation
from dbt.adapters.base import BaseRelation, available
from dbt.clients.agate_helper import DEFAULT_TYPE_TESTER
from dbt.events import AdapterLogger
from dbt.utils import executor
Expand All @@ -35,6 +35,11 @@ def date_function(cls):
def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
return "timestamp"

@classmethod
def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined]
return "float" if decimals else "integer"

def quote(self, identifier):
return "`{}`".format(identifier)

Expand Down Expand Up @@ -277,3 +282,9 @@ def get_rows_different_sql(
)

return sql

def valid_incremental_strategies(self):
"""The set of standard builtin strategies which this adapter supports out-of-the-box.
Not used to validate custom strategies defined by end users.
"""
return ["delete+insert"]
4 changes: 0 additions & 4 deletions dbt/include/tidb/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@

{% endmacro %}

{% macro tidb__current_timestamp() -%}
current_timestamp()
{%- endmacro %}

{% macro tidb__rename_relation(from_relation, to_relation) -%}
{#
tidb rename fails when the relation already exists, so a 2-step process is needed:
Expand Down
30 changes: 30 additions & 0 deletions dbt/include/tidb/macros/materializations/incremental/helpers.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,33 @@
{% macro tidb__get_incremental_default_sql(arg_dict) %}
Daemonxiao marked this conversation as resolved.
Show resolved Hide resolved
{% if arg_dict["unique_key"] %}
{% do return(get_incremental_delete_insert_sql(arg_dict)) %}
{% else %}
{% do return(get_incremental_append_sql(arg_dict)) %}
{% endif %}
{% endmacro %}

{% macro get_incremental_delete_insert_sql(arg_dict) %}
{% set target_relation = arg_dict["target_relation"] %}
{% set tmp_relation = arg_dict["temp_relation"] %}
{% set unique_key = arg_dict["unique_key"] %}
{% set dest_columns = arg_dict["dest_columns"] %}
-- use get_delete_insert_merge_sql after support multi sql
-- we will delete then insert now
{% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %}
{% call statement("pre_main") %}
{{ build_sql }}
{% endcall %}
{% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %}
{% endmacro %}

{% macro get_incremental_append_sql(arg_dict) %}
{% set target_relation = arg_dict["target_relation"] %}
{% set tmp_relation = arg_dict["temp_relation"] %}
{% set unique_key = arg_dict["unique_key"] %}
{% set dest_columns = arg_dict["dest_columns"] %}
{% do return(incremental_insert(target_relation, tmp_relation, unique_key, dest_columns)) %}
{% endmacro %}

-- need to support unique_key is sequence
{% macro incremental_delete(target, source, unique_key, dest_columns) %}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

{% materialization incremental, adapter='tidb' %}

{% set unique_key = config.get('unique_key') %}
Expand Down Expand Up @@ -34,13 +33,12 @@
from_relation=tmp_relation,
to_relation=target_relation) %}
{% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
-- use get_delete_insert_merge_sql after support multi sql
-- we will delete then insert now
{% set build_sql = incremental_delete(target_relation, tmp_relation, unique_key, dest_columns) %}
{% call statement("pre_main") %}
{{ build_sql }}
{% endcall %}
{% set build_sql = incremental_insert(target_relation, tmp_relation, unique_key, dest_columns) %}
{#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
{% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
{% set incremental_predicates = config.get('incremental_predicates', none) %}
{% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
{% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'predicates': incremental_predicates }) %}
{% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
{% endif %}

{% call statement("main") %}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@

{% macro tidb__snapshot_string_as_time(timestamp) -%}
{%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%}
{{ return(result) }}
{%- endmacro %}

{% materialization snapshot, adapter='tidb' %}
{%- set config = model['config'] -%}

Expand Down
12 changes: 12 additions & 0 deletions dbt/include/tidb/macros/utils/timestamps.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{% macro tidb__current_timestamp() -%}
current_timestamp()
{%- endmacro %}

{% macro tidb__snapshot_string_as_time(timestamp) -%}
{%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%}
{{ return(result) }}
{%- endmacro %}

{% macro tidb__current_timestamp_backcompat() -%}
current_timestamp()
{%- endmacro %}
4 changes: 2 additions & 2 deletions requirements_dev.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
dbt-core~=1.2.0
dbt-core~=1.3.1
mysql-connector-python>=8.0.0,<8.1
pytest~=7.0
markupsafe==2.0.1
dbt-tests-adapter~=1.2.0
dbt-tests-adapter~=1.3.1
pytest-dotenv
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
long_description = f.read()

package_name = "dbt-tidb"
package_version = "1.2.0"
dbt_core_version = "1.2.0"
package_version = "1.3.0"
dbt_core_version = "1.3.1"
description = """The TiDB adapter plugin for dbt"""

setup(
Expand Down
16 changes: 16 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,16 @@ You can also install tidb with [TiUP playground](https://docs.pingcap.com/tidb/s
tiup playground ${version}
```

## Set TiDB variables
TiDB use `SYSTEM` as the default value of `time_zone`. For more information, see [TiDB Time Zone Support](https://docs.pingcap.com/tidb/stable/configure-time-zone).

To pass the `CurrentTimestamp` tests, you should set `time_zone` to `UTC` first.

```sql
mysql -u <user_name> -h <host> -P <port> -p
mysql> set @@global.time_zone='UTC';
```

## Use pytest to test

If you specify a package, all Python files under the package will be tested. Don't forget to configure PYTHONPATH:
Expand All @@ -57,6 +67,12 @@ If you specify a package, all Python files under the package will be tested. Don
PYTHONPATH=. pytest tests/functional/adapter/tidb/basic
# utils
PYTHONPATH=. pytest tests/functional/adapter/tidb/utils
# concurrentcy
PYTHONPATH=. pytest tests/functional/adapter/tidb/concurrency
# ephemeral
PYTHONPATH=. pytest tests/functional/adapter/tidb/ephemeral
# incremental
PYTHONPATH=. pytest tests/functional/adapter/tidb/incremental
```

## Test grant
Expand Down
7 changes: 3 additions & 4 deletions tests/functional/adapter/tidb/basic/test_tidb.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@
from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection
from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate
from dbt.tests.adapter.basic.expected_catalog import no_stats, base_expected_catalog
from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange

from dbt.tests.util import run_dbt, check_relations_equal
from dbt.tests.adapter.incremental.test_incremental_unique_id import (
BaseIncrementalUniqueKey,
)


class TestEmptyMyAdapter(BaseEmpty):
Expand Down Expand Up @@ -83,5 +82,5 @@ def expected_catalog(self, project):
)


class TestIncrementalUniqueKey(BaseIncrementalUniqueKey):
class TestIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange):
Daemonxiao marked this conversation as resolved.
Show resolved Hide resolved
pass
25 changes: 25 additions & 0 deletions tests/functional/adapter/tidb/concurrency/test_concurrency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import pytest

from dbt.tests.util import run_dbt, check_relations_equal, rm_file, write_file
from dbt.tests.adapter.concurrency.test_concurrency import (
BaseConcurrency,
seeds__update_csv,
)


class TestConcurrencyTiDB(BaseConcurrency):
def test_conncurrency_tidb(self, project):
run_dbt(["seed", "--select", "seed"])
results = run_dbt(["run"], expect_pass=False)
assert len(results) == 7
check_relations_equal(
project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]
)

rm_file(project.project_root, "seeds", "seed.csv")
write_file(seeds__update_csv, project.project_root + "/seeds", "seed.csv")
results = run_dbt(["run"], expect_pass=False)
assert len(results) == 7
check_relations_equal(
project.adapter, ["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"]
)
15 changes: 15 additions & 0 deletions tests/functional/adapter/tidb/ephemeral/test_ephemeral.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import pytest

from dbt.tests.util import run_dbt, check_relations_equal
from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti


class TestEphemeralMultiTiDB(BaseEphemeralMulti):
def test_ephemeral_multi_snowflake(self, project):
run_dbt(["seed"])
results = run_dbt(["run"])
assert len(results) == 3
check_relations_equal(project.adapter, ["seed", "dependent"])
# TiDB does not support double dependent
# check_relations_equal(project.adapter, ["seed", "double_dependent"])
check_relations_equal(project.adapter, ["seed", "super_dependent"])
9 changes: 9 additions & 0 deletions tests/functional/adapter/tidb/incremental/test_incremental.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import pytest

from dbt.tests.adapter.incremental.test_incremental_unique_id import (
BaseIncrementalUniqueKey,
)


class TestIncrementalUniqueKeyTiDB(BaseIncrementalUniqueKey):
pass
43 changes: 43 additions & 0 deletions tests/functional/adapter/tidb/utils/data_types/test_data_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import pytest

from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt
from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean
from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric
from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString
from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp


@pytest.mark.skip(reason="TiDB does not support cast as int")
class TestTypeIntTiDB(BaseTypeInt):
pass


@pytest.mark.skip(reason="TiDB does not support cast as bigint")
class TestTypeBigIntTiDB(BaseTypeBigInt):
pass


@pytest.mark.skip(reason="TiDB does not support cast as boolean")
class TestTypeBooleanTiDB(BaseTypeBoolean):
pass


@pytest.mark.skip(reason="TiDB does not support numeric type")
class TestTypeNumericTiDb(BaseTypeNumeric):
pass


@pytest.mark.skip(reason="TiDB does not support cast as text")
class TestTypeStringTiDB(BaseTypeString):
pass


class TestTypeFloatTiDB(BaseTypeFloat):
pass


@pytest.mark.skip(reason="TiDB does not support cast as timestamp")
class TestTypeTimestampTiDB(BaseTypeTimestamp):
pass
34 changes: 0 additions & 34 deletions tests/functional/adapter/tidb/utils/fixture_bool_or.py

This file was deleted.

1 change: 1 addition & 0 deletions tests/functional/adapter/tidb/utils/set_timezone.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
set @@global.time_zone='UTC';