support dbt v1.2

.github/workflows/main.yml

@@ -6,46 +6,142 @@ on:
     branches: main
 
 jobs:
-  build:
-    name: Python 3.8 | TiDB nightly | Ubuntu latest
+  tidb_nightly:
+    name: Python ${{ matrix.python-version }} | TiDB nightly
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+
     services:
       tidb_nightly:
         image: pingcap/tidb:nightly
         ports:
           - 4000:4000
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements_dev.txt
+
+      - name: Run tests
+        run: |
+          mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql
+          export DBT_TEST_USER_1=user1
+          export DBT_TEST_USER_2=user2
+          export DBT_TEST_USER_3=user3
+          PYTHONPATH=. pytest tests/functional/adapter/tidb  
+
+  tidb_5_3:
+    name: Python ${{ matrix.python-version }} | TiDB 5.3
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+
+    services:
+      tidb_nightly:
+        image: pingcap/tidb:v5.3.2
+        ports:
+          - 4000:4000
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements_dev.txt
+
+      - name: Run tests
+        run: |
+          mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb/grant/create_user.sql
+          export DBT_TEST_USER_1=user1
+          export DBT_TEST_USER_2=user2
+          export DBT_TEST_USER_3=user3
+          PYTHONPATH=. pytest tests/functional/adapter/tidb  
+
+  tidb_5_1:
+    name: Python ${{ matrix.python-version }} | TiDB 5.1
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+
+    services:
       tidb_5.1:
-        image: pingcap/tidb:v5.1.0
+        image: pingcap/tidb:v5.1.4
         ports:
-          - 4001:4000
+          - 4000:4000
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements_dev.txt
+
+      - name: Run tests
+        run: |
+          mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb5_1/grant/create_user.sql
+          export DBT_TEST_USER_1=user1
+          export DBT_TEST_USER_2=user2
+          export DBT_TEST_USER_3=user3
+          PYTHONPATH=. pytest tests/functional/adapter/tidb5_1
+
+  tidb_4_0:
+    name: Python ${{ matrix.python-version }} | TiDB 4.0
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+
+    services:
       tidb_4.0:
-        image: pingcap/tidb:v4.0.0
+        image: pingcap/tidb:v4.0.16
         ports:
-          - 4002:4000
+          - 4000:4000
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-
-    - name: Install requirements
-      run: |
-        pip install -r requirements_dev.txt
-
-    - name: Run tests tidb_nightly
-      run: |
-        PYTHONPATH="${PYTHONPATH}:dbt" pytest test/integration/tidb.dbtspec
-
-    - name: Run tests tidb_v5.0.1 up
-      run: |
-        PYTHONPATH="${PYTHONPATH}:dbt" pytest test/integration/tidb_v5.1-v5.2.dbtspec
-
-    - name: Run tests tidb_v4.0.0 up
-      run: |
-        PYTHONPATH="${PYTHONPATH}:dbt" pytest test/integration/tidb_v4.0-v5.0.dbtspec
-
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install requirements
+        run: |
+          pip install -r requirements_dev.txt
+
+      - name: Run tests
+        run: |
+          mysql -P4000 -uroot -h127.0.0.1 < tests/functional/adapter/tidb4_0/grant/create_user.sql
+          export DBT_TEST_USER_1=user1
+          export DBT_TEST_USER_2=user2
+          export DBT_TEST_USER_3=user3
+          PYTHONPATH=. pytest tests/functional/adapter/tidb4_0

.gitignore

@@ -13,3 +13,4 @@ dbt-integration-tests
 test/integration/.user.yml
 .DS_Store
 .vscode
+logs

README.md

@@ -8,12 +8,13 @@
 The `dbt-tidb` package contains all of the code enabling [dbt](https://getdbt.com) to work with 
 [TiDB](https://en.pingcap.com/tidb/).
 
-This repository is based on [dbt-mysql](https://github.com/dbeatty10/dbt-mysql).
+This repository is partly based on [dbt-mysql](https://github.com/dbeatty10/dbt-mysql).
 Thanks to them for their excellent work.
 
 ## Table of Contents
  * [Installation](#installation)
  * [Supported features](#supported-features)
+ * [Supported functions](#supported-functions)
  * [Profile Configuration](#profile-configuration)
  * [Database User Privileges](#database-user-privileges)
  * [Running Tests](#running-tests)
@@ -35,17 +36,19 @@ $ pip install dbt-tidb
 ```
 ## Supported features
 
-|    TiDB 4.X    | TiDB 5.0 ~ 5.2 |   TiDB >= 5.3    |            Feature             |
-|:--------------:|:--------------:|:----------------:|:------------------------------:|
-|       ✅        |       ✅       |        ✅        |     Table materialization      |
-|       ✅        |       ✅       |        ✅        |      View materialization      |
-|       ✅        |       ✅       |        ✅        |  Incremental materialization   |
-|       ❌        |       ❌       |        ✅        |   Ephemeral materialization    |
-|       ✅        |       ✅       |        ✅        |             Seeds              |
-|       ✅        |       ✅       |        ✅        |            Sources             |
-|       ✅        |       ✅       |        ✅        |       Custom data tests        |
-|       ✅        |       ✅       |        ✅        |         Docs generate          |
-|       ❌        |       ❌       |        ✅        |           Snapshots            |
+|    TiDB 4.X    | TiDB 5.0 ~ 5.2 |   TiDB >= 5.3    |           Feature           |
+|:--------------:|:--------------:|:----------------:|:---------------------------:|
+|       ✅        |       ✅       |        ✅        |    Table materialization    |
+|       ✅        |       ✅       |        ✅        |    View materialization     |
+|       ❌        |       ❌       |        ✅        | Incremental materialization |
+|       ❌        |       ✅       |        ✅        |  Ephemeral materialization  |
+|       ✅        |       ✅       |        ✅        |            Seeds            |
+|       ✅        |       ✅       |        ✅        |           Sources           |
+|       ✅        |       ✅       |        ✅        |      Custom data tests      |
+|       ✅        |       ✅       |        ✅        |        Docs generate        |
+|       ❌        |       ❌       |        ✅        |          Snapshots          |
+|       ✅        |       ✅       |        ✅        |      Connection retry       |
+|       ✅        |       ✅       |        ✅        |            Grant            |
 
 Note: 
 
@@ -55,7 +58,32 @@ Note:
 * TiDB 4.X does not support using SQL func in `CREATE VIEW`, avoid it in your SQL code. 
   You can find more detail [here](https://github.com/pingcap/tidb/pull/27252).
 
-### Profile Configuration
+## Supported functions
+
+cross-db macros are moved from dbt-utils into dbt-core, so you can use the following functions directly, see [dbt-util](https://github.com/dbt-labs/dbt-utils) on how to use them.
+- bool_or
+- cast_bool_to_text
+- dateadd
+- datediff
+- date_trunc
+- hash
+- safe_cast
+- split_part
+- last_day
+- cast_bool_to_text
+- concat
+- escape_single_quotes
+- except
+- intersect
+- length
+- position
+- replace
+- right
+- listagg (not support yet)
+
+> pay attention that datediff is a little different from dbt-util that it will round down rather than round up.
+
+## Profile Configuration
 
 TiDB targets should be set up using the following configuration in your `profiles.yml` file.
 
@@ -72,18 +100,20 @@ your_profile_name:
       schema: database_name
       username: tidb_username
       password: tidb_password
+      retries: 2
 ```
 
-| Option          | Description                                            | Required? | Example                        |
-| --------------- | ------------------------------------------------------ |-----------|--------------------------------|
-| type            | The specific adapter to use                            | Required  | `tidb`                         |
-| server          | The server (hostname) to connect to                    | Required  | `yourorg.tidb.com`             |
-| port            | The port to use                                        | Required  | `4000`                         |
-| schema          | Specify the schema (database) to build models into     | Required  | `analytics`                    |
-| username        | The username to use to connect to the server           | Required  | `dbt_admin`                    |
-| password        | The password to use for authenticating to the server   | Required  | `correct-horse-battery-staple` |
+| Option           | Description                                           | Required? | Example                        |
+|------------------|-------------------------------------------------------|-----------|--------------------------------|
+| type             | The specific adapter to use                           | Required  | `tidb`                         |
+| server           | The server (hostname) to connect to                   | Required  | `yourorg.tidb.com`             |
+| port             | The port to use                                       | Required  | `4000`                         |
+| schema           | Specify the schema (database) to build models into    | Required  | `analytics`                    |
+| username         | The username to use to connect to the server          | Required  | `dbt_admin`                    |
+| password         | The password to use for authenticating to the server  | Required  | `correct-horse-battery-staple` |
+| retries          | The retry times for connection to TiDB (1 in default) | Optional  | `2`                            |
 
-### Database User Privileges
+## Database User Privileges
 
 Your database user would be able to have some abilities to read or write, such as `SELECT`, `CREATE`, and so on.
 You can find some help [here](https://docs.pingcap.com/tidb/v4.0/privilege-management) with TiDB privileges management.
@@ -100,14 +130,14 @@ You can find some help [here](https://docs.pingcap.com/tidb/v4.0/privilege-manag
 | SHOW VIEW              |
 | SUPER                  |
 
-### Running Tests
+## Running Tests
 
 See [test/README.md](test/README.md) for details on running the integration tests.
 
-### Example
+## Example
 
 Click [here](https://github.com/pingcap/dbt-tidb/wiki/Primer-Tutorial-%7C-How-to-use-dbt-with-TiDB) to see a simple example about using dbt with dbt-tidb.
 
-### Contributing
+## Contributing
 
 Welcome to contribute for dbt-tidb. See [Contributing Guide](CONTRIBUTING.md) for more information.

dbt/adapters/tidb/connections.py

@@ -17,12 +17,13 @@
 @dataclass(init=False)
 class TiDBCredentials(Credentials):
     server: str
+    schema: str
     port: Optional[int] = None
     database: Optional[str] = None
-    schema: str
     username: Optional[str] = None
     password: Optional[str] = None
     charset: Optional[str] = None
+    retries: int = 1
 
     _ALIASES = {
         "UID": "username",
@@ -85,36 +86,27 @@ def open(cls, connection):
         kwargs["host"] = credentials.server
         kwargs["user"] = credentials.username
         kwargs["passwd"] = credentials.password
+        kwargs["buffered"] = True
 
         if credentials.port:
             kwargs["port"] = credentials.port
 
-        try:
-            connection.handle = mysql.connector.connect(**kwargs)
-            connection.state = 'open'
-        except mysql.connector.Error:
-
-            try:
-                logger.debug("Failed connection without supplying the `database`. "
-                             "Trying again with `database` included.")
-
-                # Try again with the database included
-                kwargs["database"] = credentials.schema
-
-                connection.handle = mysql.connector.connect(**kwargs)
-                connection.state = 'open'
-            except mysql.connector.Error as e:
-
-                logger.debug("Got an error when attempting to open a tidb "
-                             "connection: '{}'"
-                             .format(e))
-
-                connection.handle = None
-                connection.state = 'fail'
-
-                raise dbt.exceptions.FailedToConnectException(str(e))
-
-        return connection
+        def connect():
+            handle = mysql.connector.connect(**kwargs)
+            return handle
+
+        # we just retry for any error now
+        retryable_exceptions = [
+                 mysql.connector.Error
+        ]
+
+        return cls.retry_connection(
+            connection,
+            connect=connect,
+            logger=logger,
+            retry_limit=credentials.retries,
+            retryable_exceptions=retryable_exceptions,
+        )
 
     @classmethod
     def get_credentials(cls, credentials):