Destination TiDB: install normalization (#27025)

* install normalization * changelog * add to codeowners * update changelog messages
airbytehq · Jun 6, 2023 · ae43522 · ae43522
1 parent bd22b25
commit ae43522
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 3 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -48,3 +48,4 @@
 /airbyte-integrations/connectors/destination-rockset/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations
diff --git a/airbyte-integrations/connectors/destination-tidb/Dockerfile b/airbyte-integrations/connectors/destination-tidb/Dockerfile
@@ -8,8 +8,28 @@
 # Please reach out to the Connectors Operations team if you have any question.
 FROM airbyte/integration-base-java:dev AS build
 
+RUN yum install -y python3 python3-devel jq sshpass git && \
+  alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
+  python -m ensurepip --upgrade && \
+  pip3 install dbt-tidb==1.0.1
+
+# Luckily, none of normalization's files conflict with destination-tidb's files :)
+# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
+COPY --from=airbyte/normalization-tidb:dev /airbyte /airbyte
+# Install python dependencies
+WORKDIR /airbyte/base_python_structs
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code/dbt-template/
+# Download external dbt dependencies
+# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
+RUN pip3 install "urllib3<2"
+RUN dbt deps
+
 WORKDIR /airbyte
 ENV APPLICATION destination-tidb
+ENV AIRBYTE_NORMALIZATION_INTEGRATION tidb
 
 COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
@@ -22,5 +42,8 @@ ENV APPLICATION destination-tidb
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/destination-tidb
+
+ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
+ENTRYPOINT ["/airbyte/run_with_normalization.sh"]
diff --git a/airbyte-integrations/connectors/destination-tidb/build.gradle b/airbyte-integrations/connectors/destination-tidb/build.gradle
@@ -24,3 +24,7 @@ dependencies {
     integrationTestJavaImplementation libs.connectors.testcontainers.tidb
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
 }
+
+tasks.named("airbyteDocker") {
+    dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerTiDB
+}
diff --git a/airbyte-integrations/connectors/destination-tidb/metadata.yaml b/airbyte-integrations/connectors/destination-tidb/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 06ec60c7-7468-45c0-91ac-174f6e1a788b
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   dockerRepository: airbyte/destination-tidb
   githubIssueLabel: destination-tidb
   icon: tidb.svg

diff --git a/docs/integrations/destinations/tidb.md b/docs/integrations/destinations/tidb.md
@@ -99,6 +99,7 @@ Using this feature requires additional configuration, when creating the destinat
 
 | Version | Date       | Pull Request                                               | Subject                               |
 |:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------|
+| 0.1.3   | 2023-06-05 | [\#27025](https://github.com/airbytehq/airbyte/pull/27025) | Internal code change for future development (install normalization packages inside connector) |
 | 0.1.2   | 2023-05-23 | [\#19109](https://github.com/airbytehq/airbyte/pull/19109) | Enabled Append Dedub mode             |
 | 0.1.1   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
 | 0.1.0   | 2022-08-12 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592) | Added TiDB destination.               |
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
@@ -581,7 +581,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor
         "dockerfile": "tidb.Dockerfile",
         "dbt_adapter": "dbt-tidb==1.0.1",
         "integration_name": "tidb",
-        "supports_in_connector_normalization": False,
+        "supports_in_connector_normalization": True,
         "yum_packages": [],
     },
 }