Destination Clickhouse: install normalization (#27036)

* install normalization * changelog * update changelog messages
airbytehq · Jun 5, 2023 · 3aa4512 · 3aa4512
1 parent 3476a81
commit 3aa4512
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 4 deletions.
diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile
@@ -8,6 +8,25 @@
 # Please reach out to the Connectors Operations team if you have any question.
 FROM airbyte/integration-base-java:dev AS build
 
+RUN yum install -y python3 python3-devel jq sshpass git && \
+  alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
+  python -m ensurepip --upgrade && \
+  pip3 install dbt-clickhouse>=1.4.0
+
+# Luckily, none of normalization's files conflict with destination-clickhouse's files :)
+# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
+COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte
+# Install python dependencies
+WORKDIR /airbyte/base_python_structs
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code/dbt-template/
+# Download external dbt dependencies
+# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
+RUN pip3 install "urllib3<2"
+RUN dbt deps
+
 WORKDIR /airbyte
 
 ENV APPLICATION destination-clickhouse-strict-encrypt
@@ -21,8 +40,12 @@ FROM airbyte/integration-base-java:dev
 WORKDIR /airbyte
 
 ENV APPLICATION destination-clickhouse-strict-encrypt
+ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.3
+LABEL io.airbyte.version=0.2.4
 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt
+
+ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
+ENTRYPOINT ["/airbyte/run_with_normalization.sh"]
diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle
@@ -28,3 +28,7 @@ dependencies {
     integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
 }
+
+tasks.named("airbyteDocker") {
+    dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse
+}
diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml
@@ -7,7 +7,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.2.4
   dockerRepository: airbyte/destination-clickhouse-strict-encrypt
   githubIssueLabel: destination-clickhouse
   icon: clickhouse.svg

diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile
@@ -8,9 +8,29 @@
 # Please reach out to the Connectors Operations team if you have any question.
 FROM airbyte/integration-base-java:dev AS build
 
+RUN yum install -y python3 python3-devel jq sshpass git && \
+  alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
+  python -m ensurepip --upgrade && \
+  pip3 install dbt-clickhouse>=1.4.0
+
+# Luckily, none of normalization's files conflict with destination-clickhouse's files :)
+# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
+COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte
+# Install python dependencies
+WORKDIR /airbyte/base_python_structs
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code
+RUN pip3 install .
+WORKDIR /airbyte/normalization_code/dbt-template/
+# Download external dbt dependencies
+# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
+RUN pip3 install "urllib3<2"
+RUN dbt deps
+
 WORKDIR /airbyte
 
 ENV APPLICATION destination-clickhouse
+ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse
 
 COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
@@ -24,5 +44,8 @@ ENV APPLICATION destination-clickhouse
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.3
+LABEL io.airbyte.version=0.2.4
 LABEL io.airbyte.name=airbyte/destination-clickhouse
+
+ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
+ENTRYPOINT ["/airbyte/run_with_normalization.sh"]
diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle
@@ -30,3 +30,7 @@ dependencies {
     integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse
     integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
 }
+
+tasks.named("airbyteDocker") {
+    dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse
+}
diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.2.4
   dockerRepository: airbyte/destination-clickhouse
   githubIssueLabel: destination-clickhouse
   icon: clickhouse.svg

diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md
@@ -80,6 +80,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t
 
 | Version | Date       | Pull Request                                               | Subject                                                             |
 |:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------|
+| 0.2.4   | 2023-06-05 | [\#27036](https://github.com/airbytehq/airbyte/pull/27036) | Internal code change for future development (install normalization packages inside connector) |
 | 0.2.3   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                               |
 | 0.2.2   | 2023-02-21 | [\#21509](https://github.com/airbytehq/airbyte/pull/21509) | Compatibility update with security patch for strict encrypt version |
 | 0.2.1   | 2022-12-06 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update dbt version to 1.3.1                                         |