Skip to content

Commit

Permalink
Destination Mssql: install normalization (#27034)
Browse files Browse the repository at this point in the history 
* install in mssql

* changelog+metadata

* update changelog message
  • Loading branch information
@edgao
edgao committed Jun 6, 2023
1 parent 3aa4512 commit bd22b25
Show file tree
Hide file tree
Showing 8 changed files with 62 additions and 6 deletions.
25 changes: 24 additions & 1 deletion airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,29 @@
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build

RUN yum install -y python3 python3-devel jq sshpass git && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-sqlserver==1.0.0

# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps

WORKDIR /airbyte

ENV APPLICATION destination-mssql-strict-encrypt
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql

COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar

Expand All @@ -24,5 +44,8 @@ ENV APPLICATION destination-mssql-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.23
LABEL io.airbyte.version=0.1.24
LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt

ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]
4 changes: 4 additions & 0 deletions airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,7 @@ dependencies {
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
}

tasks.named("airbyteDocker") {
dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql
}
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.1.23
dockerImageTag: 0.1.24
dockerRepository: airbyte/destination-mssql-strict-encrypt
githubIssueLabel: destination-mssql
icon: mssql.svg
Expand Down
25 changes: 24 additions & 1 deletion airbyte-integrations/connectors/destination-mssql/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,29 @@
# Please reach out to the Connectors Operations team if you have any question.
FROM airbyte/integration-base-java:dev AS build

RUN yum install -y python3 python3-devel jq sshpass git && \
alternatives --install /usr/bin/python python /usr/bin/python3 60 && \
python -m ensurepip --upgrade && \
pip3 install dbt-sqlserver==1.0.0

# Luckily, none of normalization's files conflict with destination-mssql's files :)
# We don't enforce that in any way, but hopefully we're only living in this state for a short time.
COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte
# Install python dependencies
WORKDIR /airbyte/base_python_structs
RUN pip3 install .
WORKDIR /airbyte/normalization_code
RUN pip3 install .
WORKDIR /airbyte/normalization_code/dbt-template/
# Download external dbt dependencies
# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15
RUN pip3 install "urllib3<2"
RUN dbt deps

WORKDIR /airbyte

ENV APPLICATION destination-mssql
ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql

COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar

Expand All @@ -24,5 +44,8 @@ ENV APPLICATION destination-mssql

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.23
LABEL io.airbyte.version=0.1.24
LABEL io.airbyte.name=airbyte/destination-mssql

ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh"
ENTRYPOINT ["/airbyte/run_with_normalization.sh"]
4 changes: 4 additions & 0 deletions airbyte-integrations/connectors/destination-mssql/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,7 @@ dependencies {
implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs)
}

tasks.named("airbyteDocker") {
dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql
}
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-mssql/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
dockerImageTag: 0.1.23
dockerImageTag: 0.1.24
dockerRepository: airbyte/destination-mssql
githubIssueLabel: destination-mssql
icon: mssql.svg
Expand Down
4 changes: 3 additions & 1 deletion docs/integrations/destinations/mssql.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Make sure your SQL Server database can be accessed by Airbyte. If your database
#### **Permissions**

You need a user configured in SQL Server that can create tables and write rows. We highly recommend creating an Airbyte-specific user for this purpose.
In order to allow for normalization, please grant ALTER permissions for the user configured.
In order to allow for normalization, please grant ALTER permissions for the user configured.

#### Target Database

Expand Down Expand Up @@ -116,6 +116,7 @@ Using this feature requires additional configuration, when creating the source.

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
| 0.1.24 | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector) |
| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 |
| 0.1.20 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
Expand All @@ -142,6 +143,7 @@ Using this feature requires additional configuration, when creating the source.

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
| 0.1.24 | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector) |
| 0.1.23 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
| 0.1.22 | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889 |
| 0.1.21 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage |
Expand Down
2 changes: 1 addition & 1 deletion tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ def with_integration_base_java(context: PipelineContext, build_platform: Platfor
"dockerfile": "mssql.Dockerfile",
"dbt_adapter": "dbt-sqlserver==1.0.0",
"integration_name": "mssql",
"supports_in_connector_normalization": False,
"supports_in_connector_normalization": True,
"yum_packages": [],
},
"destination-mysql": {
Expand Down

0 comments on commit bd22b25

Please sign in to comment.