From 4d23fed25739f39c50dce00e704e7bfbebf27539 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 17 Aug 2023 07:59:29 -0700 Subject: [PATCH 1/3] ensure raw datasset created --- .../connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-bigquery/metadata.yaml | 2 +- .../bigquery/typing_deduping/BigQuerySqlGenerator.java | 7 ++++++- docs/integrations/destinations/bigquery.md | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 1ad45dce5a2aa..a9d4680cef4ac 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -47,7 +47,7 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.7.8 +LABEL io.airbyte.version=1.7.9 LABEL io.airbyte.name=airbyte/destination-bigquery ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 6a5a7890116c4..8054db48e3e79 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 1.7.8 + dockerImageTag: 1.7.9 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java index 205796b353698..f2934c5621365 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java @@ -604,11 +604,16 @@ private String wrapAndQuote(final String namespace, final String tableName) { @Override public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { return new StringSubstitutor(Map.of( + "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), + "dataset_location", datasetLocation, "v2_raw_table", streamId.rawTableId(QUOTE), "v1_raw_table", wrapAndQuote(namespace, tableName) ) ).replace( - """ + """ + CREATE SCHEMA IF NOT EXISTS ${raw_namespace} + OPTIONS(location="${dataset_location}"); + CREATE OR REPLACE TABLE ${v2_raw_table} ( _airbyte_raw_id STRING, _airbyte_data JSON, diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index c7a4abe9009f2..6824b09c74d7e 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -135,6 +135,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| +| 1.7.8 | 2023-08-17 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure raw dataset created | | 1.7.8 | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging. | | 1.7.7 | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns | | 1.7.6 | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic | From 271e9f288b5b610e924e920f768c49f402084856 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 17 Aug 2023 08:00:47 -0700 Subject: [PATCH 2/3] update changelog --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 6824b09c74d7e..2cb65b59e8c55 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -135,7 +135,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| -| 1.7.8 | 2023-08-17 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure raw dataset created | +| 1.7.8 | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created | | 1.7.8 | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging. | | 1.7.7 | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns | | 1.7.6 | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic | From cc989025c4c40dafd114725775cfaed7654a6da1 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Thu, 17 Aug 2023 08:24:41 -0700 Subject: [PATCH 3/3] version bump --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 2cb65b59e8c55..14af33534b489 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -135,7 +135,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| -| 1.7.8 | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created | +| 1.7.9 | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created | | 1.7.8 | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging. | | 1.7.7 | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns | | 1.7.6 | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic |