From 907bbe441b9c3c55c05b86adba5c117029d44187 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Wed, 13 Jul 2022 19:06:14 +0300 Subject: [PATCH 1/8] quote this.schema Signed-off-by: Sergey Chvalyuk --- .../bases/base-normalization/Dockerfile | 2 +- .../transform_catalog/destination_name_transformer.py | 10 ++++++---- .../transform_catalog/stream_processor.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index a0d6d3a80faf0..2cbda1fe6f599 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py index 29db8e3ff7e39..f10f7d06e9822 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py @@ -181,13 +181,15 @@ def __normalize_identifier_name( return f"'{result}'" return result - def apply_quote(self, input: str) -> str: + def apply_quote(self, input: str, literal=True) -> str: + if literal: + input = f"'{input}'" if self.destination_type == DestinationType.ORACLE: # Oracle dbt lib doesn't implemented adapter quote yet. - return f"quote('{input}')" + return f"quote({input})" elif self.destination_type == DestinationType.CLICKHOUSE: - return f"quote('{input}')" - return f"adapter.quote('{input}')" + return f"quote({input})" + return f"adapter.quote({input})" def __normalize_naming_conventions(self, input_name: str, is_column: bool = False) -> str: result = input_name diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 544b030dbedb1..7bd6fa9604b48 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -1209,7 +1209,7 @@ def add_to_outputs( quoted_unique_key=self.get_unique_key(in_jinja=True), active_row_column_name=active_row_column_name, normalized_at_incremental_clause=self.get_incremental_clause_for_column( - "this.schema + '.' + " + self.name_transformer.apply_quote(final_table_name), + self.name_transformer.apply_quote('this.schema', literal=False) + " + '.' + " + self.name_transformer.apply_quote(final_table_name), self.get_normalized_at(in_jinja=True), ), unique_key_reference=unique_key_reference, From f114106baac532ffa39f73f614e4f51b313492aa Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Wed, 13 Jul 2022 19:18:24 +0300 Subject: [PATCH 2/8] basic-normalization.md updated Signed-off-by: Sergey Chvalyuk --- docs/understanding-airbyte/basic-normalization.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 8a739b1198342..26a0987e2c876 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,6 +353,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | +| | 0.2.7 | 2022-07-13 | [\#14683](https://github.com/airbytehq/airbyte/pull/14683) | Quote schema name to allow reserved keywords | | | 0.2.6 | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894) | Fix incorrect jinja2 macro `json_extract_array` call | | | 0.2.5 | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 | | | 0.2.4 | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846) | CDC correctly deletes propagates deletions to final tables | From 759ea2eac41f0c297d5c3dccd022d8011088a0f7 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Wed, 13 Jul 2022 19:20:11 +0300 Subject: [PATCH 3/8] NormalizationRunnerFactory.java updated Signed-off-by: Sergey Chvalyuk --- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 9a0246a2ae441..33d1f0f4c4b7c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.6"; + public static final String NORMALIZATION_VERSION = "0.2.7"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() From cc87c2875f4868656e2df12e49294aacdd6184c1 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 14 Jul 2022 09:28:22 +0300 Subject: [PATCH 4/8] format code Signed-off-by: Sergey Chvalyuk --- .../normalization/transform_catalog/stream_processor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 7bd6fa9604b48..08f8a30504842 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -1209,7 +1209,9 @@ def add_to_outputs( quoted_unique_key=self.get_unique_key(in_jinja=True), active_row_column_name=active_row_column_name, normalized_at_incremental_clause=self.get_incremental_clause_for_column( - self.name_transformer.apply_quote('this.schema', literal=False) + " + '.' + " + self.name_transformer.apply_quote(final_table_name), + self.name_transformer.apply_quote("this.schema", literal=False) + + " + '.' + " + + self.name_transformer.apply_quote(final_table_name), self.get_normalized_at(in_jinja=True), ), unique_key_reference=unique_key_reference, From 392863da88d1d53de42192a47fad068062d83197 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sun, 24 Jul 2022 15:01:40 +0300 Subject: [PATCH 5/8] lint code Signed-off-by: Sergey Chvalyuk --- .../normalization/transform_catalog/stream_processor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 08f8a30504842..ca8a20ca8f643 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -1209,9 +1209,10 @@ def add_to_outputs( quoted_unique_key=self.get_unique_key(in_jinja=True), active_row_column_name=active_row_column_name, normalized_at_incremental_clause=self.get_incremental_clause_for_column( - self.name_transformer.apply_quote("this.schema", literal=False) - + " + '.' + " - + self.name_transformer.apply_quote(final_table_name), + "{}.{}".format( + self.name_transformer.apply_quote("this.schema", literal=False), + self.name_transformer.apply_quote(final_table_name), + ), self.get_normalized_at(in_jinja=True), ), unique_key_reference=unique_key_reference, From 743163dfa1d289445d5ad5ecc74a32ff4b9d7bae Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sun, 24 Jul 2022 15:05:41 +0300 Subject: [PATCH 6/8] lint code (2) Signed-off-by: Sergey Chvalyuk --- .../normalization/transform_catalog/stream_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index ca8a20ca8f643..4c35762625dd8 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -1209,7 +1209,7 @@ def add_to_outputs( quoted_unique_key=self.get_unique_key(in_jinja=True), active_row_column_name=active_row_column_name, normalized_at_incremental_clause=self.get_incremental_clause_for_column( - "{}.{}".format( + "{} + '.' + {}".format( self.name_transformer.apply_quote("this.schema", literal=False), self.name_transformer.apply_quote(final_table_name), ), From 755598ab6135438f570878eb1d4ae201271de628 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Wed, 27 Jul 2022 12:15:52 +0300 Subject: [PATCH 7/8] basic-normalization.md updated Signed-off-by: Sergey Chvalyuk --- docs/understanding-airbyte/basic-normalization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 634bfab4b6c0d..a34b7d776cf07 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,9 +353,9 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------|:----------------------|:-----------| :--- |:---------------------------------------------------------------------------| -| | 0.2.12 | 2022-07-27 | [\#14683](https://github.com/airbytehq/airbyte/pull/14683) | Quote schema name to allow reserved keywords | -| | 0.2.11 | 2022-07-26 | [\#13591](https://github.com/airbytehq/airbyte/pull/13591) | Updated support for integer columns. | +| | 0.2.13 | 2022-07-27 | [\#14683](https://github.com/airbytehq/airbyte/pull/14683) | Quote schema name to allow reserved keywords | | | 0.2.12 | 2022-07-26 | [\#14362](https://github.com/airbytehq/airbyte/pull/14362) | Handle timezone in date-time format. Parse date correct in clickhouse. | +| | 0.2.11 | 2022-07-26 | [\#13591](https://github.com/airbytehq/airbyte/pull/13591) | Updated support for integer columns. | | | 0.2.10 | 2022-07-18 | [\#14792](https://github.com/airbytehq/airbyte/pull/14792) | Add support for key pair auth for snowflake | | | 0.2.9 | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485) | BigQuery partition pruning otimization | | | 0.2.8 | 2022-07-13 | [\#14522](https://github.com/airbytehq/airbyte/pull/14522) | BigQuery replaces `NULL` array entries with the string value `"NULL"` | From 675248fe7e2d1d0b4d595d7a93d3ddfe4ddb3254 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 28 Jul 2022 00:52:50 +0300 Subject: [PATCH 8/8] NormalizationRunnerFactory.java updated Signed-off-by: Sergey Chvalyuk --- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 3291c7e099098..e726ca3dc4850 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.12"; + public static final String NORMALIZATION_VERSION = "0.2.13"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder()