Normalization: Fix incorrect jinja2 macro json_extract_array call (#…

…13894) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Jun 19, 2022 · 49d181a · 49d181a
1 parent b338014
commit 49d181a
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 2 deletions.
diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile
@@ -28,5 +28,5 @@ WORKDIR /airbyte
 ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
 ENTRYPOINT ["/airbyte/entrypoint.sh"]
 
-LABEL io.airbyte.version=0.2.5
+LABEL io.airbyte.version=0.2.6
 LABEL io.airbyte.name=airbyte/normalization
diff --git a/...s/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql b/...s/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql
@@ -244,7 +244,7 @@
 {%- endmacro %}
 
 {% macro default__json_extract_string_array(json_column, json_path_list, normalized_json_path) -%}
-    json_extract_array({{ json_column }}, {{ format_json_path(json_path_list) }})
+    {{ json_extract_array(json_column, json_path_list, normalized_json_path) }}
 {%- endmacro %}
 
 # https://cloud.google.com/bigquery/docs/reference/standard-sql/json_functions#json_extract_string_array

diff --git a/...ase-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json b/...ase-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json
@@ -272,6 +272,29 @@
       "sync_mode": "incremental",
       "cursor_field": [],
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "arrays",
+        "json_schema": {
+          "type": ["null", "object"],
+          "properties": {
+            "array_of_strings": {
+              "type": ["null", "array"],
+              "items": {
+                "type": ["null", "string"]
+              }
+            }
+          }
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": []
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": [],
+      "destination_sync_mode": "overwrite",
+      "primary_key": []
     }
   ]
 }
diff --git a/...ase-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt b/...ase-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt
@@ -15,3 +15,4 @@
 
 {"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":1, "children": [{"ab_id": 1, "owner": {"owner_id": 1, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 2, "owner": {"owner_id": 2, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}}
 {"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":2, "children": [{"ab_id": 3, "owner": {"owner_id": 3, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 4, "owner": {"owner_id": 4, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}}
+{"type":"RECORD","record":{"stream":"arrays","emitted_at":1602638599000,"data":{"array_of_strings":["string1","string2","string3"]}}}
diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md
@@ -353,6 +353,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need
 
 | Airbyte Version | Normalization Version | Date | Pull Request | Subject |
 |:----------------| :--- | :--- | :--- | :--- |
+|                 | 0.2.6 | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894) | Fix incorrect jinja2 macro `json_extract_array` call |
 |                 | 0.2.5 | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 |
 |                 | 0.2.4 | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846) | CDC correctly deletes propagates deletions to final tables |
 |                 | 0.2.3 | 2022-06-10 | [\#11204](https://github.com/airbytehq/airbyte/pull/11204) | MySQL: add support for SSh tunneling |