🐛 Source PostgreSQL: Fixed PostgreSQL CDC numeric handling (#4568)

* 🐛 Source PostgreSQL: Fixed PostgreSQL CDC numeric handling * Rm unnecessary tests for non-CDC * Add more values to test in postgres cdc numeric * Fix comment so it points to psql docs instead * Oops, added to the wrong test file * Add sourceType as well * Fix precision on tests
airbytehq · Jul 6, 2021 · 17cf7bc · 17cf7bc
1 parent 6532774
commit 17cf7bc
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 3 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
   "name": "Postgres",
   "dockerRepository": "airbyte/source-postgres",
-  "dockerImageTag": "0.3.4",
+  "dockerImageTag": "0.3.5",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres",
   "icon": "postgresql.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -46,7 +46,7 @@
 - sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   name: Postgres
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.3.4
+  dockerImageTag: 0.3.5
   documentationUrl: https://hub.docker.com/r/airbyte/source-postgres
   icon: postgresql.svg
 - sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003

diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.4
+LABEL io.airbyte.version=0.3.5
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/...stgres/src/main/java/io/airbyte/integrations/source/postgres/DebeziumRecordPublisher.java b/...stgres/src/main/java/io/airbyte/integrations/source/postgres/DebeziumRecordPublisher.java
@@ -156,6 +156,13 @@ protected static Properties getDebeziumProperties(JsonNode config, ConfiguredAir
     props.setProperty("slot.name", config.get("replication_method").get("replication_slot").asText());
     props.setProperty("publication.name", config.get("replication_method").get("publication").asText());
 
+    // By default "decimal.handing.mode=precise" which's caused returning this value as a binary.
+    // The "double" type may cause a loss of precision, so set Debezium's config to store it as a String
+    // explicitly in its Kafka messages for more details see:
+    // https://debezium.io/documentation/reference/1.4/connectors/postgresql.html#postgresql-decimal-types
+    // https://debezium.io/documentation/faq/#how_to_retrieve_decimal_field_from_binary_representation
+    props.setProperty("decimal.handling.mode", "string");
+
     // table selection
     final String tableWhitelist = getTableWhitelist(catalog);
     props.setProperty("table.include.list", tableWhitelist);

diff --git a/...integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceComprehensiveTest.java b/...integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceComprehensiveTest.java
@@ -356,6 +356,15 @@ protected void initTests() {
             .addExpectedValues("99999", "5.1", "0", null)
             .build());
 
+    addDataTypeTestData(
+        TestDataHolder.builder()
+            .sourceType("numeric")
+            .fullSourceDataType("numeric(13,4)")
+            .airbyteType(JsonSchemaPrimitive.NUMBER)
+            .addInsertValues("0.1880", "10.0000", "5213.3468", "null")
+            .addExpectedValues("0.188", "10.0", "5213.3468", null)
+            .build());
+
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("smallint")