Destination CDK: Increase json deserialization limit (#38763)

airbytehq · Jun 7, 2024 · c30119b · c30119b
1 parent 8b76e89
commit c30119b
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 8 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -174,6 +174,7 @@ corresponds to that version.
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.36.8  | 2024-06-07 | [\#38763](https://github.com/airbytehq/airbyte/pull/38763) | Increase Jackson message length limit |
 | 0.36.7  | 2024-06-06 | [\#39220](https://github.com/airbytehq/airbyte/pull/39220) | Handle null messages in ConnectorExceptionUtil |
 | 0.36.6  | 2024-06-05 | [\#39106](https://github.com/airbytehq/airbyte/pull/39106) | Skip write to storage with 0 byte file                                                                                                                         |
 | 0.36.5  | 2024-06-01 | [\#38792](https://github.com/airbytehq/airbyte/pull/38792) | Throw config exception if no selectable table exists in user provided schemas                                                                                  |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.36.7
+version=0.36.8
diff --git a/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/json/Jsons.kt b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/kotlin/io/airbyte/commons/json/Jsons.kt
@@ -4,6 +4,7 @@
 package io.airbyte.commons.json
 
 import com.fasterxml.jackson.core.JsonProcessingException
+import com.fasterxml.jackson.core.StreamReadConstraints
 import com.fasterxml.jackson.core.type.TypeReference
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
 import com.fasterxml.jackson.core.util.Separators
@@ -28,17 +29,26 @@ private val LOGGER = KotlinLogging.logger {}
 
 object Jsons {
 
+    // allow jackson to deserialize anything under 50 MiB
+    // (the default, at time of writing 2024-05-29, with jackson 2.15.2, is 20 MiB)
+    private const val JSON_MAX_LENGTH = 50 * 1024 * 1024
+    private val STREAM_READ_CONSTRAINTS =
+        StreamReadConstraints.builder().maxStringLength(JSON_MAX_LENGTH).build()
+
     // Object Mapper is thread-safe
-    private val OBJECT_MAPPER: ObjectMapper = MoreMappers.initMapper()
+    private val OBJECT_MAPPER: ObjectMapper =
+        MoreMappers.initMapper().also {
+            it.factory.setStreamReadConstraints(STREAM_READ_CONSTRAINTS)
+        }
 
     // sort of a hotfix; I don't know how bad the performance hit is so not turning this on by
     // default
     // at time of writing (2023-08-18) this is only used in tests, so we don't care.
-    private val OBJECT_MAPPER_EXACT: ObjectMapper = MoreMappers.initMapper()
-
-    init {
-        OBJECT_MAPPER_EXACT.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
-    }
+    private val OBJECT_MAPPER_EXACT: ObjectMapper =
+        MoreMappers.initMapper().also {
+            it.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
+            it.factory.setStreamReadConstraints(STREAM_READ_CONSTRAINTS)
+        }
 
     private val YAML_OBJECT_MAPPER: ObjectMapper = MoreMappers.initYamlMapper(YAMLFactory())
     private val OBJECT_WRITER: ObjectWriter = OBJECT_MAPPER.writer(JsonPrettyPrinter())
@@ -399,7 +409,7 @@ object Jsons {
                 } else {
                     return@toMap prefix
                 }
-            }
+            },
         )
     }