From f81604d4acae46edf55fe09c9e4d091a7adb0399 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 6 Jan 2020 13:06:05 +0900 Subject: [PATCH] Upgrade Avro to 1.9.1 (#5938) ### Motivation Currently, Pulsar uses Avro 1.8.2, a version released two years ago. The latest version of Avro is 1.9.1, which uses FasterXML's Jackson 2.x instead of Codehaus's Jackson 1.x. Jackson is prone to security issues, so we should not keep using older versions. https://blog.godatadriven.com/apache-avro-1-9-release ### Modifications Avro 1.9 has some major changes: - The library used to handle logical datetime values has changed from Joda-Time to JSR-310 (https://github.com/apache/avro/pull/631) - Namespaces no longer include "$" when generating schemas containing inner classes using ReflectData (https://github.com/apache/avro/pull/283) - Validation of default values has been enabled (https://github.com/apache/avro/pull/288). This results in a validation error when parsing the following schema: ```json { "name": "fieldName", "type": [ "null", "string" ], "default": "defaultValue" } ``` The default value of a nullable field must be null (cf. https://issues.apache.org/jira/browse/AVRO-1803), and the default value of the field as above is actually null. However, this PR disables the validation in order to maintain the traditional behavior. --- pulsar-sql/presto-distribution/LICENSE | 12 +++--------- .../pulsar/sql/presto/PulsarConnectorUtils.java | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 601dba5..d130836 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -209,6 +209,7 @@ The Apache Software License, Version 2.0 * Jackson - jackson-annotations-2.8.11.jar + - jackson-core-2.8.11.jar - jackson-databind-2.8.11.4.jar - jackson-dataformat-smile-2.8.11.jar - jackson-datatype-guava-2.8.11.jar @@ -217,9 +218,6 @@ The Apache Software License, Version 2.0 - jackson-datatype-jdk8-2.8.11.jar - jackson-datatype-joda-2.8.11.jar - jackson-datatype-jsr310-2.8.11.jar - - jackson-core-2.8.11.jar - - jackson-core-asl-1.9.13.jar - - jackson-mapper-asl-1.9.13.jar - jackson-dataformat-yaml-2.8.11.jar * Guava - guava-25.1-jre.jar @@ -355,8 +353,8 @@ The Apache Software License, Version 2.0 * Apache XBean :: Reflect - xbean-reflect-3.4.jar * Avro - - avro-1.8.2.jar - - avro-protobuf-1.8.2.jar + - avro-1.9.1.jar + - avro-protobuf-1.9.1.jar * Javax - javax.inject-1.jar - javax.inject-1.jar @@ -475,8 +473,6 @@ BSD License - asm-util-6.0.jar * JLine - jline-2.14.6.jar - * ParaNamer Core - - paranamer-2.7.jar * Zstd-jni - zstd-jni-1.3.7-3.jar @@ -545,8 +541,6 @@ Public Domain (CC0) -- licenses/LICENSE-CC0.txt - HdrHistogram-2.1.9.jar * AOP Alliance - aopalliance-1.0.jar - * XZ For Java - - xz-1.5.jar * Reactive Streams - reactive-streams-1.0.2.jar diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java index 58794c0..03a3a58 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java @@ -34,6 +34,7 @@ public class PulsarConnectorUtils { public static Schema parseSchema(String schemaJson) { Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); return parser.parse(schemaJson); }