From 31713823fc68a16c3936a9dd21bf76c4fbbeb65d Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Fri, 10 May 2024 17:59:27 +0200 Subject: [PATCH 1/8] fix: ParquetMetadata must convert to json --- parquet-hadoop/pom.xml | 5 +++ .../hadoop/metadata/ColumnChunkMetaData.java | 2 + .../hadoop/metadata/ParquetMetadata.java | 34 ++++++++++----- .../hadoop/metadata/StatisticsSerializer.java | 41 +++++++++++++++++++ .../TestParquetMetadataConverter.java | 15 ++++++- parquet-jackson/pom.xml | 12 ++++++ pom.xml | 1 + 7 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index 1c9bd4ff6e..be2649832b 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -118,6 +118,11 @@ jackson-core ${jackson.version} + + ${jackson.datatype.groupId} + jackson-datatype-jdk8 + ${jackson-modules-java8.version} + ${jackson.groupId} jackson-databind diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java index 3dac15ba7c..bb6fb868fe 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java @@ -22,6 +22,7 @@ import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; import static org.apache.parquet.format.Util.readColumnMetaData; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Set; @@ -380,6 +381,7 @@ public PrimitiveType getPrimitiveType() { /** * @return the stats for this column */ + @JsonSerialize(using = StatisticsSerializer.class) public abstract Statistics getStatistics(); /** diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java index e30e872a6a..8ddf6ecf57 100755 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java @@ -19,6 +19,9 @@ package org.apache.parquet.hadoop.metadata; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; @@ -32,6 +35,13 @@ public class ParquetMetadata { private static final ObjectMapper objectMapper = new ObjectMapper(); + // Enable FAIL_ON_EMPTY_BEANS on objectmapper. Without this feature parquet-casdacing tests fail, + // because LogicalTypeAnnotation implementations are classes without any property. + static { + objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + objectMapper.registerModule(new Jdk8Module()); + } + /** * @param parquetMetaData an instance of parquet metadata to convert * @return the json representation @@ -50,19 +60,23 @@ public static String toPrettyJSON(ParquetMetadata parquetMetaData) { private static String toJSON(ParquetMetadata parquetMetaData, boolean isPrettyPrint) { try (StringWriter stringWriter = new StringWriter()) { + Object objectToPrint; + if (parquetMetaData.getFileMetaData() == null + || parquetMetaData.getFileMetaData().getEncryptionType() + == FileMetaData.EncryptionType.UNENCRYPTED) { + objectToPrint = parquetMetaData; + } else { + objectToPrint = parquetMetaData.getFileMetaData(); + } + + ObjectWriter writer; if (isPrettyPrint) { - Object objectToPrint; - if (parquetMetaData.getFileMetaData() == null - || parquetMetaData.getFileMetaData().getEncryptionType() - == FileMetaData.EncryptionType.UNENCRYPTED) { - objectToPrint = parquetMetaData; - } else { - objectToPrint = parquetMetaData.getFileMetaData(); - } - objectMapper.writerWithDefaultPrettyPrinter().writeValue(stringWriter, objectToPrint); + writer = objectMapper.writerWithDefaultPrettyPrinter(); } else { - objectMapper.writeValue(stringWriter, parquetMetaData); + writer = objectMapper.writer(); } + + writer.writeValue(stringWriter, objectToPrint); return stringWriter.toString(); } catch (IOException e) { throw new RuntimeException(e); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java new file mode 100644 index 0000000000..943faf8139 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.metadata; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import java.io.IOException; +import org.apache.parquet.column.statistics.Statistics; + +class StatisticsSerializer extends JsonSerializer> { + @Override + public void serialize(Statistics statistics, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeFieldName("min"); + jsonGenerator.writeString(statistics.minAsString()); + jsonGenerator.writeFieldName("max"); + jsonGenerator.writeString(statistics.maxAsString()); + jsonGenerator.writeFieldName("null_count"); + jsonGenerator.writeNumber(statistics.getNumNulls()); + jsonGenerator.writeEndObject(); + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index 4dcede624f..af86ca94a2 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -638,7 +638,12 @@ public void randomTestFilterMetaData() { public void testNullFieldMetadataDebugLogging() { MessageType schema = parseMessageType("message test { optional binary some_null_field; }"); org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = - new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap(), null); + new org.apache.parquet.hadoop.metadata.FileMetaData( + schema, + new HashMap<>(), + null, + org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.UNENCRYPTED, + null); List blockMetaDataList = new ArrayList(); BlockMetaData blockMetaData = new BlockMetaData(); blockMetaData.addColumn(createColumnChunkMetaData()); @@ -662,7 +667,13 @@ private ColumnChunkMetaData createColumnChunkMetaData() { PrimitiveTypeName t = PrimitiveTypeName.BINARY; ColumnPath p = ColumnPath.get("foo"); CompressionCodecName c = CompressionCodecName.GZIP; - BinaryStatistics s = new BinaryStatistics(); + Statistics s = Statistics.createStats(Types.required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("str")); + byte[] min = new byte[904]; + byte[] max = new byte[2388]; + s.updateStats(Binary.fromConstantByteArray(min)); + s.updateStats(Binary.fromConstantByteArray(max)); ColumnChunkMetaData md = ColumnChunkMetaData.get(p, t, c, e, s, 0, 0, 0, 0, 0); return md; } diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 538295bc1e..47b1aab7d0 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -43,6 +43,11 @@ jackson-databind ${jackson-databind.version} + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${jackson-modules-java8.version} + @@ -70,6 +75,7 @@ ${jackson.groupId}:* + ${jackson.datatype.groupId}:* @@ -79,6 +85,12 @@ ** + + ${jackson.datatype.groupId}:* + + ** + + diff --git a/pom.xml b/pom.xml index 7dd7433129..ea071108c4 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,7 @@ com.fasterxml.jackson 2.17.0 2.17.0 + 2.17.0 0.21.0 1.3.2 2.30.0 From 271bdb00e909bd3d5d718026af23fdd552fdd321 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 15 May 2024 11:43:27 +0200 Subject: [PATCH 2/8] Drop statistics logging --- .../hadoop/metadata/ColumnChunkMetaData.java | 3 +- .../hadoop/metadata/StatisticsSerializer.java | 41 ------------------- 2 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java index bb6fb868fe..0efb51a4b4 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java @@ -381,7 +381,7 @@ public PrimitiveType getPrimitiveType() { /** * @return the stats for this column */ - @JsonSerialize(using = StatisticsSerializer.class) + @JsonIgnore public abstract Statistics getStatistics(); /** @@ -389,6 +389,7 @@ public PrimitiveType getPrimitiveType() { * * @return the size stats for this column */ + @JsonIgnore public SizeStatistics getSizeStatistics() { throw new UnsupportedOperationException("SizeStatistics is not implemented"); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java deleted file mode 100644 index 943faf8139..0000000000 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.hadoop.metadata; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; -import org.apache.parquet.column.statistics.Statistics; - -class StatisticsSerializer extends JsonSerializer> { - @Override - public void serialize(Statistics statistics, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException { - jsonGenerator.writeStartObject(); - jsonGenerator.writeFieldName("min"); - jsonGenerator.writeString(statistics.minAsString()); - jsonGenerator.writeFieldName("max"); - jsonGenerator.writeString(statistics.maxAsString()); - jsonGenerator.writeFieldName("null_count"); - jsonGenerator.writeNumber(statistics.getNumNulls()); - jsonGenerator.writeEndObject(); - } -} From c4fee1e7ab2fb49b4f0816b6a5789b8a138780c3 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 15 May 2024 11:43:58 +0200 Subject: [PATCH 3/8] Drop duplicated type information --- .../apache/parquet/hadoop/metadata/ColumnChunkMetaData.java | 3 ++- .../apache/parquet/hadoop/metadata/ColumnChunkProperties.java | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java index 0efb51a4b4..14a949b0e0 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java @@ -22,7 +22,7 @@ import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; import static org.apache.parquet.format.Util.readColumnMetaData; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Set; @@ -339,6 +339,7 @@ public ColumnPath getPath() { * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead. */ @Deprecated + @JsonIgnore public PrimitiveTypeName getType() { decryptIfNeeded(); return properties.getType(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java index 3b0a33b144..026e37a1c0 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkProperties.java @@ -18,6 +18,7 @@ */ package org.apache.parquet.hadoop.metadata; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Arrays; import java.util.Set; import org.apache.parquet.column.Encoding; @@ -76,6 +77,7 @@ public ColumnPath getPath() { * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead. */ @Deprecated + @JsonIgnore public PrimitiveTypeName getType() { return type.getPrimitiveTypeName(); } From ed9bee0e6b959313e76e8d36eea4b1380fec3315 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 15 May 2024 11:44:29 +0200 Subject: [PATCH 4/8] Drop logging file decryptor info --- .../parquet/hadoop/metadata/FileMetaData.java | 2 ++ .../TestParquetMetadataConverter.java | 34 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java index c608cd405f..4143dd805a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/FileMetaData.java @@ -20,6 +20,7 @@ import static java.util.Collections.unmodifiableMap; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.io.Serializable; import java.util.Map; import java.util.Objects; @@ -109,6 +110,7 @@ public String getCreatedBy() { return createdBy; } + @JsonIgnore public InternalFileDecryptor getFileDecryptor() { return fileDecryptor; } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index af86ca94a2..d1177158bd 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -87,6 +87,10 @@ import org.apache.parquet.column.statistics.LongStatistics; import org.apache.parquet.column.statistics.SizeStatistics; import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.crypto.DecryptionPropertiesFactory; +import org.apache.parquet.crypto.EncryptionPropertiesFactory; +import org.apache.parquet.crypto.FileDecryptionProperties; +import org.apache.parquet.crypto.InternalFileDecryptor; import org.apache.parquet.example.Paper; import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroup; @@ -635,7 +639,7 @@ public void randomTestFilterMetaData() { } @Test - public void testNullFieldMetadataDebugLogging() { + public void testFieldMetadataDebugLogging() { MessageType schema = parseMessageType("message test { optional binary some_null_field; }"); org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = new org.apache.parquet.hadoop.metadata.FileMetaData( @@ -644,7 +648,7 @@ public void testNullFieldMetadataDebugLogging() { null, org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.UNENCRYPTED, null); - List blockMetaDataList = new ArrayList(); + List blockMetaDataList = new ArrayList<>(); BlockMetaData blockMetaData = new BlockMetaData(); blockMetaData.addColumn(createColumnChunkMetaData()); blockMetaDataList.add(blockMetaData); @@ -652,6 +656,32 @@ public void testNullFieldMetadataDebugLogging() { ParquetMetadata.toJSON(metadata); } + @Test + public void testEncryptedFieldMetadataDebugLogging() { + Configuration conf = new Configuration(); + conf.set( + EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME, + "org.apache.parquet.crypto.SampleDecryptionPropertiesFactory"); + DecryptionPropertiesFactory decryptionPropertiesFactory = DecryptionPropertiesFactory.loadFactory(conf); + FileDecryptionProperties decryptionProperties = + decryptionPropertiesFactory.getFileDecryptionProperties(conf, null); + + MessageType schema = parseMessageType("message test { optional binary some_null_field; }"); + + org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = + new org.apache.parquet.hadoop.metadata.FileMetaData( + schema, + new HashMap<>(), + null, + org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.ENCRYPTED_FOOTER, + new InternalFileDecryptor(decryptionProperties)); + + List blockMetaDataList = new ArrayList<>(); + ParquetMetadata metadata = new ParquetMetadata(fileMetaData, blockMetaDataList); + ParquetMetadata.toJSON(metadata); + System.out.println(ParquetMetadata.toPrettyJSON(metadata)); + } + @Test public void testMetadataToJson() { ParquetMetadata metadata = new ParquetMetadata(null, null); From 114600624882392947057c582cc79209caea3870 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 15 May 2024 11:51:06 +0200 Subject: [PATCH 5/8] Add comment for jackson datatype jdk8 --- .../org/apache/parquet/hadoop/metadata/ParquetMetadata.java | 5 +++-- parquet-jackson/pom.xml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java index 8ddf6ecf57..640ecfba1b 100755 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java @@ -35,10 +35,11 @@ public class ParquetMetadata { private static final ObjectMapper objectMapper = new ObjectMapper(); - // Enable FAIL_ON_EMPTY_BEANS on objectmapper. Without this feature parquet-casdacing tests fail, - // because LogicalTypeAnnotation implementations are classes without any property. static { + // Enable FAIL_ON_EMPTY_BEANS on objectmapper. Without this feature parquet-casdacing tests fail, + // because LogicalTypeAnnotation implementations are classes without any property. objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + // Add support for Java 8 Optional objectMapper.registerModule(new Jdk8Module()); } diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 47b1aab7d0..e847b795dd 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -43,6 +43,7 @@ jackson-databind ${jackson-databind.version} + com.fasterxml.jackson.datatype jackson-datatype-jdk8 From 1ad883fd461d85be4042d786f6d827ac5cb838b8 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Wed, 15 May 2024 11:54:24 +0200 Subject: [PATCH 6/8] Revert statistics test --- .../format/converter/TestParquetMetadataConverter.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index d1177158bd..2cffb51860 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -697,13 +697,7 @@ private ColumnChunkMetaData createColumnChunkMetaData() { PrimitiveTypeName t = PrimitiveTypeName.BINARY; ColumnPath p = ColumnPath.get("foo"); CompressionCodecName c = CompressionCodecName.GZIP; - Statistics s = Statistics.createStats(Types.required(PrimitiveTypeName.BINARY) - .as(LogicalTypeAnnotation.stringType()) - .named("str")); - byte[] min = new byte[904]; - byte[] max = new byte[2388]; - s.updateStats(Binary.fromConstantByteArray(min)); - s.updateStats(Binary.fromConstantByteArray(max)); + BinaryStatistics s = new BinaryStatistics(); ColumnChunkMetaData md = ColumnChunkMetaData.get(p, t, c, e, s, 0, 0, 0, 0, 0); return md; } From 327e9d175ed4efd8da1c88e822305cf62fd06c30 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 16 May 2024 16:33:30 +0200 Subject: [PATCH 7/8] Add explicit lib dependency --- parquet-hadoop/pom.xml | 11 ++++++++--- parquet-jackson/pom.xml | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index be2649832b..7e0e8bebee 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -119,15 +119,20 @@ ${jackson.version} - ${jackson.datatype.groupId} - jackson-datatype-jdk8 - ${jackson-modules-java8.version} + ${jackson.groupId} + jackson-annotations + ${jackson.version} ${jackson.groupId} jackson-databind ${jackson-databind.version} + + ${jackson.datatype.groupId} + jackson-datatype-jdk8 + ${jackson-modules-java8.version} + org.xerial.snappy snappy-java diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index e847b795dd..23ee80fbd7 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -38,6 +38,11 @@ jackson-core ${jackson.version} + + ${jackson.groupId} + jackson-annotations + ${jackson.version} + com.fasterxml.jackson.core jackson-databind From a04d1dba8ef05e939107ac0c465534f8da9c1f04 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Thu, 16 May 2024 16:36:50 +0200 Subject: [PATCH 8/8] Sync jackson versions --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5bc4ecc6e0..ec45db0898 100644 --- a/pom.xml +++ b/pom.xml @@ -67,9 +67,9 @@ com.fasterxml.jackson.core com.fasterxml.jackson.datatype com.fasterxml.jackson - 2.17.0 + 2.17.1 2.17.1 - 2.17.0 + 2.17.1 0.21.0 1.3.2 2.30.0