PARQUET-2468: ParquetMetadata must convert to json

parquet-hadoop/pom.xml

@@ -118,6 +118,11 @@
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${jackson.datatype.groupId}</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+      <version>${jackson-modules-java8.version}</version>
+    </dependency>
     <dependency>
       <groupId>${jackson.groupId}</groupId>
       <artifactId>jackson-databind</artifactId>

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java

@@ -22,6 +22,7 @@
 import static org.apache.parquet.column.Encoding.RLE_DICTIONARY;
 import static org.apache.parquet.format.Util.readColumnMetaData;
 
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.Set;
@@ -380,6 +381,7 @@ public PrimitiveType getPrimitiveType() {
   /**
    * @return the stats for this column
    */
+  @JsonSerialize(using = StatisticsSerializer.class)
   public abstract Statistics getStatistics();
 
   /**

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ParquetMetadata.java

@@ -19,6 +19,9 @@
 package org.apache.parquet.hadoop.metadata;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
@@ -32,6 +35,13 @@ public class ParquetMetadata {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  // Enable FAIL_ON_EMPTY_BEANS on objectmapper. Without this feature parquet-casdacing tests fail,
+  // because LogicalTypeAnnotation implementations are classes without any property.
+  static {
+    objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
+    objectMapper.registerModule(new Jdk8Module());
+  }
+
   /**
    * @param parquetMetaData an instance of parquet metadata to convert
    * @return the json representation
@@ -50,19 +60,23 @@ public static String toPrettyJSON(ParquetMetadata parquetMetaData) {
 
   private static String toJSON(ParquetMetadata parquetMetaData, boolean isPrettyPrint) {
     try (StringWriter stringWriter = new StringWriter()) {
+      Object objectToPrint;
+      if (parquetMetaData.getFileMetaData() == null
+          || parquetMetaData.getFileMetaData().getEncryptionType()
+              == FileMetaData.EncryptionType.UNENCRYPTED) {
+        objectToPrint = parquetMetaData;
+      } else {
+        objectToPrint = parquetMetaData.getFileMetaData();
+      }
+
+      ObjectWriter writer;
       if (isPrettyPrint) {
-        Object objectToPrint;
-        if (parquetMetaData.getFileMetaData() == null
-            || parquetMetaData.getFileMetaData().getEncryptionType()
-                == FileMetaData.EncryptionType.UNENCRYPTED) {
-          objectToPrint = parquetMetaData;
-        } else {
-          objectToPrint = parquetMetaData.getFileMetaData();
-        }
-        objectMapper.writerWithDefaultPrettyPrinter().writeValue(stringWriter, objectToPrint);
+        writer = objectMapper.writerWithDefaultPrettyPrinter();
       } else {
-        objectMapper.writeValue(stringWriter, parquetMetaData);
+        writer = objectMapper.writer();
       }
+
+      writer.writeValue(stringWriter, objectToPrint);
       return stringWriter.toString();
     } catch (IOException e) {
       throw new RuntimeException(e);

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/StatisticsSerializer.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop.metadata;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import java.io.IOException;
+import org.apache.parquet.column.statistics.Statistics;
+
+class StatisticsSerializer extends JsonSerializer<Statistics<?>> {
+  @Override
+  public void serialize(Statistics<?> statistics, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
+      throws IOException {
+    jsonGenerator.writeStartObject();
+    jsonGenerator.writeFieldName("min");
+    jsonGenerator.writeString(statistics.minAsString());
+    jsonGenerator.writeFieldName("max");
+    jsonGenerator.writeString(statistics.maxAsString());
+    jsonGenerator.writeFieldName("null_count");
+    jsonGenerator.writeNumber(statistics.getNumNulls());
+    jsonGenerator.writeEndObject();
+  }
+}

parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java

@@ -638,7 +638,12 @@ public void randomTestFilterMetaData() {
   public void testNullFieldMetadataDebugLogging() {
     MessageType schema = parseMessageType("message test { optional binary some_null_field; }");
     org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
-        new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap<String, String>(), null);
+        new org.apache.parquet.hadoop.metadata.FileMetaData(
+            schema,
+            new HashMap<>(),
+            null,
+            org.apache.parquet.hadoop.metadata.FileMetaData.EncryptionType.UNENCRYPTED,
+            null);
     List<BlockMetaData> blockMetaDataList = new ArrayList<BlockMetaData>();
     BlockMetaData blockMetaData = new BlockMetaData();
     blockMetaData.addColumn(createColumnChunkMetaData());
@@ -662,7 +667,13 @@ private ColumnChunkMetaData createColumnChunkMetaData() {
     PrimitiveTypeName t = PrimitiveTypeName.BINARY;
     ColumnPath p = ColumnPath.get("foo");
     CompressionCodecName c = CompressionCodecName.GZIP;
-    BinaryStatistics s = new BinaryStatistics();
+    Statistics<?> s = Statistics.createStats(Types.required(PrimitiveTypeName.BINARY)
+        .as(LogicalTypeAnnotation.stringType())
+        .named("str"));
+    byte[] min = new byte[904];
+    byte[] max = new byte[2388];
+    s.updateStats(Binary.fromConstantByteArray(min));
+    s.updateStats(Binary.fromConstantByteArray(max));
     ColumnChunkMetaData md = ColumnChunkMetaData.get(p, t, c, e, s, 0, 0, 0, 0, 0);
     return md;
   }

parquet-jackson/pom.xml

@@ -43,6 +43,11 @@
       <artifactId>jackson-databind</artifactId>
       <version>${jackson-databind.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+      <version>${jackson-modules-java8.version}</version>
+    </dependency>
   </dependencies>
 
   <properties>
@@ -70,6 +75,7 @@
               <artifactSet>
                 <includes>
                   <include>${jackson.groupId}:*</include>
+                  <include>${jackson.datatype.groupId}:*</include>
                 </includes>
               </artifactSet>
               <filters>
@@ -79,6 +85,12 @@
                     <include>**</include>
                   </includes>
                 </filter>
+                <filter>
+                  <artifact>${jackson.datatype.groupId}:*</artifact>
+                  <includes>
+                    <include>**</include>
+                  </includes>
+                </filter>
               </filters>
               <relocations>
                 <relocation>

pom.xml

@@ -69,6 +69,7 @@
     <jackson.package>com.fasterxml.jackson</jackson.package>
     <jackson.version>2.17.0</jackson.version>
     <jackson-databind.version>2.17.0</jackson-databind.version>
+    <jackson-modules-java8.version>2.17.0</jackson-modules-java8.version>
     <japicmp.version>0.21.0</japicmp.version>
     <javax.annotation.version>1.3.2</javax.annotation.version>
     <spotless.version>2.30.0</spotless.version>