PARQUET-1341: Fix null count stats in unsigned-sort columns. (apache#499

) * Fix null count stats in unsigned-sort columns. * Fix test case for old min/max values and unsigned ordering.
RMS · Aug 18, 2018 · 0694a27 · 0694a27
1 parent 08a82c2
commit 0694a27
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/...et-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/...et-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -621,9 +621,6 @@ public static org.apache.parquet.column.statistics.Statistics fromParquetStatist
           statsBuilder.withMin(min);
           statsBuilder.withMax(max);
         }
-        if (formatStats.isSetNull_count()) {
-          statsBuilder.withNumNulls(formatStats.null_count);
-        }
       } else {
         boolean isSet = formatStats.isSetMax() && formatStats.isSetMin();
         boolean maxEqualsMin = isSet ? Arrays.equals(formatStats.getMin(), formatStats.getMax()) : false;
@@ -639,11 +636,12 @@ public static org.apache.parquet.column.statistics.Statistics fromParquetStatist
             statsBuilder.withMin(formatStats.min.array());
             statsBuilder.withMax(formatStats.max.array());
           }
-          if (formatStats.isSetNull_count()) {
-            statsBuilder.withNumNulls(formatStats.null_count);
-          }
         }
       }
+
+      if (formatStats.isSetNull_count()) {
+        statsBuilder.withNumNulls(formatStats.null_count);
+      }
     }
     return statsBuilder.build();
   }

diff --git a/...adoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/...adoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -617,7 +617,9 @@ public void testIgnoreStatsWithSignedSortOrder() {
         StatsHelper.V1.toParquetStatistics(stats),
         binaryType);
 
-    Assert.assertTrue("Stats should be empty: " + convertedStats, convertedStats.isEmpty());
+    Assert.assertFalse("Stats should not include min/max: " + convertedStats, convertedStats.hasNonNullValue());
+    Assert.assertTrue("Stats should have null count: " + convertedStats, convertedStats.isNumNullsSet());
+    Assert.assertEquals("Stats should have 3 nulls: " + convertedStats, 3L, convertedStats.getNumNulls());
   }
 
   @Test