diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 478b78dea3e3..e726f0d2d5ca 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -564,6 +564,56 @@ acceptedBreaks: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME" justification: "Removed deprecated functionality for partition stats" + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNanValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNullValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::tightBounds()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method int org.apache.iceberg.TrackedFile::formatVersion()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method java.lang.Integer org.apache.iceberg.FieldStats::avgValueSizeInBytes()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy(java.util.Set)" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.FieldStats org.apache.iceberg.FieldStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.types.Types.StructType org.apache.iceberg.ContentStats::type()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." - code: "java.method.removed" old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" justification: "Deprecated for removal in 1.12.0" diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java index 75ca9d5835bc..17f3873d70af 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java @@ -127,7 +127,7 @@ private Expression bindUnaryOperation(StructType struct, BoundTerm boundTerm) switch (op()) { case IS_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysFalse(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysTrue(); @@ -135,7 +135,7 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { return new BoundUnaryPredicate<>(Operation.IS_NULL, boundTerm); case NOT_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysTrue(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysFalse(); @@ -158,11 +158,6 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { } } - private boolean allAncestorFieldsAreRequired(StructType struct, int fieldId) { - return TypeUtil.ancestorFields(struct.asSchema(), fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - private boolean floatingType(Type.TypeID typeID) { return Type.TypeID.DOUBLE.equals(typeID) || Type.TypeID.FLOAT.equals(typeID); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index a3bee3e3d860..5f7ad3310c55 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -273,6 +273,25 @@ public static List ancestorFields(Schema schema, int fieldId) return parents; } + /** + * Returns whether a field may contain null values. + * + *

A field may be null if it is optional or if any field that contains it is optional. A + * required field nested in an optional struct is null whenever that struct is null. + * + * @param schema The schema that contains the field ID + * @param fieldId The field ID to check + * @return true if the field may be null, false if it cannot be null + * @throws IllegalArgumentException if the field ID is not present in the schema + */ + public static boolean isNullable(Schema schema, int fieldId) { + Types.NestedField field = schema.findField(fieldId); + Preconditions.checkArgument(field != null, "Cannot find field with ID: %s", fieldId); + + return field.isOptional() + || ancestorFields(schema, fieldId).stream().anyMatch(Types.NestedField::isOptional); + } + /** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a type. * diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 8d652df0df03..ad29d85efe0d 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -57,8 +57,16 @@ import org.apache.iceberg.util.UnicodeUtil; import org.junit.jupiter.api.Test; -public class TestInclusiveMetricsEvaluator { - private static final Schema SCHEMA = +/** + * Tests inclusive evaluation of expressions using file statistics. + * + *

Subclasses evaluate the same expressions against a different representation of file statistics + * by overriding the file methods and {@link #shouldRead(Schema, Expression, boolean, Object)}. + * + * @param the type of file that carries the statistics under test + */ +public class TestInclusiveMetricsEvaluator { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), optional(2, "no_stats", Types.IntegerType.get()), @@ -75,7 +83,7 @@ public class TestInclusiveMetricsEvaluator { optional(13, "no_nan_stats", Types.DoubleType.get()), optional(14, "some_empty", Types.StringType.get())); - private static final Schema NESTED_SCHEMA = + protected static final Schema NESTED_SCHEMA = new Schema( required( 100, @@ -90,8 +98,10 @@ public class TestInclusiveMetricsEvaluator { required(104, "required_street2", Types.StringType.get()), optional(105, "optional_street2", Types.StringType.get())))); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final Schema FLOAT_SCHEMA = new Schema(required(1, "f", Types.FloatType.get())); + + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -210,9 +220,9 @@ public class TestInclusiveMetricsEvaluator { Row.of(), 10, // any value counts, including nulls - ImmutableMap.of(100, 5L, 101, 5L, 102, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(102, 5L, 103, 5L, 104, 5L, 105, 5L), // null value counts - ImmutableMap.of(100, 0L, 101, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(103, 5L, 104, 5L, 105, 5L), // nan value counts null, // lower bounds @@ -220,78 +230,234 @@ public class TestInclusiveMetricsEvaluator { // upper bounds null); + private static final DataFile MISSING_STATS = new TestDataFile("file.parquet", Row.of(), 50); + + private static final DataFile EMPTY_FILE = new TestDataFile("file.parquet", Row.of(), 0); + + private static final DataFile RANGE_OF_VALUES = + new TestDataFile( + "range_of_values.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); + + private static final DataFile SINGLE_VALUE_FILE = + new TestDataFile( + "single_value.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + + // some_empty is optional because a required column cannot contain nulls + private static final DataFile SINGLE_VALUE_WITH_NULLS = + new TestDataFile( + "single_value_nulls.avro", + Row.of(), + 10, + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), + ImmutableMap.of(14, 0L), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); + + private static final DataFile SINGLE_VALUE_WITH_NAN = + new TestDataFile( + "single_value_nan.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 2L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); + + private static final DataFile SINGLE_VALUE_NAN_BOUNDS = + new TestDataFile( + "single_value_nan_bounds.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); + + private static final Map FLOAT_BOUND = + ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 0L), + FLOAT_BOUND, + FLOAT_BOUND); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE_WITH_NAN = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 1L), // contains a NaN value + FLOAT_BOUND, + FLOAT_BOUND); + + protected boolean shouldRead(Schema schema, Expression expr, boolean caseSensitive, F testFile) { + return new InclusiveMetricsEvaluator(schema, expr, caseSensitive).eval((DataFile) testFile); + } + + protected boolean shouldRead(Schema schema, Expression expr, F testFile) { + return shouldRead(schema, expr, true, testFile); + } + + protected F file() { + return asFile(FILE); + } + + protected F file2() { + return asFile(FILE_2); + } + + protected F file3() { + return asFile(FILE_3); + } + + protected F file4() { + return asFile(FILE_4); + } + + protected F file5() { + return asFile(FILE_5); + } + + protected F file6() { + return asFile(FILE_6); + } + + protected F missingStats() { + return asFile(MISSING_STATS); + } + + protected F emptyFile() { + return asFile(EMPTY_FILE); + } + + protected F rangeOfValues() { + return asFile(RANGE_OF_VALUES); + } + + protected F singleValueFile() { + return asFile(SINGLE_VALUE_FILE); + } + + protected F singleValueWithNulls() { + return asFile(SINGLE_VALUE_WITH_NULLS); + } + + protected F singleValueWithNaN() { + return asFile(SINGLE_VALUE_WITH_NAN); + } + + protected F singleValueNaNBounds() { + return asFile(SINGLE_VALUE_NAN_BOUNDS); + } + + protected F singleFloatValueFile() { + return asFile(SINGLE_FLOAT_VALUE_FILE); + } + + protected F singleFloatValueFileWithNaN() { + return asFile(SINGLE_FLOAT_VALUE_FILE_WITH_NAN); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; + } + @Test public void testAllNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("all_nulls"), file()); assertThat(shouldRead).as("Should skip: no non-null value in all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: equal on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: startsWith on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notStartsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should read: notStartsWith on all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("some_nulls"), file()); assertThat(shouldRead) .as("Should read: column with some nulls contains a non-null value") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("no_nulls"), file()); assertThat(shouldRead).as("Should read: non-null column contains a non-null value").isTrue(); } @Test public void testNoNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNull("all_nulls"), file()); assertThat(shouldRead).as("Should read: at least one null value in all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("some_nulls"), file()); assertThat(shouldRead).as("Should read: column with some nulls contains a null value").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("no_nulls"), file()); assertThat(shouldRead).as("Should skip: non-null column contains no null values").isFalse(); } @Test public void testIsNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNaN("all_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("some_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in some nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nans"), file()); assertThat(shouldRead).as("Should skip: no-nans column contains no nan values").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nulls_double"), file()); assertThat(shouldRead).as("Should skip: all-null column doesn't contain nan value").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nans_v1_stats"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one nan value in nan and nulls only column") .isTrue(); @@ -299,35 +465,35 @@ public void testIsNaN() { @Test public void testNotNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNaN("all_nans"), file()); assertThat(shouldRead) .as("Should skip: column with all nans will not contain non-nan") .isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("some_nans"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in some nan column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nans"), file()); assertThat(shouldRead).as("Should read: at least one non-nan value in no nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nulls_double"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in all null column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nans_v1_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one null value in nan and nulls only column") .isTrue(); @@ -335,25 +501,22 @@ public void testNotNaN() { @Test public void testRequiredColumn() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("required")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("required"), file()); assertThat(shouldRead).as("Should read: required columns are always non-null").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("required")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("required"), file()); assertThat(shouldRead).as("Should skip: required columns are always non-null").isFalse(); } @Test public void testMissingColumn() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, lessThan("missing", 5)).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, lessThan("missing", 5), file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'missing'"); } @Test public void testMissingStats() { - DataFile missingStats = new TestDataFile("file.parquet", Row.of(), 50); - Expression[] exprs = new Expression[] { lessThan("no_stats", 5), @@ -369,15 +532,13 @@ public void testMissingStats() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(missingStats); + boolean shouldRead = shouldRead(SCHEMA, expr, missingStats()); assertThat(shouldRead).as("Should read when missing stats for expr: " + expr).isTrue(); } } @Test public void testZeroRecordFile() { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); - Expression[] exprs = new Expression[] { lessThan("id", 5), @@ -393,7 +554,7 @@ public void testZeroRecordFile() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(empty); + boolean shouldRead = shouldRead(SCHEMA, expr, emptyFile()); assertThat(shouldRead).as("Should never read 0-record file: " + expr).isFalse(); } } @@ -401,13 +562,10 @@ public void testZeroRecordFile() { @Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: not(false)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25))) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should skip: not(true)").isFalse(); } @@ -415,28 +573,24 @@ public void testNot() { public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MIN_VALUE - 30))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MIN_VALUE - 30)), + file()); assertThat(shouldRead).as("Should skip: and(false, true)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: and(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE))) - .eval(FILE); + shouldRead( + SCHEMA, + and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE)), + file()); assertThat(shouldRead).as("Should read: and(true, true)").isTrue(); } @@ -444,398 +598,312 @@ public void testAnd() { public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: or(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE - 19))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE - 19)), + file()); assertThat(shouldRead).as("Should read: or(false, true)").isTrue(); } @Test public void testIntegerLt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range below lower bound (30 is not < 30)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE + 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerLtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range above upper bound (79 is not > 79)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6)) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (80 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); } @Test public void testIntegerNotEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 1)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE - 4))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE - 4)), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 1)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 6))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 6)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseInsensitiveIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false, file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseSensitiveIntegerNotEqRewritten() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", 5)), true).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, not(equal("ID", 5)), true, file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'ID'"); } @Test public void testStringStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1str1x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "3str3x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("some_empty", "房东整租霍"), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("some_empty", "房东整租霍"), true, file()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", ""), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", ""), true, file()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", aboveMax), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); } @Test public void testStringNotStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1str1x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "3str3x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", aboveMax), true) - .eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abc"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abc"), true, file5()); assertThat(shouldRead).as("Should not read: all strings start with prefix").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abcd"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abcd"), true, file5()); assertThat(shouldRead).as("Should not read: lower shorter than prefix, cannot match").isTrue(); } @Test public void testIntegerIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should not read: id below lower bound (5 < 30, 6 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound (28 < 30, 29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should not read: id above upper bound (80 > 79, 81 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should not read: id above upper bound (85 > 79, 86 > 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should skip: in on all nulls column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on no nulls column").isTrue(); // should read as the number of elements in the in expression is too big @@ -843,131 +911,97 @@ public void testIntegerIn() { for (int id = -400; id <= 0; id++) { ids.add(id); } - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("id", ids)).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", ids), file()); assertThat(shouldRead).as("Should read: large in expression").isTrue(); } @Test public void testIntegerNotIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should read: id below lower bound (5 < 30, 6 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound (28 < 30, 29 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should read: id above upper bound (80 > 79, 81 > 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should read: id above upper bound (85 > 79, 86 > 79)").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on all nulls column").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue(); } @Test public void testIsNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address"), file6()); assertThat(shouldRead).as("Should not read: required_address is required").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.required_street1 is required") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.optional_street1 is optional") .isTrue(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address")).eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.optional_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); } @Test public void testNotNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address"), file6()); assertThat(shouldRead).as("Should read: required_address is required").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.required_street1 is required") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.optional_street1 is optional") .isFalse(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address")).eval(FILE_6); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); + assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.optional_street2"), file6()); assertThat(shouldRead) .as("Should not read: optional_address.optional_street2 is optional") .isFalse(); @@ -975,207 +1009,66 @@ public void testNotNullInNestedStruct() { @Test public void testNotEqSingleValueWithoutNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 0L), - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFile())) .as("Should skip: file contains no values not equal to 1.0") .isFalse(); } @Test public void testNotEqSingleValueWithNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 1L), // contains a NaN value - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFileWithNaN())) .as("Should read: file contains a NaN value not equal to 1.0") .isTrue(); } @Test public void testNotEqWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "aaa")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notEqual("required", "aaa"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "abc"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value equal to literal") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "def")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "def"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not equal to literal") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notEqual("some_empty", "abc"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueWithNaN()); assertThat(shouldRead).as("Should read: file has NaN values which match != predicate").isTrue(); - DataFile singleValueNaNBounds = - new TestDataFile( - "single_value_nan_bounds.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueNaNBounds); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueNaNBounds()); assertThat(shouldRead).as("Should read: bounds are NaN").isTrue(); } @Test public void testNotInWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "aaa", "bbb")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notIn("required", "aaa", "bbb"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "abc", "def"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value in exclusion list") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "def", "ghi")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "def", "ghi"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not in exclusion list") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notIn("some_empty", "abc", "def"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nans", 5.0F, 10.0F)) - .eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notIn("no_nans", 5.0F, 10.0F), singleValueWithNaN()); assertThat(shouldRead) .as("Should read: file has NaN values which match NOT IN predicate") .isTrue(); diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index b7da4b3108e6..872bccb987c4 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -971,6 +971,105 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + @Test + public void isNullableWithUnknownFieldId() { + Schema schema = new Schema(required(1, "id", IntegerType.get())); + + assertThatThrownBy(() -> TypeUtil.isNullable(schema, 2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot find field with ID: 2"); + } + + @Test + public void isNullableWithTopLevelFields() { + Schema schema = + new Schema( + required(1, "id", IntegerType.get()), optional(2, "data", Types.StringType.get())); + + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + } + + @Test + public void isNullableWithNestedStructs() { + Schema schema = + new Schema( + required( + 1, + "required_location", + Types.StructType.of( + required(3, "required_lat", Types.DoubleType.get()), + optional(4, "optional_lon", Types.DoubleType.get()), + required( + 5, + "required_inner", + Types.StructType.of(required(6, "required_zip", IntegerType.get()))))), + optional( + 2, + "optional_location", + Types.StructType.of( + required(7, "required_lat", Types.DoubleType.get()), + required( + 8, + "required_inner", + Types.StructType.of(required(9, "required_zip", IntegerType.get())))))); + + // a required field is not null when every field that contains it is required + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 6)).isFalse(); + + // an optional field is null regardless of the fields that contain it + assertThat(TypeUtil.isNullable(schema, 4)).isTrue(); + + // a required field nested in an optional struct is null when the struct is null + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 8)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 9)).isTrue(); + } + + @Test + public void isNullableWithListsAndMaps() { + Schema schema = + new Schema( + required( + 1, + "required_points", + Types.ListType.ofRequired( + 4, Types.StructType.of(required(5, "required_x", Types.LongType.get())))), + optional( + 2, + "optional_points", + Types.ListType.ofOptional( + 6, Types.StructType.of(required(7, "required_x", Types.LongType.get())))), + required( + 3, + "locations", + Types.MapType.ofRequired( + 8, + 9, + Types.StringType.get(), + Types.StructType.of(required(10, "required_lat", Types.DoubleType.get()))))); + + // a required element of a required list is not null, nor is anything it contains + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 4)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + + // an optional element is null, as is anything it contains + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 6)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + + // required map keys and values are not null + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 8)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 9)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 10)).isFalse(); + } + @Test public void testIndexStatsNames() { Schema schema = diff --git a/core/src/main/java/org/apache/iceberg/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java index aa7a65ac934d..51bfa52a67ad 100644 --- a/core/src/main/java/org/apache/iceberg/ContentStats.java +++ b/core/src/main/java/org/apache/iceberg/ContentStats.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.iceberg.types.Types; -interface ContentStats { +public interface ContentStats { /** Returns an iterable of the {@link FieldStats} held by this container struct */ Iterable> fieldStats(); diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java index 495dddba913d..693765a8977b 100644 --- a/core/src/main/java/org/apache/iceberg/FieldStats.java +++ b/core/src/main/java/org/apache/iceberg/FieldStats.java @@ -20,7 +20,7 @@ import org.apache.iceberg.types.Types; -interface FieldStats { +public interface FieldStats { /** The field ID of the statistic */ int fieldId(); diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 545275002c9a..47decef0ce90 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,7 +147,10 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - field.isOptional(), field.type(), baseId, metricsConfig.columnMode(id)); + TypeUtil.isNullable(tableSchema, id), + field.type(), + baseId, + metricsConfig.columnMode(id)); if (fieldStruct != null) { fieldStructs.add(optional(baseId, fieldName, fieldStruct)); @@ -178,7 +181,11 @@ public static Types.StructType statsReadSchema(Schema tableSchema, IterableThis evaluation is inclusive: it returns true if a file may match and false if it cannot + * match. + * + *

Files are passed to {@link #eval(TrackedFile)}, which returns true if the file may contain + * matching rows and false if the file cannot contain matching rows. Files may be skipped if and + * only if the return value of {@code eval} is false. + * + *

Due to the comparison implementation of ORC stats, for float/double columns in ORC files, if + * the first value in a file is NaN, metrics of this file will report NaN for both upper and lower + * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly + * checks for NaN is necessary in order to not skip files that may contain matching data. + */ +public class InclusiveStatsEvaluator { + private final Expression expr; + private final Set neverNullIds; + + public InclusiveStatsEvaluator(Schema schema, Expression unbound) { + this(schema, unbound, true); + } + + public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + Types.StructType struct = schema.asStruct(); + this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + this.neverNullIds = + neverNullIds( + schema, Binder.boundReferences(struct, Collections.singletonList(expr), caseSensitive)); + } + + /** + * Returns the IDs of the referenced fields that cannot contain null values. + * + *

Stats omit the null count for such fields, which must not be confused with an unknown count. + */ + private static Set neverNullIds(Schema schema, Set referencedIds) { + ImmutableSet.Builder neverNull = ImmutableSet.builder(); + + for (int id : referencedIds) { + if (!TypeUtil.isNullable(schema, id)) { + neverNull.add(id); + } + } + + return neverNull.build(); + } + + /** + * Test whether the file may contain records that match the expression. + * + * @param file a tracked file + * @return false if the file cannot contain rows that match the expression, true otherwise. + */ + public boolean eval(TrackedFile file) { + return new StatsEvalVisitor().eval(file); + } + + private class StatsEvalVisitor extends InclusiveEvalVisitor { + private ContentStats stats = null; + + private boolean eval(TrackedFile file) { + if (file.recordCount() == 0) { + return ROWS_CANNOT_MATCH; + } + + if (file.recordCount() < 0) { + // imported Avro files may have an incorrect -1 row count + return ROWS_MIGHT_MATCH; + } + + if (null == file.contentStats()) { + return ROWS_MIGHT_MATCH; + } + + this.stats = file.contentStats(); + + return ExpressionVisitors.visitEvaluator(expr, this); + } + + @Override + protected boolean mayContainNull(int id) { + if (neverNullIds.contains(id)) { + return false; + } + + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNullValueCount() + || fieldStats.nullValueCount() != 0; + } + + @Override + protected boolean containsNullsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNullValueCount() + && fieldStats.valueCount() - fieldStats.nullValueCount() == 0; + } + + @Override + protected boolean mayContainNaN(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNanValueCount() + || fieldStats.nanValueCount() != 0; + } + + @Override + protected boolean containsNaNsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNanValueCount() + && fieldStats.valueCount() - fieldStats.nanValueCount() == 0; + } + + @Override + protected T lowerBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.lowerBound() : null; + } + + @Override + protected T upperBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.upperBound() : null; + } + + @Override + protected T extractLowerBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.lowerBound() != null) { + VariantObject fieldLowerBounds = fieldStats.lowerBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldLowerBounds.get(bound.path()), bound.type()); + } + + return null; + } + + @Override + protected T extractUpperBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.upperBound() != null) { + VariantObject fieldUpperBounds = fieldStats.upperBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldUpperBounds.get(bound.path()), bound.type()); + } + + return null; + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java new file mode 100644 index 000000000000..a15a199c1f33 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.InclusiveStatsEvaluator; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +/** Runs the inclusive evaluation tests against {@link ContentStats} tracked by a file. */ +class TestInclusiveStatsEvaluator extends TestInclusiveMetricsEvaluator { + private static final int FORMAT_VERSION_V4 = 4; + + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema( + SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + + // stats are tracked for the leaf fields, not for the address structs themselves + private static final Types.StructType NESTED_STATS_TYPE = + StatsUtil.statsReadSchema(NESTED_SCHEMA, ImmutableList.of(102, 103, 104, 105)); + + private static final Types.StructType FLOAT_STATS_TYPE = + StatsUtil.statsReadSchema(FLOAT_SCHEMA, ImmutableList.of(1)); + + @Override + protected boolean shouldRead( + Schema schema, Expression expr, boolean caseSensitive, TrackedFile testFile) { + return new InclusiveStatsEvaluator(schema, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + stats(1, INT_MIN_VALUE, INT_MAX_VALUE, null, null, null), + stats(4, null, null, 50L, 50L, null), + stats(5, null, null, 50L, 10L, null), + stats(6, null, null, 50L, 0L, null), + stats(7, null, null, 50L, null, 50L), + stats(8, null, null, 50L, null, 10L), + stats(9, null, null, 50L, null, 0L), + stats(10, null, null, 50L, 50L, null), + stats(11, Float.NaN, Float.NaN, 50L, 0L, null), + stats(12, Double.NaN, Double.NaN, 50L, 1L, null), + stats(13, null, null, 50L, null, null), + stats(14, "", "房东整租霍营小区二层两居室", 50L, 0L, null))); + } + + @Override + protected TrackedFile file2() { + return trackedFile( + "file_2.avro", 50, contentStats(STATS_TYPE, stats(3, "aa", "dC", 50L, null, null))); + } + + @Override + protected TrackedFile file3() { + return trackedFile( + "file_3.avro", 50, contentStats(STATS_TYPE, stats(3, "1str1", "3str3", 50L, null, null))); + } + + @Override + protected TrackedFile file4() { + return trackedFile( + "file_4.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "イロハニホヘト", 50L, null, null))); + } + + @Override + protected TrackedFile file5() { + return trackedFile( + "file_5.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "abcdefghi", 50L, null, null))); + } + + @Override + protected TrackedFile file6() { + return trackedFile( + "file_6.avro", + 10, + contentStats( + NESTED_STATS_TYPE, + stats(NESTED_STATS_TYPE, 102, null, null, 5L, null, null), + stats(NESTED_STATS_TYPE, 103, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 104, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 105, null, null, 5L, 5L, null))); + } + + @Override + protected TrackedFile missingStats() { + return trackedFile("file.parquet", 50, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile rangeOfValues() { + return trackedFile( + "range_of_values.avro", + 10, + contentStats(STATS_TYPE, stats(3, "aaa", "zzz", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueFile() { + return trackedFile( + "single_value.avro", 10, contentStats(STATS_TYPE, stats(3, "abc", "abc", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueWithNulls() { + return trackedFile( + "single_value_nulls.avro", + 10, + contentStats(STATS_TYPE, stats(14, "abc", "abc", 10L, 2L, null))); + } + + @Override + protected TrackedFile singleValueWithNaN() { + return trackedFile( + "single_value_nan.avro", 10, contentStats(STATS_TYPE, stats(9, 5.0f, 5.0f, 10L, 0L, 2L))); + } + + @Override + protected TrackedFile singleValueNaNBounds() { + return trackedFile( + "single_value_nan_bounds.avro", + 10, + contentStats(STATS_TYPE, stats(9, Float.NaN, Float.NaN, 10L, 0L, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFile() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFileWithNaN() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 1L))); + } + + @Test + void fileWithoutContentStats() { + TrackedFile file = trackedFile("file.avro", 50, null); + + assertThat(shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file)) + .as("Should read: file does not track content stats") + .isTrue(); + } + + private static FieldStats stats( + int fieldId, Object lower, Object upper, Long valueCount, Long nullCount, Long nanCount) { + return stats(STATS_TYPE, fieldId, lower, upper, valueCount, nullCount, nanCount); + } + + /** Returns the stats for a field, where a null metric is one that the column does not track. */ + private static FieldStats stats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + Types.StructType type = statsType.field(StatsUtil.toBaseId(fieldId)).type().asStructType(); + FieldStatsStruct stats = new FieldStatsStruct<>(type); + set(stats, StatsUtil.LOWER_BOUND_NAME, lower); + set(stats, StatsUtil.UPPER_BOUND_NAME, upper); + set(stats, "value_count", valueCount); + set(stats, "null_value_count", nullCount); + set(stats, "nan_value_count", nanCount); + + return stats; + } + + private static void set(FieldStatsStruct stats, String metric, Object value) { + if (value == null) { + return; + } + + List fields = stats.type().fields(); + for (int pos = 0; pos < fields.size(); pos += 1) { + if (fields.get(pos).name().equals(metric)) { + stats.set(pos, value); + return; + } + } + + throw new IllegalArgumentException("Cannot set " + metric + ": not tracked by " + stats.type()); + } + + private static ContentStats contentStats( + Types.StructType statsType, FieldStats... fieldStats) { + ContentStatsStruct stats = new ContentStatsStruct(statsType); + for (FieldStats field : fieldStats) { + stats.setStats(field.fieldId(), field); + } + + return stats; + } + + private static TrackedFile trackedFile(String location, long recordCount, ContentStats stats) { + return new TrackedFileStruct( + null, + FileContent.DATA, + FORMAT_VERSION_V4, + location, + FileFormat.fromFileName(location), + null, + recordCount, + 1024L, + null, + stats, + null, + null, + null, + null, + null, + null); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index e145dd61e14c..f8fa596fcc61 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -395,12 +395,14 @@ public void testContentStatsReadSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( @@ -518,12 +520,14 @@ public void testContentStatsWriteSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( diff --git a/format/spec.md b/format/spec.md index be28a6c5f000..5faba5dcfc69 100644 --- a/format/spec.md +++ b/format/spec.md @@ -799,6 +799,8 @@ In Iceberg v4, statistics are stored in typed fields grouped in a struct that co Field-level structs in `content_stats` are based on the corresponding table field's type, requirement, and ID (`field-id`). +Stats are tracked for primitive and `variant` fields. Stats are not tracked for `struct`, `list`, and `map` fields, or for any field contained in a `list` or `map` because it may be repeated. + Field stats structs are assigned a range of 200 IDs, starting at `10_000 + 200 * field-id`. The first ID in the range (`base-id`) is the ID of the struct field in `content_stats`. Fields within the stats struct are assigned IDs from the range by adding an offset to the `base-id`. For example, the stats struct for table field 2 uses IDs in the range `[10_400, 10_599]`, the field within `content_stats` uses the `base-id`, ID `10_400`, and its `lower_bound` field (offset 1) uses ID `10_401`. Content stats must be resolved by ID; field names used for stats structs are informational. The recommended name for each field is the full name of the field in the table schema. @@ -820,10 +822,12 @@ Each stats struct holds statistics for one table field. It may contain the follo | _optional_ | 2 | `upper_bound` | Field type or `geo_upper` | all primitives or `variant` | Upper bound stored as the field's type, or `geo_upper` for geo types | | _optional_ | 3 | `tight_bounds` | `boolean` | all except `geometry`, `geography`, `variant` | When true, `lower_bound` and `upper_bound` must be equal to the min and max values | | _optional_ | 4 | `value_count` | `long` | all | Number of values in the column (including null and NaN values) | -| _optional_ | 5 | `null_value_count` | `long` | optional fields | Number of null values in the column | +| _optional_ | 5 | `null_value_count` | `long` | fields that may be null | Number of null values in the column | | _optional_ | 6 | `nan_value_count` | `long` | `float`, `double` | Number of NaN values in the column | | _optional_ | 7 | `avg_value_size_in_bytes` | `int` | `string`, `binary`, `variant` | Avg value size in memory (uncompressed) in bytes over non-null values to estimate memory consumption | +A field may be null if it is optional or if any field that contains it is optional. For example, a `required` field nested in an `optional` struct is null whenever the struct is null. + For example, stats for a `required` `int` field named `id` with field-id `2` are stored using: ``` @@ -833,7 +837,7 @@ For example, stats for a `required` `int` field named `id` with field-id `2` are 10_403: optional boolean tight_bounds; 10_404: optional long value_count; - // null_value_count is only used for optional fields + // null_value_count is only used for fields that may be null // nan_value_count is only used for float and double // avg_value_size_in_bytes is only used for variable length types }