From 219f1e9c1728c7771ae7d2fb80227a6ebed9b077 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Tue, 28 Jul 2026 18:04:29 +0200 Subject: [PATCH 1/7] Core: Add content stats evaluator --- .../TestInclusiveMetricsEvaluator.java | 799 ++++++++---------- .../java/org/apache/iceberg/ContentStats.java | 2 +- .../java/org/apache/iceberg/FieldStats.java | 2 +- .../java/org/apache/iceberg/TrackedFile.java | 2 +- .../expressions/InclusiveStatsEvaluator.java | 159 ++++ .../iceberg/TestInclusiveStatsEvaluator.java | 252 ++++++ 6 files changed, 760 insertions(+), 456 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java create mode 100644 core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 8d652df0df03..4a0101c06aa0 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -57,8 +57,16 @@ import org.apache.iceberg.util.UnicodeUtil; import org.junit.jupiter.api.Test; -public class TestInclusiveMetricsEvaluator { - private static final Schema SCHEMA = +/** + * Tests inclusive evaluation of expressions using file statistics. + * + *

Subclasses evaluate the same expressions against a different representation of file statistics + * by overriding the file methods and {@link #shouldRead(Schema, Expression, boolean, Object)}. + * + * @param the type of file that carries the statistics under test + */ +public class TestInclusiveMetricsEvaluator { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), optional(2, "no_stats", Types.IntegerType.get()), @@ -75,7 +83,7 @@ public class TestInclusiveMetricsEvaluator { optional(13, "no_nan_stats", Types.DoubleType.get()), optional(14, "some_empty", Types.StringType.get())); - private static final Schema NESTED_SCHEMA = + protected static final Schema NESTED_SCHEMA = new Schema( required( 100, @@ -90,8 +98,10 @@ public class TestInclusiveMetricsEvaluator { required(104, "required_street2", Types.StringType.get()), optional(105, "optional_street2", Types.StringType.get())))); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final Schema FLOAT_SCHEMA = new Schema(required(1, "f", Types.FloatType.get())); + + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -220,78 +230,234 @@ public class TestInclusiveMetricsEvaluator { // upper bounds null); + private static final DataFile MISSING_STATS = new TestDataFile("file.parquet", Row.of(), 50); + + private static final DataFile EMPTY_FILE = new TestDataFile("file.parquet", Row.of(), 0); + + private static final DataFile RANGE_OF_VALUES = + new TestDataFile( + "range_of_values.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); + + private static final DataFile SINGLE_VALUE_FILE = + new TestDataFile( + "single_value.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, 0L), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + + // some_empty is optional because a required column cannot contain nulls + private static final DataFile SINGLE_VALUE_WITH_NULLS = + new TestDataFile( + "single_value_nulls.avro", + Row.of(), + 10, + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), + ImmutableMap.of(14, 0L), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); + + private static final DataFile SINGLE_VALUE_WITH_NAN = + new TestDataFile( + "single_value_nan.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 2L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); + + private static final DataFile SINGLE_VALUE_NAN_BOUNDS = + new TestDataFile( + "single_value_nan_bounds.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); + + private static final Map FLOAT_BOUND = + ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 0L), + FLOAT_BOUND, + FLOAT_BOUND); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE_WITH_NAN = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 1L), // contains a NaN value + FLOAT_BOUND, + FLOAT_BOUND); + + protected boolean shouldRead(Schema schema, Expression expr, boolean caseSensitive, F testFile) { + return new InclusiveMetricsEvaluator(schema, expr, caseSensitive).eval((DataFile) testFile); + } + + protected boolean shouldRead(Schema schema, Expression expr, F testFile) { + return shouldRead(schema, expr, true, testFile); + } + + protected F file() { + return asFile(FILE); + } + + protected F file2() { + return asFile(FILE_2); + } + + protected F file3() { + return asFile(FILE_3); + } + + protected F file4() { + return asFile(FILE_4); + } + + protected F file5() { + return asFile(FILE_5); + } + + protected F file6() { + return asFile(FILE_6); + } + + protected F missingStats() { + return asFile(MISSING_STATS); + } + + protected F emptyFile() { + return asFile(EMPTY_FILE); + } + + protected F rangeOfValues() { + return asFile(RANGE_OF_VALUES); + } + + protected F singleValueFile() { + return asFile(SINGLE_VALUE_FILE); + } + + protected F singleValueWithNulls() { + return asFile(SINGLE_VALUE_WITH_NULLS); + } + + protected F singleValueWithNaN() { + return asFile(SINGLE_VALUE_WITH_NAN); + } + + protected F singleValueNaNBounds() { + return asFile(SINGLE_VALUE_NAN_BOUNDS); + } + + protected F singleFloatValueFile() { + return asFile(SINGLE_FLOAT_VALUE_FILE); + } + + protected F singleFloatValueFileWithNaN() { + return asFile(SINGLE_FLOAT_VALUE_FILE_WITH_NAN); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; + } + @Test public void testAllNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("all_nulls"), file()); assertThat(shouldRead).as("Should skip: no non-null value in all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: equal on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: startsWith on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notStartsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should read: notStartsWith on all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("some_nulls"), file()); assertThat(shouldRead) .as("Should read: column with some nulls contains a non-null value") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("no_nulls"), file()); assertThat(shouldRead).as("Should read: non-null column contains a non-null value").isTrue(); } @Test public void testNoNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNull("all_nulls"), file()); assertThat(shouldRead).as("Should read: at least one null value in all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("some_nulls"), file()); assertThat(shouldRead).as("Should read: column with some nulls contains a null value").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("no_nulls"), file()); assertThat(shouldRead).as("Should skip: non-null column contains no null values").isFalse(); } @Test public void testIsNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNaN("all_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("some_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in some nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nans"), file()); assertThat(shouldRead).as("Should skip: no-nans column contains no nan values").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nulls_double"), file()); assertThat(shouldRead).as("Should skip: all-null column doesn't contain nan value").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nans_v1_stats"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one nan value in nan and nulls only column") .isTrue(); @@ -299,35 +465,35 @@ public void testIsNaN() { @Test public void testNotNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNaN("all_nans"), file()); assertThat(shouldRead) .as("Should skip: column with all nans will not contain non-nan") .isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("some_nans"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in some nan column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nans"), file()); assertThat(shouldRead).as("Should read: at least one non-nan value in no nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nulls_double"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in all null column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nans_v1_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one null value in nan and nulls only column") .isTrue(); @@ -335,25 +501,22 @@ public void testNotNaN() { @Test public void testRequiredColumn() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("required")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("required"), file()); assertThat(shouldRead).as("Should read: required columns are always non-null").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("required")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("required"), file()); assertThat(shouldRead).as("Should skip: required columns are always non-null").isFalse(); } @Test public void testMissingColumn() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, lessThan("missing", 5)).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, lessThan("missing", 5), file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'missing'"); } @Test public void testMissingStats() { - DataFile missingStats = new TestDataFile("file.parquet", Row.of(), 50); - Expression[] exprs = new Expression[] { lessThan("no_stats", 5), @@ -369,15 +532,13 @@ public void testMissingStats() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(missingStats); + boolean shouldRead = shouldRead(SCHEMA, expr, missingStats()); assertThat(shouldRead).as("Should read when missing stats for expr: " + expr).isTrue(); } } @Test public void testZeroRecordFile() { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); - Expression[] exprs = new Expression[] { lessThan("id", 5), @@ -393,7 +554,7 @@ public void testZeroRecordFile() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(empty); + boolean shouldRead = shouldRead(SCHEMA, expr, emptyFile()); assertThat(shouldRead).as("Should never read 0-record file: " + expr).isFalse(); } } @@ -401,13 +562,10 @@ public void testZeroRecordFile() { @Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: not(false)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25))) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should skip: not(true)").isFalse(); } @@ -415,28 +573,24 @@ public void testNot() { public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MIN_VALUE - 30))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MIN_VALUE - 30)), + file()); assertThat(shouldRead).as("Should skip: and(false, true)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: and(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE))) - .eval(FILE); + shouldRead( + SCHEMA, + and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE)), + file()); assertThat(shouldRead).as("Should read: and(true, true)").isTrue(); } @@ -444,398 +598,312 @@ public void testAnd() { public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: or(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE - 19))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE - 19)), + file()); assertThat(shouldRead).as("Should read: or(false, true)").isTrue(); } @Test public void testIntegerLt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range below lower bound (30 is not < 30)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE + 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerLtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range above upper bound (79 is not > 79)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6)) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (80 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); } @Test public void testIntegerNotEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 1)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE - 4))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE - 4)), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 1)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 6))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 6)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseInsensitiveIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false, file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseSensitiveIntegerNotEqRewritten() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", 5)), true).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, not(equal("ID", 5)), true, file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'ID'"); } @Test public void testStringStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1str1x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "3str3x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("some_empty", "房东整租霍"), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("some_empty", "房东整租霍"), true, file()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", ""), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", ""), true, file()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", aboveMax), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); } @Test public void testStringNotStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1str1x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "3str3x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", aboveMax), true) - .eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abc"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abc"), true, file5()); assertThat(shouldRead).as("Should not read: all strings start with prefix").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abcd"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abcd"), true, file5()); assertThat(shouldRead).as("Should not read: lower shorter than prefix, cannot match").isTrue(); } @Test public void testIntegerIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should not read: id below lower bound (5 < 30, 6 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound (28 < 30, 29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should not read: id above upper bound (80 > 79, 81 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should not read: id above upper bound (85 > 79, 86 > 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should skip: in on all nulls column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on no nulls column").isTrue(); // should read as the number of elements in the in expression is too big @@ -843,131 +911,97 @@ public void testIntegerIn() { for (int id = -400; id <= 0; id++) { ids.add(id); } - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("id", ids)).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", ids), file()); assertThat(shouldRead).as("Should read: large in expression").isTrue(); } @Test public void testIntegerNotIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should read: id below lower bound (5 < 30, 6 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound (28 < 30, 29 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should read: id above upper bound (80 > 79, 81 > 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should read: id above upper bound (85 > 79, 86 > 79)").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on all nulls column").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue(); } @Test public void testIsNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address"), file6()); assertThat(shouldRead).as("Should not read: required_address is required").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.required_street1 is required") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.optional_street1 is optional") .isTrue(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address")).eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.optional_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); } @Test public void testNotNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address"), file6()); assertThat(shouldRead).as("Should read: required_address is required").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.required_street1 is required") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.optional_street1 is optional") .isFalse(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address")).eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.optional_street2"), file6()); assertThat(shouldRead) .as("Should not read: optional_address.optional_street2 is optional") .isFalse(); @@ -975,207 +1009,66 @@ public void testNotNullInNestedStruct() { @Test public void testNotEqSingleValueWithoutNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 0L), - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFile())) .as("Should skip: file contains no values not equal to 1.0") .isFalse(); } @Test public void testNotEqSingleValueWithNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 1L), // contains a NaN value - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFileWithNaN())) .as("Should read: file contains a NaN value not equal to 1.0") .isTrue(); } @Test public void testNotEqWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "aaa")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notEqual("required", "aaa"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "abc"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value equal to literal") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "def")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "def"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not equal to literal") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notEqual("some_empty", "abc"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueWithNaN()); assertThat(shouldRead).as("Should read: file has NaN values which match != predicate").isTrue(); - DataFile singleValueNaNBounds = - new TestDataFile( - "single_value_nan_bounds.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueNaNBounds); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueNaNBounds()); assertThat(shouldRead).as("Should read: bounds are NaN").isTrue(); } @Test public void testNotInWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "aaa", "bbb")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notIn("required", "aaa", "bbb"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "abc", "def"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value in exclusion list") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "def", "ghi")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "def", "ghi"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not in exclusion list") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notIn("some_empty", "abc", "def"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nans", 5.0F, 10.0F)) - .eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notIn("no_nans", 5.0F, 10.0F), singleValueWithNaN()); assertThat(shouldRead) .as("Should read: file has NaN values which match NOT IN predicate") .isTrue(); diff --git a/core/src/main/java/org/apache/iceberg/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java index aa7a65ac934d..51bfa52a67ad 100644 --- a/core/src/main/java/org/apache/iceberg/ContentStats.java +++ b/core/src/main/java/org/apache/iceberg/ContentStats.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.iceberg.types.Types; -interface ContentStats { +public interface ContentStats { /** Returns an iterable of the {@link FieldStats} held by this container struct */ Iterable> fieldStats(); diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java index 495dddba913d..693765a8977b 100644 --- a/core/src/main/java/org/apache/iceberg/FieldStats.java +++ b/core/src/main/java/org/apache/iceberg/FieldStats.java @@ -20,7 +20,7 @@ import org.apache.iceberg.types.Types; -interface FieldStats { +public interface FieldStats { /** The field ID of the statistic */ int fieldId(); diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 9aaeae48d688..374808b7cb53 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -25,7 +25,7 @@ import org.apache.iceberg.types.Types; /** A file tracked by a manifest. */ -interface TrackedFile { +public interface TrackedFile { Types.NestedField TRACKING = Types.NestedField.required( 147, "tracking", Tracking.schema(), "Tracking information for this entry"); diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java new file mode 100644 index 000000000000..6aeae63b91b7 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.apache.iceberg.expressions.Expressions.rewriteNot; + +import org.apache.iceberg.ContentStats; +import org.apache.iceberg.FieldStats; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantObject; + +/** + * Evaluates an {@link Expression} on a {@link TrackedFile} to test whether rows in the file may + * match. + * + *

This evaluation is inclusive: it returns true if a file may match and false if it cannot + * match. + * + *

Files are passed to {@link #eval(TrackedFile)}, which returns true if the file may contain + * matching rows and false if the file cannot contain matching rows. Files may be skipped if and + * only if the return value of {@code eval} is false. + * + *

Due to the comparison implementation of ORC stats, for float/double columns in ORC files, if + * the first value in a file is NaN, metrics of this file will report NaN for both upper and lower + * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly + * checks for NaN is necessary in order to not skip files that may contain matching data. + */ +public class InclusiveStatsEvaluator { + private final Expression expr; + + public InclusiveStatsEvaluator(Schema schema, Expression unbound) { + this(schema, unbound, true); + } + + public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + Types.StructType struct = schema.asStruct(); + this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + } + + /** + * Test whether the file may contain records that match the expression. + * + * @param file a tracked file + * @return false if the file cannot contain rows that match the expression, true otherwise. + */ + public boolean eval(TrackedFile file) { + return new MetricsEvalVisitor().eval(file); + } + + private class MetricsEvalVisitor extends InclusiveEvalVisitor { + private ContentStats stats = null; + + private boolean eval(TrackedFile file) { + if (file.recordCount() == 0) { + return ROWS_CANNOT_MATCH; + } + + if (file.recordCount() < 0) { + // imported Avro files may have an incorrect -1 row count + return ROWS_MIGHT_MATCH; + } + + if (null == file.contentStats()) { + return ROWS_MIGHT_MATCH; + } + + this.stats = file.contentStats(); + + return ExpressionVisitors.visitEvaluator(expr, this); + } + + @Override + protected boolean mayContainNull(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNullValueCount() + || fieldStats.nullValueCount() != 0; + } + + @Override + protected boolean containsNullsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNullValueCount() + && fieldStats.valueCount() - fieldStats.nullValueCount() == 0; + } + + @Override + protected boolean mayContainNaN(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNanValueCount() + || fieldStats.nanValueCount() != 0; + } + + @Override + protected boolean containsNaNsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNanValueCount() + && fieldStats.valueCount() - fieldStats.nanValueCount() == 0; + } + + @Override + protected T lowerBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.lowerBound() : null; + } + + @Override + protected T upperBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.upperBound() : null; + } + + @Override + protected T extractLowerBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.lowerBound() != null) { + VariantObject fieldLowerBounds = fieldStats.lowerBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldLowerBounds.get(bound.path()), bound.type()); + } + + return null; + } + + @Override + protected T extractUpperBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.upperBound() != null) { + VariantObject fieldUpperBounds = fieldStats.upperBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldUpperBounds.get(bound.path()), bound.type()); + } + + return null; + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java new file mode 100644 index 000000000000..a15a199c1f33 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.InclusiveStatsEvaluator; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +/** Runs the inclusive evaluation tests against {@link ContentStats} tracked by a file. */ +class TestInclusiveStatsEvaluator extends TestInclusiveMetricsEvaluator { + private static final int FORMAT_VERSION_V4 = 4; + + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema( + SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + + // stats are tracked for the leaf fields, not for the address structs themselves + private static final Types.StructType NESTED_STATS_TYPE = + StatsUtil.statsReadSchema(NESTED_SCHEMA, ImmutableList.of(102, 103, 104, 105)); + + private static final Types.StructType FLOAT_STATS_TYPE = + StatsUtil.statsReadSchema(FLOAT_SCHEMA, ImmutableList.of(1)); + + @Override + protected boolean shouldRead( + Schema schema, Expression expr, boolean caseSensitive, TrackedFile testFile) { + return new InclusiveStatsEvaluator(schema, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + stats(1, INT_MIN_VALUE, INT_MAX_VALUE, null, null, null), + stats(4, null, null, 50L, 50L, null), + stats(5, null, null, 50L, 10L, null), + stats(6, null, null, 50L, 0L, null), + stats(7, null, null, 50L, null, 50L), + stats(8, null, null, 50L, null, 10L), + stats(9, null, null, 50L, null, 0L), + stats(10, null, null, 50L, 50L, null), + stats(11, Float.NaN, Float.NaN, 50L, 0L, null), + stats(12, Double.NaN, Double.NaN, 50L, 1L, null), + stats(13, null, null, 50L, null, null), + stats(14, "", "房东整租霍营小区二层两居室", 50L, 0L, null))); + } + + @Override + protected TrackedFile file2() { + return trackedFile( + "file_2.avro", 50, contentStats(STATS_TYPE, stats(3, "aa", "dC", 50L, null, null))); + } + + @Override + protected TrackedFile file3() { + return trackedFile( + "file_3.avro", 50, contentStats(STATS_TYPE, stats(3, "1str1", "3str3", 50L, null, null))); + } + + @Override + protected TrackedFile file4() { + return trackedFile( + "file_4.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "イロハニホヘト", 50L, null, null))); + } + + @Override + protected TrackedFile file5() { + return trackedFile( + "file_5.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "abcdefghi", 50L, null, null))); + } + + @Override + protected TrackedFile file6() { + return trackedFile( + "file_6.avro", + 10, + contentStats( + NESTED_STATS_TYPE, + stats(NESTED_STATS_TYPE, 102, null, null, 5L, null, null), + stats(NESTED_STATS_TYPE, 103, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 104, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 105, null, null, 5L, 5L, null))); + } + + @Override + protected TrackedFile missingStats() { + return trackedFile("file.parquet", 50, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile rangeOfValues() { + return trackedFile( + "range_of_values.avro", + 10, + contentStats(STATS_TYPE, stats(3, "aaa", "zzz", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueFile() { + return trackedFile( + "single_value.avro", 10, contentStats(STATS_TYPE, stats(3, "abc", "abc", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueWithNulls() { + return trackedFile( + "single_value_nulls.avro", + 10, + contentStats(STATS_TYPE, stats(14, "abc", "abc", 10L, 2L, null))); + } + + @Override + protected TrackedFile singleValueWithNaN() { + return trackedFile( + "single_value_nan.avro", 10, contentStats(STATS_TYPE, stats(9, 5.0f, 5.0f, 10L, 0L, 2L))); + } + + @Override + protected TrackedFile singleValueNaNBounds() { + return trackedFile( + "single_value_nan_bounds.avro", + 10, + contentStats(STATS_TYPE, stats(9, Float.NaN, Float.NaN, 10L, 0L, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFile() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFileWithNaN() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 1L))); + } + + @Test + void fileWithoutContentStats() { + TrackedFile file = trackedFile("file.avro", 50, null); + + assertThat(shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file)) + .as("Should read: file does not track content stats") + .isTrue(); + } + + private static FieldStats stats( + int fieldId, Object lower, Object upper, Long valueCount, Long nullCount, Long nanCount) { + return stats(STATS_TYPE, fieldId, lower, upper, valueCount, nullCount, nanCount); + } + + /** Returns the stats for a field, where a null metric is one that the column does not track. */ + private static FieldStats stats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + Types.StructType type = statsType.field(StatsUtil.toBaseId(fieldId)).type().asStructType(); + FieldStatsStruct stats = new FieldStatsStruct<>(type); + set(stats, StatsUtil.LOWER_BOUND_NAME, lower); + set(stats, StatsUtil.UPPER_BOUND_NAME, upper); + set(stats, "value_count", valueCount); + set(stats, "null_value_count", nullCount); + set(stats, "nan_value_count", nanCount); + + return stats; + } + + private static void set(FieldStatsStruct stats, String metric, Object value) { + if (value == null) { + return; + } + + List fields = stats.type().fields(); + for (int pos = 0; pos < fields.size(); pos += 1) { + if (fields.get(pos).name().equals(metric)) { + stats.set(pos, value); + return; + } + } + + throw new IllegalArgumentException("Cannot set " + metric + ": not tracked by " + stats.type()); + } + + private static ContentStats contentStats( + Types.StructType statsType, FieldStats... fieldStats) { + ContentStatsStruct stats = new ContentStatsStruct(statsType); + for (FieldStats field : fieldStats) { + stats.setStats(field.fieldId(), field); + } + + return stats; + } + + private static TrackedFile trackedFile(String location, long recordCount, ContentStats stats) { + return new TrackedFileStruct( + null, + FileContent.DATA, + FORMAT_VERSION_V4, + location, + FileFormat.fromFileName(location), + null, + recordCount, + 1024L, + null, + stats, + null, + null, + null, + null, + null, + null); + } +} From 1b5943be1e9ac21642a52fc5e8bb7f9b86a82af2 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 12:42:36 +0200 Subject: [PATCH 2/7] Spec, Core: Track null counts for required fields in optional structs The content stats schema omitted null_value_count whenever a field was required, but a required field nested in an optional struct is null on every row where its parent is null, and Parquet reports that count from the leaf column's definition levels. Include the count for any field that may be null, either because it or one of its ancestors is optional. --- .../java/org/apache/iceberg/StatsUtil.java | 29 ++++++++++++++++--- .../org/apache/iceberg/TestStatsUtil.java | 4 +++ format/spec.md | 8 +++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 545275002c9a..58a866b205ec 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,7 +147,10 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - field.isOptional(), field.type(), baseId, metricsConfig.columnMode(id)); + isNullable(tableSchema, parentIndex, id), + field.type(), + baseId, + metricsConfig.columnMode(id)); if (fieldStruct != null) { fieldStructs.add(optional(baseId, fieldName, fieldStruct)); @@ -178,7 +181,11 @@ public static Types.StructType statsReadSchema(Schema tableSchema, Iterable parentIndex, int id) { + Integer currentId = id; + while (currentId != null) { + if (schema.findField(currentId).isOptional()) { + return true; + } + + currentId = parentIndex.get(currentId); + } + + return false; + } + /** Return whether a field has one value or may be repeated in map or list. */ private static boolean isScalar(Schema schema, Map parentIndex, int id) { Integer currentId = id; diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index e145dd61e14c..f8fa596fcc61 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -395,12 +395,14 @@ public void testContentStatsReadSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( @@ -518,12 +520,14 @@ public void testContentStatsWriteSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( diff --git a/format/spec.md b/format/spec.md index be28a6c5f000..5faba5dcfc69 100644 --- a/format/spec.md +++ b/format/spec.md @@ -799,6 +799,8 @@ In Iceberg v4, statistics are stored in typed fields grouped in a struct that co Field-level structs in `content_stats` are based on the corresponding table field's type, requirement, and ID (`field-id`). +Stats are tracked for primitive and `variant` fields. Stats are not tracked for `struct`, `list`, and `map` fields, or for any field contained in a `list` or `map` because it may be repeated. + Field stats structs are assigned a range of 200 IDs, starting at `10_000 + 200 * field-id`. The first ID in the range (`base-id`) is the ID of the struct field in `content_stats`. Fields within the stats struct are assigned IDs from the range by adding an offset to the `base-id`. For example, the stats struct for table field 2 uses IDs in the range `[10_400, 10_599]`, the field within `content_stats` uses the `base-id`, ID `10_400`, and its `lower_bound` field (offset 1) uses ID `10_401`. Content stats must be resolved by ID; field names used for stats structs are informational. The recommended name for each field is the full name of the field in the table schema. @@ -820,10 +822,12 @@ Each stats struct holds statistics for one table field. It may contain the follo | _optional_ | 2 | `upper_bound` | Field type or `geo_upper` | all primitives or `variant` | Upper bound stored as the field's type, or `geo_upper` for geo types | | _optional_ | 3 | `tight_bounds` | `boolean` | all except `geometry`, `geography`, `variant` | When true, `lower_bound` and `upper_bound` must be equal to the min and max values | | _optional_ | 4 | `value_count` | `long` | all | Number of values in the column (including null and NaN values) | -| _optional_ | 5 | `null_value_count` | `long` | optional fields | Number of null values in the column | +| _optional_ | 5 | `null_value_count` | `long` | fields that may be null | Number of null values in the column | | _optional_ | 6 | `nan_value_count` | `long` | `float`, `double` | Number of NaN values in the column | | _optional_ | 7 | `avg_value_size_in_bytes` | `int` | `string`, `binary`, `variant` | Avg value size in memory (uncompressed) in bytes over non-null values to estimate memory consumption | +A field may be null if it is optional or if any field that contains it is optional. For example, a `required` field nested in an `optional` struct is null whenever the struct is null. + For example, stats for a `required` `int` field named `id` with field-id `2` are stored using: ``` @@ -833,7 +837,7 @@ For example, stats for a `required` `int` field named `id` with field-id `2` are 10_403: optional boolean tight_bounds; 10_404: optional long value_count; - // null_value_count is only used for optional fields + // null_value_count is only used for fields that may be null // nan_value_count is only used for float and double // avg_value_size_in_bytes is only used for variable length types } From 1c4c47f4e78408569d9693ee56b0797d122fedcc Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:10:48 +0200 Subject: [PATCH 3/7] metrics are not tracked for structs --- .../iceberg/expressions/TestInclusiveMetricsEvaluator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 4a0101c06aa0..ad29d85efe0d 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -220,9 +220,9 @@ public class TestInclusiveMetricsEvaluator { Row.of(), 10, // any value counts, including nulls - ImmutableMap.of(100, 5L, 101, 5L, 102, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(102, 5L, 103, 5L, 104, 5L, 105, 5L), // null value counts - ImmutableMap.of(100, 0L, 101, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(103, 5L, 104, 5L, 105, 5L), // nan value counts null, // lower bounds @@ -996,7 +996,7 @@ public void testNotNullInNestedStruct() { // read optional_address and its nested fields shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); From fedb144438eb16c949b328566e07a64626f73676 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:25:15 +0200 Subject: [PATCH 4/7] Core: Treat missing null count as zero for required fields Content stats omit null_value_count for a field that cannot be null, but InclusiveStatsEvaluator read the missing count as unknown and stopped pruning files for notEqual, notIn, and notStartsWith. The v3 metrics maps carry an explicit zero from Parquet, so the same filters pruned there. Detect the fields that cannot be null the same way expression binding does, by requiring the field and all of its ancestors to be required, and report no nulls for them. --- .../expressions/InclusiveStatsEvaluator.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java index 6aeae63b91b7..9ea270329abd 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -20,10 +20,14 @@ import static org.apache.iceberg.expressions.Expressions.rewriteNot; +import java.util.Collections; +import java.util.Set; import org.apache.iceberg.ContentStats; import org.apache.iceberg.FieldStats; import org.apache.iceberg.Schema; import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.variants.Variant; import org.apache.iceberg.variants.VariantObject; @@ -46,6 +50,7 @@ */ public class InclusiveStatsEvaluator { private final Expression expr; + private final Set neverNullIds; public InclusiveStatsEvaluator(Schema schema, Expression unbound) { this(schema, unbound, true); @@ -54,6 +59,32 @@ public InclusiveStatsEvaluator(Schema schema, Expression unbound) { public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { Types.StructType struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + this.neverNullIds = + neverNullIds( + schema, Binder.boundReferences(struct, Collections.singletonList(expr), caseSensitive)); + } + + /** + * Returns the IDs of the referenced fields that cannot contain null values. + * + *

Stats omit the null count for such fields, which must not be confused with an unknown count. + */ + private static Set neverNullIds(Schema schema, Set referencedIds) { + ImmutableSet.Builder neverNull = ImmutableSet.builder(); + + for (int id : referencedIds) { + Types.NestedField field = schema.findField(id); + if (field != null && field.isRequired() && allAncestorFieldsAreRequired(schema, id)) { + neverNull.add(id); + } + } + + return neverNull.build(); + } + + private static boolean allAncestorFieldsAreRequired(Schema schema, int fieldId) { + return TypeUtil.ancestorFields(schema, fieldId).stream() + .allMatch(Types.NestedField::isRequired); } /** @@ -90,6 +121,10 @@ private boolean eval(TrackedFile file) { @Override protected boolean mayContainNull(int id) { + if (neverNullIds.contains(id)) { + return false; + } + FieldStats fieldStats = stats.statsFor(id); return fieldStats == null || !fieldStats.hasNullValueCount() From 6c13ff5fbb74bef8be18ed12d28d506f1fbd0108 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:57:19 +0200 Subject: [PATCH 5/7] accept revapi failures --- .palantir/revapi.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 478b78dea3e3..e726f0d2d5ca 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -564,6 +564,56 @@ acceptedBreaks: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME" justification: "Removed deprecated functionality for partition stats" + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNanValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNullValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::tightBounds()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method int org.apache.iceberg.TrackedFile::formatVersion()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method java.lang.Integer org.apache.iceberg.FieldStats::avgValueSizeInBytes()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy(java.util.Set)" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.FieldStats org.apache.iceberg.FieldStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.types.Types.StructType org.apache.iceberg.ContentStats::type()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." - code: "java.method.removed" old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" justification: "Deprecated for removal in 1.12.0" From c5abd31bab8981943d0deb611415be11e7a64124 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 16:05:23 +0200 Subject: [PATCH 6/7] updates --- .../apache/iceberg/expressions/InclusiveStatsEvaluator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java index 9ea270329abd..3de33f5d01de 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -94,10 +94,10 @@ private static boolean allAncestorFieldsAreRequired(Schema schema, int fieldId) * @return false if the file cannot contain rows that match the expression, true otherwise. */ public boolean eval(TrackedFile file) { - return new MetricsEvalVisitor().eval(file); + return new StatsEvalVisitor().eval(file); } - private class MetricsEvalVisitor extends InclusiveEvalVisitor { + private class StatsEvalVisitor extends InclusiveEvalVisitor { private ContentStats stats = null; private boolean eval(TrackedFile file) { From 5193e21c54625a2da21a3c2fe18565e01e9b2f18 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 16:15:01 +0200 Subject: [PATCH 7/7] API, Core: Unify isNullable() into TypeUtil --- .../iceberg/expressions/UnboundPredicate.java | 9 +- .../org/apache/iceberg/types/TypeUtil.java | 19 ++++ .../apache/iceberg/types/TestTypeUtil.java | 99 +++++++++++++++++++ .../java/org/apache/iceberg/StatsUtil.java | 18 +--- .../expressions/InclusiveStatsEvaluator.java | 8 +- 5 files changed, 123 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java index 75ca9d5835bc..17f3873d70af 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java @@ -127,7 +127,7 @@ private Expression bindUnaryOperation(StructType struct, BoundTerm boundTerm) switch (op()) { case IS_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysFalse(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysTrue(); @@ -135,7 +135,7 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { return new BoundUnaryPredicate<>(Operation.IS_NULL, boundTerm); case NOT_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysTrue(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysFalse(); @@ -158,11 +158,6 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { } } - private boolean allAncestorFieldsAreRequired(StructType struct, int fieldId) { - return TypeUtil.ancestorFields(struct.asSchema(), fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - private boolean floatingType(Type.TypeID typeID) { return Type.TypeID.DOUBLE.equals(typeID) || Type.TypeID.FLOAT.equals(typeID); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index a3bee3e3d860..5f7ad3310c55 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -273,6 +273,25 @@ public static List ancestorFields(Schema schema, int fieldId) return parents; } + /** + * Returns whether a field may contain null values. + * + *

A field may be null if it is optional or if any field that contains it is optional. A + * required field nested in an optional struct is null whenever that struct is null. + * + * @param schema The schema that contains the field ID + * @param fieldId The field ID to check + * @return true if the field may be null, false if it cannot be null + * @throws IllegalArgumentException if the field ID is not present in the schema + */ + public static boolean isNullable(Schema schema, int fieldId) { + Types.NestedField field = schema.findField(fieldId); + Preconditions.checkArgument(field != null, "Cannot find field with ID: %s", fieldId); + + return field.isOptional() + || ancestorFields(schema, fieldId).stream().anyMatch(Types.NestedField::isOptional); + } + /** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a type. * diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index b7da4b3108e6..872bccb987c4 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -971,6 +971,105 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + @Test + public void isNullableWithUnknownFieldId() { + Schema schema = new Schema(required(1, "id", IntegerType.get())); + + assertThatThrownBy(() -> TypeUtil.isNullable(schema, 2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot find field with ID: 2"); + } + + @Test + public void isNullableWithTopLevelFields() { + Schema schema = + new Schema( + required(1, "id", IntegerType.get()), optional(2, "data", Types.StringType.get())); + + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + } + + @Test + public void isNullableWithNestedStructs() { + Schema schema = + new Schema( + required( + 1, + "required_location", + Types.StructType.of( + required(3, "required_lat", Types.DoubleType.get()), + optional(4, "optional_lon", Types.DoubleType.get()), + required( + 5, + "required_inner", + Types.StructType.of(required(6, "required_zip", IntegerType.get()))))), + optional( + 2, + "optional_location", + Types.StructType.of( + required(7, "required_lat", Types.DoubleType.get()), + required( + 8, + "required_inner", + Types.StructType.of(required(9, "required_zip", IntegerType.get())))))); + + // a required field is not null when every field that contains it is required + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 6)).isFalse(); + + // an optional field is null regardless of the fields that contain it + assertThat(TypeUtil.isNullable(schema, 4)).isTrue(); + + // a required field nested in an optional struct is null when the struct is null + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 8)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 9)).isTrue(); + } + + @Test + public void isNullableWithListsAndMaps() { + Schema schema = + new Schema( + required( + 1, + "required_points", + Types.ListType.ofRequired( + 4, Types.StructType.of(required(5, "required_x", Types.LongType.get())))), + optional( + 2, + "optional_points", + Types.ListType.ofOptional( + 6, Types.StructType.of(required(7, "required_x", Types.LongType.get())))), + required( + 3, + "locations", + Types.MapType.ofRequired( + 8, + 9, + Types.StringType.get(), + Types.StructType.of(required(10, "required_lat", Types.DoubleType.get()))))); + + // a required element of a required list is not null, nor is anything it contains + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 4)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + + // an optional element is null, as is anything it contains + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 6)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + + // required map keys and values are not null + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 8)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 9)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 10)).isFalse(); + } + @Test public void testIndexStatsNames() { Schema schema = diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 58a866b205ec..47decef0ce90 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,7 +147,7 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - isNullable(tableSchema, parentIndex, id), + TypeUtil.isNullable(tableSchema, id), field.type(), baseId, metricsConfig.columnMode(id)); @@ -182,7 +182,7 @@ public static Types.StructType statsReadSchema(Schema tableSchema, Iterable parentIndex, int id) { - Integer currentId = id; - while (currentId != null) { - if (schema.findField(currentId).isOptional()) { - return true; - } - - currentId = parentIndex.get(currentId); - } - - return false; - } - /** Return whether a field has one value or may be repeated in map or list. */ private static boolean isScalar(Schema schema, Map parentIndex, int id) { Integer currentId = id; diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java index 3de33f5d01de..97e94eeba927 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -73,8 +73,7 @@ private static Set neverNullIds(Schema schema, Set referencedI ImmutableSet.Builder neverNull = ImmutableSet.builder(); for (int id : referencedIds) { - Types.NestedField field = schema.findField(id); - if (field != null && field.isRequired() && allAncestorFieldsAreRequired(schema, id)) { + if (!TypeUtil.isNullable(schema, id)) { neverNull.add(id); } } @@ -82,11 +81,6 @@ private static Set neverNullIds(Schema schema, Set referencedI return neverNull.build(); } - private static boolean allAncestorFieldsAreRequired(Schema schema, int fieldId) { - return TypeUtil.ancestorFields(schema, fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - /** * Test whether the file may contain records that match the expression. *