From 6971935997ef77c23c998865429e71686f9bb0b6 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Mon, 6 Oct 2025 11:34:40 -0700 Subject: [PATCH 01/15] Add date->timestamp promotion logic for v3 only This adds promotion logic for date->timestamp, as well as proper plumbing to ensure that these promotions only occur for Table Format v3. --- .../iceberg/types/CheckCompatibility.java | 65 +++++++++++++++++-- .../org/apache/iceberg/types/TypeUtil.java | 18 +++++ .../iceberg/types/TestReadabilityChecks.java | 2 +- .../apache/iceberg/types/TestTypeUtil.java | 25 +++++++ .../java/org/apache/iceberg/SchemaUpdate.java | 28 ++++++-- .../org/apache/iceberg/TableProperties.java | 2 + .../iceberg/schema/UnionByNameVisitor.java | 28 ++++++-- .../org/apache/iceberg/TestSchemaUpdate.java | 57 ++++++++++------ 8 files changed, 186 insertions(+), 39 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 725f7f42562e..b4ae083baeec 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -38,7 +38,20 @@ public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor writeCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return writeCompatibilityErrors(readSchema, writeSchema, true); + return writeCompatibilityErrors(readSchema, writeSchema, true, 2); + } + + /** + * Returns a list of compatibility errors for writing with the given write schema. This includes + * nullability: writing optional (nullable) values to a required field is an error. + * + * @param readSchema a read schema + * @param writeSchema a write schema + * @return a list of error details, or an empty list if there are no compatibility problems + */ + public static List writeCompatibilityErrors( + Schema readSchema, Schema writeSchema, int formatVersion) { + return writeCompatibilityErrors(readSchema, writeSchema, true, formatVersion); } /** @@ -53,7 +66,24 @@ public static List writeCompatibilityErrors(Schema readSchema, Schema wr */ public static List writeCompatibilityErrors( Schema readSchema, Schema writeSchema, boolean checkOrdering) { - return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, checkOrdering, true)); + return writeCompatibilityErrors(readSchema, writeSchema, checkOrdering, 2); + } + + /** + * Returns a list of compatibility errors for writing with the given write schema. This includes + * nullability: writing optional (nullable) values to a required field is an error Optionally this + * method allows case where input schema has different ordering than table schema. + * + * @param readSchema a read schema + * @param writeSchema a write schema + * @param checkOrdering If false, allow input schema to have different ordering than table schema + * @param formatVersion the table format version + * @return a list of error details, or an empty list if there are no compatibility problems + */ + public static List writeCompatibilityErrors( + Schema readSchema, Schema writeSchema, boolean checkOrdering, int formatVersion) { + return TypeUtil.visit( + readSchema, new CheckCompatibility(writeSchema, checkOrdering, true, formatVersion)); } /** @@ -70,7 +100,13 @@ public static List writeCompatibilityErrors( */ public static List typeCompatibilityErrors( Schema readSchema, Schema writeSchema, boolean checkOrdering) { - return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, checkOrdering, false)); + return typeCompatibilityErrors(readSchema, writeSchema, checkOrdering, 1); + } + + public static List typeCompatibilityErrors( + Schema readSchema, Schema writeSchema, boolean checkOrdering, int formatVersion) { + return TypeUtil.visit( + readSchema, new CheckCompatibility(writeSchema, checkOrdering, false, formatVersion)); } /** @@ -84,7 +120,13 @@ public static List typeCompatibilityErrors( * @return a list of error details, or an empty list if there are no compatibility problems */ public static List typeCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, true, false)); + return typeCompatibilityErrors(readSchema, writeSchema, 2); + } + + public static List typeCompatibilityErrors( + Schema readSchema, Schema writeSchema, int formatVersion) { + return TypeUtil.visit( + readSchema, new CheckCompatibility(writeSchema, true, false, formatVersion)); } /** @@ -95,7 +137,13 @@ public static List typeCompatibilityErrors(Schema readSchema, Schema wri * @return a list of error details, or an empty list if there are no compatibility problems */ public static List readCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, false, true)); + return readCompatibilityErrors(readSchema, writeSchema, 2); + } + + public static List readCompatibilityErrors( + Schema readSchema, Schema writeSchema, int formatVersion) { + return TypeUtil.visit( + readSchema, new CheckCompatibility(writeSchema, false, true, formatVersion)); } private static final ImmutableList NO_ERRORS = ImmutableList.of(); @@ -103,14 +151,17 @@ public static List readCompatibilityErrors(Schema readSchema, Schema wri private final Schema schema; private final boolean checkOrdering; private final boolean checkNullability; + private final int formatVersion; // the current file schema, maintained while traversing a write schema private Type currentType; - private CheckCompatibility(Schema schema, boolean checkOrdering, boolean checkNullability) { + private CheckCompatibility( + Schema schema, boolean checkOrdering, boolean checkNullability, int formatVersion) { this.schema = schema; this.checkOrdering = checkOrdering; this.checkNullability = checkNullability; + this.formatVersion = formatVersion; } @Override @@ -273,7 +324,7 @@ public List primitive(Type.PrimitiveType readPrimitive) { currentType.typeId().toString().toLowerCase(Locale.ENGLISH), readPrimitive)); } - if (!TypeUtil.isPromotionAllowed(currentType.asPrimitiveType(), readPrimitive)) { + if (!TypeUtil.isPromotionAllowed(currentType.asPrimitiveType(), readPrimitive, formatVersion)) { return ImmutableList.of( String.format(": %s cannot be promoted to %s", currentType, readPrimitive)); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index b1c556be0667..892fcc3595aa 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -424,6 +424,11 @@ public static Type find(Type type, Predicate predicate) { } public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { + return TypeUtil.isPromotionAllowed(from, to, 2); + } + + public static boolean isPromotionAllowed( + Type from, Type.PrimitiveType to, Integer formatVersion) { // Warning! Before changing this function, make sure that the type change doesn't introduce // compatibility problems in partitioning. if (from.equals(to)) { @@ -431,6 +436,19 @@ public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { } switch (from.typeId()) { + case DATE: + if (formatVersion < 3) { + return false; + } else if (to.typeId() == Type.TypeID.TIMESTAMP) { + // Timezone types cannot be promoted. + Types.TimestampType toTs = (Types.TimestampType) to; + return Types.TimestampType.withoutZone().equals(toTs); + } else if (to.typeId() == Type.TypeID.TIMESTAMP_NANO) { + // Timezone types cannot be promoted. + Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; + return Types.TimestampNanoType.withoutZone().equals(toTs); + } + // fall through case INTEGER: return to.typeId() == Type.TypeID.LONG; diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 20299cdafce2..21335977be9c 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -70,7 +70,7 @@ public void testPrimitiveTypes() { CheckCompatibility.writeCompatibilityErrors( new Schema(required(1, "to_field", to)), fromSchema); - if (TypeUtil.isPromotionAllowed(from, to)) { + if (TypeUtil.isPromotionAllowed(from, to, 2)) { assertThat(errors).as("Should produce 0 error messages").isEmpty(); } else { assertThat(errors).hasSize(1); diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index d4742f518754..f4ee4accf4be 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -945,4 +945,29 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 16)).containsExactly(pointsElement, points); assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + + @Test + public void testDateToTimestampPromotion() { + // Format version < 3 should not be accepted. + assertThat( + TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withoutZone(), 2)) + .isFalse(); + // Timezone should not be accepted. + assertThat(TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withZone(), 3)) + .isFalse(); + // Timezone nano should not be accepted. + assertThat( + TypeUtil.isPromotionAllowed( + Types.DateType.get(), Types.TimestampNanoType.withZone(), 3)) + .isFalse(); + // Timestamp without timezone should be accepted. + assertThat( + TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withoutZone(), 3)) + .isTrue(); + // Timestamp nano without timezone should be accepted. + assertThat( + TypeUtil.isPromotionAllowed( + Types.DateType.get(), Types.TimestampNanoType.withoutZone(), 3)) + .isTrue(); + } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index e42df2fe5ed3..75ba3761cb5a 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -66,6 +66,7 @@ class SchemaUpdate implements UpdateSchema { private boolean allowIncompatibleChanges = false; private Set identifierFieldNames; private boolean caseSensitive = true; + private final int formatVersion; SchemaUpdate(TableOperations ops) { this(ops, ops.current()); @@ -73,20 +74,35 @@ class SchemaUpdate implements UpdateSchema { /** For testing only. */ SchemaUpdate(Schema schema, int lastColumnId) { - this(null, null, schema, lastColumnId); + this(null, null, schema, lastColumnId, TableProperties.DEFAULT_FORMAT_VERSION); } - private SchemaUpdate(TableOperations ops, TableMetadata base) { - this(ops, base, base.schema(), base.lastColumnId()); + /** For testing only. */ + SchemaUpdate(Schema schema, int lastColumnId, int formatVersion) { + this(null, null, schema, lastColumnId, formatVersion); } - private SchemaUpdate(TableOperations ops, TableMetadata base, Schema schema, int lastColumnId) { + private SchemaUpdate(TableOperations ops, TableMetadata base) { + this( + ops, + base, + base.schema(), + base.lastColumnId(), + PropertyUtil.propertyAsInt( + base.properties(), + TableProperties.FORMAT_VERSION, + TableProperties.DEFAULT_FORMAT_VERSION)); + } + + private SchemaUpdate( + TableOperations ops, TableMetadata base, Schema schema, int lastColumnId, int formatVersion) { this.ops = ops; this.base = base; this.schema = schema; this.lastColumnId = lastColumnId; this.idToParent = Maps.newHashMap(TypeUtil.indexParents(schema.asStruct())); this.identifierFieldNames = schema.identifierFieldNames(); + this.formatVersion = formatVersion; } @Override @@ -282,7 +298,7 @@ public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { } Preconditions.checkArgument( - TypeUtil.isPromotionAllowed(field.type(), newType), + TypeUtil.isPromotionAllowed(field.type(), newType, formatVersion), "Cannot change column type: %s: %s -> %s", name, field.type(), @@ -375,7 +391,7 @@ public UpdateSchema moveAfter(String name, String afterName) { @Override public UpdateSchema unionByNameWith(Schema newSchema) { - UnionByNameVisitor.visit(this, schema, newSchema, caseSensitive); + UnionByNameVisitor.visit(this, schema, newSchema, caseSensitive, formatVersion); return this; } diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 64be3db498dc..ca903982471d 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -41,6 +41,8 @@ private TableProperties() {} */ public static final String FORMAT_VERSION = "format-version"; + public static final int DEFAULT_FORMAT_VERSION = 2; + /** Reserved table property for table UUID. */ public static final String UUID = "uuid"; diff --git a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java index c3b9a50b2081..491778abe19c 100644 --- a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java @@ -36,11 +36,14 @@ public class UnionByNameVisitor extends SchemaWithPartnerVisitor newType:int returns true, meaning it is ignorable // existingType:int -> newType:long returns false, meaning it is not ignorable return newType.isPrimitiveType() - && TypeUtil.isPromotionAllowed(newType, existingType.asPrimitiveType()); + && TypeUtil.isPromotionAllowed(newType, existingType.asPrimitiveType(), formatVersion); } else { // Complex -> Complex return !newType.isPrimitiveType(); diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index fb942dde2aa2..fd8f987721ed 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -77,11 +77,13 @@ public class TestSchemaUpdate { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get())); private static final Set ALL_IDS = ImmutableSet.copyOf(TypeUtil.getProjectedIds(SCHEMA)); - private static final int SCHEMA_LAST_COLUMN_ID = 23; + private static final int SCHEMA_LAST_COLUMN_ID = 25; @Test public void testNoChanges() { @@ -193,13 +195,17 @@ public void testUpdateTypes() { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.TimestampType.withoutZone()), + optional(25, "date_to_time_nano", Types.TimestampNanoType.withoutZone())); Schema updated = - new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) + new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID, 3) .updateColumn("id", Types.LongType.get()) .updateColumn("locations.lat", Types.DoubleType.get()) .updateColumn("locations.long", Types.DoubleType.get()) + .updateColumn("date_to_time", Types.TimestampType.withoutZone()) + .updateColumn("date_to_time_nano", Types.TimestampNanoType.withoutZone()) .apply(); assertThat(updated.asStruct()).isEqualTo(expected); @@ -326,7 +332,9 @@ public void testUpdateTypesCaseInsensitive() { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get())); Schema updated = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -432,7 +440,9 @@ public void testRename() { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get())); Schema renamed = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -489,7 +499,9 @@ public void testRenameCaseInsensitive() { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get())); Schema renamed = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -532,7 +544,7 @@ public void testAddFields() { Types.StructType.of( required(12, "lat", Types.FloatType.get()), required(13, "long", Types.FloatType.get()), - optional(25, "alt", Types.FloatType.get()))), + optional(27, "alt", Types.FloatType.get()))), "map of address to coordinate"), optional( 5, @@ -542,8 +554,8 @@ public void testAddFields() { Types.StructType.of( required(15, "x", Types.LongType.get()), required(16, "y", Types.LongType.get()), - optional(26, "z", Types.LongType.get()), - optional(27, "t.t", Types.LongType.get()))), + optional(28, "z", Types.LongType.get()), + optional(29, "t.t", Types.LongType.get()))), "2-D cartesian points"), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get())), optional( @@ -551,7 +563,9 @@ public void testAddFields() { "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), "string map of properties"), - optional(24, "toplevel", Types.DecimalType.of(9, 2))); + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get()), + optional(26, "toplevel", Types.DecimalType.of(9, 2))); Schema added = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -901,9 +915,9 @@ public void testMixedChanges() { required(23, "zip", Types.IntegerType.get())), Types.StructType.of( required(12, "latitude", Types.DoubleType.get(), "latitude"), - optional(25, "alt", Types.FloatType.get()), + optional(27, "alt", Types.FloatType.get()), required( - 28, "description", Types.StringType.get(), "Location description"))), + 30, "description", Types.StringType.get(), "Location description"))), "map of address to coordinate"), optional( 5, @@ -913,11 +927,13 @@ public void testMixedChanges() { Types.StructType.of( optional(15, "X", Types.LongType.get()), required(16, "y.y", Types.LongType.get()), - optional(26, "z", Types.LongType.get()), - optional(27, "t.t", Types.LongType.get(), "name with '.'"))), + optional(28, "z", Types.LongType.get()), + optional(29, "t.t", Types.LongType.get(), "name with '.'"))), "2-D cartesian points"), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get())), - optional(24, "toplevel", Types.DecimalType.of(9, 2))); + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get()), + optional(26, "toplevel", Types.DecimalType.of(9, 2))); Schema updated = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -1002,7 +1018,7 @@ public void testDeleteThenAddNested() { "preferences", Types.StructType.of( optional(9, "feature2", Types.BooleanType.get()), - optional(24, "feature1", Types.BooleanType.get())), + optional(26, "feature1", Types.BooleanType.get())), "struct of named boolean options"), required( 4, @@ -1033,7 +1049,9 @@ public void testDeleteThenAddNested() { 7, "properties", Types.MapType.ofOptional(18, 19, Types.StringType.get(), Types.StringType.get()), - "string map of properties")); + "string map of properties"), + optional(24, "date_to_time", Types.DateType.get()), + optional(25, "date_to_time_nano", Types.DateType.get())); Schema updatedNested = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID) @@ -2204,8 +2222,7 @@ public void testDeleteContainingNestedIdentifierFieldColumnsFails() { new SchemaUpdate(newSchema, SCHEMA_LAST_COLUMN_ID + 2).deleteColumn("out").apply()) .isInstanceOf(IllegalArgumentException.class) .hasMessage( - "Cannot delete field 24: out: required struct<25: nested: required string> " - + "as it will delete nested identifier field 25: nested: required string"); + "Cannot delete field 26: out: required struct<27: nested: required string> as it will delete nested identifier field 27: nested: required string"); } @Test From 2168586c4f17c15a30aba00c36971a9a7c65a478 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 9 Oct 2025 11:39:00 -0700 Subject: [PATCH 02/15] wip --- .../iceberg/types/CheckCompatibility.java | 4 +- .../org/apache/iceberg/types/TypeUtil.java | 8 ++-- .../iceberg/types/TestReadabilityChecks.java | 2 +- .../apache/iceberg/types/TestTypeUtil.java | 14 +++--- .../java/org/apache/iceberg/SchemaUpdate.java | 14 +++++- .../iceberg/schema/UnionByNameVisitor.java | 3 +- .../iceberg/TestSchemaUnionByFieldName.java | 22 ++++++++++ .../org/apache/iceberg/TestSchemaUpdate.java | 44 +++++++++++++++++++ 8 files changed, 99 insertions(+), 12 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index b4ae083baeec..69fd1f3e4901 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -47,6 +47,7 @@ public static List writeCompatibilityErrors(Schema readSchema, Schema wr * * @param readSchema a read schema * @param writeSchema a write schema + * @param formatVersion the table format version * @return a list of error details, or an empty list if there are no compatibility problems */ public static List writeCompatibilityErrors( @@ -324,7 +325,8 @@ public List primitive(Type.PrimitiveType readPrimitive) { currentType.typeId().toString().toLowerCase(Locale.ENGLISH), readPrimitive)); } - if (!TypeUtil.isPromotionAllowed(currentType.asPrimitiveType(), readPrimitive, formatVersion)) { + if (!TypeUtil.isPromotionAllowed( + currentType.asPrimitiveType(), readPrimitive, formatVersion, false)) { return ImmutableList.of( String.format(": %s cannot be promoted to %s", currentType, readPrimitive)); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 892fcc3595aa..0487ce17492e 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -424,11 +424,11 @@ public static Type find(Type type, Predicate predicate) { } public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { - return TypeUtil.isPromotionAllowed(from, to, 2); + return TypeUtil.isPromotionAllowed(from, to, 2, false); } public static boolean isPromotionAllowed( - Type from, Type.PrimitiveType to, Integer formatVersion) { + Type from, Type.PrimitiveType to, Integer formatVersion, boolean sourceIdReference) { // Warning! Before changing this function, make sure that the type change doesn't introduce // compatibility problems in partitioning. if (from.equals(to)) { @@ -439,6 +439,8 @@ public static boolean isPromotionAllowed( case DATE: if (formatVersion < 3) { return false; + } else if (sourceIdReference) { + return false; } else if (to.typeId() == Type.TypeID.TIMESTAMP) { // Timezone types cannot be promoted. Types.TimestampType toTs = (Types.TimestampType) to; @@ -448,7 +450,7 @@ public static boolean isPromotionAllowed( Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; return Types.TimestampNanoType.withoutZone().equals(toTs); } - // fall through + return false; case INTEGER: return to.typeId() == Type.TypeID.LONG; diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 21335977be9c..9572920b157d 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -70,7 +70,7 @@ public void testPrimitiveTypes() { CheckCompatibility.writeCompatibilityErrors( new Schema(required(1, "to_field", to)), fromSchema); - if (TypeUtil.isPromotionAllowed(from, to, 2)) { + if (TypeUtil.isPromotionAllowed(from, to, 2, false)) { assertThat(errors).as("Should produce 0 error messages").isEmpty(); } else { assertThat(errors).hasSize(1); diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index f4ee4accf4be..64f7f79091d3 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -950,24 +950,28 @@ public void ancestorFieldsInNestedSchema() { public void testDateToTimestampPromotion() { // Format version < 3 should not be accepted. assertThat( - TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withoutZone(), 2)) + TypeUtil.isPromotionAllowed( + Types.DateType.get(), Types.TimestampType.withoutZone(), 2, false)) .isFalse(); // Timezone should not be accepted. - assertThat(TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withZone(), 3)) + assertThat( + TypeUtil.isPromotionAllowed( + Types.DateType.get(), Types.TimestampType.withZone(), 3, false)) .isFalse(); // Timezone nano should not be accepted. assertThat( TypeUtil.isPromotionAllowed( - Types.DateType.get(), Types.TimestampNanoType.withZone(), 3)) + Types.DateType.get(), Types.TimestampNanoType.withZone(), 3, false)) .isFalse(); // Timestamp without timezone should be accepted. assertThat( - TypeUtil.isPromotionAllowed(Types.DateType.get(), Types.TimestampType.withoutZone(), 3)) + TypeUtil.isPromotionAllowed( + Types.DateType.get(), Types.TimestampType.withoutZone(), 3, false)) .isTrue(); // Timestamp nano without timezone should be accepted. assertThat( TypeUtil.isPromotionAllowed( - Types.DateType.get(), Types.TimestampNanoType.withoutZone(), 3)) + Types.DateType.get(), Types.TimestampNanoType.withoutZone(), 3, false)) .isTrue(); } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 75ba3761cb5a..4de2d8681dd8 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -72,6 +72,11 @@ class SchemaUpdate implements UpdateSchema { this(ops, ops.current()); } + /** For testing only. */ + SchemaUpdate(TableMetadata base) { + this(null, base, base.schema(), base.lastColumnId(), TableProperties.DEFAULT_FORMAT_VERSION); + } + /** For testing only. */ SchemaUpdate(Schema schema, int lastColumnId) { this(null, null, schema, lastColumnId, TableProperties.DEFAULT_FORMAT_VERSION); @@ -297,8 +302,15 @@ public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { return this; } + // If field is listed in source-ids, we need to flag it for promoting date -> timestamp. + List partitionFields = + this.base != null + ? this.base.spec().getFieldsBySourceId(field.fieldId()) + : Lists.newArrayList(); + Preconditions.checkArgument( - TypeUtil.isPromotionAllowed(field.type(), newType, formatVersion), + TypeUtil.isPromotionAllowed( + field.type(), newType, formatVersion, !partitionFields.isEmpty()), "Cannot change column type: %s: %s -> %s", name, field.type(), diff --git a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java index 491778abe19c..4d11531f135c 100644 --- a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java @@ -222,7 +222,8 @@ private boolean isIgnorableTypeUpdate(Type existingType, Type newType) { // existingType:long -> newType:int returns true, meaning it is ignorable // existingType:int -> newType:long returns false, meaning it is not ignorable return newType.isPrimitiveType() - && TypeUtil.isPromotionAllowed(newType, existingType.asPrimitiveType(), formatVersion); + && TypeUtil.isPromotionAllowed( + newType, existingType.asPrimitiveType(), formatVersion, false); } else { // Complex -> Complex return !newType.isPrimitiveType(); diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java index a9255e4125fb..898522a13d3b 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java @@ -413,6 +413,28 @@ public void testTypePromoteDecimalToFixedScaleWithWiderPrecision() { assertThat(applied.asStruct()).isEqualTo(newSchema.asStruct()); } + @Test + // date -> Can promote to timestamp + public void testTypePromoteDateToTimestamp() { + Schema currentSchema = new Schema(required(1, "aCol", DateType.get())); + Schema newSchema = new Schema(required(1, "aCol", TimestampType.withoutZone())); + + Schema applied = new SchemaUpdate(currentSchema, 1).unionByNameWith(newSchema).apply(); + assertThat(applied.asStruct()).isEqualTo(newSchema.asStruct()); + assertThat(applied.asStruct().fields()).hasSize(1); + assertThat(applied.asStruct().fields().get(0).type()).isEqualTo(TimestampType.withoutZone()); + } + + @Test + public void testTypePromoteDateToTimestampWithZone() { + Schema currentSchema = new Schema(required(1, "aCol", DateType.get())); + Schema newSchema = new Schema(required(1, "aCol", TimestampType.withZone())); + + assertThatThrownBy(() -> new SchemaUpdate(currentSchema, 1).unionByNameWith(newSchema).apply()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change column type: aCol: date -> timestamptz"); + } + @Test public void testAddPrimitiveToNestedStruct() { Schema schema = diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index fd8f987721ed..f6b3340d6407 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -2596,4 +2596,48 @@ public void testCaseInsensitiveMoveAfterNewlyAddedField() { assertThat(actual.asStruct()).isEqualTo(expected.asStruct()); } + + @Test + public void testDateToTimestampPromotionNotAllowedInV2() { + Schema schema = new Schema(required(1, "col", Types.DateType.get())); + // v2 format does not allow date to timestamp promotion + assertThatThrownBy( + () -> + new SchemaUpdate(schema, 1, 2) + .updateColumn("col", Types.TimestampType.withoutZone())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change column type: col: date -> timestamp"); + } + + @Test + public void testDateToTimestampTzPromotionNotAllowedInV3() { + Schema schema = new Schema(required(1, "col", Types.DateType.get())); + // v3 format does not allow date to timestamptz promotion + assertThatThrownBy( + () -> + new SchemaUpdate(schema, 1, 3).updateColumn("col", Types.TimestampType.withZone())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change column type: col: date -> timestamptz"); + } + + @Test + public void testUpdatePartitionedDateToTimestampV3Fails() { + Schema schema = + new Schema( + required(1, "id", Types.IntegerType.get()), required(2, "ts", Types.DateType.get())); + + PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); + + TableMetadata metadata = + TableMetadata.newTableMetadata(schema, spec, "file:/tmp", java.util.Collections.emptyMap()); + + assertThatThrownBy( + () -> + new SchemaUpdate(metadata) + .updateColumn("ts", Types.TimestampType.withoutZone()) + .apply()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change column type: ts: date -> timestamp"); + } + } } From b1496ba9bfda50268aefe8765fdecd136005d406 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 9 Oct 2025 14:35:57 -0700 Subject: [PATCH 03/15] pr comments --- api/src/main/java/org/apache/iceberg/types/TypeUtil.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 0487ce17492e..dbeee47b0961 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -423,6 +423,12 @@ public static Type find(Type type, Predicate predicate) { return visit(type, new FindTypeVisitor(predicate)); } + /** + * @deprecated will be removed in 2.0.0, use {@link #isPromotionAllowed(Type, Type.PrimitiveType, + * Integer, boolean)} instead. This method does not take advantage of table format or source + * id references + */ + @Deprecated public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { return TypeUtil.isPromotionAllowed(from, to, 2, false); } From 08503d98585a83c73c09aad10316818b77a2cd64 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Fri, 10 Oct 2025 15:22:38 -0700 Subject: [PATCH 04/15] update to only use bucket --- .../java/org/apache/iceberg/SchemaUpdate.java | 8 +++-- .../org/apache/iceberg/TestSchemaUpdate.java | 29 +++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 4de2d8681dd8..a8fae6d07b12 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -74,7 +74,7 @@ class SchemaUpdate implements UpdateSchema { /** For testing only. */ SchemaUpdate(TableMetadata base) { - this(null, base, base.schema(), base.lastColumnId(), TableProperties.DEFAULT_FORMAT_VERSION); + this(null, base, base.schema(), base.lastColumnId(), base.formatVersion()); } /** For testing only. */ @@ -308,9 +308,11 @@ public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { ? this.base.spec().getFieldsBySourceId(field.fieldId()) : Lists.newArrayList(); + boolean isBucketPartitioned = + partitionFields.stream().anyMatch(pf -> pf.transform().toString().startsWith("bucket[")); + Preconditions.checkArgument( - TypeUtil.isPromotionAllowed( - field.type(), newType, formatVersion, !partitionFields.isEmpty()), + TypeUtil.isPromotionAllowed(field.type(), newType, formatVersion, isBucketPartitioned), "Cannot change column type: %s: %s -> %s", name, field.type(), diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index f6b3340d6407..fce4c7d0639a 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Set; import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -2621,7 +2622,7 @@ public void testDateToTimestampTzPromotionNotAllowedInV3() { } @Test - public void testUpdatePartitionedDateToTimestampV3Fails() { + public void testUpdatePartitionedDateToTimestampV3Succeeds() { Schema schema = new Schema( required(1, "id", Types.IntegerType.get()), required(2, "ts", Types.DateType.get())); @@ -2629,7 +2630,31 @@ public void testUpdatePartitionedDateToTimestampV3Fails() { PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); TableMetadata metadata = - TableMetadata.newTableMetadata(schema, spec, "file:/tmp", java.util.Collections.emptyMap()); + TableMetadata.newTableMetadata( + schema, spec, "file:/tmp", ImmutableMap.of("format-version", "3")); + + Schema updated = + new SchemaUpdate(metadata).updateColumn("ts", Types.TimestampType.withoutZone()).apply(); + + Schema expected = + new Schema( + required(1, "id", Types.IntegerType.get()), + required(2, "ts", Types.TimestampType.withoutZone())); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + public void testUpdatePartitionedDateToTimestampV3Fails() { + Schema schema = + new Schema( + required(1, "id", Types.IntegerType.get()), required(2, "ts", Types.DateType.get())); + + PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("ts", 4).build(); + + TableMetadata metadata = + TableMetadata.newTableMetadata( + schema, spec, "file:/tmp", ImmutableMap.of("format-version", "3")); assertThatThrownBy( () -> From 4198beceb9c2d8945dcc7ab40ea4e522346b081f Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Fri, 10 Oct 2025 15:44:40 -0700 Subject: [PATCH 05/15] metrics fix --- .../data/TestMetricsRowGroupFilter.java | 98 +++++++++++++++++++ .../iceberg/parquet/ParquetConversions.java | 17 ++++ 2 files changed, 115 insertions(+) diff --git a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java index 3cb46b309d82..e6ca01b26c8e 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java +++ b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java @@ -1000,6 +1000,104 @@ public void testParquetTypePromotion() { assertThat(shouldRead).as("Should succeed with promoted schema").isTrue(); } + @TestTemplate + public void testParquetDateToTimestampPromotion() throws IOException { + assumeThat(format).as("Only valid for Parquet").isEqualTo(FileFormat.PARQUET); + + Schema dateSchema = new Schema(required(1, "dt", Types.DateType.get())); + List records = Lists.newArrayList(); + GenericRecord record1 = GenericRecord.create(dateSchema); + record1.setField("dt", java.time.LocalDate.parse("2018-01-01")); + records.add(record1); + GenericRecord record2 = GenericRecord.create(dateSchema); + record2.setField("dt", java.time.LocalDate.parse("2018-01-31")); + records.add(record2); + File parquetFile = writeParquetFile("test-date-promotion", dateSchema, records); + + try (ParquetFileReader reader = + ParquetFileReader.open(parquetInputFile(Files.localInput(parquetFile)))) { + assertThat(reader.getRowGroups()).as("Should create only one row group").hasSize(1); + BlockMetaData rowGroup = reader.getRowGroups().get(0); + MessageType parquetSchema = reader.getFileMetaData().getSchema(); + + Schema promotedSchema = new Schema(required(1, "dt", Types.TimestampType.withoutZone())); + boolean shouldRead = + new ParquetMetricsRowGroupFilter( + promotedSchema, + lessThan( + "dt", + java.time.LocalDateTime.parse("2018-01-15T12:00:00") + .toInstant(java.time.ZoneOffset.UTC) + .toEpochMilli() + * 1000), + true) + .shouldRead(parquetSchema, rowGroup); + assertThat(shouldRead).as("Should read: one possible date").isTrue(); + + shouldRead = + new ParquetMetricsRowGroupFilter( + promotedSchema, + lessThan( + "dt", + java.time.LocalDateTime.parse("2017-12-01T12:00:00") + .toInstant(java.time.ZoneOffset.UTC) + .toEpochMilli() + * 1000), + true) + .shouldRead(parquetSchema, rowGroup); + assertThat(shouldRead).as("Should not read: date range below lower bound").isFalse(); + } + } + + @TestTemplate + public void testParquetDateToTimestampNanoPromotion() throws IOException { + assumeThat(format).as("Only valid for Parquet").isEqualTo(FileFormat.PARQUET); + + Schema dateSchema = new Schema(required(1, "dt", Types.DateType.get())); + List records = Lists.newArrayList(); + GenericRecord record1 = GenericRecord.create(dateSchema); + record1.setField("dt", java.time.LocalDate.parse("2018-01-01")); + records.add(record1); + GenericRecord record2 = GenericRecord.create(dateSchema); + record2.setField("dt", java.time.LocalDate.parse("2018-01-31")); + records.add(record2); + File parquetFile = writeParquetFile("test-date-promotion", dateSchema, records); + + try (ParquetFileReader reader = + ParquetFileReader.open(parquetInputFile(Files.localInput(parquetFile)))) { + assertThat(reader.getRowGroups()).as("Should create only one row group").hasSize(1); + BlockMetaData rowGroup = reader.getRowGroups().get(0); + MessageType parquetSchema = reader.getFileMetaData().getSchema(); + + Schema promotedSchema = new Schema(required(1, "dt", Types.TimestampNanoType.withoutZone())); + boolean shouldRead = + new ParquetMetricsRowGroupFilter( + promotedSchema, + lessThan( + "dt", + java.time.LocalDateTime.parse("2018-01-15T12:00:00") + .toInstant(java.time.ZoneOffset.UTC) + .toEpochMilli() + * 1000), + true) + .shouldRead(parquetSchema, rowGroup); + assertThat(shouldRead).as("Should read: one possible date").isTrue(); + + shouldRead = + new ParquetMetricsRowGroupFilter( + promotedSchema, + lessThan( + "dt", + java.time.LocalDateTime.parse("2017-12-01T12:00:00") + .toInstant(java.time.ZoneOffset.UTC) + .toEpochMilli() + * 1000), + true) + .shouldRead(parquetSchema, rowGroup); + assertThat(shouldRead).as("Should not read: date range below lower bound").isFalse(); + } + } + @TestTemplate public void testTransformFilter() { assumeThat(format).isEqualTo(FileFormat.PARQUET); diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index 7e8e6d77de5c..cade9959b15f 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -23,10 +23,13 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.UUID; import java.util.function.Function; import org.apache.iceberg.types.Type; import org.apache.iceberg.util.UUIDUtil; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; @@ -72,6 +75,7 @@ static T convertValue(Type type, PrimitiveType parquetType, Object value) { } } + @SuppressWarnings("checkstyle:CyclomaticComplexity") static Function converterFromParquet( PrimitiveType parquetType, Type icebergType) { Function fromParquet = converterFromParquet(parquetType); @@ -82,6 +86,19 @@ static Function converterFromParquet( } else if (icebergType.typeId() == Type.TypeID.DOUBLE && parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FLOAT) { return value -> ((Float) fromParquet.apply(value)).doubleValue(); + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP + && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if(logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType).isAdjustedToUTC()) { + return fromParquet; + } + return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toMicros(1); + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if(logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType).isAdjustedToUTC()) { + return fromParquet; + } + return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toNanos(1); } else if (icebergType.typeId() == Type.TypeID.UUID) { return binary -> UUIDUtil.convert(((Binary) binary).toByteBuffer()); } From 879f0cca492c15573fca6f1cadc3069902d12b8c Mon Sep 17 00:00:00 2001 From: Alex Stephen <1325798+rambleraptor@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:54:27 -0800 Subject: [PATCH 06/15] Update api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java Co-authored-by: emkornfield --- .../main/java/org/apache/iceberg/types/CheckCompatibility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 69fd1f3e4901..2d5f8d7febce 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -73,7 +73,7 @@ public static List writeCompatibilityErrors( /** * Returns a list of compatibility errors for writing with the given write schema. This includes * nullability: writing optional (nullable) values to a required field is an error Optionally this - * method allows case where input schema has different ordering than table schema. + * configuring whether different orderings between schema is considered an error. * * @param readSchema a read schema * @param writeSchema a write schema From 2608fa4d54612249425cdfd48ef39dbbefed5e53 Mon Sep 17 00:00:00 2001 From: Alex Stephen <1325798+rambleraptor@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:54:58 -0800 Subject: [PATCH 07/15] Update api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java Co-authored-by: emkornfield --- .../main/java/org/apache/iceberg/types/CheckCompatibility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 2d5f8d7febce..6190a920c9c2 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -72,7 +72,7 @@ public static List writeCompatibilityErrors( /** * Returns a list of compatibility errors for writing with the given write schema. This includes - * nullability: writing optional (nullable) values to a required field is an error Optionally this + * nullability: writing optional (nullable) values to a required field is an error. This method allows * configuring whether different orderings between schema is considered an error. * * @param readSchema a read schema From 17778331d260468a851b8b80f82db36c391d4478 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 19 Nov 2025 12:11:23 -0800 Subject: [PATCH 08/15] PR comments --- .../iceberg/types/CheckCompatibility.java | 11 ++++--- .../org/apache/iceberg/types/TypeUtil.java | 32 +++++++++++-------- .../iceberg/schema/UnionByNameVisitor.java | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 6190a920c9c2..30a4c8bb125d 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -29,6 +29,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor> { + private static final int DEFAULT_FORMAT_VERSION = 2; /** * Returns a list of compatibility errors for writing with the given write schema. This includes * nullability: writing optional (nullable) values to a required field is an error. @@ -38,7 +39,7 @@ public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor writeCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return writeCompatibilityErrors(readSchema, writeSchema, true, 2); + return writeCompatibilityErrors(readSchema, writeSchema, true, DEFAULT_FORMAT_VERSION); } /** @@ -52,7 +53,7 @@ public static List writeCompatibilityErrors(Schema readSchema, Schema wr */ public static List writeCompatibilityErrors( Schema readSchema, Schema writeSchema, int formatVersion) { - return writeCompatibilityErrors(readSchema, writeSchema, true, formatVersion); + return writeCompatibilityErrors(readSchema, writeSchema, /*checkOrdering=*/true, formatVersion); } /** @@ -67,7 +68,7 @@ public static List writeCompatibilityErrors( */ public static List writeCompatibilityErrors( Schema readSchema, Schema writeSchema, boolean checkOrdering) { - return writeCompatibilityErrors(readSchema, writeSchema, checkOrdering, 2); + return writeCompatibilityErrors(readSchema, writeSchema, checkOrdering, DEFAULT_FORMAT_VERSION); } /** @@ -121,7 +122,7 @@ public static List typeCompatibilityErrors( * @return a list of error details, or an empty list if there are no compatibility problems */ public static List typeCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return typeCompatibilityErrors(readSchema, writeSchema, 2); + return typeCompatibilityErrors(readSchema, writeSchema, DEFAULT_FORMAT_VERSION); } public static List typeCompatibilityErrors( @@ -138,7 +139,7 @@ public static List typeCompatibilityErrors( * @return a list of error details, or an empty list if there are no compatibility problems */ public static List readCompatibilityErrors(Schema readSchema, Schema writeSchema) { - return readCompatibilityErrors(readSchema, writeSchema, 2); + return readCompatibilityErrors(readSchema, writeSchema, DEFAULT_FORMAT_VERSION); } public static List readCompatibilityErrors( diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index dbeee47b0961..eecc3cea1ed4 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -433,6 +433,23 @@ public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { return TypeUtil.isPromotionAllowed(from, to, 2, false); } + private static boolean handleDateType(Type from, Type.PrimitiveType to, Integer formatVersion, boolean sourceIdReference) { + if (formatVersion < 3) { + return false; + } else if (sourceIdReference) { + return false; + } else if (to.typeId() == Type.TypeID.TIMESTAMP) { + // Timezone types cannot be promoted. + Types.TimestampType toTs = (Types.TimestampType) to; + return Types.TimestampType.withoutZone().equals(toTs); + } else if (to.typeId() == Type.TypeID.TIMESTAMP_NANO) { + // Timezone types cannot be promoted. + Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; + return Types.TimestampNanoType.withoutZone().equals(toTs); + } + return false; + } + public static boolean isPromotionAllowed( Type from, Type.PrimitiveType to, Integer formatVersion, boolean sourceIdReference) { // Warning! Before changing this function, make sure that the type change doesn't introduce @@ -443,20 +460,7 @@ public static boolean isPromotionAllowed( switch (from.typeId()) { case DATE: - if (formatVersion < 3) { - return false; - } else if (sourceIdReference) { - return false; - } else if (to.typeId() == Type.TypeID.TIMESTAMP) { - // Timezone types cannot be promoted. - Types.TimestampType toTs = (Types.TimestampType) to; - return Types.TimestampType.withoutZone().equals(toTs); - } else if (to.typeId() == Type.TypeID.TIMESTAMP_NANO) { - // Timezone types cannot be promoted. - Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; - return Types.TimestampNanoType.withoutZone().equals(toTs); - } - return false; + return handleDateType(from, to, formatVersion, sourceIdReference); case INTEGER: return to.typeId() == Type.TypeID.LONG; diff --git a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java index 4d11531f135c..f1cc980627c9 100644 --- a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java @@ -223,7 +223,7 @@ private boolean isIgnorableTypeUpdate(Type existingType, Type newType) { // existingType:int -> newType:long returns false, meaning it is not ignorable return newType.isPrimitiveType() && TypeUtil.isPromotionAllowed( - newType, existingType.asPrimitiveType(), formatVersion, false); + newType, existingType.asPrimitiveType(), formatVersion, /*sourceIdReference*/ false); } else { // Complex -> Complex return !newType.isPrimitiveType(); From 1952a69446459de23b17e0318c99953b98a0c038 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 19 Nov 2025 12:15:54 -0800 Subject: [PATCH 09/15] PR comments --- .../iceberg/types/CheckCompatibility.java | 10 +++--- .../org/apache/iceberg/types/TypeUtil.java | 31 ++++++++++--------- .../iceberg/parquet/ParquetConversions.java | 27 +++++++++------- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 30a4c8bb125d..38f1c014889a 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -29,7 +29,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; public class CheckCompatibility extends TypeUtil.CustomOrderSchemaVisitor> { - private static final int DEFAULT_FORMAT_VERSION = 2; + private static final int DEFAULT_FORMAT_VERSION = 2; + /** * Returns a list of compatibility errors for writing with the given write schema. This includes * nullability: writing optional (nullable) values to a required field is an error. @@ -53,7 +54,8 @@ public static List writeCompatibilityErrors(Schema readSchema, Schema wr */ public static List writeCompatibilityErrors( Schema readSchema, Schema writeSchema, int formatVersion) { - return writeCompatibilityErrors(readSchema, writeSchema, /*checkOrdering=*/true, formatVersion); + return writeCompatibilityErrors( + readSchema, writeSchema, /* checkOrdering= */ true, formatVersion); } /** @@ -73,8 +75,8 @@ public static List writeCompatibilityErrors( /** * Returns a list of compatibility errors for writing with the given write schema. This includes - * nullability: writing optional (nullable) values to a required field is an error. This method allows - * configuring whether different orderings between schema is considered an error. + * nullability: writing optional (nullable) values to a required field is an error. This method + * allows configuring whether different orderings between schema is considered an error. * * @param readSchema a read schema * @param writeSchema a write schema diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index eecc3cea1ed4..287cb768b676 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -433,21 +433,22 @@ public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { return TypeUtil.isPromotionAllowed(from, to, 2, false); } - private static boolean handleDateType(Type from, Type.PrimitiveType to, Integer formatVersion, boolean sourceIdReference) { - if (formatVersion < 3) { - return false; - } else if (sourceIdReference) { - return false; - } else if (to.typeId() == Type.TypeID.TIMESTAMP) { - // Timezone types cannot be promoted. - Types.TimestampType toTs = (Types.TimestampType) to; - return Types.TimestampType.withoutZone().equals(toTs); - } else if (to.typeId() == Type.TypeID.TIMESTAMP_NANO) { - // Timezone types cannot be promoted. - Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; - return Types.TimestampNanoType.withoutZone().equals(toTs); - } + private static boolean handleDateType( + Type.PrimitiveType to, Integer formatVersion, boolean sourceIdReference) { + if (formatVersion < 3) { return false; + } else if (sourceIdReference) { + return false; + } else if (to.typeId() == Type.TypeID.TIMESTAMP) { + // Timezone types cannot be promoted. + Types.TimestampType toTs = (Types.TimestampType) to; + return Types.TimestampType.withoutZone().equals(toTs); + } else if (to.typeId() == Type.TypeID.TIMESTAMP_NANO) { + // Timezone types cannot be promoted. + Types.TimestampNanoType toTs = (Types.TimestampNanoType) to; + return Types.TimestampNanoType.withoutZone().equals(toTs); + } + return false; } public static boolean isPromotionAllowed( @@ -460,7 +461,7 @@ public static boolean isPromotionAllowed( switch (from.typeId()) { case DATE: - return handleDateType(from, to, formatVersion, sourceIdReference); + return handleDateType(to, formatVersion, sourceIdReference); case INTEGER: return to.typeId() == Type.TypeID.LONG; diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index cade9959b15f..cc2837ef83a1 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -23,8 +23,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeUnit; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.apache.iceberg.types.Type; import org.apache.iceberg.util.UUIDUtil; @@ -88,17 +88,22 @@ static Function converterFromParquet( return value -> ((Float) fromParquet.apply(value)).doubleValue(); } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - if(logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType).isAdjustedToUTC()) { - return fromParquet; - } + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation + && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType) + .isAdjustedToUTC()) { + return fromParquet; + } return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toMicros(1); - } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - if(logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType).isAdjustedToUTC()) { - return fromParquet; - } - return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toNanos(1); + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO + && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { + LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); + if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation + && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType) + .isAdjustedToUTC()) { + return fromParquet; + } + return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toNanos(1); } else if (icebergType.typeId() == Type.TypeID.UUID) { return binary -> UUIDUtil.convert(((Binary) binary).toByteBuffer()); } From 7e1519803d9a53b0f1f15431e2c1a571e74bb4de Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 19 Nov 2025 12:17:19 -0800 Subject: [PATCH 10/15] last PR comment --- .../main/java/org/apache/iceberg/types/CheckCompatibility.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 38f1c014889a..d10f65d3fcc9 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -78,7 +78,7 @@ public static List writeCompatibilityErrors( * nullability: writing optional (nullable) values to a required field is an error. This method * allows configuring whether different orderings between schema is considered an error. * - * @param readSchema a read schema + * @param readSchema a read schema - input schema * @param writeSchema a write schema * @param checkOrdering If false, allow input schema to have different ordering than table schema * @param formatVersion the table format version From 4287d2cc7e89d61e581824cd3fb431f97db419ef Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Wed, 19 Nov 2025 13:49:31 -0800 Subject: [PATCH 11/15] checkstyle issues --- .../java/org/apache/iceberg/schema/UnionByNameVisitor.java | 2 +- .../org/apache/iceberg/data/TestMetricsRowGroupFilter.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java index f1cc980627c9..a89ebc1bda82 100644 --- a/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/UnionByNameVisitor.java @@ -223,7 +223,7 @@ private boolean isIgnorableTypeUpdate(Type existingType, Type newType) { // existingType:int -> newType:long returns false, meaning it is not ignorable return newType.isPrimitiveType() && TypeUtil.isPromotionAllowed( - newType, existingType.asPrimitiveType(), formatVersion, /*sourceIdReference*/ false); + newType, existingType.asPrimitiveType(), formatVersion, /* sourceIdReference*/ false); } else { // Complex -> Complex return !newType.isPrimitiveType(); diff --git a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java index e6ca01b26c8e..9dab304d38b2 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java +++ b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java @@ -1018,7 +1018,7 @@ public void testParquetDateToTimestampPromotion() throws IOException { ParquetFileReader.open(parquetInputFile(Files.localInput(parquetFile)))) { assertThat(reader.getRowGroups()).as("Should create only one row group").hasSize(1); BlockMetaData rowGroup = reader.getRowGroups().get(0); - MessageType parquetSchema = reader.getFileMetaData().getSchema(); + parquetSchema = reader.getFileMetaData().getSchema(); Schema promotedSchema = new Schema(required(1, "dt", Types.TimestampType.withoutZone())); boolean shouldRead = @@ -1067,7 +1067,7 @@ public void testParquetDateToTimestampNanoPromotion() throws IOException { ParquetFileReader.open(parquetInputFile(Files.localInput(parquetFile)))) { assertThat(reader.getRowGroups()).as("Should create only one row group").hasSize(1); BlockMetaData rowGroup = reader.getRowGroups().get(0); - MessageType parquetSchema = reader.getFileMetaData().getSchema(); + parquetSchema = reader.getFileMetaData().getSchema(); Schema promotedSchema = new Schema(required(1, "dt", Types.TimestampNanoType.withoutZone())); boolean shouldRead = From c47864162a81f503f94169c45064979221702253 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Tue, 27 Jan 2026 20:17:06 -0800 Subject: [PATCH 12/15] rebase issue --- core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index fce4c7d0639a..af41244ca313 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -2664,5 +2664,4 @@ public void testUpdatePartitionedDateToTimestampV3Fails() { .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot change column type: ts: date -> timestamp"); } - } } From ff99d95a86b66737d0077fa1dd0e73a9a242d42a Mon Sep 17 00:00:00 2001 From: Daniel Rodrigues Date: Wed, 18 Mar 2026 20:40:35 +0100 Subject: [PATCH 13/15] Testing timestamp promotion fixes --- .../iceberg/TestSchemaUnionByFieldName.java | 2 +- .../iceberg/parquet/ParquetConversions.java | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java index 898522a13d3b..c83ac4dc3145 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUnionByFieldName.java @@ -419,7 +419,7 @@ public void testTypePromoteDateToTimestamp() { Schema currentSchema = new Schema(required(1, "aCol", DateType.get())); Schema newSchema = new Schema(required(1, "aCol", TimestampType.withoutZone())); - Schema applied = new SchemaUpdate(currentSchema, 1).unionByNameWith(newSchema).apply(); + Schema applied = new SchemaUpdate(currentSchema, 1, 3).unionByNameWith(newSchema).apply(); assertThat(applied.asStruct()).isEqualTo(newSchema.asStruct()); assertThat(applied.asStruct().fields()).hasSize(1); assertThat(applied.asStruct().fields().get(0).type()).isEqualTo(TimestampType.withoutZone()); diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index cc2837ef83a1..d3bf75e79088 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -86,23 +86,9 @@ static Function converterFromParquet( } else if (icebergType.typeId() == Type.TypeID.DOUBLE && parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FLOAT) { return value -> ((Float) fromParquet.apply(value)).doubleValue(); - } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP - && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation - && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType) - .isAdjustedToUTC()) { - return fromParquet; - } + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP && parquetType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toMicros(1); - } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO - && parquetType.getOriginalType() == org.apache.parquet.schema.OriginalType.DATE) { - LogicalTypeAnnotation logicalType = parquetType.getLogicalTypeAnnotation(); - if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation - && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType) - .isAdjustedToUTC()) { - return fromParquet; - } + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO && parquetType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toNanos(1); } else if (icebergType.typeId() == Type.TypeID.UUID) { return binary -> UUIDUtil.convert(((Binary) binary).toByteBuffer()); From 91bf1da6be7f31249a75c5654534323322b1cb9f Mon Sep 17 00:00:00 2001 From: Daniel Rodrigues Date: Wed, 18 Mar 2026 20:45:22 +0100 Subject: [PATCH 14/15] run Spotless Apply --- .../org/apache/iceberg/parquet/ParquetConversions.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java index d3bf75e79088..4db683058fde 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetConversions.java @@ -86,9 +86,13 @@ static Function converterFromParquet( } else if (icebergType.typeId() == Type.TypeID.DOUBLE && parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FLOAT) { return value -> ((Float) fromParquet.apply(value)).doubleValue(); - } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP && parquetType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP + && parquetType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toMicros(1); - } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO && parquetType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + } else if (icebergType.typeId() == Type.TypeID.TIMESTAMP_NANO + && parquetType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return value -> (long) ((Integer) fromParquet.apply(value)) * TimeUnit.DAYS.toNanos(1); } else if (icebergType.typeId() == Type.TypeID.UUID) { return binary -> UUIDUtil.convert(((Binary) binary).toByteBuffer()); From 67fdea378150bffc1a178ce5ba52d7ffddbea37c Mon Sep 17 00:00:00 2001 From: Daniel Rodrigues Date: Thu, 19 Mar 2026 13:47:41 +0100 Subject: [PATCH 15/15] trigger GitHub actions