diff --git a/data/src/test/java/org/apache/iceberg/data/DataGenerators.java b/data/src/test/java/org/apache/iceberg/data/DataGenerators.java index a78dccd05204..0e8978e4c09e 100644 --- a/data/src/test/java/org/apache/iceberg/data/DataGenerators.java +++ b/data/src/test/java/org/apache/iceberg/data/DataGenerators.java @@ -251,9 +251,6 @@ public Schema schema() { } // Generator for reader default-value tests across primitive types. - // FIXED is excluded: Spark's InternalRowConverter expects a ByteBuffer but the generator produces - // byte[] (ClassCastException). - // TODO: include FIXED once Spark's converter handles it. static class PrimitiveDefaults implements DataGenerator { static final Schema READ_SCHEMA = new Schema( @@ -302,7 +299,18 @@ static class PrimitiveDefaults implements DataGenerator { 14, "time_with_default", Types.TimeType.get(), - Literal.of(DateTimeUtil.isoTimeToMicros("23:59:59.999999")))); + Literal.of(DateTimeUtil.isoTimeToMicros("23:59:59.999999"))), + optionalWithDefault( + 15, + "fixed_with_default", + Types.FixedType.ofLength(16), + Literal.of( + ByteBuffer.wrap( + new byte[] { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + })) + .to(Types.FixedType.ofLength(16)))); static final Schema WRITE_SCHEMA = new Schema(required(1, "id", Types.LongType.get())); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java index b7ba7a34309a..a6c19fc49a1f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java @@ -27,13 +27,13 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; @@ -79,13 +79,7 @@ private static Object convert(Type type, Object value) { : ChronoUnit.MICROS.between(EPOCH, ((LocalDateTime) value).atZone(ZoneId.of("UTC"))); case STRING -> UTF8String.fromString((String) value); case UUID -> UTF8String.fromString(value.toString()); - case FIXED, BINARY -> { - ByteBuffer buffer = (ByteBuffer) value; - yield Arrays.copyOfRange( - buffer.array(), - buffer.arrayOffset() + buffer.position(), - buffer.arrayOffset() + buffer.remaining()); - } + case FIXED, BINARY -> toByteArray(value); case DECIMAL -> Decimal.apply((BigDecimal) value); case STRUCT -> convert((Types.StructType) type, (Record) value); case LIST -> @@ -112,4 +106,15 @@ private static Object convert(Type type, Object value) { "Unsupported type for conversion to InternalRow: " + type); }; } + + private static byte[] toByteArray(Object value) { + if (value instanceof byte[] bytes) { + return bytes; + } else if (value instanceof ByteBuffer buffer) { + return ByteBuffers.toByteArray(buffer); + } + + throw new UnsupportedOperationException( + "Unsupported binary value class: " + value.getClass().getName()); + } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java index fcf1751c2ddd..4f964f8a74e6 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java @@ -34,11 +34,7 @@ public class TestSparkFormatModel extends BaseFormatModelTests { private static final Set UNSUPPORTED_TYPE_IDS = - Set.of( - Type.TypeID.TIME, - Type.TypeID.TIMESTAMP_NANO, - // TODO: Remove once FIXED is working on TCK - Type.TypeID.FIXED); + Set.of(Type.TypeID.TIME, Type.TypeID.TIMESTAMP_NANO); @Override protected Collection unsupportedTypeIds() { diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java index b7ba7a34309a..a6c19fc49a1f 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java @@ -27,13 +27,13 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; @@ -79,13 +79,7 @@ private static Object convert(Type type, Object value) { : ChronoUnit.MICROS.between(EPOCH, ((LocalDateTime) value).atZone(ZoneId.of("UTC"))); case STRING -> UTF8String.fromString((String) value); case UUID -> UTF8String.fromString(value.toString()); - case FIXED, BINARY -> { - ByteBuffer buffer = (ByteBuffer) value; - yield Arrays.copyOfRange( - buffer.array(), - buffer.arrayOffset() + buffer.position(), - buffer.arrayOffset() + buffer.remaining()); - } + case FIXED, BINARY -> toByteArray(value); case DECIMAL -> Decimal.apply((BigDecimal) value); case STRUCT -> convert((Types.StructType) type, (Record) value); case LIST -> @@ -112,4 +106,15 @@ private static Object convert(Type type, Object value) { "Unsupported type for conversion to InternalRow: " + type); }; } + + private static byte[] toByteArray(Object value) { + if (value instanceof byte[] bytes) { + return bytes; + } else if (value instanceof ByteBuffer buffer) { + return ByteBuffers.toByteArray(buffer); + } + + throw new UnsupportedOperationException( + "Unsupported binary value class: " + value.getClass().getName()); + } } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java index fcf1751c2ddd..4f964f8a74e6 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java @@ -34,11 +34,7 @@ public class TestSparkFormatModel extends BaseFormatModelTests { private static final Set UNSUPPORTED_TYPE_IDS = - Set.of( - Type.TypeID.TIME, - Type.TypeID.TIMESTAMP_NANO, - // TODO: Remove once FIXED is working on TCK - Type.TypeID.FIXED); + Set.of(Type.TypeID.TIME, Type.TypeID.TIMESTAMP_NANO); @Override protected Collection unsupportedTypeIds() { diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java index b7ba7a34309a..a6c19fc49a1f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/InternalRowConverter.java @@ -27,13 +27,13 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; @@ -79,13 +79,7 @@ private static Object convert(Type type, Object value) { : ChronoUnit.MICROS.between(EPOCH, ((LocalDateTime) value).atZone(ZoneId.of("UTC"))); case STRING -> UTF8String.fromString((String) value); case UUID -> UTF8String.fromString(value.toString()); - case FIXED, BINARY -> { - ByteBuffer buffer = (ByteBuffer) value; - yield Arrays.copyOfRange( - buffer.array(), - buffer.arrayOffset() + buffer.position(), - buffer.arrayOffset() + buffer.remaining()); - } + case FIXED, BINARY -> toByteArray(value); case DECIMAL -> Decimal.apply((BigDecimal) value); case STRUCT -> convert((Types.StructType) type, (Record) value); case LIST -> @@ -112,4 +106,15 @@ private static Object convert(Type type, Object value) { "Unsupported type for conversion to InternalRow: " + type); }; } + + private static byte[] toByteArray(Object value) { + if (value instanceof byte[] bytes) { + return bytes; + } else if (value instanceof ByteBuffer buffer) { + return ByteBuffers.toByteArray(buffer); + } + + throw new UnsupportedOperationException( + "Unsupported binary value class: " + value.getClass().getName()); + } } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java index fcf1751c2ddd..4f964f8a74e6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkFormatModel.java @@ -34,11 +34,7 @@ public class TestSparkFormatModel extends BaseFormatModelTests { private static final Set UNSUPPORTED_TYPE_IDS = - Set.of( - Type.TypeID.TIME, - Type.TypeID.TIMESTAMP_NANO, - // TODO: Remove once FIXED is working on TCK - Type.TypeID.FIXED); + Set.of(Type.TypeID.TIME, Type.TypeID.TIMESTAMP_NANO); @Override protected Collection unsupportedTypeIds() {