From 9360f3a7b33a65e183e126fd73d8c6d0f60b80f1 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 11:40:03 +0000 Subject: [PATCH 1/8] Add vector type support for Direct Access API inserts Enables inserting vector data types through the relational Direct Access API with full validation: - Type checking ensures vectors match column precision (FLOAT/16, HALF/32, DOUBLE/64) - Dimension validation verifies vector size matches schema definition - Null/empty vector handling for optional vector columns - Comprehensive test coverage via InsertVectorTest for all insert scenarios Also refactors vector parsing logic into VectorUtils.parseVector() to eliminate code duplication across MessageTuple and FRL. --- .../foundationdb/record/util/VectorUtils.java | 7 +- .../relational/recordlayer/MessageTuple.java | 28 ++-- .../recordlayer/RecordTypeTable.java | 61 +++++-- .../recordlayer/InsertVectorTest.java | 155 ++++++++++++++++++ .../foundationdb/relational/server/FRL.java | 17 +- 5 files changed, 221 insertions(+), 47 deletions(-) create mode 100644 fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java index 4610cc06e4..2f63451696 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java @@ -47,6 +47,11 @@ public static int getVectorPrecision(@Nonnull final RealVector vector) { @Nonnull public static RealVector parseVector(@Nonnull final ByteString byteString, @Nonnull final Type.Vector vectorType) { final var precision = vectorType.getPrecision(); + return parseVector(byteString, precision); + } + + @Nonnull + public static RealVector parseVector(@Nonnull final ByteString byteString, int precision) { if (precision == 16) { return HalfRealVector.fromBytes(byteString.toByteArray()); } @@ -56,7 +61,7 @@ public static RealVector parseVector(@Nonnull final ByteString byteString, @Nonn if (precision == 64) { return DoubleRealVector.fromBytes(byteString.toByteArray()); } - throw new RecordCoreException("unexpected vector type " + vectorType); + throw new RecordCoreException("unexpected vector precision " + precision); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index 1276192c5d..bdad654d4a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleFieldsProto; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; +import com.apple.foundationdb.record.util.VectorUtils; import com.apple.foundationdb.relational.api.exceptions.InvalidColumnReferenceException; import com.google.protobuf.ByteString; import com.google.protobuf.DescriptorProtos; @@ -63,7 +64,11 @@ public Object getObject(int position) throws InvalidColumnReferenceException { if (fieldOptions.hasVectorOptions()) { final var precision = fieldOptions.getVectorOptions().getPrecision(); final var byteStringFieldValue = (ByteString)fieldValue; - return getVectorFromBytes(byteStringFieldValue, precision); + if (byteStringFieldValue.isEmpty()) { + return null; + } else { + return VectorUtils.parseVector(byteStringFieldValue, precision); + } } if (fieldDescriptor.isRepeated()) { final var list = (List) fieldValue; @@ -76,21 +81,6 @@ public Object getObject(int position) throws InvalidColumnReferenceException { } } - @Nonnull - private static RealVector getVectorFromBytes(@Nonnull final ByteString byteString, int precision) { - final var bytes = byteString.toByteArray(); - if (precision == 64) { - return DoubleRealVector.fromBytes(bytes); - } - if (precision == 32) { - return FloatRealVector.fromBytes(bytes); - } - if (precision == 16) { - return HalfRealVector.fromBytes(bytes); - } - throw new RecordCoreException("unexpected vector precision " + precision); - } - public static Object sanitizeField(@Nonnull final Object field, @Nonnull final DescriptorProtos.FieldOptions fieldOptions) { if (field instanceof Message && ((Message) field).getDescriptorForType().equals(TupleFieldsProto.UUID.getDescriptor())) { return TupleFieldsHelper.fromProto((Message) field, TupleFieldsProto.UUID.getDescriptor()); @@ -103,7 +93,11 @@ public static Object sanitizeField(@Nonnull final Object field, @Nonnull final D final var fieldVectorOptionsMaybe = fieldOptions.getExtension(RecordMetaDataOptionsProto.field); if (fieldVectorOptionsMaybe.hasVectorOptions()) { final var precision = fieldVectorOptionsMaybe.getVectorOptions().getPrecision(); - return getVectorFromBytes(byteString, precision); + if (byteString.isEmpty()) { + return null; + } else { + return VectorUtils.parseVector(byteString, precision); + } } return byteString.toByteArray(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java index c3833ad4cd..432b740aba 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java @@ -21,9 +21,14 @@ package com.apple.foundationdb.relational.recordlayer; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.AbstractRealVector; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleRange; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; @@ -57,6 +62,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; @@ -195,7 +201,7 @@ public static Message toDynamicMessage(RelationalStruct struct, Descriptors.Desc final var maybeEnumValue = struct.getString(i + 1); if (maybeEnumValue != null) { final var valueDescriptor = fd.getEnumType().findValueByName(maybeEnumValue); - Assert.thatUnchecked(valueDescriptor != null, ErrorCode.CANNOT_CONVERT_TYPE, "Invalid enum value: %s", maybeEnumValue); + Assert.that(valueDescriptor != null, ErrorCode.CANNOT_CONVERT_TYPE, "Invalid enum value: %s", maybeEnumValue); builder.setField(fd, valueDescriptor); } continue; @@ -209,7 +215,11 @@ public static Message toDynamicMessage(RelationalStruct struct, Descriptors.Desc case STRING: final var obj = struct.getObject(i + 1); if (obj != null) { - builder.setField(fd, obj); + try { + builder.setField(fd, obj); + } catch (IllegalArgumentException ex) { + throw new RelationalException("Unexpected Column type " + struct.getMetaData().getColumnTypeName(i) + " for column " + columnName, ErrorCode.CANNOT_CONVERT_TYPE, ex); + } } break; case BYTES: @@ -237,24 +247,47 @@ public static Message toDynamicMessage(RelationalStruct struct, Descriptors.Desc } } } else { - if (fd.getType() == Descriptors.FieldDescriptor.Type.MESSAGE) { - Assert.thatUnchecked(NullableArrayUtils.isWrappedArrayDescriptor(fd.getMessageType())); - // wrap array in a struct and call toDynamicMessage again - final var wrapper = new ImmutableRowStruct(new ArrayRow(array), RelationalStructMetaData.of( - DataType.StructType.from("STRUCT", List.of( - DataType.StructType.Field.from(NullableArrayUtils.REPEATED_FIELD_NAME, array.getMetaData().asRelationalType(), 0) - ), true))); - builder.setField(fd, toDynamicMessage(wrapper, fd.getMessageType())); - } else { - Assert.failUnchecked("Field Type expected to be of Type ARRAY but is actually " + fd.getType()); + Assert.that(fd.getType() == Descriptors.FieldDescriptor.Type.MESSAGE, ErrorCode.CANNOT_CONVERT_TYPE, + "Field Type expected to be of Type ARRAY but is actually " + fd.getType()); + Assert.that(NullableArrayUtils.isWrappedArrayDescriptor(fd.getMessageType())); + // wrap array in a struct and call toDynamicMessage again + final var wrapper = new ImmutableRowStruct(new ArrayRow(array), RelationalStructMetaData.of( + DataType.StructType.from("STRUCT", List.of( + DataType.StructType.Field.from(NullableArrayUtils.REPEATED_FIELD_NAME, array.getMetaData().asRelationalType(), 0) + ), true))); + builder.setField(fd, toDynamicMessage(wrapper, fd.getMessageType())); + } + break; + case VECTOR: + final var vector = struct.getObject(i + 1); + if (vector != null) { + Assert.that(vector instanceof AbstractRealVector, ErrorCode.CANNOT_CONVERT_TYPE, + "Field Type expected to be of Type VECTOR but is actually " + fd.getType()); + final var fieldOptionMaybe = Optional.ofNullable(fd.getOptions()).map(f -> f.getExtension(RecordMetaDataOptionsProto.field)); + Assert.that(fieldOptionMaybe.isPresent() && fieldOptionMaybe.get().hasVectorOptions(), ErrorCode.CANNOT_CONVERT_TYPE, "Cannot insert non vector type into vector column"); + final var vectorOptions = fieldOptionMaybe.get().getVectorOptions(); + Assert.that(vectorOptions.getDimensions() == ((AbstractRealVector)vector).getNumDimensions(), ErrorCode.CANNOT_CONVERT_TYPE, "Wrong number of dimension for vector"); + final int precision = vectorOptions.getPrecision(); + switch (precision) { + case 16: + Assert.that(vector instanceof HalfRealVector, ErrorCode.CANNOT_CONVERT_TYPE, "Wrong precision for vector"); + break; + case 32: + Assert.that(vector instanceof FloatRealVector, ErrorCode.CANNOT_CONVERT_TYPE, "Wrong precision for vector"); + break; + case 64: + Assert.that(vector instanceof DoubleRealVector, ErrorCode.CANNOT_CONVERT_TYPE, "Wrong precision for vector"); + break; + default: } + builder.setField(fd, ((AbstractRealVector) vector).getRawData()); } break; case NULL: break; default: - Assert.failUnchecked(ErrorCode.INTERNAL_ERROR, (String.format(Locale.ROOT, "Unexpected Column type <%s> for column <%s>", - struct.getMetaData().getColumnType(i), columnName))); + Assert.fail(ErrorCode.INTERNAL_ERROR, (String.format(Locale.ROOT, "Unexpected Column type <%s> for column <%s>", + struct.getMetaData().getColumnTypeName(i), columnName))); break; } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java new file mode 100644 index 0000000000..1df769c252 --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java @@ -0,0 +1,155 @@ +/* + * InsertTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer; + +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.relational.api.EmbeddedRelationalStruct; +import com.apple.foundationdb.relational.api.KeySet; +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.RelationalConnection; +import com.apple.foundationdb.relational.api.RelationalResultSet; +import com.apple.foundationdb.relational.api.RelationalStatement; +import com.apple.foundationdb.relational.api.RelationalStruct; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.utils.RelationalAssertions; +import com.apple.foundationdb.relational.utils.ResultSetAssert; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.sql.DriverManager; +import java.sql.SQLException; + +public class InsertVectorTest { + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + @RegisterExtension + @Order(1) + public final SimpleDatabaseRule database = new SimpleDatabaseRule(InsertVectorTest.class, "CREATE TABLE V(PK INTEGER, V1 VECTOR(4, FLOAT), V2 VECTOR(3, HALF), V3 VECTOR(2, DOUBLE), PRIMARY KEY(PK))"); + + @Test + void insertNulls() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).build(); + s.executeInsert("V", record); + try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { + ResultSetAssert.assertThat(rs) + .hasNextRow() + .hasColumn("PK", 0) + .hasColumn("V1", null) + .hasColumn("V2", null) + .hasColumn("V3", null) + .hasNoNextRow(); + } + } + } + } + + @Test + void partialInsert() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})).build(); + s.executeInsert("V", record); + try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { + ResultSetAssert.assertThat(rs) + .hasNextRow() + .hasColumn("PK", 0) + .hasColumn("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})) + .hasColumn("V2", null) + .hasColumn("V3", null) + .hasNoNextRow(); + } + } + } + } + + @Test + void fullInsert() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0) + .addObject("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})) + .addObject("V2", new HalfRealVector(new int[]{1, 2, 3})) + .addObject("V3", new DoubleRealVector(new double[]{1d, 2d})) + .build(); + s.executeInsert("V", record); + try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { + ResultSetAssert.assertThat(rs) + .hasNextRow() + .hasColumn("PK", 0) + .hasColumn("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})) + .hasColumn("V2", new HalfRealVector(new int[]{1, 2, 3})) + .hasColumn("V3", new DoubleRealVector(new double[]{1d, 2d})) + .hasNoNextRow(); + } + } + } + } + + @Test + void insertWrongDimensionFails() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[] {1f, 2f, 3f, 4f, 5f})).build(); + RelationalAssertions.assertThrowsSqlException( + () -> s.executeInsert("V", record)) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + } + } + + @Test + void insertWrongPrecisionFails() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new DoubleRealVector(new double[] {1d, 2d, 3d, 4d})).build(); + RelationalAssertions.assertThrowsSqlException( + () -> s.executeInsert("V", record)) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + } + } + + @Test + void insertWrongTypeFails() throws SQLException { + try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { + conn.setSchema("TEST_SCHEMA"); + try (RelationalStatement s = conn.createStatement()) { + RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addInt("V1", 42).build(); + RelationalAssertions.assertThrowsSqlException( + () -> s.executeInsert("V", record)) + .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); + } + } + } +} diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java index 5aeb02ec36..3c31f29d97 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace; +import com.apple.foundationdb.record.util.VectorUtils; import com.apple.foundationdb.relational.api.EmbeddedRelationalDriver; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; @@ -256,20 +257,6 @@ private static void setStatementOptions(final @Nullable Options options, final S } } - @Nonnull - public static RealVector parseVector(@Nonnull final ByteString byteString, int precision) { - if (precision == 16) { - return HalfRealVector.fromBytes(byteString.toByteArray()); - } - if (precision == 32) { - return FloatRealVector.fromBytes(byteString.toByteArray()); - } - if (precision == 64) { - return DoubleRealVector.fromBytes(byteString.toByteArray()); - } - throw new RecordCoreException("unexpected vector type with precision " + precision); - } - private static void addPreparedStatementParameter(@Nonnull RelationalPreparedStatement relationalPreparedStatement, @Nonnull Parameter parameter, int index) throws SQLException { final var oneOfValue = parameter.getParameter(); @@ -288,7 +275,7 @@ private static void addPreparedStatementParameter(@Nonnull RelationalPreparedSta } else if (oneOfValue.hasBinary()) { if (parameter.hasMetadata() && parameter.getMetadata().hasVectorMetadata()) { final var vectorProtoType = parameter.getMetadata().getVectorMetadata(); - relationalPreparedStatement.setObject(index, parseVector(oneOfValue.getBinary(), vectorProtoType.getPrecision())); + relationalPreparedStatement.setObject(index, VectorUtils.parseVector(oneOfValue.getBinary(), vectorProtoType.getPrecision())); } else { relationalPreparedStatement.setBytes(index, oneOfValue.getBinary().toByteArray()); } From b5eae702c01cb7daf25288fad289ad0e624587af Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 11:44:37 +0000 Subject: [PATCH 2/8] Fix checks --- .../relational/recordlayer/MessageTuple.java | 5 ---- .../recordlayer/InsertVectorTest.java | 24 +++++++++---------- .../foundationdb/relational/server/FRL.java | 6 ----- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index bdad654d4a..fe96f4869b 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -21,11 +21,6 @@ package com.apple.foundationdb.relational.recordlayer; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.linear.DoubleRealVector; -import com.apple.foundationdb.linear.FloatRealVector; -import com.apple.foundationdb.linear.HalfRealVector; -import com.apple.foundationdb.linear.RealVector; -import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleFieldsProto; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java index 1df769c252..49451129c9 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/InsertVectorTest.java @@ -55,8 +55,8 @@ void insertNulls() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).build(); - s.executeInsert("V", record); + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).build(); + s.executeInsert("V", rec); try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { ResultSetAssert.assertThat(rs) .hasNextRow() @@ -75,8 +75,8 @@ void partialInsert() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})).build(); - s.executeInsert("V", record); + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})).build(); + s.executeInsert("V", rec); try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { ResultSetAssert.assertThat(rs) .hasNextRow() @@ -95,12 +95,12 @@ void fullInsert() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0) + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0) .addObject("V1", new FloatRealVector(new float[]{1f, 2f, 3f, 4f})) .addObject("V2", new HalfRealVector(new int[]{1, 2, 3})) .addObject("V3", new DoubleRealVector(new double[]{1d, 2d})) .build(); - s.executeInsert("V", record); + s.executeInsert("V", rec); try (RelationalResultSet rs = s.executeGet("V", new KeySet().setKeyColumn("PK", 0), Options.NONE)) { ResultSetAssert.assertThat(rs) .hasNextRow() @@ -119,9 +119,9 @@ void insertWrongDimensionFails() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[] {1f, 2f, 3f, 4f, 5f})).build(); + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new FloatRealVector(new float[] {1f, 2f, 3f, 4f, 5f})).build(); RelationalAssertions.assertThrowsSqlException( - () -> s.executeInsert("V", record)) + () -> s.executeInsert("V", rec)) .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); } } @@ -132,9 +132,9 @@ void insertWrongPrecisionFails() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new DoubleRealVector(new double[] {1d, 2d, 3d, 4d})).build(); + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addObject("V1", new DoubleRealVector(new double[] {1d, 2d, 3d, 4d})).build(); RelationalAssertions.assertThrowsSqlException( - () -> s.executeInsert("V", record)) + () -> s.executeInsert("V", rec)) .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); } } @@ -145,9 +145,9 @@ void insertWrongTypeFails() throws SQLException { try (RelationalConnection conn = DriverManager.getConnection(database.getConnectionUri().toString()).unwrap(RelationalConnection.class)) { conn.setSchema("TEST_SCHEMA"); try (RelationalStatement s = conn.createStatement()) { - RelationalStruct record = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addInt("V1", 42).build(); + RelationalStruct rec = EmbeddedRelationalStruct.newBuilder().addInt("PK", 0).addInt("V1", 42).build(); RelationalAssertions.assertThrowsSqlException( - () -> s.executeInsert("V", record)) + () -> s.executeInsert("V", rec)) .hasErrorCode(ErrorCode.CANNOT_CONVERT_TYPE); } } diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java index 3c31f29d97..f3a74b53d5 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java @@ -21,11 +21,6 @@ package com.apple.foundationdb.relational.server; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.linear.DoubleRealVector; -import com.apple.foundationdb.linear.FloatRealVector; -import com.apple.foundationdb.linear.HalfRealVector; -import com.apple.foundationdb.linear.RealVector; -import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace; @@ -56,7 +51,6 @@ import com.apple.foundationdb.relational.recordlayer.ddl.RecordLayerMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.query.cache.RelationalPlanCache; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; -import com.google.protobuf.ByteString; import javax.annotation.Nonnull; import javax.annotation.Nullable; From d0a43e84bfbe34e16647c9aca792a86b63fa823f Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 11:47:12 +0000 Subject: [PATCH 3/8] Fix teamscale issues --- .../foundationdb/relational/recordlayer/MessageTuple.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index fe96f4869b..b2eb275661 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -57,11 +57,11 @@ public Object getObject(int position) throws InvalidColumnReferenceException { final var fieldOptions = fieldDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.field); final var fieldValue = message.getField(message.getDescriptorForType().getFields().get(position)); if (fieldOptions.hasVectorOptions()) { - final var precision = fieldOptions.getVectorOptions().getPrecision(); final var byteStringFieldValue = (ByteString)fieldValue; if (byteStringFieldValue.isEmpty()) { return null; } else { + final var precision = fieldOptions.getVectorOptions().getPrecision(); return VectorUtils.parseVector(byteStringFieldValue, precision); } } @@ -87,10 +87,10 @@ public static Object sanitizeField(@Nonnull final Object field, @Nonnull final D final var byteString = (ByteString) field; final var fieldVectorOptionsMaybe = fieldOptions.getExtension(RecordMetaDataOptionsProto.field); if (fieldVectorOptionsMaybe.hasVectorOptions()) { - final var precision = fieldVectorOptionsMaybe.getVectorOptions().getPrecision(); if (byteString.isEmpty()) { return null; } else { + final var precision = fieldVectorOptionsMaybe.getVectorOptions().getPrecision(); return VectorUtils.parseVector(byteString, precision); } } From b1b03735f93f13ae7d5469865ac2285774267371 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 11:53:04 +0000 Subject: [PATCH 4/8] Add defensive assert --- .../foundationdb/relational/recordlayer/RecordTypeTable.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java index 432b740aba..2552027118 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java @@ -279,6 +279,7 @@ public static Message toDynamicMessage(RelationalStruct struct, Descriptors.Desc Assert.that(vector instanceof DoubleRealVector, ErrorCode.CANNOT_CONVERT_TYPE, "Wrong precision for vector"); break; default: + Assert.fail(ErrorCode.INTERNAL_ERROR, "Unknown precision for vector"); } builder.setField(fd, ((AbstractRealVector) vector).getRawData()); } From a469ea2dfa9aa4d1504c46808a34606d961baaaf Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 13:31:27 +0000 Subject: [PATCH 5/8] Fix missing import --- .../main/java/com/apple/foundationdb/relational/server/FRL.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java index f3a74b53d5..96f3fdd2d4 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.server; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace; From 73ab83aef845ca40ff0f086498b59ff446723ace Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Fri, 7 Nov 2025 14:01:59 +0000 Subject: [PATCH 6/8] Fix teamscale --- .../foundationdb/relational/recordlayer/RecordTypeTable.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java index 2552027118..e418b82a4a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordTypeTable.java @@ -280,6 +280,7 @@ public static Message toDynamicMessage(RelationalStruct struct, Descriptors.Desc break; default: Assert.fail(ErrorCode.INTERNAL_ERROR, "Unknown precision for vector"); + break; } builder.setField(fd, ((AbstractRealVector) vector).getRawData()); } From 0c6727427bb9f37b997adc85bb4678d81a2e9e73 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 7 Nov 2025 14:45:11 +0000 Subject: [PATCH 7/8] Adapt test behavior. - before current version, (particularly, version 4.8.11.0) we allow inserting vectors of incompatible precision and dimensions. - fix the tests to reflect that. --- yaml-tests/src/test/resources/vector.yamsql | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index 5926309379..479d27b1f2 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -367,63 +367,108 @@ test_block: # Wrong number of dimensions - HALF vector with 2 dimensions into table expecting 3 - - query: insert into tHalfVector values (400, !! !v16 [1.1, 2.2] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - HALF vector with 4 dimensions into table expecting 3 - - query: insert into tHalfVector values (401, !! !v16 [1.1, 2.2, 3.3, 4.4] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - FLOAT vector with 2 dimensions into table expecting 3 - - query: insert into tFloatVector values (402, !! !v32 [1.1, 2.2] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - FLOAT vector with 4 dimensions into table expecting 3 - - query: insert into tFloatVector values (403, !! !v32 [1.1, 2.2, 3.3, 4.4] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - DOUBLE vector with 2 dimensions into table expecting 3 - - query: insert into tDoubleVector values (404, !! !v64 [1.1, 2.2] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - DOUBLE vector with 4 dimensions into table expecting 3 - - query: insert into tDoubleVector values (405, !! !v64 [1.1, 2.2, 3.3, 4.4] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - FLOAT vector into HALF table - - query: insert into tHalfVector values (406, !! !v32 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - DOUBLE vector into HALF table - - query: insert into tHalfVector values (407, !! !v64 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - HALF vector into FLOAT table - - query: insert into tFloatVector values (408, !! !v16 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - DOUBLE vector into FLOAT table - - query: insert into tFloatVector values (409, !! !v64 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - HALF vector into DOUBLE table - - query: insert into tDoubleVector values (410, !! !v16 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong precision - FLOAT vector into DOUBLE table - - query: insert into tDoubleVector values (411, !! !v32 [1.1, 2.2, 3.3] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - HALF vector with 1 dimension into table expecting 3 - - query: insert into tHalfVector values (412, !! !v16 [1.1] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - FLOAT vector with 5 dimensions into table expecting 3 - - query: insert into tFloatVector values (413, !! !v32 [1.1, 2.2, 3.3, 4.4, 5.5] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Wrong number of dimensions - DOUBLE vector with 0 dimensions into table expecting 3 - - query: insert into tDoubleVector values (414, !! !v64 [] !!) + - initialVersionAtLeast: !current_version - error: "22000" + - initialVersionLessThan: !current_version + - count: 1 # Tests for tWithStruct (struct with HALF vector field) # Insert struct with vector - From b9b095b5a3ec1afc56ef27c92d23d00c5dc04861 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 7 Nov 2025 15:06:02 +0000 Subject: [PATCH 8/8] Add tests for inserting `null` vector. - fix a problem with JDBC handling of `null` vector value. --- .../jdbc/RelationalResultSetFacade.java | 9 +++++++-- yaml-tests/src/test/resources/vector.yamsql | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java index 56415bec14..07d683d33b 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java @@ -352,8 +352,13 @@ public Object getObject(int oneBasedColumn) throws SQLException { case ENUM: o = getString(oneBasedColumn); break; - case VECTOR: - o = TypeConversion.parseVector(getBytes(oneBasedColumn), ((DataType.VectorType)relationalType).getPrecision()); + case VECTOR: { + final var bytes = getBytes(oneBasedColumn); + if (wasNull()) { + return null; + } + o = TypeConversion.parseVector(bytes, ((DataType.VectorType)relationalType).getPrecision()); + } break; default: throw new SQLException("Unsupported type " + type); diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index 479d27b1f2..a585d768f2 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -186,6 +186,24 @@ test_block: - - query: insert into tDoubleVector values (200, null) - count: 1 + - + - query: select * from tHalfVector where a = 200 + - initialVersionAtLeast: !current_version + - result: [{200, !null _}] + - initialVersionLessThan: !current_version + - error: "XXXXX" + - + - query: select * from tFloatVector where a = 200 + - initialVersionAtLeast: !current_version + - result: [{200, !null _}] + - initialVersionLessThan: !current_version + - error: "XXXXX" + - + - query: select * from tDoubleVector where a = 200 + - initialVersionAtLeast: !current_version + - result: [{200, !null _}] + - initialVersionLessThan: !current_version + - error: "XXXXX" # IS NULL tests for NULL vectors - - query: select b is null from tHalfVector where a = 200