From 85c5cf7e2ce329b7e61827b981001a972aeec548 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 23 Oct 2025 12:48:30 +0100 Subject: [PATCH 01/15] Support Vector data type in SQL. --- .../query/plan/cascades/typing/Type.java | 198 +++++++++++++++++- .../plan/cascades/typing/TypeRepository.java | 3 +- .../plan/cascades/values/FieldValue.java | 5 + .../values/RecordConstructorValue.java | 3 + .../foundationdb/record/util/VectorUtils.java | 61 ++++++ .../main/proto/record_metadata_options.proto | 5 + .../src/main/proto/record_query_plan.proto | 8 + fdb-relational-api/fdb-relational-api.gradle | 1 + .../relational/api/metadata/DataType.java | 92 +++++++- .../src/main/antlr/RelationalLexer.g4 | 2 + .../src/main/antlr/RelationalParser.g4 | 12 +- .../relational/recordlayer/MessageTuple.java | 36 +++- ...RecordLayerStoreSchemaTemplateCatalog.java | 10 +- .../recordlayer/metadata/DataTypeUtils.java | 10 + .../recordlayer/query/ParseHelpers.java | 4 + .../recordlayer/query/SemanticAnalyzer.java | 131 ++++++++++++ .../query/visitors/BaseVisitor.java | 6 - .../query/visitors/DdlVisitor.java | 33 +-- .../query/visitors/DelegatingVisitor.java | 13 +- .../query/visitors/TypedVisitor.java | 4 - .../recordlayer/util/ExceptionUtil.java | 7 +- .../recordlayer/query/StandardQueryTests.java | 44 ++++ .../fdb-relational-grpc.gradle | 1 + .../jdbc/RelationalResultSetFacade.java | 40 +++- .../relational/jdbc/TypeConversion.java | 23 ++ .../grpc/relational/jdbc/v1/column.proto | 9 +- .../proto/grpc/relational/jdbc/v1/jdbc.proto | 18 +- .../fdb-relational-jdbc.gradle | 1 + .../relational/jdbc/ParameterHelper.java | 105 ++++++++-- .../relational/jdbc/ParameterHelperTest.java | 2 +- .../foundationdb/relational/server/FRL.java | 28 ++- .../relational/yamltests/CustomTag.java | 43 ++++ .../yamltests/CustomYamlConstructor.java | 13 +- .../relational/yamltests/Matchers.java | 46 ++++ .../yamltests/command/QueryInterpreter.java | 18 +- .../src/test/java/YamlIntegrationTests.java | 5 + yaml-tests/src/test/resources/vector.yamsql | 42 ++++ 37 files changed, 1003 insertions(+), 79 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java create mode 100644 yaml-tests/src/test/resources/vector.yamsql diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 127efbc757..ed94a95ec4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -20,10 +20,12 @@ package com.apple.foundationdb.record.query.plan.cascades.typing; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanSerializable; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleFieldsProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.planprotos.PType; @@ -38,6 +40,7 @@ import com.apple.foundationdb.record.planprotos.PType.PRelationType; import com.apple.foundationdb.record.planprotos.PType.PTypeCode; import com.apple.foundationdb.record.planprotos.PType.PUuidType; +import com.apple.foundationdb.record.planprotos.PType.PVectorType; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion; import com.apple.foundationdb.record.query.plan.cascades.Narrowable; import com.apple.foundationdb.record.query.plan.cascades.NullableArrayTypeUtils; @@ -47,6 +50,7 @@ import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization; import com.apple.foundationdb.record.util.ProtoUtils; +import com.apple.foundationdb.record.util.VectorUtils; import com.apple.foundationdb.util.StringUtils; import com.google.auto.service.AutoService; import com.google.common.annotations.VisibleForTesting; @@ -160,6 +164,10 @@ default boolean isRecord() { return getTypeCode().equals(TypeCode.RECORD); } + default boolean isVector() { + return getTypeCode().equals(TypeCode.VECTOR); + } + /** * Checks whether a {@link Type} is {@link Relation}. * @@ -416,12 +424,18 @@ static List fromTyped(@Nonnull List typedList) { private static Type fromProtoType(@Nullable Descriptors.GenericDescriptor descriptor, @Nonnull Descriptors.FieldDescriptor.Type protoType, @Nonnull FieldDescriptorProto.Label protoLabel, + @Nullable DescriptorProtos.FieldOptions fieldOptions, boolean isNullable) { - final var typeCode = TypeCode.fromProtobufType(protoType); + final var typeCode = TypeCode.fromProtobufFieldDescriptor(protoType, fieldOptions); if (protoLabel == FieldDescriptorProto.Label.LABEL_REPEATED) { // collection type - return fromProtoTypeToArray(descriptor, protoType, typeCode, false); + return fromProtoTypeToArray(descriptor, protoType, typeCode, fieldOptions, false); } else if (typeCode.isPrimitive()) { + final var fieldOptionMaybe = Optional.ofNullable(fieldOptions).map(f -> f.getExtension(RecordMetaDataOptionsProto.field)); + if (fieldOptionMaybe.isPresent() && fieldOptionMaybe.get().hasVectorOptions()) { + final var vectorOptions = fieldOptionMaybe.get().getVectorOptions(); + return Type.Vector.of(isNullable, vectorOptions.getPrecision(), vectorOptions.getDimensions()); + } return primitiveType(typeCode, isNullable); } else if (typeCode == TypeCode.ENUM) { final var enumDescriptor = (Descriptors.EnumDescriptor)Objects.requireNonNull(descriptor); @@ -432,8 +446,8 @@ private static Type fromProtoType(@Nullable Descriptors.GenericDescriptor descri if (NullableArrayTypeUtils.describesWrappedArray(messageDescriptor)) { // find TypeCode of array elements final var elementField = messageDescriptor.findFieldByName(NullableArrayTypeUtils.getRepeatedFieldName()); - final var elementTypeCode = TypeCode.fromProtobufType(elementField.getType()); - return fromProtoTypeToArray(descriptor, protoType, elementTypeCode, true); + final var elementTypeCode = TypeCode.fromProtobufFieldDescriptor(elementField.getType(), elementField.getOptions()); + return fromProtoTypeToArray(descriptor, protoType, elementTypeCode, fieldOptions, true); } else if (TupleFieldsProto.UUID.getDescriptor().equals(messageDescriptor)) { return Type.uuidType(isNullable); } else { @@ -453,7 +467,9 @@ private static Type fromProtoType(@Nullable Descriptors.GenericDescriptor descri @Nonnull private static Array fromProtoTypeToArray(@Nullable Descriptors.GenericDescriptor descriptor, @Nonnull Descriptors.FieldDescriptor.Type protoType, - @Nonnull TypeCode typeCode, boolean isNullable) { + @Nonnull TypeCode typeCode, + @Nullable DescriptorProtos.FieldOptions fieldOptions, + boolean isNullable) { if (typeCode.isPrimitive()) { final var primitiveType = primitiveType(typeCode, false); return new Array(isNullable, primitiveType); @@ -465,10 +481,10 @@ private static Array fromProtoTypeToArray(@Nullable Descriptors.GenericDescripto if (isNullable) { Descriptors.Descriptor wrappedDescriptor = ((Descriptors.Descriptor)Objects.requireNonNull(descriptor)).findFieldByName(NullableArrayTypeUtils.getRepeatedFieldName()).getMessageType(); Objects.requireNonNull(wrappedDescriptor); - return new Array(true, fromProtoType(wrappedDescriptor, Descriptors.FieldDescriptor.Type.MESSAGE, FieldDescriptorProto.Label.LABEL_OPTIONAL, false)); + return new Array(true, fromProtoType(wrappedDescriptor, Descriptors.FieldDescriptor.Type.MESSAGE, FieldDescriptorProto.Label.LABEL_OPTIONAL, fieldOptions, false)); } else { // case 2: any arbitrary sub message we don't understand - return new Array(false, fromProtoType(descriptor, protoType, FieldDescriptorProto.Label.LABEL_OPTIONAL, false)); + return new Array(false, fromProtoType(descriptor, protoType, FieldDescriptorProto.Label.LABEL_OPTIONAL, fieldOptions, false)); } } } @@ -649,6 +665,12 @@ static Type fromObject(@Nullable final Object object) { if (object instanceof DynamicMessage) { return Record.fromDescriptor(((DynamicMessage) object).getDescriptorForType()); } + if (object instanceof RealVector) { + final var vector = (RealVector)object; + final var dimensions = vector.getNumDimensions(); + final var precision = VectorUtils.getVectorPrecision(vector); + return Type.Vector.of(false, precision, dimensions); + } final var typeCode = typeCodeFromPrimitive(object); if (typeCode == TypeCode.NULL) { return Type.nullType(); @@ -709,6 +731,7 @@ enum TypeCode { INT(Integer.class, FieldDescriptorProto.Type.TYPE_INT32, true, true), LONG(Long.class, FieldDescriptorProto.Type.TYPE_INT64, true, true), STRING(String.class, FieldDescriptorProto.Type.TYPE_STRING, true, false), + VECTOR(RealVector.class, FieldDescriptorProto.Type.TYPE_BYTES, true, false), VERSION(FDBRecordVersion.class, FieldDescriptorProto.Type.TYPE_BYTES, true, false), ENUM(Enum.class, FieldDescriptorProto.Type.TYPE_ENUM, false, false), RECORD(Message.class, null, false, false), @@ -815,11 +838,12 @@ private static BiMap, TypeCode> computeClassToTypeCodeMap() { /** * Generates a {@link TypeCode} that corresponds to the given protobuf * {@link com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type}. - * @param protobufType The protobuf type. + * @param protobufType The protobuf descriptor of the type. * @return A corresponding {@link TypeCode} instance. */ @Nonnull - public static TypeCode fromProtobufType(@Nonnull final Descriptors.FieldDescriptor.Type protobufType) { + public static TypeCode fromProtobufFieldDescriptor(@Nonnull final Descriptors.FieldDescriptor.Type protobufType, + @Nullable final DescriptorProtos.FieldOptions fieldOptions) { switch (protobufType) { case DOUBLE: return TypeCode.DOUBLE; @@ -847,7 +871,15 @@ public static TypeCode fromProtobufType(@Nonnull final Descriptors.FieldDescript case MESSAGE: return TypeCode.RECORD; case BYTES: + { + if (fieldOptions != null) { + final var recordTypeOptions = fieldOptions.getExtension(RecordMetaDataOptionsProto.field); + if (recordTypeOptions.hasVectorOptions()) { + return TypeCode.VECTOR; + } + } return TypeCode.BYTES; + } default: throw new IllegalArgumentException("unknown protobuf type " + protobufType); } @@ -955,6 +987,7 @@ class Primitive implements Type { @Nonnull private final TypeCode typeCode; + @Nonnull private final Supplier hashCodeSupplier = Suppliers.memoize(this::computeHashCode); private Primitive(final boolean isNullable, @Nonnull final TypeCode typeCode) { @@ -1176,6 +1209,151 @@ public boolean equals(final Object other) { } } + class Vector implements Type { + private final boolean isNullable; + private final int precision; + private final int dimensions; + + private Vector(final boolean isNullable, final int precision, final int dimensions) { + this.isNullable = isNullable; + this.precision = precision; + this.dimensions = dimensions; + } + + @Nonnull + @SuppressWarnings("PMD.ReplaceVectorWithList") + public static Vector of(final boolean isNullable, final int precision, final int dimensions) { + return new Vector(isNullable, precision, dimensions); + } + + @Override + public TypeCode getTypeCode() { + return TypeCode.VECTOR; + } + + @Override + public boolean isPrimitive() { + return true; + } + + @Override + public boolean isNullable() { + return isNullable; + } + + @Nonnull + @Override + public Type withNullability(final boolean newIsNullable) { + if (isNullable == newIsNullable) { + return this; + } + return new Vector(newIsNullable, precision, dimensions); + } + + public int getPrecision() { + return precision; + } + + public int getDimensions() { + return dimensions; + } + + @Nonnull + @Override + public ExplainTokens describe() { + final var resultExplainTokens = new ExplainTokens(); + resultExplainTokens.addKeyword(getTypeCode().toString()); + return resultExplainTokens.addOptionalWhitespace().addOpeningParen().addOptionalWhitespace() + .addNested(new ExplainTokens().addToString(precision).addToString(", ").addToString(dimensions)).addOptionalWhitespace() + .addClosingParen(); + } + + @Override + public void addProtoField(@Nonnull final TypeRepository.Builder typeRepositoryBuilder, + @Nonnull final DescriptorProto.Builder descriptorBuilder, final int fieldNumber, + @Nonnull final String fieldName, @Nonnull final Optional typeNameOptional, + @Nonnull final FieldDescriptorProto.Label label) { + final var protoType = Objects.requireNonNull(getTypeCode().getProtoType()); + FieldDescriptorProto.Builder builder = FieldDescriptorProto.newBuilder() + .setNumber(fieldNumber) + .setName(fieldName) + .setType(protoType) + .setLabel(label); + final var fieldOptions = RecordMetaDataOptionsProto.FieldOptions.newBuilder() + .setVectorOptions( + RecordMetaDataOptionsProto.FieldOptions.VectorOptions + .newBuilder() + .setPrecision(precision) + .setDimensions(dimensions) + .build()) + .build(); + builder.getOptionsBuilder().setExtension(RecordMetaDataOptionsProto.field, fieldOptions); + typeNameOptional.ifPresent(builder::setTypeName); + descriptorBuilder.addField(builder); + } + + @Nonnull + @Override + public PType toTypeProto(@Nonnull final PlanSerializationContext serializationContext) { + return PType.newBuilder().setVectorType(toProto(serializationContext)).build(); + } + + @Nonnull + @Override + public PVectorType toProto(@Nonnull final PlanSerializationContext serializationContext) { + final PVectorType.Builder vectorTypeBuilder = PVectorType.newBuilder() + .setIsNullable(isNullable) + .setDimensions(dimensions) + .setPrecision(precision); + return vectorTypeBuilder.build(); + } + + @Nonnull + @SuppressWarnings("PMD.ReplaceVectorWithList") + public static Vector fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PVectorType vectorTypeProto) { + Verify.verify(vectorTypeProto.hasIsNullable()); + return new Vector(vectorTypeProto.getIsNullable(), vectorTypeProto.getPrecision(), vectorTypeProto.getDimensions()); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PVectorType.class; + } + + @Nonnull + @Override + @SuppressWarnings("PMD.ReplaceVectorWithList") + public Vector fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PVectorType vectorTypeProto) { + return Vector.fromProto(serializationContext, vectorTypeProto); + } + } + + @Override + public int hashCode() { + return Objects.hash(getTypeCode().name(), precision, dimensions); + } + + @Override + @SuppressWarnings("PMD.ReplaceVectorWithList") + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final Vector vector = (Vector)o; + return isNullable == vector.isNullable + && precision == vector.precision + && dimensions == vector.dimensions; + } + } + /** * The none type is an unresolved type meaning that an entity returning a none type should resolve the * type to a regular type as the runtime does not support a none-typed data producer. Only the empty array constant @@ -2250,10 +2428,12 @@ public static Record fromFieldDescriptorsMap(final boolean isNullable, @Nonnull final var fieldsBuilder = ImmutableList.builder(); for (final var entry : Objects.requireNonNull(fieldDescriptorMap).entrySet()) { final var fieldDescriptor = entry.getValue(); + final var fieldOptions = fieldDescriptor.getOptions(); fieldsBuilder.add( new Field(fromProtoType(getTypeSpecificDescriptor(fieldDescriptor), fieldDescriptor.getType(), fieldDescriptor.toProto().getLabel(), + fieldOptions, !fieldDescriptor.isRequired()), Optional.of(entry.getKey()), Optional.of(fieldDescriptor.getNumber()))); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/TypeRepository.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/TypeRepository.java index c8aa4d2d2e..30e6de5c3d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/TypeRepository.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/TypeRepository.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.query.plan.cascades.typing; +import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleFieldsProto; import com.google.common.base.Preconditions; import com.google.common.base.Verify; @@ -67,7 +68,7 @@ public class TypeRepository { public static final TypeRepository EMPTY_SCHEMA = empty(); @Nonnull - public static final List DEPENDENCIES = List.of(TupleFieldsProto.getDescriptor()); + public static final List DEPENDENCIES = List.of(TupleFieldsProto.getDescriptor(), RecordMetaDataOptionsProto.getDescriptor()); @Nonnull private final FileDescriptorSet fileDescSet; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java index 9d8d5ba679..d307751e85 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java @@ -44,6 +44,7 @@ import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence.Precedence; +import com.apple.foundationdb.record.util.VectorUtils; import com.google.auto.service.AutoService; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -188,6 +189,10 @@ private static Object unwrapPrimitive(@Nonnull Type type, @Nullable Object field } else if (type.isUuid()) { Verify.verify(fieldValue instanceof UUID); return fieldValue; + } else if (type.isVector()) { + Verify.verify(fieldValue instanceof ByteString); + final var byteString = (ByteString) fieldValue; + return VectorUtils.parseVector(byteString, (Type.Vector)type); } else { // This also may need to turn ByteString's into byte[] for Type.TypeCode.BYTES return fieldValue; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java index 7839dd254c..70fb80495e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; @@ -224,6 +225,8 @@ private static Object protoObjectForPrimitive(@Nonnull Type type, @Nonnull Objec } } else if (type.getTypeCode() == Type.TypeCode.VERSION) { return ZeroCopyByteString.wrap(((FDBRecordVersion)field).toBytes(false)); + } else if (type.getTypeCode() == Type.TypeCode.VECTOR) { + return ZeroCopyByteString.wrap(((RealVector)field).getRawData()); } return field; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java new file mode 100644 index 0000000000..f3180b9863 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java @@ -0,0 +1,61 @@ +/* + * VectorUtils.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.util; + +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.google.protobuf.ByteString; + +import javax.annotation.Nonnull; + +public class VectorUtils { + public static int getVectorPrecision(@Nonnull final RealVector vector) { + if (vector instanceof DoubleRealVector) { + return 64; + } + if (vector instanceof FloatRealVector) { + return 32; + } + if (vector instanceof HalfRealVector) { + return 16; + } + throw new RecordCoreException("unexpected vector type " + vector.getClass()); + } + + @Nonnull + public static RealVector parseVector(@Nonnull final ByteString byteString, @Nonnull final Type.Vector vectorType) { + final var precision = vectorType.getPrecision(); + if (precision == 16) { + return HalfRealVector.fromBytes(byteString.toByteArray()); + } + if (precision == 32) { + return FloatRealVector.fromBytes(byteString.toByteArray()); + } + if (precision == 64) { + return DoubleRealVector.fromBytes(byteString.toByteArray()); + } + throw new RecordCoreException("unexpected vector type " + vectorType); + } +} diff --git a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto index 26ecbeb261..0395c07fac 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto @@ -60,6 +60,11 @@ message FieldOptions { repeated Index.Option options = 3; // Note: there is no way to specify these in a .proto file. } optional IndexOption index = 3; + message VectorOptions { + optional int32 precision = 1 [default = 16]; + optional int32 dimensions = 2 [default = 768]; + } + optional VectorOptions vectorOptions = 4; } extend google.protobuf.FieldOptions { diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 4839e23a63..dd799eb3c8 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -52,6 +52,7 @@ message PType { RELATION = 15; NONE = 16; UUID = 17; + VECTOR = 18; } message PPrimitiveType { @@ -75,6 +76,12 @@ message PType { // nothing } + message PVectorType { + optional bool is_nullable = 1; + optional int32 precision = 2; + optional int32 dimensions = 3; + } + message PAnyRecordType { optional bool is_nullable = 1; } @@ -123,6 +130,7 @@ message PType { PArrayType array_type = 8; PAnyRecordType any_record_type = 9; PUuidType uuid_type = 10; + PVectorType vector_type = 11; } } diff --git a/fdb-relational-api/fdb-relational-api.gradle b/fdb-relational-api/fdb-relational-api.gradle index 7521bd69d0..09635a6b98 100644 --- a/fdb-relational-api/fdb-relational-api.gradle +++ b/fdb-relational-api/fdb-relational-api.gradle @@ -56,6 +56,7 @@ buildHost=${InetAddress.getLocalHost().getCanonicalHostName()} } dependencies { + implementation project(":fdb-extensions") api(project(':fdb-java-annotations')) { exclude(group: "com.squareup", module: "javapoet") } diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java index c5bf628d33..239d351347 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java @@ -20,6 +20,10 @@ package com.apple.foundationdb.relational.api.metadata; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalStruct; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; @@ -71,6 +75,7 @@ public abstract class DataType { typeCodeJdbcTypeMap.put(Code.ENUM, Types.OTHER); typeCodeJdbcTypeMap.put(Code.UUID, Types.OTHER); typeCodeJdbcTypeMap.put(Code.BYTES, Types.BINARY); + typeCodeJdbcTypeMap.put(Code.VECTOR, Types.OTHER); typeCodeJdbcTypeMap.put(Code.VERSION, Types.BINARY); typeCodeJdbcTypeMap.put(Code.STRUCT, Types.STRUCT); typeCodeJdbcTypeMap.put(Code.ARRAY, Types.ARRAY); @@ -203,6 +208,17 @@ public static DataType getDataTypeFromObject(@Nullable Object obj) { return Primitives.STRING.type(); } else if (obj instanceof UUID) { return Primitives.UUID.type(); + } else if (obj instanceof RealVector) { + int numDimensions = ((RealVector)obj).getNumDimensions(); + if (obj instanceof HalfRealVector) { + return VectorType.of(16, numDimensions, false); + } else if (obj instanceof FloatRealVector) { + return VectorType.of(32, numDimensions, false); + } else if (obj instanceof DoubleRealVector) { + return VectorType.of(64, numDimensions, false); + } else { + throw new IllegalStateException("Unexpected vector object type: " + obj.getClass().getName()); + } } else if (obj instanceof RelationalStruct) { return ((RelationalStruct) obj).getMetaData().getRelationalDataType(); } else if (obj instanceof RelationalArray) { @@ -732,6 +748,79 @@ public String toString() { } } + public static final class VectorType extends DataType { + private final int precision; + + private final int dimensions; + + @Nonnull + private final Supplier hashCodeSupplier = Suppliers.memoize(this::computeHashCode); + + private VectorType(final boolean isNullable, int precision, int dimensions) { + super(isNullable, true, Code.VECTOR); + this.precision = precision; + this.dimensions = dimensions; + } + + @Override + public boolean isResolved() { + return true; + } + + @Nonnull + @Override + public DataType withNullable(final boolean isNullable) { + if (isNullable == this.isNullable()) { + return this; + } + return new VectorType(isNullable, precision, dimensions); + } + + @Nonnull + @Override + public DataType resolve(@Nonnull final Map resolutionMap) { + return this; + } + + public int getPrecision() { + return precision; + } + + public int getDimensions() { + return dimensions; + } + + private int computeHashCode() { + return Objects.hash(getCode(), precision, dimensions, isNullable()); + } + + @Override + public int hashCode() { + return hashCodeSupplier.get(); + } + + @Override + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final VectorType that = (VectorType)o; + return precision == that.precision + && dimensions == that.dimensions + && isNullable() == that.isNullable(); + } + + @Override + public String toString() { + return "vector(p=" + precision + ", d=" + dimensions + ")" + (isNullable() ? " ∪ ∅" : ""); + } + + @Nonnull + public static VectorType of(int precision, int dimensions, boolean isNullable) { + return new VectorType(isNullable, precision, dimensions); + } + } + public static final class VersionType extends DataType { @Nonnull private static final VersionType NOT_NULLABLE_INSTANCE = new VersionType(false); @@ -1507,7 +1596,8 @@ public enum Code { STRUCT, ARRAY, UNKNOWN, - NULL + NULL, + VECTOR, } @SuppressWarnings("PMD.AvoidFieldNameMatchingTypeName") diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index bbd31ecf1a..77a34e26f3 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -502,6 +502,7 @@ GENERAL: 'GENERAL'; GLOBAL: 'GLOBAL'; GRANTS: 'GRANTS'; GROUP_REPLICATION: 'GROUP_REPLICATION'; +HALF: 'HALF'; HANDLER: 'HANDLER'; HAS: 'HAS'; HASH: 'HASH'; @@ -734,6 +735,7 @@ USER_RESOURCES: 'USER_RESOURCES'; VALIDATION: 'VALIDATION'; VALUE: 'VALUE'; VARIABLES: 'VARIABLES'; +VECTOR: 'VECTOR'; VIEW: 'VIEW'; VIRTUAL: 'VIRTUAL'; VISIBLE: 'VISIBLE'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index e83801ba78..769f661c1b 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -138,7 +138,17 @@ columnType : primitiveType | customType=uid; primitiveType - : BOOLEAN | INTEGER | BIGINT | FLOAT | DOUBLE | STRING | BYTES | UUID; + : BOOLEAN | INTEGER | BIGINT | FLOAT | DOUBLE | STRING | BYTES | UUID | vectorType; + +vectorType + : VECTOR '(' dimensions=DECIMAL_LITERAL ',' elementType=vectorElementType ')' + ; + +vectorElementType + : FLOAT + | DOUBLE + | HALF + ; columnConstraint : nullNotnull #nullColumnConstraint diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index 18ad52c85b..5d5b9de553 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -21,6 +21,12 @@ package com.apple.foundationdb.relational.recordlayer; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordMetaDataOptionsProto; import com.apple.foundationdb.record.TupleFieldsProto; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.relational.api.exceptions.InvalidColumnReferenceException; @@ -28,6 +34,7 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Message; +import javax.annotation.Nonnull; import java.util.List; import java.util.stream.Collectors; @@ -49,19 +56,40 @@ public Object getObject(int position) throws InvalidColumnReferenceException { if (position < 0 || position >= getNumFields()) { throw InvalidColumnReferenceException.getExceptionForInvalidPositionNumber(position); } - Descriptors.FieldDescriptor fieldDescriptor = message.getDescriptorForType().getFields().get(position); + final Descriptors.FieldDescriptor fieldDescriptor = message.getDescriptorForType().getFields().get(position); + final var recordTypeOptions = fieldDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.field); + final var fieldValue = message.getField(message.getDescriptorForType().getFields().get(position)); + if (recordTypeOptions.hasVectorOptions()) { + final var precision = recordTypeOptions.getVectorOptions().getPrecision(); + final var byteStringFieldValue = (ByteString)fieldValue; + return getVectorFromBytes(byteStringFieldValue, precision); + } if (fieldDescriptor.isRepeated()) { - final var list = (List) message.getField(message.getDescriptorForType().getFields().get(position)); + final var list = (List) fieldValue; return list.stream().map(MessageTuple::sanitizeField).collect(Collectors.toList()); } if (message.hasField(fieldDescriptor)) { - final var field = message.getField(message.getDescriptorForType().getFields().get(position)); - return sanitizeField(field); + return sanitizeField(fieldValue); } else { return null; } } + @Nonnull + private static RealVector getVectorFromBytes(@Nonnull final ByteString byteString, int precision) { + final var bytes = byteString.toByteArray(); + if (precision == 64) { + return DoubleRealVector.fromBytes(bytes); + } + if (precision == 32) { + return FloatRealVector.fromBytes(bytes); + } + if (precision == 16) { + return HalfRealVector.fromBytes(bytes); + } + throw new RecordCoreException("unexpected vector precision " + precision); + } + public static Object sanitizeField(final Object field) { if (field instanceof Message && ((Message) field).getDescriptorForType().equals(TupleFieldsProto.UUID.getDescriptor())) { return TupleFieldsHelper.fromProto((Message) field, TupleFieldsProto.UUID.getDescriptor()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java index cea1c2128b..c755308d7e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java @@ -76,6 +76,14 @@ */ class RecordLayerStoreSchemaTemplateCatalog implements SchemaTemplateCatalog { + @Nonnull + private static final com.google.protobuf.ExtensionRegistry registry = com.google.protobuf.ExtensionRegistry.newInstance(); + + static { + registry.add(com.apple.foundationdb.record.RecordMetaDataOptionsProto.field); + registry.add(com.apple.foundationdb.record.RecordMetaDataOptionsProto.record); + } + @Nonnull private final RecordLayerSchema catalogSchema; @@ -210,7 +218,7 @@ private static SchemaTemplate toSchemaTemplate(@Nonnull final Message message) t // deserialization of the same message over and over again. final Descriptors.Descriptor descriptor = message.getDescriptorForType(); final ByteString bs = Assert.castUnchecked(message.getField(descriptor.findFieldByName(SchemaTemplateSystemTable.METADATA)), ByteString.class); - final RecordMetaData metaData = RecordMetaData.build(RecordMetaDataProto.MetaData.parseFrom(bs.toByteArray())); + final RecordMetaData metaData = RecordMetaData.newBuilder().setRecords(RecordMetaDataProto.MetaData.parseFrom(bs.toByteArray(), registry)).getRecordMetaData(); final String name = message.getField(descriptor.findFieldByName(SchemaTemplateSystemTable.TEMPLATE_NAME)).toString(); int templateVersion = (int) message.getField(descriptor.findFieldByName(SchemaTemplateSystemTable.TEMPLATE_VERSION)); return RecordLayerSchemaTemplate.fromRecordMetadata(metaData, name, templateVersion); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java index 2c3820be2c..f29d51158f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java @@ -61,6 +61,11 @@ public static DataType toRelationalType(@Nonnull final Type type) { final var typeCode = type.getTypeCode(); + if (typeCode == Type.TypeCode.VECTOR) { + final var vectorType = (Type.Vector)type; + return DataType.VectorType.of(vectorType.getPrecision(), vectorType.getDimensions(), vectorType.isNullable()); + } + if (typeCode == Type.TypeCode.ANY || typeCode == Type.TypeCode.NONE || typeCode == Type.TypeCode.NULL || typeCode == Type.TypeCode.UNKNOWN) { return DataType.UnknownType.instance(); } @@ -129,6 +134,11 @@ public static Type toRecordLayerType(@Nonnull final DataType type) { final var asEnum = (DataType.EnumType) type; final List enumValues = asEnum.getValues().stream().map(v -> new Type.Enum.EnumValue(v.getName(), v.getNumber())).collect(Collectors.toList()); return new Type.Enum(asEnum.isNullable(), enumValues, asEnum.getName()); + case VECTOR: + final var vectorType = (DataType.VectorType)type; + final var precision = vectorType.getPrecision(); + final var dimensions = vectorType.getDimensions(); + return Type.Vector.of(type.isNullable(), precision, dimensions); case UNKNOWN: return new Type.Any(); default: diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ParseHelpers.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ParseHelpers.java index 086d17bbc2..53b8454405 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ParseHelpers.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ParseHelpers.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.half.Half; import com.apple.foundationdb.record.query.plan.cascades.TreeLike; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.cascades.values.Value; @@ -75,6 +76,9 @@ public static Object parseDecimal(@Nonnull String valueAsString) { case 'd': // fallthrough case 'D': return Double.parseDouble(valueAsString.substring(0, lastCharIdx)); + case 'h': // fallthrough + case 'H': + return Half.valueOf(valueAsString.substring(0, lastCharIdx)); default: return Double.parseDouble(valueAsString); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index e8e4bdca2a..7d5bc06107 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -502,6 +502,137 @@ public Optional lookupNestedField(@Nonnull Identifier requestedIdent return Optional.of(nestedAttribute); } + public static final class ParsedTypeInfo { + @Nullable + private final RelationalParser.PrimitiveTypeContext primitiveTypeContext; + + @Nullable + private final Identifier customType; + + private final boolean isNullable; + + private final boolean isRepeated; + + private ParsedTypeInfo(@Nullable final RelationalParser.PrimitiveTypeContext primitiveTypeContext, + @Nullable final Identifier customType, final boolean isNullable, final boolean isRepeated) { + this.primitiveTypeContext = primitiveTypeContext; + this.customType = customType; + this.isNullable = isNullable; + this.isRepeated = isRepeated; + } + + public boolean hasPrimitiveType() { + return primitiveTypeContext != null; + } + + @Nullable + public RelationalParser.PrimitiveTypeContext getPrimitiveTypeContext() { + return primitiveTypeContext; + } + + public boolean hasCustomType() { + return customType != null; + } + + @Nullable + public Identifier getCustomType() { + return customType; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean isRepeated() { + return isRepeated; + } + + @Nonnull + public static ParsedTypeInfo ofPrimitiveType(@Nonnull final RelationalParser.PrimitiveTypeContext primitiveTypeContext, + final boolean isNullable, final boolean isRepeated) { + return new ParsedTypeInfo(primitiveTypeContext, null, isNullable, isRepeated); + } + + @Nonnull + public static ParsedTypeInfo ofCustomType(@Nonnull final Identifier customType, + final boolean isNullable, final boolean isRepeated) { + return new ParsedTypeInfo(null, customType, isNullable, isRepeated); + } + } + + @Nonnull + public DataType lookupType(@Nonnull final ParsedTypeInfo parsedTypeInfo, + @Nonnull final Function> dataTypeProvider) { + DataType type; + final var isNullable = parsedTypeInfo.isNullable(); + if (parsedTypeInfo.hasCustomType()) { + final var typeName = Assert.notNullUnchecked(parsedTypeInfo.getCustomType()).getName(); + final var maybeFound = dataTypeProvider.apply(typeName); + // if we cannot find the type now, mark it, we will try to resolve it later on via a second pass. + type = maybeFound.orElseGet(() -> DataType.UnresolvedType.of(typeName, isNullable)); + } else { + final var primitiveType = Assert.notNullUnchecked(parsedTypeInfo.getPrimitiveTypeContext()); + if (primitiveType.vectorType() != null) { + final var vectorTypeCtx = primitiveType.vectorType(); + final var vectorElementTypeCtx = vectorTypeCtx.elementType; + int precision = 16; + if (vectorElementTypeCtx.FLOAT() != null) { + precision = 32; + } else if (vectorElementTypeCtx.DOUBLE() != null) { + precision = 64; + } else { + Assert.notNullUnchecked(vectorElementTypeCtx.HALF()); + } + int length = Assert.castUnchecked(ParseHelpers.parseDecimal(vectorTypeCtx.dimensions.getText()), Integer.class); + type = DataType.VectorType.of(precision, length, isNullable); + } else { + final var primitiveTypeName = parsedTypeInfo.getPrimitiveTypeContext().getText(); + + switch (primitiveTypeName.toUpperCase(Locale.ROOT)) { + case "STRING": + type = isNullable ? DataType.Primitives.NULLABLE_STRING.type() : DataType.Primitives.STRING.type(); + break; + case "INTEGER": + type = isNullable ? DataType.Primitives.NULLABLE_INTEGER.type() : DataType.Primitives.INTEGER.type(); + break; + case "BIGINT": + type = isNullable ? DataType.Primitives.NULLABLE_LONG.type() : DataType.Primitives.LONG.type(); + break; + case "DOUBLE": + type = isNullable ? DataType.Primitives.NULLABLE_DOUBLE.type() : DataType.Primitives.DOUBLE.type(); + break; + case "BOOLEAN": + type = isNullable ? DataType.Primitives.NULLABLE_BOOLEAN.type() : DataType.Primitives.BOOLEAN.type(); + break; + case "BYTES": + type = isNullable ? DataType.Primitives.NULLABLE_BYTES.type() : DataType.Primitives.BYTES.type(); + break; + case "FLOAT": + type = isNullable ? DataType.Primitives.NULLABLE_FLOAT.type() : DataType.Primitives.FLOAT.type(); + break; + case "UUID": + type = isNullable ? DataType.Primitives.NULLABLE_UUID.type() : DataType.Primitives.UUID.type(); + break; + default: + Assert.notNullUnchecked(metadataCatalog); + // assume it is a custom type, will fail in upper layers if the type can not be resolved. + // lookup the type (Struct, Table, or Enum) in the schema template metadata under construction. + final var maybeFound = dataTypeProvider.apply(primitiveTypeName); + // if we cannot find the type now, mark it, we will try to resolve it later on via a second pass. + type = maybeFound.orElseGet(() -> DataType.UnresolvedType.of(primitiveTypeName, isNullable)); + break; + } + } + } + + if (parsedTypeInfo.isRepeated()) { + return DataType.ArrayType.from(type.withNullable(false), isNullable); + } else { + return type; + } + } + + @Nonnull public DataType lookupType(@Nonnull Identifier typeIdentifier, boolean isNullable, boolean isRepeated, @Nonnull Function> dataTypeProvider) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index d96bf8b694..04df166363 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -370,12 +370,6 @@ public DataType visitColumnType(@Nonnull RelationalParser.ColumnTypeContext ctx) return ddlVisitor.visitColumnType(ctx); } - @Nonnull - @Override - public DataType visitPrimitiveType(@Nonnull RelationalParser.PrimitiveTypeContext ctx) { - return ddlVisitor.visitPrimitiveType(ctx); - } - @Nonnull @Override public Boolean visitNullColumnConstraint(@Nonnull RelationalParser.NullColumnConstraintContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 5575626681..95220e736d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -56,7 +56,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -99,11 +98,14 @@ public static DdlVisitor of(@Nonnull BaseVisitor delegate, @Override public DataType visitFunctionColumnType(@Nonnull final RelationalParser.FunctionColumnTypeContext ctx) { final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); + final SemanticAnalyzer.ParsedTypeInfo typeInfo; if (ctx.customType != null) { final var columnType = visitUid(ctx.customType); - return semanticAnalyzer.lookupType(columnType, true, false, metadataBuilder::findType); + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofCustomType(columnType, true, false); + } else { + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.primitiveType(), true, false); } - return visitPrimitiveType(ctx.primitiveType()).withNullable(true); + return semanticAnalyzer.lookupType(typeInfo, metadataBuilder::findType); } // TODO: remove @@ -111,20 +113,14 @@ public DataType visitFunctionColumnType(@Nonnull final RelationalParser.Function @Override public DataType visitColumnType(@Nonnull RelationalParser.ColumnTypeContext ctx) { final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); + final SemanticAnalyzer.ParsedTypeInfo typeInfo; if (ctx.customType != null) { final var columnType = visitUid(ctx.customType); - return semanticAnalyzer.lookupType(columnType, false, false, metadataBuilder::findType); + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofCustomType(columnType, false, false); + } else { + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.primitiveType(), false, false); } - return visitPrimitiveType(ctx.primitiveType()); - } - - // TODO: remove - @Nonnull - @Override - public DataType visitPrimitiveType(@Nonnull RelationalParser.PrimitiveTypeContext ctx) { - final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); - final var primitiveType = Identifier.of(ctx.getText()); - return semanticAnalyzer.lookupType(primitiveType, false, false, ignored -> Optional.empty()); + return semanticAnalyzer.lookupType(typeInfo, metadataBuilder::findType); } /** @@ -149,7 +145,14 @@ public RecordLayerColumn visitColumnDefinition(@Nonnull RelationalParser.ColumnD containsNullableArray = containsNullableArray || (isRepeated && isNullable); final var columnTypeId = ctx.columnType().customType != null ? visitUid(ctx.columnType().customType) : Identifier.of(ctx.columnType().getText()); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); - final var columnType = semanticAnalyzer.lookupType(columnTypeId, isNullable, isRepeated, metadataBuilder::findType); + final SemanticAnalyzer.ParsedTypeInfo typeInfo; + if (ctx.columnType().customType != null) { + final var columnType = visitUid(ctx.columnType().customType); + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofCustomType(columnType, true, isRepeated); + } else { + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.columnType().primitiveType(), true, isRepeated); + } + final var columnType = semanticAnalyzer.lookupType(typeInfo, metadataBuilder::findType); return RecordLayerColumn.newBuilder().setName(columnId.getName()).setDataType(columnType).build(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 388befc27a..81205910a3 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -202,10 +202,21 @@ public DataType visitColumnType(@Nonnull RelationalParser.ColumnTypeContext ctx) @Nonnull @Override - public DataType visitPrimitiveType(@Nonnull RelationalParser.PrimitiveTypeContext ctx) { + public Object visitPrimitiveType(@Nonnull RelationalParser.PrimitiveTypeContext ctx) { return getDelegate().visitPrimitiveType(ctx); } + @Nonnull + @Override + public Object visitVectorType(final RelationalParser.VectorTypeContext ctx) { + return getDelegate().visitVectorType(ctx); + } + + @Override + public Object visitVectorElementType(final RelationalParser.VectorElementTypeContext ctx) { + return getDelegate().visitVectorElementType(ctx); + } + @Nonnull @Override public Boolean visitNullColumnConstraint(@Nonnull RelationalParser.NullColumnConstraintContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 7f15686ea8..0a87975e4c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -146,10 +146,6 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override DataType visitColumnType(@Nonnull RelationalParser.ColumnTypeContext ctx); - @Nonnull - @Override - DataType visitPrimitiveType(@Nonnull RelationalParser.PrimitiveTypeContext ctx); - @Nonnull @Override Boolean visitNullColumnConstraint(@Nonnull RelationalParser.NullColumnConstraintContext ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java index a56aa803a2..9e18e915bb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java @@ -59,7 +59,11 @@ private static RelationalException recordCoreToRelationalException(RecordCoreExc if (re.getCause() instanceof RelationalException) { return (RelationalException) re.getCause(); } - + try { + Thread.sleep(10000L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } ErrorCode code = ErrorCode.UNKNOWN; if (re instanceof FDBExceptions.FDBStoreTransactionTimeoutException) { code = ErrorCode.TRANSACTION_TIMEOUT; @@ -87,6 +91,7 @@ private static RelationalException recordCoreToRelationalException(RecordCoreExc @Nonnull private static ErrorCode translateErrorCode(@Nonnull final SemanticException semanticException) { final var semanticErrorCode = semanticException.getErrorCode(); + switch (semanticErrorCode) { case INCOMPATIBLE_TYPE: return ErrorCode.CANNOT_CONVERT_TYPE; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index ee3385f00f..96c1c54d4b 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -20,6 +20,11 @@ package com.apple.foundationdb.relational.recordlayer.query; +import com.apple.foundationdb.half.Half; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.EmbeddedRelationalArray; import com.apple.foundationdb.relational.api.EmbeddedRelationalStruct; @@ -49,6 +54,11 @@ import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -1588,6 +1598,40 @@ void testInsertStructWithUuidTest() throws Exception { } } + @Nonnull + private static Stream vectorTypeProvider() { + return Stream.of( + Arguments.of("vector(3, half)", new HalfRealVector(new double[]{1.1d, 1.2d, 1.3d})), + Arguments.of("vector(3, float)", new FloatRealVector(new double[]{1.1d, 1.2d, 1.3d})), + Arguments.of("vector(3, double)", new DoubleRealVector(new double[]{1.1d, 1.2d, 1.3d})) + ); + } + + @ParameterizedTest + @MethodSource("vectorTypeProvider") + void vectorInsertSelectPrepared(String vectorType, RealVector vector) throws Exception { + final String schemaTemplate = String.format("CREATE TABLE T1(pk bigint, v %s, PRIMARY KEY(pk))", vectorType); + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var insert = ddl.setSchemaAndGetConnection().prepareStatement("insert into t1 values (?pk, ?v)")) { + insert.setLong("pk", 1L); + insert.setObject("v", vector); + final var numActualInserted = insert.executeUpdate(); + Assertions.assertEquals(1, numActualInserted); + } + try (var statement = ddl.setSchemaAndGetConnection().createStatement()) { + Assertions.assertTrue(statement.execute("select * from t1")); + try (final RelationalResultSet resultSet = statement.getResultSet()) { + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .hasColumn("PK", 1L) + .hasColumn("v", vector) + .hasNoNextRow(); + } + } + } + } + // todo (yhatem) add more tests for queries w and w/o index definition. private void insertTypeConflictRecords(RelationalStatement s) throws SQLException { diff --git a/fdb-relational-grpc/fdb-relational-grpc.gradle b/fdb-relational-grpc/fdb-relational-grpc.gradle index 06715e0bcb..416a4f4bb2 100644 --- a/fdb-relational-grpc/fdb-relational-grpc.gradle +++ b/fdb-relational-grpc/fdb-relational-grpc.gradle @@ -45,6 +45,7 @@ dependencies { // Having grpc depend on api makes it so we can add utility here shared by // server and client, both of which depend on grpc and api. implementation project(":fdb-relational-api") + implementation project(":fdb-extensions") compileOnly(libs.jsr305) implementation(libs.generatedAnnotation) implementation(libs.grpc.commonProtos) diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java index bbc1635af5..251a77b6a9 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java @@ -20,6 +20,10 @@ package com.apple.foundationdb.relational.jdbc; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalResultSet; @@ -36,6 +40,7 @@ import com.apple.foundationdb.relational.util.PositionalIndex; import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings; import com.google.common.base.Suppliers; +import com.google.protobuf.ByteString; import javax.annotation.Nonnull; import java.sql.SQLException; @@ -343,13 +348,20 @@ public Object getObject(int oneBasedColumn) throws SQLException { break; case Types.OTHER: int index = PositionalIndex.toProtobuf(oneBasedColumn); - Column column = this.delegate.getRow(rowIndex).getColumns().getColumn(index); - if (column.hasUuid()) { - o = getUUID(oneBasedColumn); - } else { - // Probably an enum, it's not clear exactly how we should handle this, but we currently only have one - // thing which appears as OTHER - o = getString(oneBasedColumn); + final var relationalType = getMetaData().getRelationalDataType().getFields().get(index).getType(); + final var typeCode = relationalType.getCode(); + switch (typeCode) { + case UUID: + o = getUUID(oneBasedColumn); + break; + case ENUM: + o = getString(oneBasedColumn); + break; + case VECTOR: + o = parseVector(getBytes(oneBasedColumn), ((DataType.VectorType)relationalType).getPrecision()); + break; + default: + throw new SQLException("Unsupported type " + type); } break; default: @@ -393,4 +405,18 @@ public T unwrap(Class iface) throws SQLException { public boolean isWrapperFor(Class iface) throws SQLException { return iface.isInstance(this); } + + @Nonnull + public static RealVector parseVector(@Nonnull final byte[] bytes, int precision) throws SQLException { + if (precision == 16) { + return HalfRealVector.fromBytes(bytes); + } + if (precision == 32) { + return FloatRealVector.fromBytes(bytes); + } + if (precision == 64) { + return DoubleRealVector.fromBytes(bytes); + } + throw new SQLException("unexpected vector type with precision " + precision); + } } diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java index 1a92754030..29b7b47e4e 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.jdbc; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.ArrayMetaData; import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.Options; @@ -46,6 +47,7 @@ import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Struct; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Type; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Uuid; +import com.apple.foundationdb.relational.jdbc.grpc.v1.column.VectorMetadata; import com.apple.foundationdb.relational.util.PositionalIndex; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; @@ -190,6 +192,10 @@ private static ColumnMetadata toColumnMetadata(StructMetaData metadata, int oneB var enumMetadata = toEnumMetadata((DataType.EnumType) type); columnMetadataBuilder.setEnumMetadata(enumMetadata); break; + case VECTOR: + var vectorMetadata = toVectorMetadata((DataType.VectorType)type); + columnMetadataBuilder.setVectorMetadata(vectorMetadata); + break; default: break; } @@ -202,6 +208,10 @@ private static EnumMetadata toEnumMetadata(@Nonnull DataType.EnumType enumType) return builder.build(); } + private static VectorMetadata toVectorMetadata(@Nonnull DataType.VectorType vectorType) { + return VectorMetadata.newBuilder().setDimensions(vectorType.getDimensions()).setPrecision(vectorType.getPrecision()).build(); + } + private static Type toProtobufType(@Nonnull DataType type) { switch (type.getCode()) { case LONG: @@ -228,6 +238,8 @@ private static Type toProtobufType(@Nonnull DataType type) { return Type.ENUM; case UUID: return Type.UUID; + case VECTOR: + return Type.VECTOR; default: throw new RelationalException("not supported in toProtobuf: " + type, ErrorCode.INTERNAL_ERROR).toUncheckedWrappedException(); } @@ -471,6 +483,10 @@ private static Column toColumn(@Nonnull DataType.StructType.Field field, @Nonnul .setLeastSignificantBits(a.getLeastSignificantBits()) .build())); break; + case VECTOR: + column = toColumn(wasNull ? null : (RealVector)value, + (a, b) -> a == null ? b.clearBinary() : b.setBinary(ByteString.copyFrom(a.getRawData()))); + break; default: throw new SQLException("DataType: " + field.getType() + " not supported", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); @@ -854,6 +870,10 @@ private static DataType.EnumType getEnumDataType(@Nonnull EnumMetadata enumMetad return DataType.EnumType.from(enumMetadata.getName(), enumValues, nullable); } + private static DataType.VectorType getVectorType(@Nonnull VectorMetadata vectorMetadata, boolean nullable) { + return DataType.VectorType.of(vectorMetadata.getPrecision(), vectorMetadata.getDimensions(), nullable); + } + static DataType getDataType(@Nonnull Type type, @Nonnull ColumnMetadata columnMetadata, boolean nullable) { switch (type) { case LONG: @@ -881,6 +901,9 @@ static DataType getDataType(@Nonnull Type type, @Nonnull ColumnMetadata columnMe case ARRAY: final var arrayMetadata = columnMetadata.getArrayMetadata(); return DataType.ArrayType.from(getDataType(arrayMetadata.getType(), arrayMetadata, arrayMetadata.getNullable()), nullable); + case VECTOR: + final var vectorMetadata = columnMetadata.getVectorMetadata(); + return getVectorType(vectorMetadata, nullable); default: throw new RelationalException("Not implemeneted: " + type.name(), ErrorCode.INTERNAL_ERROR).toUncheckedWrappedException(); } diff --git a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/column.proto b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/column.proto index 1eb8c13d1d..860e219434 100644 --- a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/column.proto +++ b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/column.proto @@ -41,6 +41,7 @@ enum Type { STRUCT = 11; ARRAY = 12; NULL = 13; + VECTOR = 14; } message ColumnMetadata { @@ -52,11 +53,12 @@ message ColumnMetadata { // Indicates a column that isn't part of the DDL for a query, but is part of the returned // tuple (and therefore necessary to keep so that our positional ordering is intact) bool phantom = 5; - // If this column has a Struct or Array, or is of Enum type, their metadata is here. + // If this column has a Struct or Array, Enum, or Vector type, their metadata is here. oneof metadata { ListColumnMetadata structMetadata = 6; ColumnMetadata arrayMetadata = 7; EnumMetadata enumMetadata = 8; + VectorMetadata vectorMetadata = 10; } Type type = 9; } @@ -130,3 +132,8 @@ message EnumMetadata { string name = 1; repeated string values = 2; } + +message VectorMetadata { + int32 precision = 1; + int32 dimensions = 2; +} diff --git a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto index 995a798cb1..af4ca0dddd 100644 --- a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto +++ b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto @@ -108,11 +108,23 @@ message Parameters { // A parameter has java sql type as well as actual parameter. message Parameter { - int32 java_sql_types_code = 1; + int32 java_sql_types_code = 1; // deprecated // Reuse the column type here. It has much of what we need carrying // across parameters as well as null support, etc. - Column parameter = 2; - Type type = 3; + Column parameter = 2; // deprecated + Type type = 3; // deprecated + ParameterMetadata metadata = 4; +} + +message ParameterMetadata { + int32 java_sql_types_code = 1; + Type type = 2; + oneof metadata { + ListColumnMetadata structMetadata = 3; + ColumnMetadata arrayMetadata = 4; + EnumMetadata enumMetadata = 5; + VectorMetadata vectorMetadata = 6; + } } message Options { diff --git a/fdb-relational-jdbc/fdb-relational-jdbc.gradle b/fdb-relational-jdbc/fdb-relational-jdbc.gradle index 3754f6ec51..9142f2f08f 100644 --- a/fdb-relational-jdbc/fdb-relational-jdbc.gradle +++ b/fdb-relational-jdbc/fdb-relational-jdbc.gradle @@ -29,6 +29,7 @@ apply from: rootProject.file('gradle/publishing.gradle') dependencies { implementation project(":fdb-relational-api") implementation project(":fdb-relational-grpc") + implementation project(":fdb-extensions") implementation(libs.grpc.inprocess) implementation(libs.grpc.netty) implementation(libs.grpc.protobuf) diff --git a/fdb-relational-jdbc/src/main/java/com/apple/foundationdb/relational/jdbc/ParameterHelper.java b/fdb-relational-jdbc/src/main/java/com/apple/foundationdb/relational/jdbc/ParameterHelper.java index 10c4e6d983..ff74ddae05 100644 --- a/fdb-relational-jdbc/src/main/java/com/apple/foundationdb/relational/jdbc/ParameterHelper.java +++ b/fdb-relational-jdbc/src/main/java/com/apple/foundationdb/relational/jdbc/ParameterHelper.java @@ -20,14 +20,21 @@ package com.apple.foundationdb.relational.jdbc; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.jdbc.grpc.v1.Parameter; +import com.apple.foundationdb.relational.jdbc.grpc.v1.ParameterMetadata; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Column; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Type; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Uuid; +import com.apple.foundationdb.relational.jdbc.grpc.v1.column.VectorMetadata; import com.google.protobuf.ByteString; +import javax.annotation.Nonnull; import java.sql.Array; import java.sql.SQLException; import java.sql.Types; @@ -39,56 +46,77 @@ public class ParameterHelper { public static Parameter ofBoolean(boolean b) { return Parameter.newBuilder() - .setType(Type.BOOLEAN) - .setJavaSqlTypesCode(Types.BOOLEAN) + .setType(Type.BOOLEAN) // deprecated + .setJavaSqlTypesCode(Types.BOOLEAN) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.BOOLEAN) + .setJavaSqlTypesCode(Types.BOOLEAN)) .setParameter(Column.newBuilder().setBoolean(b)) .build(); } public static Parameter ofInt(int i) { return Parameter.newBuilder() - .setType(Type.INTEGER) - .setJavaSqlTypesCode(Types.INTEGER) + .setType(Type.INTEGER) // deprecated + .setJavaSqlTypesCode(Types.INTEGER) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.INTEGER) + .setJavaSqlTypesCode(Types.INTEGER)) .setParameter(Column.newBuilder().setInteger(i)) .build(); } public static Parameter ofLong(long l) { return Parameter.newBuilder() - .setType(Type.LONG) - .setJavaSqlTypesCode(Types.BIGINT) + .setType(Type.LONG) // deprecated + .setJavaSqlTypesCode(Types.BIGINT) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.LONG) + .setJavaSqlTypesCode(Types.BIGINT)) .setParameter(Column.newBuilder().setLong(l)) .build(); } public static Parameter ofFloat(float f) { return Parameter.newBuilder() - .setType(Type.FLOAT) - .setJavaSqlTypesCode(Types.FLOAT) + .setType(Type.FLOAT) // deprecated + .setJavaSqlTypesCode(Types.FLOAT) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.FLOAT) + .setJavaSqlTypesCode(Types.FLOAT)) .setParameter(Column.newBuilder().setFloat(f)) .build(); } public static Parameter ofDouble(double d) { return Parameter.newBuilder() - .setType(Type.DOUBLE) - .setJavaSqlTypesCode(Types.DOUBLE) + .setType(Type.DOUBLE) // deprecated + .setJavaSqlTypesCode(Types.DOUBLE) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.DOUBLE) + .setJavaSqlTypesCode(Types.DOUBLE)) .setParameter(Column.newBuilder().setDouble(d)) .build(); } public static Parameter ofString(String s) { return Parameter.newBuilder() - .setType(Type.STRING) - .setJavaSqlTypesCode(Types.VARCHAR) + .setType(Type.STRING) // deprecated + .setJavaSqlTypesCode(Types.VARCHAR) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.STRING) + .setJavaSqlTypesCode(Types.VARCHAR)) .setParameter(Column.newBuilder().setString(s)) .build(); } public static Parameter ofUUID(UUID id) { return Parameter.newBuilder() - .setType(Type.UUID) - .setJavaSqlTypesCode(Types.OTHER) + .setType(Type.UUID) // deprecated + .setJavaSqlTypesCode(Types.OTHER) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.UUID) + .setJavaSqlTypesCode(Types.OTHER)) .setParameter(Column.newBuilder().setUuid(Uuid.newBuilder() .setMostSignificantBits(id.getMostSignificantBits()) .setLeastSignificantBits(id.getLeastSignificantBits()) @@ -96,18 +124,50 @@ public static Parameter ofUUID(UUID id) { .build(); } + public static Parameter ofVector(RealVector vector) throws SQLException { + return Parameter.newBuilder() + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.VECTOR) + .setVectorMetadata(VectorMetadata.newBuilder() + .setDimensions(vector.getNumDimensions()) + .setPrecision(getVectorPrecision(vector)).build()) + .setJavaSqlTypesCode(Types.OTHER)) + // TODO use PB ZeroCopyByteString. + .setParameter(Column.newBuilder().setBinary(ByteString.copyFrom(vector.getRawData()))) + .build(); + } + + public static int getVectorPrecision(@Nonnull final RealVector vector) throws SQLException { + if (vector instanceof DoubleRealVector) { + return 64; + } + if (vector instanceof FloatRealVector) { + return 32; + } + if (vector instanceof HalfRealVector) { + return 16; + } + throw new SQLException("unexpected vector type " + vector.getClass()); + } + public static Parameter ofBytes(byte[] bytes) { return Parameter.newBuilder() - .setType(Type.BYTES) - .setJavaSqlTypesCode(Types.BINARY) + .setType(Type.BYTES) // deprecated + .setJavaSqlTypesCode(Types.BINARY) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.BYTES) + .setJavaSqlTypesCode(Types.BINARY)) .setParameter(Column.newBuilder().setBinary(ByteString.copyFrom(bytes))) .build(); } public static Parameter ofNull(int sqlType) { return Parameter.newBuilder() - .setType(Type.NULL) - .setJavaSqlTypesCode(Types.NULL) + .setType(Type.NULL) // deprecated + .setJavaSqlTypesCode(Types.NULL) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.NULL) + .setJavaSqlTypesCode(Types.NULL)) .setParameter(Column.newBuilder().setNullType(sqlType)) .build(); } @@ -122,8 +182,11 @@ public static Parameter ofArray(final Array a) throws SQLException { throw new SQLException("Array type not supported: " + a.getClass().getName(), ErrorCode.INVALID_PARAMETER.getErrorCode()); } return Parameter.newBuilder() - .setType(Type.ARRAY) - .setJavaSqlTypesCode(Types.ARRAY) + .setType(Type.ARRAY) // deprecated + .setJavaSqlTypesCode(Types.ARRAY) // deprecated + .setMetadata(ParameterMetadata.newBuilder() + .setType(Type.ARRAY) + .setJavaSqlTypesCode(Types.ARRAY)) .setParameter(Column.newBuilder() .setArray(com.apple.foundationdb.relational.jdbc.grpc.v1.column.Array.newBuilder() .setElementType(a.getBaseType()) @@ -156,6 +219,8 @@ public static Parameter ofObject(Object x) throws SQLException { return ofString((String)x); case UUID: return ofUUID((UUID)x); + case VECTOR: + return ofVector((RealVector)x); case NULL: return ofNull(type.getJdbcSqlCode()); // TODO: This would be generic null... default: diff --git a/fdb-relational-jdbc/src/test/java/com/apple/foundationdb/relational/jdbc/ParameterHelperTest.java b/fdb-relational-jdbc/src/test/java/com/apple/foundationdb/relational/jdbc/ParameterHelperTest.java index a774b11e80..ca7d779a4a 100644 --- a/fdb-relational-jdbc/src/test/java/com/apple/foundationdb/relational/jdbc/ParameterHelperTest.java +++ b/fdb-relational-jdbc/src/test/java/com/apple/foundationdb/relational/jdbc/ParameterHelperTest.java @@ -115,7 +115,7 @@ private static void checkValue(Object value, Parameter actual, Type expectedType private static void checkValue(Object value, Parameter actual, Type expectedType, Function typeChecker, Function typeGetter, @Nullable Consumer customAssert) { - Assertions.assertEquals(expectedType, actual.getType()); + Assertions.assertEquals(expectedType, actual.getMetadata().getType()); Assertions.assertTrue(actual.hasParameter()); if (value != null) { Assertions.assertTrue(typeChecker.apply(actual.getParameter())); diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java index 3bc7f273ef..1018574d32 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java @@ -21,10 +21,15 @@ package com.apple.foundationdb.relational.server; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.relational.api.EmbeddedRelationalDriver; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; @@ -38,6 +43,7 @@ import com.apple.foundationdb.relational.api.Transaction; import com.apple.foundationdb.relational.api.catalog.StoreCatalog; import com.apple.foundationdb.relational.api.exceptions.RelationalException; +import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metrics.NoOpMetricRegistry; import com.apple.foundationdb.relational.jdbc.TypeConversion; import com.apple.foundationdb.relational.jdbc.grpc.v1.Parameter; @@ -51,6 +57,7 @@ import com.apple.foundationdb.relational.recordlayer.ddl.RecordLayerMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.query.cache.RelationalPlanCache; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; +import com.google.protobuf.ByteString; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -251,6 +258,20 @@ private static void setStatementOptions(final @Nullable Options options, final S } } + @Nonnull + public static RealVector parseVector(@Nonnull final ByteString byteString, int precision) { + if (precision == 16) { + return HalfRealVector.fromBytes(byteString.toByteArray()); + } + if (precision == 32) { + return FloatRealVector.fromBytes(byteString.toByteArray()); + } + if (precision == 64) { + return DoubleRealVector.fromBytes(byteString.toByteArray()); + } + throw new RecordCoreException("unexpected vector type with precision " + precision); + } + private static void addPreparedStatementParameter(@Nonnull RelationalPreparedStatement relationalPreparedStatement, @Nonnull Parameter parameter, int index) throws SQLException { final var oneOfValue = parameter.getParameter(); @@ -267,7 +288,12 @@ private static void addPreparedStatementParameter(@Nonnull RelationalPreparedSta } else if (oneOfValue.hasBoolean()) { relationalPreparedStatement.setBoolean(index, oneOfValue.getBoolean()); } else if (oneOfValue.hasBinary()) { - relationalPreparedStatement.setBytes(index, oneOfValue.getBinary().toByteArray()); + if (parameter.hasMetadata() && parameter.getMetadata().hasVectorMetadata()) { + final var vectorProtoType = parameter.getMetadata().getVectorMetadata(); + relationalPreparedStatement.setObject(index, parseVector(oneOfValue.getBinary(), vectorProtoType.getPrecision())); + } else { + relationalPreparedStatement.setBytes(index, oneOfValue.getBinary().toByteArray()); + } } else if (oneOfValue.hasNullType()) { relationalPreparedStatement.setNull(index, oneOfValue.getNullType()); } else if (oneOfValue.hasUuid()) { diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java index aa109962dc..99553a31fb 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java @@ -20,6 +20,12 @@ package com.apple.foundationdb.relational.yamltests; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.relational.util.Assert; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + import javax.annotation.Nonnull; import java.util.UUID; @@ -131,4 +137,41 @@ public String toString() { return "!not_null"; } } + + static final class Vector16Field { + @Nonnull + private final SequenceNode yamlNode; + + HalfRealVector vector; + + Vector16Field(@Nonnull final Node node) { + this.yamlNode = Assert.castUnchecked(node, SequenceNode.class); + } + + @Nonnull + public Matchers.ResultSetMatchResult matchWith(@Nonnull Object other, int rowNumber, @Nonnull String cellRef) { + HalfRealVector thisVector = (HalfRealVector)Matchers.constructVectorFromString(16, Assert.castUnchecked(yamlNode, SequenceNode.class)); + if (other instanceof HalfRealVector) { + if (other.equals(thisVector)) { + return Matchers.ResultSetMatchResult.success(); + } + } + // not matched + return Matchers.ResultSetMatchResult.fail("vector mismatch at row " + rowNumber + ", " + cellRef + " expected: " + this + ", got: " + other); + } + + @Override + public String toString() { + return prettyPrintYamlNode(yamlNode); + } + + @Nonnull + private static String prettyPrintYamlNode(@Nonnull final SequenceNode sequenceNode) { + final var stringBuilder = new StringBuilder(); + stringBuilder.append(sequenceNode.getTag()).append(" ["); + sequenceNode.getValue().forEach(v -> stringBuilder.append(Assert.castUnchecked(v, ScalarNode.class).getValue())); + stringBuilder.append("]"); + return stringBuilder.toString(); + } + } } diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java index 8b58f36467..387ebbab07 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java @@ -28,11 +28,14 @@ import com.apple.foundationdb.relational.yamltests.block.TransactionSetupsBlock; import com.apple.foundationdb.relational.yamltests.command.Command; import com.apple.foundationdb.relational.yamltests.command.QueryConfig; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.constructor.AbstractConstruct; import org.yaml.snakeyaml.constructor.SafeConstructor; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; import org.yaml.snakeyaml.nodes.Tag; import javax.annotation.Nonnull; @@ -56,6 +59,7 @@ public CustomYamlConstructor(LoaderOptions loaderOptions) { yamlConstructors.put(new Tag("!null"), new ConstructNullPlaceholder()); yamlConstructors.put(new Tag("!not_null"), new ConstructNotNull()); yamlConstructors.put(new Tag("!current_version"), new ConstructCurrentVersion()); + yamlConstructors.put(new Tag("!v16"), new ConstructVector16Field()); //blocks requireLineNumber.add(PreambleBlock.OPTIONS); @@ -175,12 +179,19 @@ private static class ConstructUuidField extends AbstractConstruct { @Override public Object construct(Node node) { if (!(node instanceof ScalarNode)) { - Assert.failUnchecked("The value of uuid (!sc) tag must be a scalar, however '" + node + "' is found!"); + Assert.failUnchecked("The value of uuid (!uuid) tag must be a scalar, however '" + node + "' is found!"); } return new CustomTag.UuidField(((ScalarNode) node).getValue()); } } + private static class ConstructVector16Field extends AbstractConstruct { + @Override + public Object construct(Node node) { + return new CustomTag.Vector16Field(node); + } + } + private static class ConstructNullPlaceholder extends AbstractConstruct { @Override public Object construct(Node node) { diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java index 4bc08f2661..9166f07fc5 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java @@ -20,8 +20,13 @@ package com.apple.foundationdb.relational.yamltests; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.util.pair.ImmutablePair; import com.apple.foundationdb.record.util.pair.Pair; +import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalResultSet; @@ -30,10 +35,13 @@ import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings; import com.google.common.collect.HashMultiset; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Multiset; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import de.vandermeer.asciitable.AsciiTable; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -576,6 +584,9 @@ private static ResultSetMatchResult matchField(@Nullable final Object expected, if (expected instanceof CustomTag.UuidField) { return ((CustomTag.UuidField) expected).matchWith(actual, rowNumber, cellRef); } + if (expected instanceof CustomTag.Vector16Field) { + return ((CustomTag.Vector16Field) expected).matchWith(actual, rowNumber, cellRef); + } // (nested) message if (expected instanceof Map) { @@ -675,4 +686,39 @@ private static ResultSetMatchResult matchIntField(@Nonnull final Integer expecte } return ResultSetMatchResult.fail(String.format(Locale.ROOT, "cell mismatch at row: %d cellRef: %s%n expected 🟢 does not match 🟡.%n🟢 %s (Integer) %n🟡 %s (%s)", rowNumber, cellRef, expected, actual, actual.getClass().getSimpleName())); } + + @Nonnull + public static RealVector constructVectorFromString(int precision, @Nonnull final SequenceNode yamlElementsNode) { + final var elements = yamlElementsNode.getValue().stream().map(v -> Assert.castUnchecked(v, ScalarNode.class).getValue()) + .collect(ImmutableList.toImmutableList()); + + // Handle empty vector case + if (elements.isEmpty()) { + switch (precision) { + case 16: + return new HalfRealVector(new double[0]); + case 32: + return new FloatRealVector(new double[0]); + case 64: + return new DoubleRealVector(new double[0]); + default: + throw new IllegalArgumentException("Unsupported vector precision: " + precision + ". Expected 16, 32, or 64."); + } + } + + // Split by comma and parse each element as double + final double[] values = elements.stream().map(Double::parseDouble).mapToDouble(d -> d).toArray(); + + // Create appropriate vector based on precision + switch (precision) { + case 16: + return new HalfRealVector(values); + case 32: + return new FloatRealVector(values); + case 64: + return new DoubleRealVector(values); + default: + throw new IllegalArgumentException("Unsupported vector precision: " + precision + ". Expected 16, 32, or 64."); + } + } } diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java index 4449cf1cc5..f17ea72b08 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java @@ -49,6 +49,8 @@ import java.util.UUID; import java.util.stream.Collectors; +import static com.apple.foundationdb.relational.yamltests.Matchers.constructVectorFromString; + /** * {@link QueryInterpreter} interprets the query that is provided in the YAML testing framework using the * {@link QueryCommand} command. The query in the framework is expected to be a valid query string based on the @@ -96,6 +98,7 @@ private static final class QueryParameterYamlConstructor extends SafeConstructor private static final Tag IN_LIST_TAG = new Tag("!in"); private static final Tag NULL_TAG = new Tag("!n"); private static final Tag UUID_TAG = new Tag("!uuid"); + private static final Tag VECTOR16_TAG = new Tag("!v16"); private QueryParameterYamlConstructor(LoaderOptions loaderOptions) { super(loaderOptions); @@ -105,13 +108,15 @@ private QueryParameterYamlConstructor(LoaderOptions loaderOptions) { this.yamlConstructors.put(IN_LIST_TAG, new ConstructInList()); this.yamlConstructors.put(NULL_TAG, new ConstructNull()); this.yamlConstructors.put(UUID_TAG, new ConstructUuid()); + this.yamlConstructors.put(VECTOR16_TAG, new ConstructVector16()); this.yamlConstructors.put(new Tag("!l"), new CustomYamlConstructor.ConstructLong()); } @Override public Parameter constructObject(Node node) { - if (node.getTag().equals(RANDOM_TAG) || node.getTag().equals(ARRAY_GENERATOR_TAG) || node.getTag().equals(IN_LIST_TAG) || node.getTag().equals(NULL_TAG) || node.getTag().equals(UUID_TAG)) { + if (node.getTag().equals(RANDOM_TAG) || node.getTag().equals(ARRAY_GENERATOR_TAG) || node.getTag().equals(IN_LIST_TAG) + || node.getTag().equals(NULL_TAG) || node.getTag().equals(UUID_TAG) || node.getTag().equals(VECTOR16_TAG)) { return (Parameter) super.constructObject(node); } else if (node instanceof SequenceNode) { // simple list @@ -207,6 +212,17 @@ public Object construct(Node node) { return new PrimitiveParameter(UUID.fromString(((ScalarNode) node).getValue())); } } + + /** + * Constructor for Vector16 (HalfVector) literal value. + */ + private static class ConstructVector16 extends AbstractConstruct { + + @Override + public Object construct(Node node) { + return new PrimitiveParameter(constructVectorFromString(16, Assert.castUnchecked(node, SequenceNode.class))); + } + } } private QueryInterpreter(int lineNumber, @Nonnull String query, @Nonnull final YamlExecutionContext executionContext) { diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index f0e8becebc..8977a36e12 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -331,4 +331,9 @@ public void serializationOptions(YamlTest.Runner runner) throws Exception { public void castTests(YamlTest.Runner runner) throws Exception { runner.runYamsql("cast-tests.yamsql"); } + + @TestTemplate + public void vectorTests(YamlTest.Runner runner) throws Exception { + runner.runYamsql("vector.yamsql"); + } } diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql new file mode 100644 index 0000000000..c52b919579 --- /dev/null +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -0,0 +1,42 @@ +# +# vector.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +options: + supported_version: !current_version +--- +schema_template: + create table tv(a bigint, b vector(3, half), primary key(a)) + create table tv16x3(a bigint, b vector(3, half), primary key(a)) +--- +test_block: + preset: single_repetition_ordered + options: + statement_type: prepared + name: basic-vector-tests + tests: +# - +# # inserting vector literal is not yet supported. +# - query: insert into tv values (100, [1.3, 1.4, 1.6]) +# - error: "XX000" + - + - query: insert into tv values (100, !! !v16 [1.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select * from tv + - result: [{100, !v16 [1.3, 1.4, 1.6]}] From 8194dba06f26344f22e609364a95deead2ba5147 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 23 Oct 2025 17:18:52 +0100 Subject: [PATCH 02/15] Support testing different flavours of vectors in YAML. - prepared, over JDBC. --- .../query/visitors/DdlVisitor.java | 1 - .../recordlayer/util/ExceptionUtil.java | 5 - .../recordlayer/query/StandardQueryTests.java | 1 - .../jdbc/RelationalResultSetFacade.java | 1 - .../foundationdb/relational/server/FRL.java | 2 - .../relational/yamltests/CustomTag.java | 177 ------------------ .../yamltests/CustomYamlConstructor.java | 84 +-------- .../relational/yamltests/Matchers.java | 41 ++-- .../yamltests/command/QueryInterpreter.java | 34 +++- .../yamltests/tags/AbstractVectorTag.java | 73 ++++++++ .../yamltests/tags/CurrentVersionTag.java | 60 ++++++ .../relational/yamltests/tags/CustomTag.java | 40 ++++ .../relational/yamltests/tags/IgnoreTag.java | 63 +++++++ .../relational/yamltests/tags/IsNullTag.java | 85 +++++++++ .../relational/yamltests/tags/LongTag.java | 64 +++++++ .../relational/yamltests/tags/Matchable.java | 46 +++++ .../relational/yamltests/tags/NotNullTag.java | 64 +++++++ .../yamltests/tags/StringContainsTag.java | 100 ++++++++++ .../relational/yamltests/tags/UuidTag.java | 102 ++++++++++ .../yamltests/tags/Vector16Tag.java | 61 ++++++ .../yamltests/tags/Vector32Tag.java | 61 ++++++ .../yamltests/tags/Vector64Tag.java | 61 ++++++ .../yamltests/tags/package-info.java | 25 +++ yaml-tests/src/test/resources/vector.yamsql | 21 ++- yaml-tests/yaml-tests.gradle | 2 + 25 files changed, 974 insertions(+), 300 deletions(-) delete mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/AbstractVectorTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CurrentVersionTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CustomTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IgnoreTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IsNullTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/LongTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Matchable.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/NotNullTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/StringContainsTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/UuidTag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector16Tag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector32Tag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector64Tag.java create mode 100644 yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/package-info.java diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 95220e736d..3846f44a86 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -143,7 +143,6 @@ public RecordLayerColumn visitColumnDefinition(@Nonnull RelationalParser.ColumnD // but a way to represent it in RecordMetadata. Assert.thatUnchecked(isRepeated || isNullable, ErrorCode.UNSUPPORTED_OPERATION, "NOT NULL is only allowed for ARRAY column type"); containsNullableArray = containsNullableArray || (isRepeated && isNullable); - final var columnTypeId = ctx.columnType().customType != null ? visitUid(ctx.columnType().customType) : Identifier.of(ctx.columnType().getText()); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); final SemanticAnalyzer.ParsedTypeInfo typeInfo; if (ctx.columnType().customType != null) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java index 9e18e915bb..65035feaf2 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/ExceptionUtil.java @@ -59,11 +59,6 @@ private static RelationalException recordCoreToRelationalException(RecordCoreExc if (re.getCause() instanceof RelationalException) { return (RelationalException) re.getCause(); } - try { - Thread.sleep(10000L); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } ErrorCode code = ErrorCode.UNKNOWN; if (re instanceof FDBExceptions.FDBStoreTransactionTimeoutException) { code = ErrorCode.TRANSACTION_TIMEOUT; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 96c1c54d4b..67f96de47a 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -20,7 +20,6 @@ package com.apple.foundationdb.relational.recordlayer.query; -import com.apple.foundationdb.half.Half; import com.apple.foundationdb.linear.DoubleRealVector; import com.apple.foundationdb.linear.FloatRealVector; import com.apple.foundationdb.linear.HalfRealVector; diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java index 251a77b6a9..712036f0f8 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java @@ -40,7 +40,6 @@ import com.apple.foundationdb.relational.util.PositionalIndex; import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings; import com.google.common.base.Suppliers; -import com.google.protobuf.ByteString; import javax.annotation.Nonnull; import java.sql.SQLException; diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java index 1018574d32..5aeb02ec36 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/FRL.java @@ -29,7 +29,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseFactory; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpace; -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.relational.api.EmbeddedRelationalDriver; import com.apple.foundationdb.relational.api.KeySet; import com.apple.foundationdb.relational.api.Options; @@ -43,7 +42,6 @@ import com.apple.foundationdb.relational.api.Transaction; import com.apple.foundationdb.relational.api.catalog.StoreCatalog; import com.apple.foundationdb.relational.api.exceptions.RelationalException; -import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metrics.NoOpMetricRegistry; import com.apple.foundationdb.relational.jdbc.TypeConversion; import com.apple.foundationdb.relational.jdbc.grpc.v1.Parameter; diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java deleted file mode 100644 index 99553a31fb..0000000000 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomTag.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * CustomTag.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.relational.yamltests; - -import com.apple.foundationdb.linear.HalfRealVector; -import com.apple.foundationdb.relational.util.Assert; -import org.yaml.snakeyaml.nodes.Node; -import org.yaml.snakeyaml.nodes.ScalarNode; -import org.yaml.snakeyaml.nodes.SequenceNode; - -import javax.annotation.Nonnull; -import java.util.UUID; - -@SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") -public abstract class CustomTag { - - static final class Ignore extends CustomTag { - static final Ignore INSTANCE = new Ignore(); - - private Ignore() { - } - - @Override - public String toString() { - return "!ignore"; - } - } - - static final class StringContains { - @Nonnull - private final String value; - - StringContains(@Nonnull final String value) { - this.value = value; - } - - @Nonnull - public String getValue() { - return value; - } - - @Nonnull - public Matchers.ResultSetMatchResult matchWith(@Nonnull Object other, int rowNumber, @Nonnull String cellRef) { - if (other instanceof String) { - final var otherStr = (String) other; - if (otherStr.contains(value)) { - return Matchers.ResultSetMatchResult.success(); - } else { - return Matchers.ResultSetMatchResult.fail("String mismatch at row: " + rowNumber + " cellRef: " + cellRef + "\n The string '" + otherStr + "' does not contain '" + value + "'"); - } - } else { - return Matchers.ResultSetMatchResult.fail("expected to match against a " + String.class.getSimpleName() + " value, however we got " + other + " which is " + other.getClass().getSimpleName()); - } - } - - @Override - public String toString() { - return "!sc " + value; - } - } - - static final class UuidField { - @Nonnull - private final String value; - - UuidField(@Nonnull final String value) { - this.value = value; - } - - @Nonnull - public String getValue() { - return value; - } - - @Nonnull - public Matchers.ResultSetMatchResult matchWith(@Nonnull Object other, int rowNumber, @Nonnull String cellRef) { - UUID otherUUID; - try { - otherUUID = UUID.fromString(value); - } catch (IllegalArgumentException e) { - return Matchers.ResultSetMatchResult.fail("Provided string is not a valid UUID format at row: " + rowNumber + " cellRef: " + cellRef + "\n. Provided String '" + value + "'"); - - } - if (other instanceof UUID) { - if (otherUUID.equals(UUID.fromString(value))) { - return Matchers.ResultSetMatchResult.success(); - } - } - // not matched - return Matchers.ResultSetMatchResult.fail("UUID mismatch at row: " + rowNumber + " cellRef: " + cellRef + "\n The actual '" + other + "' does not match UUID '" + value + "'"); - } - - @Override - public String toString() { - return "!uuid " + value; - } - } - - static final class NullPlaceholder { - static final NullPlaceholder INSTANCE = new NullPlaceholder(); - - private NullPlaceholder() { - } - - @Override - public String toString() { - return "!null"; - } - } - - static final class NotNull { - static final NotNull INSTANCE = new NotNull(); - - private NotNull() { - } - - @Override - public String toString() { - return "!not_null"; - } - } - - static final class Vector16Field { - @Nonnull - private final SequenceNode yamlNode; - - HalfRealVector vector; - - Vector16Field(@Nonnull final Node node) { - this.yamlNode = Assert.castUnchecked(node, SequenceNode.class); - } - - @Nonnull - public Matchers.ResultSetMatchResult matchWith(@Nonnull Object other, int rowNumber, @Nonnull String cellRef) { - HalfRealVector thisVector = (HalfRealVector)Matchers.constructVectorFromString(16, Assert.castUnchecked(yamlNode, SequenceNode.class)); - if (other instanceof HalfRealVector) { - if (other.equals(thisVector)) { - return Matchers.ResultSetMatchResult.success(); - } - } - // not matched - return Matchers.ResultSetMatchResult.fail("vector mismatch at row " + rowNumber + ", " + cellRef + " expected: " + this + ", got: " + other); - } - - @Override - public String toString() { - return prettyPrintYamlNode(yamlNode); - } - - @Nonnull - private static String prettyPrintYamlNode(@Nonnull final SequenceNode sequenceNode) { - final var stringBuilder = new StringBuilder(); - stringBuilder.append(sequenceNode.getTag()).append(" ["); - sequenceNode.getValue().forEach(v -> stringBuilder.append(Assert.castUnchecked(v, ScalarNode.class).getValue())); - stringBuilder.append("]"); - return stringBuilder.toString(); - } - } -} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java index 387ebbab07..1041e7b146 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/CustomYamlConstructor.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.yamltests; +import com.apple.foundationdb.record.util.ServiceLoaderProvider; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.yamltests.block.PreambleBlock; @@ -28,15 +29,11 @@ import com.apple.foundationdb.relational.yamltests.block.TransactionSetupsBlock; import com.apple.foundationdb.relational.yamltests.command.Command; import com.apple.foundationdb.relational.yamltests.command.QueryConfig; -import com.google.common.base.Verify; -import com.google.common.collect.ImmutableList; +import com.apple.foundationdb.relational.yamltests.tags.CustomTag; import org.yaml.snakeyaml.LoaderOptions; -import org.yaml.snakeyaml.constructor.AbstractConstruct; import org.yaml.snakeyaml.constructor.SafeConstructor; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.ScalarNode; -import org.yaml.snakeyaml.nodes.SequenceNode; -import org.yaml.snakeyaml.nodes.Tag; import javax.annotation.Nonnull; import java.util.ArrayList; @@ -52,14 +49,9 @@ public class CustomYamlConstructor extends SafeConstructor { public CustomYamlConstructor(LoaderOptions loaderOptions) { super(loaderOptions); - yamlConstructors.put(new Tag("!ignore"), new ConstructIgnore()); - yamlConstructors.put(new Tag("!l"), new ConstructLong()); - yamlConstructors.put(new Tag("!sc"), new ConstructStringContains()); - yamlConstructors.put(new Tag("!uuid"), new ConstructUuidField()); - yamlConstructors.put(new Tag("!null"), new ConstructNullPlaceholder()); - yamlConstructors.put(new Tag("!not_null"), new ConstructNotNull()); - yamlConstructors.put(new Tag("!current_version"), new ConstructCurrentVersion()); - yamlConstructors.put(new Tag("!v16"), new ConstructVector16Field()); + // custom tags + final var loader = ServiceLoaderProvider.load(CustomTag.class); + loader.forEach(customTag -> customTag.accept(yamlConstructors::put)); //blocks requireLineNumber.add(PreambleBlock.OPTIONS); @@ -147,70 +139,4 @@ public String toString() { return object + "@line:" + lineNumber; } } - - private static class ConstructIgnore extends AbstractConstruct { - @Override - public Object construct(Node node) { - return CustomTag.Ignore.INSTANCE; - } - } - - public static class ConstructLong extends AbstractConstruct { - @Override - public Object construct(Node node) { - if (!(node instanceof ScalarNode)) { - Assert.failUnchecked("The value of the long (!l) tag must be a scalar, however '" + node + "' is found!"); - } - return Long.valueOf(((ScalarNode) node).getValue()); - } - } - - private static class ConstructStringContains extends AbstractConstruct { - @Override - public Object construct(Node node) { - if (!(node instanceof ScalarNode)) { - Assert.failUnchecked("The value of the string-contains (!sc) tag must be a scalar, however '" + node + "' is found!"); - } - return new CustomTag.StringContains(((ScalarNode) node).getValue()); - } - } - - private static class ConstructUuidField extends AbstractConstruct { - @Override - public Object construct(Node node) { - if (!(node instanceof ScalarNode)) { - Assert.failUnchecked("The value of uuid (!uuid) tag must be a scalar, however '" + node + "' is found!"); - } - return new CustomTag.UuidField(((ScalarNode) node).getValue()); - } - } - - private static class ConstructVector16Field extends AbstractConstruct { - @Override - public Object construct(Node node) { - return new CustomTag.Vector16Field(node); - } - } - - private static class ConstructNullPlaceholder extends AbstractConstruct { - @Override - public Object construct(Node node) { - return CustomTag.NullPlaceholder.INSTANCE; - } - } - - private static class ConstructNotNull extends AbstractConstruct { - @Override - public Object construct(Node node) { - return CustomTag.NotNull.INSTANCE; - } - } - - private static class ConstructCurrentVersion extends AbstractConstruct { - - @Override - public Object construct(Node node) { - return PreambleBlock.CurrentVersion.INSTANCE; - } - } } diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java index 9166f07fc5..eef794e99b 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/Matchers.java @@ -27,6 +27,9 @@ import com.apple.foundationdb.record.util.pair.ImmutablePair; import com.apple.foundationdb.record.util.pair.Pair; import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.yamltests.tags.IgnoreTag; +import com.apple.foundationdb.relational.yamltests.tags.Matchable; +import com.apple.foundationdb.relational.yamltests.tags.IsNullTag; import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalResultSet; @@ -241,7 +244,7 @@ public static T notNull(@Nullable final T object, @Nonnull final String desc @Nonnull public static Object valueElseKey(@Nonnull final Map.Entry entry) { if (isNull(entry.getKey()) && isNull(entry.getValue())) { - fail(String.format(Locale.ROOT, "encountered YAML-style 'null' which is not supported, consider using '%s' instead", CustomTag.NullPlaceholder.INSTANCE)); + fail(String.format(Locale.ROOT, "encountered YAML-style 'null' which is not supported, consider using '%s' instead", IsNullTag.usage())); } return entry.getValue() == null ? entry.getKey() : entry.getValue(); } @@ -406,7 +409,7 @@ public String toString() { } public static Pair matchResultSet(final Object expected, final RelationalResultSet actual, final boolean isExpectedOrdered) throws SQLException { - if (expected instanceof CustomTag.Ignore) { + if (expected instanceof IgnoreTag) { return ImmutablePair.of(ResultSetMatchResult.success(), null); } if (expected == null) { @@ -558,34 +561,18 @@ private static ResultSetMatchResult matchField(@Nullable final Object expected, @Nullable final Object actual, int rowNumber, @Nonnull String cellRef) throws SQLException { - // the test does not care about the incoming value. - if (expected instanceof CustomTag.Ignore) { + if (expected == null && actual == null) { return ResultSetMatchResult.success(); } - final var expectedIsNull = expected instanceof CustomTag.NullPlaceholder; - - if (expectedIsNull && actual == null) { - return ResultSetMatchResult.success(); - } - if (expectedIsNull || actual == null) { - if (expectedIsNull) { - return ResultSetMatchResult.fail("actual result set is non-NULL, expecting NULL result set"); - } else { - return ResultSetMatchResult.fail("actual result set is NULL, expecting non-NULL result set"); - } - } - if (expected instanceof CustomTag.NotNull) { - // Actual value is not null, which is all the test cares about - return ResultSetMatchResult.success(); - } - if (expected instanceof CustomTag.StringContains) { - return ((CustomTag.StringContains) expected).matchWith(actual, rowNumber, cellRef); + if (expected == null) { + return ResultSetMatchResult.fail("actual result set is non-NULL, expecting NULL result set"); } - if (expected instanceof CustomTag.UuidField) { - return ((CustomTag.UuidField) expected).matchWith(actual, rowNumber, cellRef); + if (!(expected instanceof IsNullTag.IsNullMatcher) && actual == null) { + return ResultSetMatchResult.fail("actual result set is NULL, expecting non-NULL result set"); } - if (expected instanceof CustomTag.Vector16Field) { - return ((CustomTag.Vector16Field) expected).matchWith(actual, rowNumber, cellRef); + + if (expected instanceof Matchable) { + return ((Matchable)expected).matches(actual, rowNumber, cellRef); } // (nested) message @@ -662,7 +649,7 @@ private static ResultSetMatchResult matchField(@Nullable final Object expected, if (Objects.equals(expected, actual)) { return ResultSetMatchResult.success(); } else { - return ResultSetMatchResult.fail(String.format(Locale.ROOT, "cell mismatch at row: %d cellRef: %s%n expected 🟢 does not match 🟡.%n🟢 %s (%s)%n🟡 %s (%s)", rowNumber, cellRef, expected, expected == null ? "NULL" : expected.getClass().getSimpleName(), actual, actual.getClass().getSimpleName())); + return ResultSetMatchResult.fail(String.format(Locale.ROOT, "cell mismatch at row: %d cellRef: %s%n expected 🟢 does not match 🟡.%n🟢 %s (%s)%n🟡 %s (%s)", rowNumber, cellRef, expected, expected.getClass().getSimpleName(), actual, actual.getClass().getSimpleName())); } } diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java index f17ea72b08..1613b5c926 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryInterpreter.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.relational.yamltests.command; import com.apple.foundationdb.relational.util.Assert; -import com.apple.foundationdb.relational.yamltests.CustomYamlConstructor; import com.apple.foundationdb.relational.yamltests.YamlExecutionContext; import com.apple.foundationdb.relational.yamltests.command.parameterinjection.InListParameter; import com.apple.foundationdb.relational.yamltests.command.parameterinjection.ListParameter; @@ -29,6 +28,7 @@ import com.apple.foundationdb.relational.yamltests.command.parameterinjection.PrimitiveParameter; import com.apple.foundationdb.relational.yamltests.command.parameterinjection.TupleParameter; import com.apple.foundationdb.relational.yamltests.command.parameterinjection.UnboundParameter; +import com.apple.foundationdb.relational.yamltests.tags.LongTag; import org.apache.commons.lang3.tuple.Pair; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; @@ -99,6 +99,8 @@ private static final class QueryParameterYamlConstructor extends SafeConstructor private static final Tag NULL_TAG = new Tag("!n"); private static final Tag UUID_TAG = new Tag("!uuid"); private static final Tag VECTOR16_TAG = new Tag("!v16"); + private static final Tag VECTOR32_TAG = new Tag("!v32"); + private static final Tag VECTOR64_TAG = new Tag("!v64"); private QueryParameterYamlConstructor(LoaderOptions loaderOptions) { super(loaderOptions); @@ -109,14 +111,18 @@ private QueryParameterYamlConstructor(LoaderOptions loaderOptions) { this.yamlConstructors.put(NULL_TAG, new ConstructNull()); this.yamlConstructors.put(UUID_TAG, new ConstructUuid()); this.yamlConstructors.put(VECTOR16_TAG, new ConstructVector16()); + this.yamlConstructors.put(VECTOR32_TAG, new ConstructVector32()); + this.yamlConstructors.put(VECTOR64_TAG, new ConstructVector64()); - this.yamlConstructors.put(new Tag("!l"), new CustomYamlConstructor.ConstructLong()); + final var longTag = new LongTag(); + this.yamlConstructors.put(longTag.getTag(), longTag.getConstruct()); } @Override public Parameter constructObject(Node node) { if (node.getTag().equals(RANDOM_TAG) || node.getTag().equals(ARRAY_GENERATOR_TAG) || node.getTag().equals(IN_LIST_TAG) - || node.getTag().equals(NULL_TAG) || node.getTag().equals(UUID_TAG) || node.getTag().equals(VECTOR16_TAG)) { + || node.getTag().equals(NULL_TAG) || node.getTag().equals(UUID_TAG) || node.getTag().equals(VECTOR16_TAG) + || node.getTag().equals(VECTOR32_TAG) || node.getTag().equals(VECTOR64_TAG)) { return (Parameter) super.constructObject(node); } else if (node instanceof SequenceNode) { // simple list @@ -223,6 +229,28 @@ public Object construct(Node node) { return new PrimitiveParameter(constructVectorFromString(16, Assert.castUnchecked(node, SequenceNode.class))); } } + + /** + * Constructor for Vector32 (FloatVector) literal value. + */ + private static class ConstructVector32 extends AbstractConstruct { + + @Override + public Object construct(Node node) { + return new PrimitiveParameter(constructVectorFromString(32, Assert.castUnchecked(node, SequenceNode.class))); + } + } + + /** + * Constructor for Vector64 (DoubleVector) literal value. + */ + private static class ConstructVector64 extends AbstractConstruct { + + @Override + public Object construct(Node node) { + return new PrimitiveParameter(constructVectorFromString(64, Assert.castUnchecked(node, SequenceNode.class))); + } + } } private QueryInterpreter(int lineNumber, @Nonnull String query, @Nonnull final YamlExecutionContext executionContext) { diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/AbstractVectorTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/AbstractVectorTag.java new file mode 100644 index 0000000000..8702c10bb8 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/AbstractVectorTag.java @@ -0,0 +1,73 @@ +/* + * Vector16Field.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.common.base.Verify; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.stream.Collectors; + +public abstract class AbstractVectorTag implements CustomTag { + + public static final class VectorMatcher implements Matchable { + @Nonnull + private final SequenceNode yamlNode; + + private final int precision; + + public VectorMatcher(@Nonnull final Node node, int precision) { + this.yamlNode = Assert.castUnchecked(node, SequenceNode.class); + this.precision = precision; + } + + @Override + public String toString() { + return prettyPrintYamlNode(yamlNode); + } + + @Nonnull + @Override + @SuppressWarnings("unchecked") + public Matchers.ResultSetMatchResult matches(@Nullable Object other, int rowNumber, @Nonnull String cellRef) { + final var maybeNull = Matchable.shouldNotBeNull(other, rowNumber, cellRef); + if (maybeNull.isPresent()) { + return maybeNull.get(); + } + V thisVector = (V)Matchers.constructVectorFromString(precision, Assert.castUnchecked(yamlNode, SequenceNode.class)); + if (Verify.verifyNotNull(other).equals(thisVector)) { + return Matchers.ResultSetMatchResult.success(); + } + return Matchable.prettyPrintError("expected vector '" + prettyPrintYamlNode(yamlNode) + "' got '" + other + "' instead", rowNumber, cellRef); + } + + @Nonnull + private static String prettyPrintYamlNode(@Nonnull final SequenceNode sequenceNode) { + return sequenceNode.getTag() + sequenceNode.getValue().stream() + .map(v -> Assert.castUnchecked(v, ScalarNode.class).getValue()).collect(Collectors.joining(",", "[", "]")); + } + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CurrentVersionTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CurrentVersionTag.java new file mode 100644 index 0000000000..2608b828de --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CurrentVersionTag.java @@ -0,0 +1,60 @@ +/* + * CurrentVersionTag.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.yamltests.block.PreambleBlock; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public class CurrentVersionTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!current_version"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Object construct(Node node) { + return PreambleBlock.CurrentVersion.INSTANCE; + } + }; + + public CurrentVersionTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CustomTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CustomTag.java new file mode 100644 index 0000000000..5606cb15cd --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/CustomTag.java @@ -0,0 +1,40 @@ +/* + * CustomTag.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; +import java.util.function.BiConsumer; + +public interface CustomTag { + + default void accept(@Nonnull BiConsumer visitor) { + visitor.accept(getTag(), getConstruct()); + } + + @Nonnull + Tag getTag(); + + @Nonnull + Construct getConstruct(); +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IgnoreTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IgnoreTag.java new file mode 100644 index 0000000000..90ecf395a9 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IgnoreTag.java @@ -0,0 +1,63 @@ +/* + * Ignore.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public final class IgnoreTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!ignore"); + + @Nonnull + private static final Matchable INSTANCE = (other, rowNumber, cellRef) -> Matchers.ResultSetMatchResult.success(); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Object construct(Node node) { + return INSTANCE; + } + }; + + public IgnoreTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IsNullTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IsNullTag.java new file mode 100644 index 0000000000..97bab7cae1 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/IsNullTag.java @@ -0,0 +1,85 @@ +/* + * NullPlaceholder.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +@AutoService(CustomTag.class) +public final class IsNullTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!null"); + + @Nonnull + private static final Matchable INSTANCE = new IsNullMatcher(); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Object construct(Node node) { + return INSTANCE; + } + }; + + public IsNullTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } + + @Nonnull + public static String usage() { + return tag + " _"; + } + + public static final class IsNullMatcher implements Matchable { + @Nonnull + @Override + public Matchers.ResultSetMatchResult matches(@Nullable final Object other, final int rowNumber, @Nonnull final String cellRef) { + if (other == null) { + return Matchers.ResultSetMatchResult.success(); + } + return Matchable.prettyPrintError("expected NULL, got '" + other + "' instead", rowNumber, cellRef); + } + + @Override + public String toString() { + return tag.getValue(); + } + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/LongTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/LongTag.java new file mode 100644 index 0000000000..85277c4003 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/LongTag.java @@ -0,0 +1,64 @@ +/* + * LongTag.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.util.Assert; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public class LongTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!l"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Long construct(final Node node) { + if (!(node instanceof ScalarNode)) { + Assert.failUnchecked("The value of the long (!l) tag must be a scalar, however '" + node + "' is found!"); + } + return Long.valueOf(((ScalarNode) node).getValue()); + } + }; + + public LongTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Matchable.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Matchable.java new file mode 100644 index 0000000000..9a234cf56c --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Matchable.java @@ -0,0 +1,46 @@ +/* + * MatchableTag.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.yamltests.Matchers; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Optional; + +public interface Matchable { + + @Nonnull + Matchers.ResultSetMatchResult matches(@Nullable Object other, int rowNumber, @Nonnull String cellRef); + + @Nonnull + static Matchers.ResultSetMatchResult prettyPrintError(@Nonnull final String cause, int rowNumber, @Nonnull final String cellRef) { + return Matchers.ResultSetMatchResult.fail("wrong result at row: " + rowNumber + ", cell: " + cellRef + ", cause: " + cause); + } + + @Nonnull + static Optional shouldNotBeNull(@Nullable final Object object, int rowNumber, @Nonnull final String cellRef) { + if (object != null) { + return Optional.empty(); + } + return Optional.of(Matchers.ResultSetMatchResult.fail("unexpected NULL at row: " + rowNumber + ", cell: " + cellRef)); + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/NotNullTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/NotNullTag.java new file mode 100644 index 0000000000..e60dd6eea6 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/NotNullTag.java @@ -0,0 +1,64 @@ +/* + * NotNull.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public final class NotNullTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!not_null"); + + @Nonnull + private static final Matchable INSTANCE = (other, rowNumber, cellRef) -> + Matchable.shouldNotBeNull(other, rowNumber, cellRef).orElse(Matchers.ResultSetMatchResult.success()); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Matchable construct(Node node) { + return INSTANCE; + } + }; + + public NotNullTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/StringContainsTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/StringContainsTag.java new file mode 100644 index 0000000000..8d435b9900 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/StringContainsTag.java @@ -0,0 +1,100 @@ +/* + * StringContains.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +@AutoService(CustomTag.class) +public final class StringContainsTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!sc"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Matchable construct(final Node node) { + if (!(node instanceof ScalarNode)) { + Assert.failUnchecked("The value of the string-contains (!sc) tag must be a scalar, however '" + node + "' is found!"); + } + return new StringContainsMatcher(((ScalarNode) node).getValue()); + } + }; + + public StringContainsTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } + + public static final class StringContainsMatcher implements Matchable { + @Nonnull + private final String value; + + public StringContainsMatcher(@Nonnull final String value) { + this.value = value; + } + + @Nonnull + public String getValue() { + return value; + } + + @Nonnull + @Override + public Matchers.ResultSetMatchResult matches(@Nullable final Object other, int rowNumber, @Nonnull final String cellRef) { + final var maybeNull = Matchable.shouldNotBeNull(other, rowNumber, cellRef); + if (maybeNull.isPresent()) { + return maybeNull.get(); + } + if (other instanceof String) { + final var otherStr = (String)other; + if (otherStr.contains(value)) { + return Matchers.ResultSetMatchResult.success(); + } else { + + return Matchable.prettyPrintError("expected string containing '" + value + "' instead got '" + other + "'", rowNumber, cellRef); + } + } + return Matchable.prettyPrintError("expected to match against a String value, however we got " + other + " of type " + other.getClass().getSimpleName(), + rowNumber, cellRef); + } + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/UuidTag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/UuidTag.java new file mode 100644 index 0000000000..4e82f58d87 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/UuidTag.java @@ -0,0 +1,102 @@ +/* + * UuidField.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.yamltests.Matchers; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.UUID; + +@AutoService(CustomTag.class) +public final class UuidTag implements CustomTag { + + @Nonnull + private static final Tag tag = new Tag("!uuid"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + public Matchable construct(final Node node) { + if (!(node instanceof ScalarNode)) { + Assert.failUnchecked("The value of uuid (!uuid) tag must be a scalar, however '" + node + "' is found!"); + } + return new UuidMatcher(((ScalarNode) node).getValue()); + } + }; + + public UuidTag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } + + public static final class UuidMatcher implements Matchable { + @Nonnull + private final String value; + + public UuidMatcher(@Nonnull final String value) { + this.value = value; + } + + @Nonnull + public String getValue() { + return value; + } + + @Nonnull + @Override + public Matchers.ResultSetMatchResult matches(@Nullable Object other, int rowNumber, @Nonnull String cellRef) { + final var maybeNull = Matchable.shouldNotBeNull(other, rowNumber, cellRef); + if (maybeNull.isPresent()) { + return maybeNull.get(); + } + final UUID otherUUID; + try { + otherUUID = UUID.fromString(value); + } catch (IllegalArgumentException e) { + return Matchable.prettyPrintError("expected value '" + value + "' is not a proper UUID", rowNumber, cellRef); + } + if (other instanceof UUID) { + if (otherUUID.equals(UUID.fromString(value))) { + return Matchers.ResultSetMatchResult.success(); + } + } + return Matchable.prettyPrintError("expected UUID: '" + value + "' got " + other, rowNumber, cellRef); + } + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector16Tag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector16Tag.java new file mode 100644 index 0000000000..3fef71e2da --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector16Tag.java @@ -0,0 +1,61 @@ +/* + * Vector16Field.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.linear.HalfRealVector; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public final class Vector16Tag extends AbstractVectorTag { + + @Nonnull + private static final Tag tag = new Tag("!v16"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + @Nonnull + public Matchable construct(Node node) { + return new VectorMatcher(node, 16); + } + }; + + public Vector16Tag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector32Tag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector32Tag.java new file mode 100644 index 0000000000..5c436359d3 --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector32Tag.java @@ -0,0 +1,61 @@ +/* + * Vector16Field.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.linear.FloatRealVector; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public final class Vector32Tag extends AbstractVectorTag { + + @Nonnull + private static final Tag tag = new Tag("!v32"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + @Nonnull + public Matchable construct(Node node) { + return new VectorMatcher(node, 32); + } + }; + + public Vector32Tag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector64Tag.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector64Tag.java new file mode 100644 index 0000000000..25586bbb4c --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/Vector64Tag.java @@ -0,0 +1,61 @@ +/* + * Vector16Field.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.yamltests.tags; + +import com.apple.foundationdb.linear.DoubleRealVector; +import com.google.auto.service.AutoService; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Construct; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; + +import javax.annotation.Nonnull; + +@AutoService(CustomTag.class) +public final class Vector64Tag extends AbstractVectorTag { + + @Nonnull + private static final Tag tag = new Tag("!v64"); + + @Nonnull + private static final Construct CONSTRUCT_INSTANCE = new AbstractConstruct() { + @Override + @Nonnull + public Matchable construct(Node node) { + return new VectorMatcher(node, 64); + } + }; + + public Vector64Tag() { + } + + @Nonnull + @Override + public Tag getTag() { + return tag; + } + + @Nonnull + @Override + public Construct getConstruct() { + return CONSTRUCT_INSTANCE; + } +} diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/package-info.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/package-info.java new file mode 100644 index 0000000000..0799deb0ec --- /dev/null +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/tags/package-info.java @@ -0,0 +1,25 @@ +/* + * package-info.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Contains implementations of custom YAML tags used to facilitate testing. + */ + +package com.apple.foundationdb.relational.yamltests.tags; diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index c52b919579..0db851175f 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -21,8 +21,9 @@ options: supported_version: !current_version --- schema_template: - create table tv(a bigint, b vector(3, half), primary key(a)) - create table tv16x3(a bigint, b vector(3, half), primary key(a)) + create table tHalfVector(a bigint, b vector(3, half), primary key(a)) + create table tFloatVector(a bigint, b vector(3, float), primary key(a)) + create table tDoubleVector(a bigint, b vector(3, double), primary key(a)) --- test_block: preset: single_repetition_ordered @@ -35,8 +36,20 @@ test_block: # - query: insert into tv values (100, [1.3, 1.4, 1.6]) # - error: "XX000" - - - query: insert into tv values (100, !! !v16 [1.3, 1.4, 1.6] !!) + - query: insert into tHalfVector values (100, !! !v16 [1.3, 1.4, 1.6] !!) - count: 1 - - - query: select * from tv + - query: select * from tHalfVector - result: [{100, !v16 [1.3, 1.4, 1.6]}] + - + - query: insert into tFloatVector values (100, !! !v32 [15.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select * from tFloatVector + - result: [{100, !v32 [15.3, 1.4, 1.6]}] + - + - query: insert into tDoubleVector values (100, !! !v64 [1.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select * from tDoubleVector + - result: [{100, !v64 [1.3, 1.4, 1.6]}] diff --git a/yaml-tests/yaml-tests.gradle b/yaml-tests/yaml-tests.gradle index 8ed4e50c92..3046f4cac0 100644 --- a/yaml-tests/yaml-tests.gradle +++ b/yaml-tests/yaml-tests.gradle @@ -68,6 +68,8 @@ dependencies { testImplementation project(":fdb-relational-jdbc") testImplementation project(path: ":fdb-record-layer-debugger", configuration: 'tests') implementation(libs.diffutils) + compileOnly(libs.autoService) + annotationProcessor(libs.autoService) } // This "method" resolves the latest shadowJar version of the relational-server, and returns the file reference to it From 9364c55884a826a40ccdbf9c7fe44bebf65053d0 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 24 Oct 2025 15:24:11 +0100 Subject: [PATCH 03/15] Additional testing. - Fix other incorrect tests in DdlStatementParsingTest.java. --- .../relational/recordlayer/MessageTuple.java | 6 +- .../recordlayer/query/SemanticAnalyzer.java | 3 +- .../query/visitors/DdlVisitor.java | 4 +- .../api/ddl/DdlStatementParsingTest.java | 170 +++++++--- .../relational/api/ddl/DdlTestUtil.java | 292 ++++++++++++++---- 5 files changed, 361 insertions(+), 114 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index 5d5b9de553..b38e06260a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -57,10 +57,10 @@ public Object getObject(int position) throws InvalidColumnReferenceException { throw InvalidColumnReferenceException.getExceptionForInvalidPositionNumber(position); } final Descriptors.FieldDescriptor fieldDescriptor = message.getDescriptorForType().getFields().get(position); - final var recordTypeOptions = fieldDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.field); + final var fieldOptions = fieldDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.field); final var fieldValue = message.getField(message.getDescriptorForType().getFields().get(position)); - if (recordTypeOptions.hasVectorOptions()) { - final var precision = recordTypeOptions.getVectorOptions().getPrecision(); + if (fieldOptions.hasVectorOptions()) { + final var precision = fieldOptions.getVectorOptions().getPrecision(); final var byteStringFieldValue = (ByteString)fieldValue; return getVectorFromBytes(byteStringFieldValue, precision); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 7d5bc06107..a44c55035b 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -581,9 +581,10 @@ public DataType lookupType(@Nonnull final ParsedTypeInfo parsedTypeInfo, } else if (vectorElementTypeCtx.DOUBLE() != null) { precision = 64; } else { - Assert.notNullUnchecked(vectorElementTypeCtx.HALF()); + Assert.notNullUnchecked(vectorElementTypeCtx.HALF(), ErrorCode.SYNTAX_ERROR, "unsupported vector element type " + vectorTypeCtx.getText()); } int length = Assert.castUnchecked(ParseHelpers.parseDecimal(vectorTypeCtx.dimensions.getText()), Integer.class); + Assert.thatUnchecked(length > 0, ErrorCode.SYNTAX_ERROR, "vector dimension must be positive"); type = DataType.VectorType.of(precision, length, isNullable); } else { final var primitiveTypeName = parsedTypeInfo.getPrimitiveTypeContext().getText(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 3846f44a86..061b1842eb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -147,9 +147,9 @@ public RecordLayerColumn visitColumnDefinition(@Nonnull RelationalParser.ColumnD final SemanticAnalyzer.ParsedTypeInfo typeInfo; if (ctx.columnType().customType != null) { final var columnType = visitUid(ctx.columnType().customType); - typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofCustomType(columnType, true, isRepeated); + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofCustomType(columnType, isNullable, isRepeated); } else { - typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.columnType().primitiveType(), true, isRepeated); + typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.columnType().primitiveType(), isNullable, isRepeated); } final var columnType = semanticAnalyzer.lookupType(typeInfo, metadataBuilder::findType); return RecordLayerColumn.newBuilder().setName(columnId.getName()).setDataType(columnType).build(); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index 21eff80f86..6ddfe294a4 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -30,17 +30,22 @@ import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RecordContextTransaction; import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; import com.apple.foundationdb.relational.recordlayer.Utils; import com.apple.foundationdb.relational.recordlayer.ddl.AbstractMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.ddl.NoOpMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.metric.RecordLayerMetricCollector; +import com.apple.foundationdb.relational.recordlayer.query.Plan; +import com.apple.foundationdb.relational.recordlayer.query.PreparedParams; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.utils.PermutationIterator; import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; import com.apple.foundationdb.relational.utils.TestSchemas; +import com.google.common.collect.ImmutableList; import com.google.protobuf.DescriptorProtos; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assumptions; @@ -57,6 +62,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URI; +import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; import java.util.List; @@ -76,12 +82,17 @@ public class DdlStatementParsingTest { @RegisterExtension @Order(2) - public final SimpleDatabaseRule database = new SimpleDatabaseRule(DdlStatementParsingTest.class, TestSchemas.books()); + public final SimpleDatabaseRule database = new SimpleDatabaseRule(DdlStatementParsingTest.class, TestSchemas.books(), + Options.builder().withOption(Options.Name.CASE_SENSITIVE_IDENTIFIERS, true).build(), null); @RegisterExtension @Order(3) public final RelationalConnectionRule connection = new RelationalConnectionRule(database::getConnectionUri) - .withSchema("TEST_SCHEMA"); + .withSchema("TEST_SCHEMA") + .withOptions(Options.builder().withOption(Options.Name.CASE_SENSITIVE_IDENTIFIERS, true).build()); + + public DdlStatementParsingTest() throws SQLException { + } @BeforeAll public static void setup() { @@ -89,7 +100,7 @@ public static void setup() { } private static final String[] validPrimitiveDataTypes = new String[]{ - "integer", "bigint", "double", "boolean", "string", "bytes" + "integer", "bigint", "double", "boolean", "string", "bytes", "vector(3, float)", "vector(4, double)", "vector(5, half)" }; @Nonnull @@ -128,8 +139,13 @@ void shouldWorkWithInjectedFactory(@Nonnull final String query, @Nonnull final MetadataOperationsFactory metadataOperationsFactory) throws Exception { connection.setAutoCommit(false); (connection.getUnderlyingEmbeddedConnection()).createNewTransaction(); - DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), - "/DdlStatementParsingTest", metadataOperationsFactory).getPlan(query); + final var transaction = connection.getUnderlyingEmbeddedConnection().getTransaction(); + final var plan = DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), + "/DdlStatementParsingTest", metadataOperationsFactory, PreparedParams.empty(), + Options.builder().withOption(Options.Name.CASE_SENSITIVE_IDENTIFIERS, true).build()).getPlan(query); + // execute the plan so we run any extra test-driven verifications within the transactional closure. + plan.execute(Plan.ExecutionContext.of(transaction, Options.NONE, connection, + new RecordLayerMetricCollector(transaction.unwrap(RecordContextTransaction.class).getContext()))); connection.rollback(); connection.setAutoCommit(true); } @@ -149,8 +165,12 @@ void shouldFailWithInjectedQueryFactory(@Nonnull final String query, @Nullable E void shouldWorkWithInjectedQueryFactory(@Nonnull final String query, @Nonnull DdlQueryFactory queryFactory) throws Exception { connection.setAutoCommit(false); (connection.getUnderlyingEmbeddedConnection()).createNewTransaction(); - DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), + final var transaction = connection.getUnderlyingEmbeddedConnection().getTransaction(); + final var plan = DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), "/DdlStatementParsingTest", queryFactory).getPlan(query); + // execute the plan so we run any extra test-driven verifications within the transactional closure. + plan.execute(Plan.ExecutionContext.of(transaction, Options.NONE, connection, + new RecordLayerMetricCollector(transaction.unwrap(RecordContextTransaction.class).getContext()))); connection.rollback(); connection.setAutoCommit(true); } @@ -215,7 +235,7 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat DescriptorProtos.FileDescriptorProto fileDescriptorProto = getProtoDescriptor(template); Assertions.assertEquals(1, fileDescriptorProto.getEnumTypeCount(), "should have one enum defined"); fileDescriptorProto.getEnumTypeList().forEach(enumDescriptorProto -> { - Assertions.assertEquals("MY_ENUM", enumDescriptorProto.getName()); + Assertions.assertEquals("my_enum", enumDescriptorProto.getName()); Assertions.assertEquals(2, enumDescriptorProto.getValueCount()); Assertions.assertEquals(List.of("VAL_1", "VAL_2"), enumDescriptorProto.getValueList().stream() .map(DescriptorProtos.EnumValueDescriptorProto::getName) @@ -237,7 +257,7 @@ private static Stream typesMap() { Arguments.of(Types.VARCHAR, "STRING"), Arguments.of(Types.BOOLEAN, "BOOLEAN"), Arguments.of(Types.BINARY, "BYTES"), - Arguments.of(Types.STRUCT, "BAZ"), + Arguments.of(Types.STRUCT, "baz"), Arguments.of(Types.ARRAY, "STRING ARRAY") ); } @@ -280,22 +300,6 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull final SchemaT } } - private static void checkColumnNullability(@Nonnull final SchemaTemplate template, int sqlType, boolean isNullable) { - Assertions.assertInstanceOf(RecordLayerSchemaTemplate.class, template); - Assertions.assertEquals(1, ((RecordLayerSchemaTemplate) template).getTables().size(), "should have only 1 table"); - final var table = ((RecordLayerSchemaTemplate) template).findTableByName("BAR"); - Assertions.assertTrue(table.isPresent()); - final var columns = table.get().getColumns(); - Assertions.assertEquals(2, columns.size()); - final var maybeNullableArrayColumn = columns.stream().filter(c -> c.getName().equals("FOO_FIELD")).findFirst(); - Assertions.assertTrue(maybeNullableArrayColumn.isPresent()); - if (isNullable) { - Assertions.assertTrue(maybeNullableArrayColumn.get().getDataType().isNullable()); - } else { - Assertions.assertFalse(maybeNullableArrayColumn.get().getDataType().isNullable()); - } - Assertions.assertEquals(sqlType, maybeNullableArrayColumn.get().getDataType().getJdbcSqlCode()); - } @Test void failsToParseEmptyTemplateStatements() throws Exception { @@ -333,6 +337,50 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat }); } + @Nonnull + private static Stream invalidVectorTypes() { + return Stream.of( + // Zero dimensions + Arguments.of("vector(0, float)"), + // Negative dimensions + Arguments.of("vector(-1, float)"), + // Invalid element type + Arguments.of("vector(3, integer)"), + Arguments.of("vector(3, bigint)"), + Arguments.of("vector(3, string)"), + Arguments.of("vector(3, boolean)"), + Arguments.of("vector(3, bytes)"), + Arguments.of("vector(3, int)"), + // Missing dimensions + Arguments.of("vector(float)"), + // Missing element type + Arguments.of("vector(3)"), + // Empty vector + Arguments.of("vector()"), + // Wrong order (type, dimensions) + Arguments.of("vector(float, 3)"), + // Non-numeric dimensions + Arguments.of("vector(abc, float)"), + // Decimal dimensions + Arguments.of("vector(3.5, float)"), + // Multiple commas + Arguments.of("vector(3,, float)"), + // Extra parameters + Arguments.of("vector(3, float, extra)"), + // Case variations of invalid types + Arguments.of("vector(3, FLOAT32)"), + Arguments.of("vector(3, DOUBLE64)") + ); + } + + @ParameterizedTest + @MethodSource("invalidVectorTypes") + void createInvalidVectorType(String vectorType) throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TABLE test_table (id bigint, vec_col " + vectorType + ", PRIMARY KEY(id))"; + shouldFailWith(stmt, ErrorCode.SYNTAX_ERROR); + } + @ParameterizedTest @MethodSource("columnTypePermutations") void createSchemaTemplateWithOutOfOrderDefinitionsWork(List columns) throws Exception { @@ -358,14 +406,14 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat @MethodSource("columnTypePermutations") void createSchemaTemplates(List columns) throws Exception { final String columnStatement = "CREATE SCHEMA TEMPLATE test_template " + - " CREATE TYPE AS STRUCT FOO " + makeColumnDefinition(columns, false) + - " CREATE TABLE BAR (col0 bigint, col1 FOO, PRIMARY KEY(col0))"; + " CREATE TYPE AS STRUCT foo " + makeColumnDefinition(columns, false) + + " CREATE TABLE bar (col0 bigint, col1 foo, PRIMARY KEY(col0))"; shouldWorkWithInjectedFactory(columnStatement, new AbstractMetadataOperationsFactory() { @Nonnull @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - Assertions.assertEquals("TEST_TEMPLATE", template.getName(), "incorrect template name!"); + Assertions.assertEquals("test_template", template.getName(), "incorrect template name!"); DdlTestUtil.ParsedSchema schema = new DdlTestUtil.ParsedSchema(getProtoDescriptor(template)); Assertions.assertEquals(1, schema.getTables().size(), "Incorrect number of tables"); return txn -> { @@ -383,16 +431,16 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat @ParameterizedTest @MethodSource("columnTypePermutations") void createSchemaTemplateTableWithOnlyRecordType(List columns) throws Exception { - final String baseTableDef = makeColumnDefinition(columns, false).replace(")", ", SINGLE ROW ONLY)"); + final String baseTableDef = replaceLast(makeColumnDefinition(columns, false), ')', ", SINGLE ROW ONLY)"); final String columnStatement = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TABLE FOO " + baseTableDef; + "CREATE TABLE foo " + baseTableDef; shouldWorkWithInjectedFactory(columnStatement, new AbstractMetadataOperationsFactory() { @Nonnull @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - Assertions.assertEquals("TEST_TEMPLATE", template.getName(), "incorrect template name!"); + Assertions.assertEquals("test_template", template.getName(), "incorrect template name!"); DdlTestUtil.ParsedSchema schema = new DdlTestUtil.ParsedSchema(getProtoDescriptor(template)); Assertions.assertEquals(1, schema.getTables().size(), "Incorrect number of tables"); return txn -> { @@ -433,8 +481,8 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat void createSchemaTemplateWithIndex(List columns) throws Exception { final String indexColumns = String.join(",", chooseIndexColumns(columns, n -> n % 2 == 0)); final String templateStatement = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TYPE AS STRUCT FOO " + makeColumnDefinition(columns, false) + - "CREATE TABLE TBL " + makeColumnDefinition(columns, true) + + "CREATE TYPE AS STRUCT foo " + makeColumnDefinition(columns, false) + + "CREATE TABLE tbl " + makeColumnDefinition(columns, true) + "CREATE INDEX v_idx as select " + indexColumns + " from tbl order by " + indexColumns; shouldWorkWithInjectedFactory(templateStatement, new AbstractMetadataOperationsFactory() { @@ -447,7 +495,7 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat Table info = ((RecordLayerSchemaTemplate) template).getTables().stream().findFirst().orElseThrow(); Assertions.assertEquals(1, info.getIndexes().size(), "Incorrect number of indexes!"); final Index index = Assert.optionalUnchecked(info.getIndexes().stream().findFirst()); - Assertions.assertEquals("V_IDX", index.getName(), "Incorrect index name!"); + Assertions.assertEquals("v_idx", index.getName(), "Incorrect index name!"); final var actualKe = ((RecordLayerIndex) index).getKeyExpression().toKeyExpression(); List keys = null; @@ -481,8 +529,8 @@ void createSchemaTemplateWithIndexAndInclude(List columns) throws Except final List indexedColumns = chooseIndexColumns(columns, n -> n % 2 == 0); //choose every other column final List unindexedColumns = chooseIndexColumns(columns, n -> n % 2 != 0); final String templateStatement = "CREATE SCHEMA TEMPLATE test_template " + - " CREATE TYPE AS STRUCT FOO " + makeColumnDefinition(columns, false) + - " CREATE TABLE TBL " + makeColumnDefinition(columns, true) + + " CREATE TYPE AS STRUCT foo " + makeColumnDefinition(columns, false) + + " CREATE TABLE tbl " + makeColumnDefinition(columns, true) + " CREATE INDEX v_idx as select " + Stream.concat(indexedColumns.stream(), unindexedColumns.stream()).collect(Collectors.joining(",")) + " from tbl order by " + String.join(",", indexedColumns); shouldWorkWithInjectedFactory(templateStatement, new AbstractMetadataOperationsFactory() { @Nonnull @@ -493,7 +541,7 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat Table info = ((RecordLayerSchemaTemplate) template).getTables().stream().findFirst().orElseThrow(); Assertions.assertEquals(1, info.getIndexes().size(), "Incorrect number of indexes!"); final Index index = Assert.optionalUnchecked(info.getIndexes().stream().findFirst()); - Assertions.assertEquals("V_IDX", index.getName(), "Incorrect index name!"); + Assertions.assertEquals("v_idx", index.getName(), "Incorrect index name!"); RecordKeyExpressionProto.KeyExpression actualKe = ((RecordLayerIndex) index).getKeyExpression().toKeyExpression(); Assertions.assertNotNull(actualKe.getKeyWithValue(), "Null KeyExpression for included columns!"); @@ -559,7 +607,7 @@ void dropSchemaTemplates() throws Exception { @Nonnull @Override public ConstantAction getDropSchemaTemplateConstantAction(@Nonnull String templateId, boolean throwIfDoesNotExist, @Nonnull Options options) { - Assertions.assertEquals("TEST_TEMPLATE", templateId, "Incorrect schema template name!"); + Assertions.assertEquals("test_template", templateId, "Incorrect schema template name!"); called[0] = true; return txn -> { }; @@ -585,14 +633,14 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat @ParameterizedTest @MethodSource("columnTypePermutations") void createTable(List columns) throws Exception { - final String columnStatement = "CREATE SCHEMA TEMPLATE test_template CREATE TABLE FOO " + + final String columnStatement = "CREATE SCHEMA TEMPLATE test_template CREATE TABLE foo " + makeColumnDefinition(columns, true); shouldWorkWithInjectedFactory(columnStatement, new AbstractMetadataOperationsFactory() { @Nonnull @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - Assertions.assertEquals("TEST_TEMPLATE", template.getName(), "incorrect template name!"); + Assertions.assertEquals("test_template", template.getName(), "incorrect template name!"); DdlTestUtil.ParsedSchema schema = new DdlTestUtil.ParsedSchema(getProtoDescriptor(template)); Assertions.assertEquals(1, schema.getTables().size(), "Incorrect number of tables"); return txn -> { @@ -610,8 +658,11 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat @ParameterizedTest @MethodSource("columnTypePermutations") void createTableAndType(List columns) throws Exception { - final String tableDef = "CREATE TABLE tbl " + makeColumnDefinition(columns, true); final String typeDef = "CREATE TYPE AS STRUCT typ " + makeColumnDefinition(columns, false); + // current implementation of metadata prunes unused types in the serialization, this may or may not + // be something we want to commit to long term. + final var columnsWithType = ImmutableList.builder().addAll(columns).add("typ").build(); + final String tableDef = "CREATE TABLE tbl " + makeColumnDefinition(columnsWithType, true); final String templateStatement = "CREATE SCHEMA TEMPLATE test_template " + typeDef + " " + tableDef; @@ -621,12 +672,12 @@ void createTableAndType(List columns) throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - Assertions.assertEquals("TEST_TEMPLATE", template.getName(), "incorrect template name!"); + Assertions.assertEquals("test_template", template.getName(), "incorrect template name!"); DdlTestUtil.ParsedSchema schema = new DdlTestUtil.ParsedSchema(getProtoDescriptor(template)); Assertions.assertEquals(1, schema.getTables().size(), "Incorrect number of tables"); return txn -> { try { - assertColumnsMatch(schema.getTable("tbl"), columns); + assertColumnsMatch(schema.getTable("tbl"), columnsWithType); assertColumnsMatch(schema.getType("typ"), columns); } catch (Exception ve) { throw ExceptionUtil.toRelationalException(ve); @@ -645,7 +696,7 @@ void createDatabase() throws Exception { @Nonnull @Override public ConstantAction getCreateDatabaseConstantAction(@Nonnull URI dbPath, @Nonnull Options constantActionOptions) { - Assertions.assertEquals(URI.create("/DB_PATH"), dbPath, "Incorrect database path!"); + Assertions.assertEquals(URI.create("/db_path"), dbPath, "Incorrect database path!"); return NoOpMetadataOperationsFactory.INSTANCE.getCreateDatabaseConstantAction(dbPath, constantActionOptions); } }); @@ -924,7 +975,7 @@ private static List chooseIndexColumns(@Nonnull final List colum //choose every other column return IntStream.range(0, columns.size()) .filter(indexChoice) - .mapToObj(n -> "COL" + n) + .mapToObj(n -> "col" + n) .collect(Collectors.toList()); } @@ -936,4 +987,35 @@ private static void assertColumnsMatch(@Nonnull final DdlTestUtil.ParsedType typ .collect(Collectors.toList()); Assertions.assertEquals(expectedColStrings, columnStrings, "Incorrect columns for type <" + type.getName() + ">"); } + + private static void checkColumnNullability(@Nonnull final SchemaTemplate template, int sqlType, boolean isNullable) { + Assertions.assertInstanceOf(RecordLayerSchemaTemplate.class, template); + Assertions.assertEquals(1, ((RecordLayerSchemaTemplate) template).getTables().size(), "should have only 1 table"); + final var table = ((RecordLayerSchemaTemplate) template).findTableByName("bar"); + Assertions.assertTrue(table.isPresent()); + final var columns = table.get().getColumns(); + Assertions.assertEquals(2, columns.size()); + final var maybeNullableArrayColumn = columns.stream().filter(c -> c.getName().equals("foo_field")).findFirst(); + Assertions.assertTrue(maybeNullableArrayColumn.isPresent()); + if (isNullable) { + Assertions.assertTrue(maybeNullableArrayColumn.get().getDataType().isNullable()); + } else { + Assertions.assertFalse(maybeNullableArrayColumn.get().getDataType().isNullable()); + } + Assertions.assertEquals(sqlType, maybeNullableArrayColumn.get().getDataType().getJdbcSqlCode()); + } + + @Nonnull + private static String replaceLast(@Nonnull final String str, final char oldChar, @Nonnull final String replacement) { + if (str.isEmpty()) { + return str; + } + + int lastIndex = str.lastIndexOf(oldChar); + if (lastIndex == -1) { + return str; + } + + return str.substring(0, lastIndex) + replacement + str.substring(lastIndex + 1); + } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java index 31632cf034..5697cbe746 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.relational.recordlayer.query.PreparedParams; import com.apple.foundationdb.relational.util.Assert; import com.google.protobuf.DescriptorProtos; +import org.assertj.core.api.Assertions; import javax.annotation.Nonnull; import java.net.URI; @@ -104,12 +105,22 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio @Nonnull final String databaseUri, @Nonnull final MetadataOperationsFactory metadataOperationsFactory, @Nonnull final PreparedParams preparedParams) throws SQLException, RelationalException { + return getPlanGenerator(embeddedConnection, schemaTemplateName, databaseUri, metadataOperationsFactory, preparedParams, Options.NONE); + } + + @Nonnull + static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnection embeddedConnection, + @Nonnull final String schemaTemplateName, + @Nonnull final String databaseUri, + @Nonnull final MetadataOperationsFactory metadataOperationsFactory, + @Nonnull final PreparedParams preparedParams, + @Nonnull final Options options) throws SQLException, RelationalException { final var planContext = PlanContext.Builder.unapply(createVanillaPlanContext(embeddedConnection, schemaTemplateName, databaseUri, preparedParams)) .withConstantActionFactory(metadataOperationsFactory).build(); final var storeState = new RecordStoreState(null, Map.of()); try (var schema = embeddedConnection.getRecordLayerDatabase().loadSchema(embeddedConnection.getSchema())) { final var metadata = schema.loadStore().getRecordMetaData(); - return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerFactoryRegistryImpl.instance(), Options.NONE); + return PlanGenerator.create(Optional.empty(), planContext, metadata, storeState, IndexMaintainerFactoryRegistryImpl.instance(), options); } } @@ -127,52 +138,108 @@ static PlanGenerator getPlanGenerator(@Nonnull final EmbeddedRelationalConnectio } } + /** + * A wrapper around {@link DescriptorProtos.FieldDescriptorProto} that provides utility methods + * for extracting column information in a test-friendly format. + * + *

This class is primarily used in DDL parsing tests to verify that SQL column definitions + * are correctly translated into protobuf field descriptors. It translates protobuf type information + * back into SQL-like type names for comparison against expected DDL statements. + */ + public static final class ParsedColumn { - public static class ParsedColumn { - private final DescriptorProtos.FieldDescriptorProto descriptor; + @Nonnull + private final DescriptorProtos.FieldDescriptorProto fieldDescriptor; - public ParsedColumn(DescriptorProtos.FieldDescriptorProto descriptor) { - this.descriptor = descriptor; + /** + * Creates a new ParsedColumn wrapper around the given field descriptor. + * + * @param fieldDescriptor the protobuf field descriptor to wrap + */ + public ParsedColumn(@Nonnull final DescriptorProtos.FieldDescriptorProto fieldDescriptor) { + this.fieldDescriptor = fieldDescriptor; } + /** + * Returns the name of the column as defined in the field descriptor. + * + * @return the column name + */ + @Nonnull String getName() { - return descriptor.getName(); + return fieldDescriptor.getName(); } + /** + * Translates the protobuf field type back into a SQL-compatible type string. + * + *

This method maps protobuf types (TYPE_INT32, TYPE_STRING, etc.) to their SQL equivalents + * (integer, string, etc.). It also handles special cases like: + *

    + *
  • Vector types with precision and dimensions (e.g., "vector(3, float)")
    • + *
  • Array types (LABEL_REPEATED fields)
    • + *
  • Custom message and enum types
    • + *
+ * + * @return the SQL-compatible type string for this column + * @throws IllegalStateException if an unexpected protobuf type or vector precision is encountered + */ + @Nonnull String getType() { String type = ""; - switch (descriptor.getType()) { - case TYPE_INT32: - type += "INTEGER"; - break; - case TYPE_INT64: - type += "BIGINT"; - break; - case TYPE_FLOAT: - type += "FLOAT"; - break; - case TYPE_DOUBLE: - type += "DOUBLE"; - break; - case TYPE_BOOL: - type += "BOOLEAN"; - break; - case TYPE_STRING: - type += "STRING"; - break; - case TYPE_BYTES: - type += "BYTES"; - break; - case TYPE_MESSAGE: - type += "MESSAGE " + descriptor.getTypeName(); + if (fieldDescriptor.hasTypeName() && !fieldDescriptor.getTypeName().isEmpty()) { + return fieldDescriptor.getTypeName(); + } else { + switch (fieldDescriptor.getType()) { + case TYPE_INT32: + type += "integer"; + break; + case TYPE_INT64: + type += "bigint"; + break; + case TYPE_FLOAT: + type += "float"; + break; + case TYPE_DOUBLE: + type += "double"; + break; + case TYPE_BOOL: + type += "boolean"; + break; + case TYPE_STRING: + type += "string"; + break; + case TYPE_BYTES: { + final var fieldOptions = fieldDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.field); + if (fieldOptions.hasVectorOptions()) { + final var precision = fieldOptions.getVectorOptions().getPrecision(); + final var dimensions = fieldOptions.getVectorOptions().getDimensions(); + String elementType; + if (precision == 16) { + elementType = "half"; + } else if (precision == 32) { + elementType = "float"; + } else if (precision == 64) { + elementType = "double"; + } else { + throw new IllegalStateException("Unexpected vector precision " + precision); + } + type += "vector(" + dimensions + ", " + elementType + ")"; + } else { + type += "bytes"; + } + } break; - case TYPE_ENUM: - //TODO(Bfines) figure this one out - default: - throw new IllegalStateException("Unexpected descriptor java type <" + descriptor.getType()); + case TYPE_ENUM: // fallthrough + case TYPE_MESSAGE: + type += fieldDescriptor.getTypeName(); + break; + default: + throw new IllegalStateException("Unexpected descriptor type " + fieldDescriptor.getType()); + } } - if (descriptor.getLabel() == DescriptorProtos.FieldDescriptorProto.Label.LABEL_REPEATED) { + if (fieldDescriptor.getLabel() == DescriptorProtos.FieldDescriptorProto.Label.LABEL_REPEATED) { type = " array"; } @@ -180,24 +247,53 @@ String getType() { } } + /** + * A wrapper around {@link DescriptorProtos.DescriptorProto} that represents a structured type + * (such as a STRUCT) defined in a DDL schema. + * + *

This class parses a protobuf message descriptor and extracts its fields as {@link ParsedColumn} + * instances, providing a test-friendly interface for verifying DDL type definitions. It translates + * protobuf descriptors back into DDL-compatible format for comparison in tests. + */ public static class ParsedType { + + @Nonnull private final DescriptorProtos.DescriptorProto descriptor; - private List columns; + @Nonnull + private final List columns; - public ParsedType(DescriptorProtos.DescriptorProto descriptor) { + /** + * Creates a new ParsedType wrapper around the given descriptor. + * + * @param descriptor the protobuf descriptor representing this type + */ + ParsedType(@Nonnull final DescriptorProtos.DescriptorProto descriptor) { this.descriptor = descriptor; this.columns = parseColumns(); } - public String getName() { + /** + * Returns the name of this type as defined in the descriptor. + * + * @return the type name + */ + @Nonnull + String getName() { return descriptor.getName(); } + /** + * Returns the list of columns (fields) that make up this type. + * + * @return the list of parsed columns + */ + @Nonnull List getColumns() { return columns; } + @Nonnull private List parseColumns() { List cols = new ArrayList<>(descriptor.getFieldCount()); for (DescriptorProtos.FieldDescriptorProto field :descriptor.getFieldList()) { @@ -207,57 +303,108 @@ private List parseColumns() { } /** - * Get the Columns listed the way the DDL language would expect them, with 1 entry per column of the form. - * [NAME TYPE] + * Returns the columns formatted as DDL column definitions. + * + *

Each entry in the returned list is a string in the format "NAME TYPE", + * matching the way the DDL language expects column definitions. + * + * @return a list of column definition strings */ - public List getColumnStrings() { + List getColumnStrings() { return columns.stream().map(col -> col.getName() + " " + col.getType()).collect(Collectors.toList()); } } - public static class ParsedTable extends ParsedType { + /** + * A wrapper around {@link DescriptorProtos.DescriptorProto} that represents a table + * defined in a DDL schema. + * + *

This is a specialized version of {@link ParsedType} that specifically represents + * tables (as opposed to user-defined types). Tables are distinguished from types by + * their presence in the schema's union descriptor. + */ + public static final class ParsedTable extends ParsedType { + /** + * Creates a new ParsedTable wrapper around the given descriptor. + * + * @param descriptor the protobuf descriptor representing this table + */ public ParsedTable(DescriptorProtos.DescriptorProto descriptor) { super(descriptor); } } - public static class ParsedSchema { + /** + * A wrapper around {@link DescriptorProtos.FileDescriptorProto} that represents a complete + * DDL schema with tables and types. + * + *

This class parses a protobuf file descriptor and separates message types into tables + * and user-defined types. Tables are identified by their presence in the schema's union + * descriptor, while other message types are treated as custom types (e.g., STRUCTs). + * + *

This class is primarily used in DDL parsing tests to verify that entire schema + * definitions are correctly translated from SQL to protobuf and back. + */ + public static final class ParsedSchema { + + @Nonnull private final DescriptorProtos.FileDescriptorProto schemaDescriptor; + @Nonnull private List tables; + + @Nonnull private List types; - public ParsedSchema(DescriptorProtos.FileDescriptorProto schemaDescriptor) { + /** + * Creates a new ParsedSchema wrapper around the given file descriptor. + * + *

Upon construction, this immediately parses the descriptor to separate + * tables from types. + * + * @param schemaDescriptor the protobuf file descriptor representing the schema + */ + ParsedSchema(@Nonnull final DescriptorProtos.FileDescriptorProto schemaDescriptor) { this.schemaDescriptor = schemaDescriptor; buildTypesAndTables(); } + /** + * Returns the list of tables defined in this schema. + * + * @return the list of parsed tables + */ + @Nonnull List getTables() { return tables; } + /** + * Returns the list of user-defined types (non-table types) in this schema. + * + * @return the list of parsed types + */ + @Nonnull List getTypes() { return types; } private void buildTypesAndTables() { - /* - * Parses the FileDescriptor into types and tables - */ - //first, find the UnionDescriptor so that we know what are tables + // Parses the FileDescriptor into types and tables + // First, find the UnionDescriptor so that we know what are tables Set tableNames = new HashSet<>(); - for (DescriptorProtos.DescriptorProto typeDesc : schemaDescriptor.getMessageTypeList()) { - final RecordMetaDataOptionsProto.RecordTypeOptions extension = typeDesc.getOptions().getExtension(RecordMetaDataOptionsProto.record); - if (extension != null && extension.hasUsage() && extension.getUsage() == RecordMetaDataOptionsProto.RecordTypeOptions.Usage.UNION) { - //we found the Union Descriptor - for (DescriptorProtos.FieldDescriptorProto tableDescs : typeDesc.getFieldList()) { - tableNames.add(tableDescs.getTypeName()); + for (DescriptorProtos.DescriptorProto typeDescriptor : schemaDescriptor.getMessageTypeList()) { + final RecordMetaDataOptionsProto.RecordTypeOptions extension = typeDescriptor.getOptions().getExtension(RecordMetaDataOptionsProto.record); + if (extension.hasUsage() && extension.getUsage() == RecordMetaDataOptionsProto.RecordTypeOptions.Usage.UNION) { + // we found the Union Descriptor + for (DescriptorProtos.FieldDescriptorProto tableDescriptor : typeDescriptor.getFieldList()) { + tableNames.add(tableDescriptor.getTypeName()); } } } - //now parse types and tables + // now parse types and tables types = new ArrayList<>(); tables = new ArrayList<>(); @@ -270,23 +417,40 @@ private void buildTypesAndTables() { } } - public ParsedType getType(String typeName) { - for (ParsedType parsedType : types) { - if (parsedType.getName().equalsIgnoreCase(typeName)) { + /** + * Finds and returns a user-defined type by name. + * + * @param typeName the name of the type to find + * @return the ParsedType if found, null otherwise + */ + @Nonnull + @SuppressWarnings("DataFlowIssue") + ParsedType getType(@Nonnull final String typeName) { + for (final ParsedType parsedType : types) { + if (parsedType.getName().equals(typeName)) { return parsedType; } } - return null; + Assertions.fail("could not find type " + typeName); + return null; // not reachable. } - public ParsedType getTable(String tableName) { - for (ParsedType table : tables) { - if (table.getName().equalsIgnoreCase(tableName)) { + /** + * Finds and returns a table by name. + * + * @param tableName the name of the table to find + * @return the ParsedType representing the table if found, null otherwise + */ + @Nonnull + @SuppressWarnings("DataFlowIssue") + ParsedType getTable(@Nonnull final String tableName) { + for (final ParsedType table : tables) { + if (table.getName().equals(tableName)) { return table; } } - return null; + Assertions.fail("could not find table" + tableName); + return null; // not reachable. } } - } From eafb7514639c3b1159ae821b5d4cb8b9d8cb5e5a Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 24 Oct 2025 18:58:41 +0100 Subject: [PATCH 04/15] Support CAST from ARRAY to VECTOR. --- .../query/plan/cascades/values/CastValue.java | 153 +++++++++- .../foundationdb/record/util/VectorUtils.java | 3 + .../src/main/proto/record_query_plan.proto | 5 +- .../src/main/antlr/RelationalParser.g4 | 9 +- .../recordlayer/query/SemanticAnalyzer.java | 58 +--- .../query/visitors/ExpressionVisitor.java | 7 +- .../test/resources/cast-tests.metrics.binpb | 32 ++ .../test/resources/cast-tests.metrics.yaml | 24 ++ .../src/test/resources/cast-tests.yamsql | 10 + .../src/test/resources/vector.metrics.binpb | 273 ++++++++++++++++++ .../src/test/resources/vector.metrics.yaml | 205 +++++++++++++ yaml-tests/src/test/resources/vector.yamsql | 104 ++++++- 12 files changed, 814 insertions(+), 69 deletions(-) create mode 100644 yaml-tests/src/test/resources/vector.metrics.binpb create mode 100644 yaml-tests/src/test/resources/vector.metrics.yaml diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java index 2ea8dcb3fd..a5bd0b5f83 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java @@ -22,11 +22,17 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.half.Half; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.planprotos.PCastValue; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; @@ -50,11 +56,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.BiFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A value that casts an object of a type to an object of another type using SQL CAST semantics. @@ -237,7 +246,9 @@ enum PhysicalOperator { // STRING to complex types. STRING_TO_ENUM(Type.TypeCode.STRING, Type.TypeCode.ENUM, ((descriptor, in) -> PromoteValue.PhysicalOperator.stringToEnumValue((Descriptors.EnumDescriptor)descriptor, (String)in))), - STRING_TO_UUID(Type.TypeCode.STRING, Type.TypeCode.UUID, ((descriptor, in) -> PromoteValue.PhysicalOperator.stringToUuidValue((String) in))); + STRING_TO_UUID(Type.TypeCode.STRING, Type.TypeCode.UUID, ((descriptor, in) -> PromoteValue.PhysicalOperator.stringToUuidValue((String) in))), + + ARRAY_TO_VECTOR(Type.TypeCode.ARRAY, Type.TypeCode.VECTOR, CastValue::castArrayToVector); @Nonnull private final Type.TypeCode from; @@ -335,14 +346,20 @@ public Object eval(@Nonnull final FDBRecordStoreBase stor return null; } - // Special handling for array to array casts + // Special handling for array to other types. if (physicalOperator.equals(PhysicalOperator.ARRAY_TO_ARRAY)) { final Type inType = child.getResultType(); final var fromArray = (Type.Array) inType; final var toArray = (Type.Array) castToType; final var descriptor = new ArrayCastDescriptor(fromArray, toArray); return castArrayToArray(descriptor, childResult); + } else if (physicalOperator.equals(PhysicalOperator.ARRAY_TO_VECTOR)) { + final Type inType = child.getResultType(); + final var fromArray = (Type.Array) inType; + final var descriptor = new ArrayCastDescriptor(fromArray, castToType); + return castArrayToVector(descriptor, childResult); } + return physicalOperator.getCastFunction().apply(null, childResult); } @@ -527,6 +544,7 @@ public CastValue fromProto(@Nonnull final PlanSerializationContext serialization * @param in the input array to cast * @return the cast array */ + @Nullable @SuppressWarnings("unchecked") private static Object castArrayToArray(Object descriptor, Object in) { if (in == null) { @@ -537,7 +555,8 @@ private static Object castArrayToArray(Object descriptor, Object in) { final var sourceArrayType = typeDescriptor.getFromType(); final var targetArrayType = typeDescriptor.getToType(); final var sourceElementType = sourceArrayType.getElementType(); - final var targetElementType = targetArrayType.getElementType(); + Verify.verify(targetArrayType.isArray()); + final var targetElementType = ((Type.Array)targetArrayType).getElementType(); // Check for null element types if (targetElementType == null) { @@ -575,14 +594,41 @@ private static Object castArrayToArray(Object descriptor, Object in) { return resultList; } + @Nullable + @SuppressWarnings("unchecked") + private static Object castArrayToVector(Object descriptor, Object in) { + if (in == null) { + return null; + } + + final var typeDescriptor = (ArrayCastDescriptor) descriptor; + final var sourceArrayType = typeDescriptor.getFromType(); + final var targetArrayType = typeDescriptor.getToType(); + final var sourceElementType = sourceArrayType.getElementType(); + Verify.verify(targetArrayType.isVector()); + final var targetVectorType = (Type.Vector)targetArrayType; + + final var inputList = (java.util.List) in; + + if (sourceElementType == null) { + SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "Source array element type cannot be null"); + } + + if (inputList.size() != targetVectorType.getDimensions()) { + SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "Source array is not the same size of the vector"); + } + + return parseVector(inputList, targetVectorType, sourceElementType); + } + /** * Descriptor for array to array casting operations. */ - private static class ArrayCastDescriptor { + private static final class ArrayCastDescriptor { private final Type.Array fromType; - private final Type.Array toType; + private final Type toType; - public ArrayCastDescriptor(Type.Array fromType, Type.Array toType) { + public ArrayCastDescriptor(Type.Array fromType, Type toType) { this.fromType = fromType; this.toType = toType; } @@ -591,8 +637,101 @@ public Type.Array getFromType() { return fromType; } - public Type.Array getToType() { + public Type getToType() { return toType; } } + + @Nonnull + private static RealVector parseHalfVector(@Nonnull final List array, @Nonnull final Type sourceElementType) { + final var sourceTypeCode = sourceElementType.getTypeCode(); + if (sourceTypeCode == Type.TypeCode.FLOAT) { + final var halfArray = array.stream().map(obj -> Half.valueOf((Float)obj)).toArray(Half[]::new); + return new HalfRealVector(halfArray); + } + if (sourceTypeCode == Type.TypeCode.DOUBLE) { + + + final var doubleArray = array.stream().map(obj -> ((Number)obj).doubleValue()).mapToDouble(Double::doubleValue).toArray(); + return new HalfRealVector(doubleArray); + } + if (sourceTypeCode == Type.TypeCode.INT) { + final var intArray = array.stream().map(obj -> ((Number)obj).intValue()).mapToInt(Integer::intValue).toArray(); + return new HalfRealVector(intArray); + } + if (sourceTypeCode == Type.TypeCode.LONG) { + final var longArray = array.stream().map(obj -> ((Number)obj).longValue()).mapToLong(Long::longValue).toArray(); + return new HalfRealVector(longArray); + } + SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "can not cast array of " + sourceElementType + " to half vector"); + return null; // not reachable. + } + + @Nonnull + private static RealVector parseFloatVector(@Nonnull final List array, @Nonnull final Type sourceElementType) { + final var sourceTypeCode = sourceElementType.getTypeCode(); + if (sourceTypeCode == Type.TypeCode.FLOAT) { + final var floatArray = toFloatArray(array.stream().map(obj -> ((Number)obj).floatValue())); + return new FloatRealVector(floatArray); + } + if (sourceTypeCode == Type.TypeCode.DOUBLE) { + final var doubleArray = array.stream().map(obj -> ((Number)obj).doubleValue()).mapToDouble(Double::doubleValue).toArray(); + return new FloatRealVector(doubleArray); + } + if (sourceTypeCode == Type.TypeCode.INT) { + final var intArray = array.stream().map(obj -> ((Number)obj).intValue()).mapToInt(Integer::intValue).toArray(); + return new FloatRealVector(intArray); + } + if (sourceTypeCode == Type.TypeCode.LONG) { + final var longArray = array.stream().map(obj -> ((Number)obj).longValue()).mapToLong(Long::longValue).toArray(); + return new FloatRealVector(longArray); + } + SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "can not cast array of " + sourceElementType + " to float vector"); + return null; // not reachable. + } + + @Nonnull + private static RealVector parseDoubleVector(@Nonnull final List array, @Nonnull final Type sourceElementType) { + final var sourceTypeCode = sourceElementType.getTypeCode(); + if (sourceTypeCode == Type.TypeCode.DOUBLE) { + final var doubleArray = array.stream().map(obj -> ((Number)obj).doubleValue()).toArray(Double[]::new); + return new DoubleRealVector(doubleArray); + } + if (sourceTypeCode == Type.TypeCode.INT) { + final var intArray = array.stream().map(obj -> ((Number)obj).intValue()).mapToInt(Integer::intValue).toArray(); + return new DoubleRealVector(intArray); + } + if (sourceTypeCode == Type.TypeCode.LONG) { + final var longArray = array.stream().map(obj -> ((Number)obj).longValue()).mapToLong(Long::longValue).toArray(); + return new DoubleRealVector(longArray); + } + SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "can not cast array of " + sourceElementType + " to double vector"); + return null; // not reachable. + } + + + @Nonnull + private static RealVector parseVector(@Nonnull final List array, @Nonnull final Type.Vector vectorType, + @Nonnull final Type sourceElementType) { + if (vectorType.getPrecision() == 16) { + return parseHalfVector(array, sourceElementType); + } + if (vectorType.getPrecision() == 32) { + return parseFloatVector(array, sourceElementType); + } + if (vectorType.getPrecision() == 64) { + return parseDoubleVector(array, sourceElementType); + } + throw new RecordCoreException("unexpected vector type " + vectorType); + } + + @Nonnull + private static float[] toFloatArray(@Nonnull final Stream floatStream) { + List floatList = floatStream.collect(Collectors.toList()); + float[] floatArray = new float[floatList.size()]; + for (int i = 0; i < floatList.size(); i++) { + floatArray[i] = floatList.get(i); + } + return floatArray; + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java index f3180b9863..28a8ea2c8b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java @@ -29,6 +29,7 @@ import com.google.protobuf.ByteString; import javax.annotation.Nonnull; +import java.util.List; public class VectorUtils { public static int getVectorPrecision(@Nonnull final RealVector vector) { @@ -58,4 +59,6 @@ public static RealVector parseVector(@Nonnull final ByteString byteString, @Nonn } throw new RecordCoreException("unexpected vector type " + vectorType); } + + } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index dd799eb3c8..3d0e63c430 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1331,12 +1331,15 @@ message PCastValue { NULL_TO_ENUM = 33; NULL_TO_UUID = 34; - // ARRAY conversion + // ARRAY to ARRAY conversion ARRAY_TO_ARRAY = 35; // STRING to complex types. STRING_TO_ENUM = 36; STRING_TO_UUID = 37; + + // ARRAY to VECTOR conversion + ARRAY_TO_VECTOR = 38; } optional PValue child = 1; optional PType cast_to_type = 2; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 769f661c1b..7526451ee6 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -819,14 +819,7 @@ collectionOptions ; convertedDataType - : - ( - typeName=(BINARY| NCHAR) lengthOneDimension? - | typeName=CHAR lengthOneDimension? (charSet charsetName)? - | typeName=(DATE | DATETIME | TIME | JSON | INT | INTEGER | STRING | DOUBLE | BOOLEAN | FLOAT | BIGINT | BYTES | UUID) - | typeName=DECIMAL lengthTwoOptionalDimension? - | (SIGNED | UNSIGNED) INTEGER? - ) ARRAY? + : typeName=primitiveType ARRAY? ; lengthOneDimension diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index a44c55035b..e6c1e49570 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -560,6 +560,16 @@ public static ParsedTypeInfo ofCustomType(@Nonnull final Identifier customType, } } + @Nonnull + public DataType lookupBuiltInType(@Nonnull final ParsedTypeInfo parsedTypeInfo) { + Assert.thatUnchecked(!parsedTypeInfo.hasCustomType(), ErrorCode.INTERNAL_ERROR, () -> "unexpected custom type " + + Assert.notNullUnchecked(parsedTypeInfo.getCustomType()).getName()); + return lookupType(parsedTypeInfo, typeToLookUp -> { + Assert.failUnchecked("unexpected custom type " + typeToLookUp); + return Optional.empty(); + }); + } + @Nonnull public DataType lookupType(@Nonnull final ParsedTypeInfo parsedTypeInfo, @Nonnull final Function> dataTypeProvider) { @@ -633,54 +643,6 @@ public DataType lookupType(@Nonnull final ParsedTypeInfo parsedTypeInfo, } } - - @Nonnull - public DataType lookupType(@Nonnull Identifier typeIdentifier, boolean isNullable, boolean isRepeated, - @Nonnull Function> dataTypeProvider) { - DataType type; - final var typeName = typeIdentifier.getName(); - switch (typeName.toUpperCase(Locale.ROOT)) { - case "STRING": - type = isNullable ? DataType.Primitives.NULLABLE_STRING.type() : DataType.Primitives.STRING.type(); - break; - case "INTEGER": - type = isNullable ? DataType.Primitives.NULLABLE_INTEGER.type() : DataType.Primitives.INTEGER.type(); - break; - case "BIGINT": - type = isNullable ? DataType.Primitives.NULLABLE_LONG.type() : DataType.Primitives.LONG.type(); - break; - case "DOUBLE": - type = isNullable ? DataType.Primitives.NULLABLE_DOUBLE.type() : DataType.Primitives.DOUBLE.type(); - break; - case "BOOLEAN": - type = isNullable ? DataType.Primitives.NULLABLE_BOOLEAN.type() : DataType.Primitives.BOOLEAN.type(); - break; - case "BYTES": - type = isNullable ? DataType.Primitives.NULLABLE_BYTES.type() : DataType.Primitives.BYTES.type(); - break; - case "FLOAT": - type = isNullable ? DataType.Primitives.NULLABLE_FLOAT.type() : DataType.Primitives.FLOAT.type(); - break; - case "UUID": - type = isNullable ? DataType.Primitives.NULLABLE_UUID.type() : DataType.Primitives.UUID.type(); - break; - default: - Assert.notNullUnchecked(metadataCatalog); - // assume it is a custom type, will fail in upper layers if the type can not be resolved. - // lookup the type (Struct, Table, or Enum) in the schema template metadata under construction. - final var maybeFound = dataTypeProvider.apply(typeName); - // if we cannot find the type now, mark it, we will try to resolve it later on via a second pass. - type = maybeFound.orElseGet(() -> DataType.UnresolvedType.of(typeName, isNullable)); - break; - } - - if (isRepeated) { - return DataType.ArrayType.from(type.withNullable(false), isNullable); - } else { - return type; - } - } - @Nonnull private static Expressions expandStructExpression(@Nonnull Expression expression) { Assert.thatUnchecked(expression.getDataType().getCode() == DataType.Code.STRUCT, ErrorCode.INVALID_COLUMN_REFERENCE, diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index f5fbc8dca0..d449458143 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -334,12 +334,11 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa public Expression visitDataTypeFunctionCall(@Nonnull RelationalParser.DataTypeFunctionCallContext ctx) { if (ctx.CAST() != null) { final var sourceExpression = Assert.castUnchecked(ctx.expression().accept(this), Expression.class); - final var targetTypeString = ctx.convertedDataType().typeName.getText(); final var isRepeated = ctx.convertedDataType().ARRAY() != null; - final var targetDataType = getDelegate().getSemanticAnalyzer().lookupType( - Identifier.of(targetTypeString), false, isRepeated, ignored -> Optional.empty()); + final var typeInfo = SemanticAnalyzer.ParsedTypeInfo.ofPrimitiveType(ctx.convertedDataType().typeName, false, isRepeated); + // Cast does not currently support user-defined struct types. + final var targetDataType = getDelegate().getSemanticAnalyzer().lookupBuiltInType(typeInfo); final var targetType = DataTypeUtils.toRecordLayerType(targetDataType); - final var castValue = CastValue.inject(sourceExpression.getUnderlying(), targetType); return Expression.ofUnnamed(targetDataType, castValue); } diff --git a/yaml-tests/src/test/resources/cast-tests.metrics.binpb b/yaml-tests/src/test/resources/cast-tests.metrics.binpb index 15d633fb29..c28e1c5fa0 100644 --- a/yaml-tests/src/test/resources/cast-tests.metrics.binpb +++ b/yaml-tests/src/test/resources/cast-tests.metrics.binpb @@ -48,6 +48,38 @@ a 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; } k +cast-operator-testsTEXPLAIN select CAST([1.2d, 4.3d, 5.9d] AS INTEGER ARRAY) from test_cast where id = 1 += (0?8@SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(ARRAY(INT) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.ID EQUALS promote(@c20 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, INT AS NUM_COL, STRING AS STR_COL, BOOLEAN AS BOOL_COL)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TEST_CAST]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, INT AS NUM_COL, STRING AS STR_COL, BOOLEAN AS BOOL_COL)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TEST_CAST_ARRAYS, TEST_CAST]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +p +cast-operator-testsYEXPLAIN select CAST([80.90f, 25.76f, 16.7f] AS INTEGER ARRAY) from test_cast where id = 1 += (088@SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(ARRAY(INT) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.ID EQUALS promote(@c20 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, INT AS NUM_COL, STRING AS STR_COL, BOOLEAN AS BOOL_COL)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TEST_CAST]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, INT AS NUM_COL, STRING AS STR_COL, BOOLEAN AS BOOL_COL)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TEST_CAST_ARRAYS, TEST_CAST]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +k cast-operator-testsTEXPLAIN select CAST(['42', '100', '7'] AS INTEGER ARRAY) from test_cast where id = 1 = (098@SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)digraph G { fontname=courier; diff --git a/yaml-tests/src/test/resources/cast-tests.metrics.yaml b/yaml-tests/src/test/resources/cast-tests.metrics.yaml index 159ca5b506..4f741a7832 100644 --- a/yaml-tests/src/test/resources/cast-tests.metrics.yaml +++ b/yaml-tests/src/test/resources/cast-tests.metrics.yaml @@ -34,6 +34,30 @@ cast-operator-tests: insert_time_ms: 1 insert_new_count: 21 insert_reused_count: 2 +- query: EXPLAIN select CAST([1.2d, 4.3d, 5.9d] AS INTEGER ARRAY) from test_cast + where id = 1 + explain: SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) + | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0) + task_count: 230 + task_total_time_ms: 15 + transform_count: 61 + transform_time_ms: 6 + transform_yield_count: 15 + insert_time_ms: 1 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select CAST([80.90f, 25.76f, 16.7f] AS INTEGER ARRAY) from test_cast + where id = 1 + explain: SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) + | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0) + task_count: 230 + task_total_time_ms: 14 + transform_count: 61 + transform_time_ms: 7 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 - query: EXPLAIN select CAST(['42', '100', '7'] AS INTEGER ARRAY) from test_cast where id = 1 explain: SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) diff --git a/yaml-tests/src/test/resources/cast-tests.yamsql b/yaml-tests/src/test/resources/cast-tests.yamsql index 08702060b8..e600d961bd 100644 --- a/yaml-tests/src/test/resources/cast-tests.yamsql +++ b/yaml-tests/src/test/resources/cast-tests.yamsql @@ -52,6 +52,16 @@ test_block: - query: select CAST([1, 2, 3] AS DOUBLE ARRAY) from test_cast where id = 1 - explain: "SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c20, @c7, @c9) AS ARRAY(DOUBLE)) AS _0)" - result: [{[1.0, 2.0, 3.0]}] + - + # Array casting: DOUBLE array to INT array + - query: select CAST([1.2d, 4.3d, 5.9d] AS INTEGER ARRAY) from test_cast where id = 1 + - explain: "SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)" + - result: [{[1, 4, 6]}] + - + # Array casting: FLOAT array to INT array + - query: select CAST([80.90f, 25.76f, 16.7f] AS INTEGER ARRAY) from test_cast where id = 1 + - explain: "SCAN(<,>) | TFILTER TEST_CAST | FILTER _.ID EQUALS promote(@c20 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS ARRAY(INT)) AS _0)" + - result: [{[81, 26, 17]}] - # Array casting: STRING array to INT array - query: select CAST(['42', '100', '7'] AS INTEGER ARRAY) from test_cast where id = 1 diff --git a/yaml-tests/src/test/resources/vector.metrics.binpb b/yaml-tests/src/test/resources/vector.metrics.binpb new file mode 100644 index 0000000000..47c7dcd852 --- /dev/null +++ b/yaml-tests/src/test/resources/vector.metrics.binpb @@ -0,0 +1,273 @@ + +l + vector-tests\EXPLAIN select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 +ɐ = ҳ(08@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +k + vector-tests[EXPLAIN select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0T8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +h + vector-testsXEXPLAIN select cast([1.2, 4.3, 5.9] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (098@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +b + vector-testsREXPLAIN select cast([1, 4, 5] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0̪O8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +e + vector-testsUEXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0J8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +i + vector-testsYEXPLAIN select cast([1.2, 4.3, 5.9] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 += (018@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +c + vector-testsSEXPLAIN select cast([1, 4, 5] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 += (098@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +f + vector-testsVEXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 +莰= (038@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +d + vector-testsTEXPLAIN select cast([1, 4, 5] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 += ׈(0#8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(64, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +g + vector-testsWEXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 += (0ŵ(8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(64, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +e + vector-testsUEXPLAIN select cast([1, 2.5, 3L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0&8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +f + vector-testsVEXPLAIN select cast([1, 2.5, 3L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 += Δ(0 8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +n + vector-tests^EXPLAIN select cast([1000L, 2000L, 3000L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0J8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +x + vector-testshEXPLAIN select cast([1000000L, 2000000L, 3000000L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 +莰= (038@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c24 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +k + vector-tests[EXPLAIN select cast([-1.5, -2.5, -3.5] as VECTOR(3, HALF)) from tDoubleVector where a = 100 += (0%8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(16, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(16, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(16, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c27 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +o + vector-tests_EXPLAIN select cast([-1.5f, -2.5f, -3.5f] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 += (0 8@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(32, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(32, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(32, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c27 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} +m + vector-tests]EXPLAIN select cast([-100, -200, -300] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 += (08@SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(64, 3)) AS _0)digraph G { + fontname=courier; + rankdir=BT; + splines=polyline; + 1 [ label=<
Value Computation
MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(64, 3)) AS _0)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(VECTOR(64, 3) AS _0)" ]; + 2 [ label=<
Predicate Filter
WHERE q2.A EQUALS promote(@c27 AS LONG)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 3 [ label=<
Type Filter
WHERE record IS [TDOUBLEVECTOR]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS A, VECTOR(64, 3) AS B)" ]; + 4 [ label=<
Scan
range: <-∞, ∞>
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 5 [ label=<
Primary Storage
record types: [TDOUBLEVECTOR, TFLOATVECTOR, THALFVECTOR]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(RECORD)" ]; + 3 -> 2 [ label=< q2> label="q2" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 3 [ label=< q19> label="q19" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q26> label="q26" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; +} \ No newline at end of file diff --git a/yaml-tests/src/test/resources/vector.metrics.yaml b/yaml-tests/src/test/resources/vector.metrics.yaml new file mode 100644 index 0000000000..43b7b5e79f --- /dev/null +++ b/yaml-tests/src/test/resources/vector.metrics.yaml @@ -0,0 +1,205 @@ +vector-tests: +- query: EXPLAIN select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, FLOAT)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 67 + transform_count: 61 + transform_time_ms: 49 + transform_yield_count: 15 + insert_time_ms: 3 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 18 + transform_count: 61 + transform_time_ms: 9 + transform_yield_count: 15 + insert_time_ms: 1 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1.2, 4.3, 5.9] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 15 + transform_count: 61 + transform_time_ms: 8 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1, 4, 5] as VECTOR(3, HALF)) from tDoubleVector where + a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 14 + transform_count: 61 + transform_time_ms: 7 + transform_yield_count: 15 + insert_time_ms: 1 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 11 + transform_count: 61 + transform_time_ms: 6 + transform_yield_count: 15 + insert_time_ms: 1 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1.2, 4.3, 5.9] as VECTOR(3, FLOAT)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 11 + transform_count: 61 + transform_time_ms: 6 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1, 4, 5] as VECTOR(3, FLOAT)) from tDoubleVector where + a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 11 + transform_count: 61 + transform_time_ms: 6 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, FLOAT)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 17 + transform_count: 61 + transform_time_ms: 12 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1, 4, 5] as VECTOR(3, DOUBLE)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0) + task_count: 230 + task_total_time_ms: 7 + transform_count: 61 + transform_time_ms: 4 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1L, 4L, 5L] as VECTOR(3, DOUBLE)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0) + task_count: 230 + task_total_time_ms: 10 + transform_count: 61 + transform_time_ms: 5 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1, 2.5, 3L] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 10 + transform_count: 61 + transform_time_ms: 5 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1, 2.5, 3L] as VECTOR(3, FLOAT)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 8 + transform_count: 61 + transform_time_ms: 4 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1000L, 2000L, 3000L] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 11 + transform_count: 61 + transform_time_ms: 6 + transform_yield_count: 15 + insert_time_ms: 1 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([1000000L, 2000000L, 3000000L] as VECTOR(3, FLOAT)) + from tDoubleVector where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS + LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 17 + transform_count: 61 + transform_time_ms: 12 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([-1.5, -2.5, -3.5] as VECTOR(3, HALF)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS + LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(16, 3)) AS _0) + task_count: 230 + task_total_time_ms: 9 + transform_count: 61 + transform_time_ms: 5 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([-1.5f, -2.5f, -3.5f] as VECTOR(3, FLOAT)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS + LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(32, 3)) AS _0) + task_count: 230 + task_total_time_ms: 7 + transform_count: 61 + transform_time_ms: 4 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 +- query: EXPLAIN select cast([-100, -200, -300] as VECTOR(3, DOUBLE)) from tDoubleVector + where a = 100 + explain: SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS + LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(64, 3)) AS _0) + task_count: 230 + task_total_time_ms: 6 + transform_count: 61 + transform_time_ms: 3 + transform_yield_count: 15 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 2 diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index 0db851175f..837aca6642 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -29,7 +29,7 @@ test_block: preset: single_repetition_ordered options: statement_type: prepared - name: basic-vector-tests + name: vector-tests tests: # - # # inserting vector literal is not yet supported. @@ -53,3 +53,105 @@ test_block: - - query: select * from tDoubleVector - result: [{100, !v64 [1.3, 1.4, 1.6]}] + - + - query: select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1.2, 4.3, 5.9]}] + # FLOAT array to HALF vector + - + - query: select cast([1.2f, 4.3f, 5.9f] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1.2, 4.3, 5.9]}] + # DOUBLE array to HALF vector + - + - query: select cast([1.2, 4.3, 5.9] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1.2, 4.3, 5.9]}] + # INT array to HALF vector + - + - query: select cast([1, 4, 5] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1.0, 4.0, 5.0]}] + # LONG array to HALF vector + - + - query: select cast([1L, 4L, 5L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1.0, 4.0, 5.0]}] + # DOUBLE array to FLOAT vector + - + - query: select cast([1.2, 4.3, 5.9] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1.2, 4.3, 5.9]}] + # INT array to FLOAT vector + - + - query: select cast([1, 4, 5] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1.0, 4.0, 5.0]}] + # LONG array to FLOAT vector + - + - query: select cast([1L, 4L, 5L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1.0, 4.0, 5.0]}] + # DOUBLE array to DOUBLE vector + - + - query: select cast([1.2, 4.3, 5.9] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 + - result: [{!v64 [1.2, 4.3, 5.9]}] + # INT array to DOUBLE vector + - + - query: select cast([1, 4, 5] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)" + - result: [{!v64 [1.0, 4.0, 5.0]}] + # LONG array to DOUBLE vector + - + - query: select cast([1L, 4L, 5L] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(64, 3)) AS _0)" + - result: [{!v64 [1.0, 4.0, 5.0]}] + # Mix of integer and float conversions to HALF + - + - query: select cast([1, 2.5, 3L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1.0, 2.5, 3.0]}] + # Mix of integer and double conversions to FLOAT + - + - query: select cast([1, 2.5, 3L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1.0, 2.5, 3.0]}] + # Large numbers in LONG array to HALF vector + - + - query: select cast([1000L, 2000L, 3000L] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [1000.0, 2000.0, 3000.0]}] + # Large numbers in LONG array to FLOAT vector + - + - query: select cast([1000000L, 2000000L, 3000000L] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c24 AS LONG) | MAP (CAST(array(@c5, @c7, @c9) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [1000000.0, 2000000.0, 3000000.0]}] + # Negative numbers to vectors + - + - query: select cast([-1.5, -2.5, -3.5] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(16, 3)) AS _0)" + - result: [{!v16 [-1.5, -2.5, -3.5]}] + - + - query: select cast([-1.5f, -2.5f, -3.5f] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(32, 3)) AS _0)" + - result: [{!v32 [-1.5, -2.5, -3.5]}] + - + - query: select cast([-100, -200, -300] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 + - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(64, 3)) AS _0)" + - result: [{!v64 [-100.0, -200.0, -300.0]}] + # STRING array to vector should fail + - + - query: select cast(['a', 'b', 'c'] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - error: "22F3H" + # BOOLEAN array to vector should fail + - + - query: select cast([true, false, true] as VECTOR(3, HALF)) from tDoubleVector where a = 100 + - error: "22F3H" + # Mismatched dimensions should fail (array has 4 elements, vector expects 3) + - + - query: select cast([1.0, 2.0, 3.0, 4.0] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 + - error: "22F3H" + # Mismatched dimensions should fail (array has 2 elements, vector expects 3) + - + - query: select cast([1.0, 2.0] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 + - error: "22F3H" From b46710b66584e736e478e64473efa8827ac48df5 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Mon, 27 Oct 2025 18:01:14 +0000 Subject: [PATCH 05/15] WIP - update test file with more tests. --- yaml-tests/src/test/resources/vector.yamsql | 191 ++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index 837aca6642..d641e37a0a 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -139,6 +139,197 @@ test_block: - query: select cast([-100, -200, -300] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 - explain: "SCAN(<,>) | TFILTER TDOUBLEVECTOR | FILTER _.A EQUALS promote(@c27 AS LONG) | MAP (CAST(array(@c5, @c8, @c11) AS VECTOR(64, 3)) AS _0)" - result: [{!v64 [-100.0, -200.0, -300.0]}] + # IS NULL tests for vectors + - + - query: select b is null from tHalfVector where a = 100 + - result: [{false}] + - + - query: select b is null from tFloatVector where a = 100 + - result: [{false}] + - + - query: select b is null from tDoubleVector where a = 100 + - result: [{false}] + # IS NOT NULL tests for vectors + - + - query: select b is not null from tHalfVector where a = 100 + - result: [{true}] + - + - query: select b is not null from tFloatVector where a = 100 + - result: [{true}] + - + - query: select b is not null from tDoubleVector where a = 100 + - result: [{true}] + # Insert NULL vectors + - + - query: insert into tHalfVector values (200, null) + - count: 1 + - + - query: insert into tFloatVector values (200, null) + - count: 1 + - + - query: insert into tDoubleVector values (200, null) + - count: 1 + # IS NULL tests for NULL vectors + - + - query: select b is null from tHalfVector where a = 200 + - result: [{true}] + - + - query: select b is null from tFloatVector where a = 200 + - result: [{true}] + - + - query: select b is null from tDoubleVector where a = 200 + - result: [{true}] + # IS NOT NULL tests for NULL vectors + - + - query: select b is not null from tHalfVector where a = 200 + - result: [{false}] + - + - query: select b is not null from tFloatVector where a = 200 + - result: [{false}] + - + - query: select b is not null from tDoubleVector where a = 200 + - result: [{false}] + # Filter using IS NULL + - + - query: select a from tHalfVector where b is null + - result: [{200}] + - + - query: select a from tFloatVector where b is null + - result: [{200}] + - + - query: select a from tDoubleVector where b is null + - result: [{200}] + # Filter using IS NOT NULL + - + - query: select a from tHalfVector where b is not null + - result: [{100}] + - + - query: select a from tFloatVector where b is not null + - result: [{100}] + - + - query: select a from tDoubleVector where b is not null + - result: [{100}] + # Vector equality tests (VEC = VEC) + - + - query: insert into tHalfVector values (300, !! !v16 [1.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select a from tHalfVector where b = !! !v16 [1.3, 1.4, 1.6] !! + - result: [{100}, {300}] + - + - query: select a from tHalfVector where b = !! !v16 [9.9, 9.9, 9.9] !! + - result: [] + - + - query: insert into tFloatVector values (300, !! !v32 [15.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select a from tFloatVector where b = !! !v32 [15.3, 1.4, 1.6] !! + - result: [{100}, {300}] + - + - query: select a from tFloatVector where b = !! !v32 [9.9, 9.9, 9.9] !! + - result: [] + - + - query: insert into tDoubleVector values (300, !! !v64 [1.3, 1.4, 1.6] !!) + - count: 1 + - + - query: select a from tDoubleVector where b = !! !v64 [1.3, 1.4, 1.6] !! + - result: [{100}, {300}] + - + - query: select a from tDoubleVector where b = !! !v64 [9.9, 9.9, 9.9] !! + - result: [] + # Vector inequality tests (VEC != VEC) + - + - query: select a from tHalfVector where b != !! !v16 [1.3, 1.4, 1.6] !! + - result: [] + - + - query: select a from tHalfVector where b != !! !v16 [9.9, 9.9, 9.9] !! + - result: [{100}, {300}] + - + - query: select a from tFloatVector where b != !! !v32 [15.3, 1.4, 1.6] !! + - result: [] + - + - query: select a from tFloatVector where b != !! !v32 [9.9, 9.9, 9.9] !! + - result: [{100}, {300}] + - + - query: select a from tDoubleVector where b != !! !v64 [1.3, 1.4, 1.6] !! + - result: [] + - + - query: select a from tDoubleVector where b != !! !v64 [9.9, 9.9, 9.9] !! + - result: [{100}, {300}] + # Vector IS DISTINCT FROM tests + - + - query: select b is distinct from !! !v16 [1.3, 1.4, 1.6] !! from tHalfVector where a = 100 + - result: [{false}] + - + - query: select b is distinct from !! !v16 [9.9, 9.9, 9.9] !! from tHalfVector where a = 100 + - result: [{true}] + - + - query: select b is distinct from null from tHalfVector where a = 200 + - result: [{false}] + - + - query: select b is distinct from !! !v16 [1.3, 1.4, 1.6] !! from tHalfVector where a = 200 + - result: [{true}] + - + - query: select b is distinct from !! !v32 [15.3, 1.4, 1.6] !! from tFloatVector where a = 100 + - result: [{false}] + - + - query: select b is distinct from !! !v32 [9.9, 9.9, 9.9] !! from tFloatVector where a = 100 + - result: [{true}] + - + - query: select b is distinct from null from tFloatVector where a = 200 + - result: [{false}] + - + - query: select b is distinct from !! !v32 [15.3, 1.4, 1.6] !! from tFloatVector where a = 200 + - result: [{true}] + - + - query: select b is distinct from !! !v64 [1.3, 1.4, 1.6] !! from tDoubleVector where a = 100 + - result: [{false}] + - + - query: select b is distinct from !! !v64 [9.9, 9.9, 9.9] !! from tDoubleVector where a = 100 + - result: [{true}] + - + - query: select b is distinct from null from tDoubleVector where a = 200 + - result: [{false}] + - + - query: select b is distinct from !! !v64 [1.3, 1.4, 1.6] !! from tDoubleVector where a = 200 + - result: [{true}] + # Vector NOT DISTINCT FROM tests + - + - query: select b is not distinct from !! !v16 [1.3, 1.4, 1.6] !! from tHalfVector where a = 100 + - result: [{true}] + - + - query: select b is not distinct from !! !v16 [9.9, 9.9, 9.9] !! from tHalfVector where a = 100 + - result: [{false}] + - + - query: select b is not distinct from null from tHalfVector where a = 200 + - result: [{true}] + - + - query: select b is not distinct from !! !v16 [1.3, 1.4, 1.6] !! from tHalfVector where a = 200 + - result: [{false}] + - + - query: select b is not distinct from !! !v32 [15.3, 1.4, 1.6] !! from tFloatVector where a = 100 + - result: [{true}] + - + - query: select b is not distinct from !! !v32 [9.9, 9.9, 9.9] !! from tFloatVector where a = 100 + - result: [{false}] + - + - query: select b is not distinct from null from tFloatVector where a = 200 + - result: [{true}] + - + - query: select b is not distinct from !! !v32 [15.3, 1.4, 1.6] !! from tFloatVector where a = 200 + - result: [{false}] + - + - query: select b is not distinct from !! !v64 [1.3, 1.4, 1.6] !! from tDoubleVector where a = 100 + - result: [{true}] + - + - query: select b is not distinct from !! !v64 [9.9, 9.9, 9.9] !! from tDoubleVector where a = 100 + - result: [{false}] + - + - query: select b is not distinct from null from tDoubleVector where a = 200 + - result: [{true}] + - + - query: select b is not distinct from !! !v64 [1.3, 1.4, 1.6] !! from tDoubleVector where a = 200 + - result: [{false}] # STRING array to vector should fail - - query: select cast(['a', 'b', 'c'] as VECTOR(3, FLOAT)) from tDoubleVector where a = 100 From e87cacfa21e845fc6a1695e1b35fffccb4924b62 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Mon, 27 Oct 2025 19:03:05 +0000 Subject: [PATCH 06/15] Support basic comparison operators for Vector types. - fix few problems in prepared statement handling and type inference during normalization. - add more tests. --- .../expressions/LiteralKeyExpression.java | 2 +- .../record/query/expressions/Comparisons.java | 3 ++ .../query/plan/cascades/typing/Type.java | 6 ++++ .../plan/cascades/values/RelOpValue.java | 23 ++++++++++++- .../plan/serialization/PlanSerialization.java | 33 +++++++++++++++++-- .../foundationdb/record/util/VectorUtils.java | 1 - .../src/main/proto/record_query_plan.proto | 23 +++++++++++++ .../relational/api/RelationalArray.java | 11 ++++++- .../relational/api/RelationalStruct.java | 10 +++++- .../relational/api/WithMetadata.java | 33 +++++++++++++++++++ .../recordlayer/query/AstNormalizer.java | 2 +- .../recordlayer/query/Literals.java | 1 + .../recordlayer/query/LiteralsUtils.java | 10 ++++-- .../query/MutablePlanGenerationContext.java | 33 ++++++++++++++----- .../recordlayer/query/OrderedLiteral.java | 1 + .../recordlayer/query/PreparedParams.java | 19 ++++++++++- .../server/jdbc/v1/JDBCService.java | 4 +++ .../src/main/resources/log4j2.xml | 6 ++-- .../yamltests/server/ExternalServer.java | 2 +- .../src/test/resources/in-predicate.yamsql | 22 +++++++++++++ 20 files changed, 221 insertions(+), 24 deletions(-) create mode 100644 fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/WithMetadata.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index f9ad976cef..04cfce69d2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -190,7 +190,7 @@ public static RecordKeyExpressionProto.Value toProtoValue(@Nullable Object value } else if (value instanceof byte[]) { builder.setBytesValue(ZeroCopyByteString.wrap((byte[])value)); } else if (value != null) { - throw new RecordCoreException("Unsupported value type").addLogInfo( + throw new RecordCoreException("Unsupported value type " + value.getClass()).addLogInfo( "value_type", value.getClass().getName()); } return builder.build(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index 2adfe8f5cb..c2ea478553 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.Bindings; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ObjectPlanHash; @@ -224,6 +225,8 @@ public static Object toClassWithRealEquals(@Nonnull Object obj) { return obj; } else if (obj instanceof List) { return obj; + } else if (obj instanceof RealVector) { + return obj; } else { throw new RecordCoreException("Tried to compare non-comparable object " + obj.getClass()); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index ed94a95ec4..0e26c0e6eb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -1268,6 +1268,12 @@ public ExplainTokens describe() { .addClosingParen(); } + @Nonnull + @Override + public String toString() { + return describe().render(DefaultExplainFormatter.forDebugging()).toString(); + } + @Override public void addProtoField(@Nonnull final TypeRepository.Builder typeRepositoryBuilder, @Nonnull final DescriptorProto.Builder descriptorBuilder, final int fieldNumber, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java index e0fde29b52..4cb1d1cb2f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java @@ -1042,6 +1042,24 @@ private enum BinaryPhysicalOperator { NOT_DISTINCT_FROM_NN(Comparisons.Type.NOT_DISTINCT_FROM, Type.TypeCode.NULL, Type.TypeCode.NULL, (l, r) -> true), + EQ_VEC_VEC(Comparisons.Type.EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, Objects::equals), + NEQ_VEC_VEC(Comparisons.Type.NOT_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, (l, r) -> !l.equals(r)), + NEQ_VEC_NULL(Comparisons.Type.NOT_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + NEQ_NULL_VEC(Comparisons.Type.NOT_EQUALS, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), + LT_VEC_NULL(Comparisons.Type.LESS_THAN, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + LT_NULL_VEC(Comparisons.Type.LESS_THAN, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), + LTE_VEC_NULL(Comparisons.Type.LESS_THAN_OR_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + LTE_NULL_VEC(Comparisons.Type.LESS_THAN_OR_EQUALS, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), + GT_VEC_NULL(Comparisons.Type.GREATER_THAN, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + GT_NULL_VEC(Comparisons.Type.GREATER_THAN, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), + GTE_VEC_NULL(Comparisons.Type.GREATER_THAN_OR_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + GTE_NULL_VEC(Comparisons.Type.GREATER_THAN_OR_EQUALS, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), + IS_DISTINCT_FROM_NULL_VEC(Comparisons.Type.IS_DISTINCT_FROM, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> Objects.nonNull(r)), + IS_DISTINCT_FROM_VEC_NULL(Comparisons.Type.IS_DISTINCT_FROM, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> Objects.nonNull(l)), + IS_DISTINCT_FROM_VEC_VEC(Comparisons.Type.IS_DISTINCT_FROM, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, (l, r) -> !l.equals(r)), + NOT_DISTINCT_FROM_VEC_NULL(Comparisons.Type.NOT_DISTINCT_FROM, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> Objects.isNull(l)), + NOT_DISTINCT_FROM_NULL_VEC(Comparisons.Type.NOT_DISTINCT_FROM, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> Objects.isNull(r)), + NOT_DISTINCT_FROM_VEC_VEC(Comparisons.Type.NOT_DISTINCT_FROM, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, Objects::equals), ; // We can pass down UUID or String till here. @@ -1148,7 +1166,10 @@ private enum UnaryPhysicalOperator { IS_NOT_NULL_ID(Comparisons.Type.NOT_NULL, Type.TypeCode.UUID, Objects::nonNull), IS_NULL_NT(Comparisons.Type.IS_NULL, Type.TypeCode.NULL, Objects::isNull), - IS_NOT_NULL_NT(Comparisons.Type.NOT_NULL, Type.TypeCode.NULL, Objects::nonNull); + IS_NOT_NULL_NT(Comparisons.Type.NOT_NULL, Type.TypeCode.NULL, Objects::nonNull), + + IS_NULL_VECTOR(Comparisons.Type.IS_NULL, Type.TypeCode.VECTOR, Objects::isNull), + IS_NOT_NULL_VECTOR(Comparisons.Type.NOT_NULL, Type.TypeCode.VECTOR, Objects::nonNull); @Nonnull private static final Supplier> protoEnumBiMapSupplier = diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java index 849e85239d..c855b82d89 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerialization.java @@ -21,16 +21,21 @@ package com.apple.foundationdb.record.query.plan.serialization; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; import com.apple.foundationdb.record.planprotos.PComparableObject; import com.apple.foundationdb.record.planprotos.PEnumLightValue; import com.apple.foundationdb.record.planprotos.PFDBRecordVersion; +import com.apple.foundationdb.record.planprotos.PType; import com.apple.foundationdb.record.planprotos.PUUID; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.util.ProtoUtils; +import com.apple.foundationdb.record.util.VectorUtils; import com.google.common.base.Verify; import com.google.common.collect.BiMap; import com.google.common.collect.EnumBiMap; @@ -69,24 +74,36 @@ public class PlanSerialization { */ @Nonnull public static PComparableObject valueObjectToProto(@Nullable final Object object) { + return valueObjectToProto(object, null); + } + + @Nonnull + public static PComparableObject valueObjectToProto(@Nullable final Object object, @Nullable PType typeProto) { final PComparableObject.Builder builder = PComparableObject.newBuilder(); if (object instanceof Internal.EnumLite) { builder.setEnumObject(PEnumLightValue.newBuilder() - .setName(object.toString()).setNumber(((Internal.EnumLite)object).getNumber())); + .setName(object.toString()).setNumber(((Internal.EnumLite)object).getNumber())); } else if (object instanceof UUID) { final UUID uuid = (UUID)object; builder.setUuid(PUUID.newBuilder() - .setMostSigBits(uuid.getMostSignificantBits()) - .setLeastSigBits(uuid.getLeastSignificantBits())) + .setMostSigBits(uuid.getMostSignificantBits()) + .setLeastSigBits(uuid.getLeastSignificantBits())) .build(); } else if (object instanceof FDBRecordVersion) { builder.setFdbRecordVersion(PFDBRecordVersion.newBuilder() .setRawBytes(ByteString.copyFrom(((FDBRecordVersion)object).toBytes(false))).build()); } else if (object instanceof ByteString) { builder.setBytesAsByteString((ByteString)object); + } else if (object instanceof RealVector) { + Verify.verifyNotNull(typeProto); + builder.setPrimitiveObject(RecordKeyExpressionProto.Value + .newBuilder().setBytesValue(ByteString.copyFrom(((RealVector)object).getRawData())).build()); } else { builder.setPrimitiveObject(LiteralKeyExpression.toProtoValue(object)); } + if (typeProto != null) { + builder.setType(typeProto); + } return builder.build(); } @@ -102,6 +119,16 @@ public static PComparableObject valueObjectToProto(@Nullable final Object object */ @Nullable public static Object protoToValueObject(@Nonnull final PComparableObject proto) { + if (proto.hasType()) { + // the only implementation that leverages the presence of the type field is Vector. + // this can be extended in the future to cover other types as well, such as + // FDBRecordVersion, and UUID instead of having special sub-message requirements + final var type = Type.fromTypeProto(PlanSerializationContext.newForCurrentMode(), proto.getType()); + Verify.verify(type.isVector()); + final var primitiveObject = proto.getPrimitiveObject(); + return VectorUtils.parseVector(primitiveObject.getBytesValue(), (Type.Vector)type); + } + if (proto.hasEnumObject()) { final PEnumLightValue enumProto = Objects.requireNonNull(proto.getEnumObject()); Verify.verify(enumProto.hasNumber()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java index 28a8ea2c8b..4610cc06e4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/util/VectorUtils.java @@ -29,7 +29,6 @@ import com.google.protobuf.ByteString; import javax.annotation.Nonnull; -import java.util.List; public class VectorUtils { public static int getVectorPrecision(@Nonnull final RealVector vector) { diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 3d0e63c430..548cf44855 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1173,6 +1173,25 @@ message PBinaryRelOpValue { NOT_DISTINCT_FROM_NID = 492; NOT_DISTINCT_FROM_IDN = 493; NOT_DISTINCT_FROM_NN = 494; + + EQ_VEC_VEC = 495; + NEQ_VEC_VEC = 496; + NEQ_VEC_NULL = 497; + NEQ_NULL_VEC = 498; + LT_VEC_NULL = 499; + LT_NULL_VEC = 500; + LTE_VEC_NULL = 501; + LTE_NULL_VEC = 502; + GT_VEC_NULL = 503; + GT_NULL_VEC = 504; + GTE_VEC_NULL = 505; + GTE_NULL_VEC = 506; + IS_DISTINCT_FROM_NULL_VEC = 507; + IS_DISTINCT_FROM_VEC_NULL = 508; + IS_DISTINCT_FROM_VEC_VEC = 509; + NOT_DISTINCT_FROM_VEC_NULL = 510; + NOT_DISTINCT_FROM_NULL_VEC = 511; + NOT_DISTINCT_FROM_VEC_VEC = 512; } optional PRelOpValue super = 1; optional PBinaryPhysicalOperator operator = 2; @@ -1216,6 +1235,9 @@ message PUnaryRelOpValue { IS_NULL_NT = 21; IS_NOT_NULL_NT = 22; + + IS_NULL_VECTOR = 23; + IS_NOT_NULL_VECTOR = 24; } optional PRelOpValue super = 1; optional PUnaryPhysicalOperator operator = 2; @@ -2205,6 +2227,7 @@ message PComparableObject { PFDBRecordVersion fdb_record_version = 4; bytes bytes_as_byte_string = 5; } + optional PType type = 6; } // diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalArray.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalArray.java index 08d6b9800a..dfc523f261 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalArray.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalArray.java @@ -20,6 +20,9 @@ package com.apple.foundationdb.relational.api; +import com.apple.foundationdb.relational.api.metadata.DataType; + +import javax.annotation.Nonnull; import java.sql.Array; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -27,7 +30,7 @@ import java.util.ArrayList; import java.util.Map; -public interface RelationalArray extends Array, Wrapper { +public interface RelationalArray extends Array, Wrapper, WithMetadata { ArrayMetaData getMetaData() throws SQLException; @@ -97,4 +100,10 @@ default T unwrap(Class iface) throws SQLException { default boolean isWrapperFor(Class iface) throws SQLException { return iface.isInstance(this); } + + @Nonnull + @Override + default DataType getRelationalMetaData() throws SQLException { + return getMetaData().asRelationalType(); + } } diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalStruct.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalStruct.java index 71bab66022..20fc1ad7a2 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalStruct.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/RelationalStruct.java @@ -22,7 +22,9 @@ import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.InvalidColumnReferenceException; +import com.apple.foundationdb.relational.api.metadata.DataType; +import javax.annotation.Nonnull; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Struct; @@ -33,7 +35,7 @@ /** * A {@link Struct} but with metadata describing the instance. */ -public interface RelationalStruct extends Struct, Wrapper { +public interface RelationalStruct extends Struct, Wrapper, WithMetadata { StructMetaData getMetaData() throws SQLException; @@ -138,6 +140,12 @@ default boolean isWrapperFor(Class iface) throws SQLException { return iface.isInstance(this); } + @Nonnull + @Override + default DataType getRelationalMetaData() throws SQLException { + return getMetaData().getRelationalDataType(); + } + /** * Utility for figuring out one-based-position of a column when passed columnName. * @param columnName Name to lookup. diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/WithMetadata.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/WithMetadata.java new file mode 100644 index 0000000000..13c955b302 --- /dev/null +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/WithMetadata.java @@ -0,0 +1,33 @@ +/* + * WithMetadata.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.api; + +import com.apple.foundationdb.relational.api.metadata.DataType; + +import javax.annotation.Nonnull; +import java.sql.SQLException; + +public interface WithMetadata { + + @Nonnull + DataType getRelationalMetaData() throws SQLException; + +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java index 35c18b6952..ae5f465e3c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java @@ -556,7 +556,7 @@ private void processLiteral(@Nonnull final Object literal, final int tokenIndex, @Nullable final Integer unnamedParameterIndex, @Nullable final String parameterName) { if (allowLiteralAddition) { queryHasherContextBuilder.getLiteralsBuilder() - .addLiteral(Type.any(), literal, unnamedParameterIndex, parameterName, tokenIndex); + .addLiteral(Type.fromObject(literal), literal, unnamedParameterIndex, parameterName, tokenIndex); } if (allowTokenAddition) { final String canonicalName = parameterName == null ? "?" : "?" + parameterName; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java index 64f39bf1d6..c0a8989076 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LiteralsUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LiteralsUtils.java index a467b3a927..de001d36fe 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LiteralsUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LiteralsUtils.java @@ -21,6 +21,8 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.RealVector; +import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization; @@ -76,13 +78,17 @@ public static LiteralObject objectToLiteralObjectProto(@Nonnull final Type type, final var message = (Message) object; builder.setRecordObject(message.toByteString()); } else if (type.isArray()) { - final var elementType = Objects.requireNonNull(((Type.Array) type).getElementType()); - final var array = (List) object; + final var elementType = Objects.requireNonNull(((Type.Array)type).getElementType()); + final var array = (List)object; final var arrayBuilder = LiteralObject.Array.newBuilder(); for (final Object element : array) { arrayBuilder.addElementObjects(objectToLiteralObjectProto(elementType, element)); } builder.setArrayObject(arrayBuilder.build()); + } else if (type.isVector() || object instanceof RealVector) { + final var typeProto = type.isVector() ? type.toTypeProto(PlanSerializationContext.newForCurrentMode()) + : Type.fromObject(object).toTypeProto(PlanSerializationContext.newForCurrentMode()); + builder.setScalarObject(PlanSerialization.valueObjectToProto(object, typeProto)); } else { // scalar builder.setScalarObject(PlanSerialization.valueObjectToProto(object)); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java index db31f0e630..232f7c786c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalStruct; +import com.apple.foundationdb.relational.api.WithMetadata; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.util.Assert; @@ -247,21 +248,21 @@ private Value processQueryLiteralOrParameter(@Nonnull Type type, @Nonnull private Value processPreparedStatementParameter(@Nullable Object param, - @Nullable Type type, + @Nonnull Type type, @Nullable Integer unnamedParameterIndex, @Nullable String parameterName, int tokenIndex) { if (param instanceof Array) { - Assert.thatUnchecked(type == null || type.isArray(), DATATYPE_MISMATCH, "Array type field required as prepared statement parameter"); + Assert.thatUnchecked(type.isArray(), DATATYPE_MISMATCH, "Array type field required as prepared statement parameter instead of " + type); return processPreparedStatementArrayParameter((Array)param, (Type.Array)type, unnamedParameterIndex, parameterName, tokenIndex); } else if (param instanceof Struct) { - Assert.thatUnchecked(type == null || type.isRecord(), DATATYPE_MISMATCH, "Required type field required as prepared statement parameter"); + Assert.thatUnchecked(type.isRecord(), DATATYPE_MISMATCH, "Required type field required as prepared statement parameter instead of " + type); return processPreparedStatementStructParameter((Struct)param, (Type.Record)type, unnamedParameterIndex, parameterName, tokenIndex); } else if (param instanceof byte[]) { return processQueryLiteralOrParameter(Type.primitiveType(Type.TypeCode.BYTES), ZeroCopyByteString.wrap((byte[])param), unnamedParameterIndex, parameterName, tokenIndex); } else { - return processQueryLiteralOrParameter(type == null ? Type.any() : type, param, unnamedParameterIndex, parameterName, tokenIndex); + return processQueryLiteralOrParameter(type, param, unnamedParameterIndex, parameterName, tokenIndex); } } @@ -465,15 +466,31 @@ public void importAuxiliaryLiterals(@Nonnull final Literals auxiliaryLiterals) { public Value processNamedPreparedParam(@Nonnull String param, int tokenIndex) { final var value = preparedParams.namedParamValue(param); //TODO type should probably be Type.any() instead of null - return processPreparedStatementParameter(value, null, null, param, tokenIndex); + return processPreparedStatementParameter(value, getObjectType(value), null, param, tokenIndex); } @Nonnull public Value processUnnamedPreparedParam(int tokenIndex) { // TODO (Make prepared statement parameters stateless) final int currentUnnamedParameterIndex = preparedParams.currentUnnamedParamIndex(); - final var param = preparedParams.nextUnnamedParamValue(); - //TODO type should probably be Type.any() instead of null - return processPreparedStatementParameter(param, null, currentUnnamedParameterIndex, null, tokenIndex); + final Object param; + if (!shouldProcessLiteral()) { + param = PreparedParams.copyOf(preparedParams, true).nextUnnamedParamValue(); + } else { + param = preparedParams.nextUnnamedParamValue(); + } + return processPreparedStatementParameter(param, getObjectType(param), currentUnnamedParameterIndex, null, tokenIndex); + } + + @Nonnull + private static Type getObjectType(@Nullable final Object object) { + if (object instanceof WithMetadata) { + try { + return DataTypeUtils.toRecordLayerType(((WithMetadata)object).getRelationalMetaData()); + } catch (SQLException e) { + throw new RelationalException(e).toUncheckedWrappedException(); + } + } + return Type.fromObject(object); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java index 935671eff7..f170f66fa9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.relational.continuation.TypedQueryArgument; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PreparedParams.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PreparedParams.java index 089ade9cd6..3b31aba969 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PreparedParams.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PreparedParams.java @@ -55,6 +55,14 @@ private PreparedParams(@Nonnull Map unnamedParams, this.namedParams = namedParameters; } + private PreparedParams(@Nonnull Map unnamedParams, + @Nonnull Map namedParameters, + int nextParam) { + this.unnamedParams = unnamedParams; + this.namedParams = namedParameters; + this.nextParam = nextParam; + } + public int currentUnnamedParamIndex() { return nextParam; } @@ -102,6 +110,15 @@ public static PreparedParams ofNamed(@Nonnull Map parameters) { @Nonnull public static PreparedParams copyOf(@Nonnull PreparedParams other) { - return new PreparedParams(other.unnamedParams, other.namedParams); + return copyOf(other, false); + } + + @Nonnull + public static PreparedParams copyOf(@Nonnull PreparedParams other, boolean withCurrentUnnamedParamIndex) { + if (withCurrentUnnamedParamIndex) { + return new PreparedParams(other.unnamedParams, other.namedParams, other.currentUnnamedParamIndex()); + } else { + return new PreparedParams(other.unnamedParams, other.namedParams); + } } } diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java index 8eaa01cd7e..53a08fd756 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java @@ -100,8 +100,12 @@ public void execute(StatementRequest request, StreamObserver responseObserver.onNext(statementResponseBuilder.build()); responseObserver.onCompleted(); } catch (SQLException e) { + System.out.println("WHOOOOPS"); + e.printStackTrace(); responseObserver.onError(StatusProto.toStatusRuntimeException(GrpcSQLExceptionUtil.create(e))); } catch (RuntimeException e) { + System.out.println("WHOOOOPSY DAISY"); + e.printStackTrace(); throw handleUncaughtException(e); } } diff --git a/fdb-relational-server/src/main/resources/log4j2.xml b/fdb-relational-server/src/main/resources/log4j2.xml index 0b4610117f..d5e5efd7e3 100644 --- a/fdb-relational-server/src/main/resources/log4j2.xml +++ b/fdb-relational-server/src/main/resources/log4j2.xml @@ -19,17 +19,17 @@ limitations under the License. --> - + - + - + diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java index 76413c49c5..9d061d354a 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java @@ -44,7 +44,7 @@ public class ExternalServer { private static final Logger logger = LogManager.getLogger(ExternalServer.class); public static final String EXTERNAL_SERVER_PROPERTY_NAME = "yaml_testing_external_server"; - private static final boolean SAVE_SERVER_OUTPUT = false; + private static final boolean SAVE_SERVER_OUTPUT = true; @Nonnull private final File serverJar; diff --git a/yaml-tests/src/test/resources/in-predicate.yamsql b/yaml-tests/src/test/resources/in-predicate.yamsql index 64eb0a6c5b..baa026e3d2 100644 --- a/yaml-tests/src/test/resources/in-predicate.yamsql +++ b/yaml-tests/src/test/resources/in-predicate.yamsql @@ -249,6 +249,7 @@ test_block: # "expected" as the point of error is different in both the cases. # https://github.com/FoundationDB/fdb-record-layer/issues/3583 test_block: + preset: single_repetition_ordered tests: - # LONG value matched against IN list of LONG and DOUBLE values @@ -257,29 +258,50 @@ test_block: # clear whether we want to support it as it would probably # make plan lookup in the cache much more expensive. - query: select a, b from ta where b in (1, 3.0, 5, 7.0) + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in ('foo' , 35) + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in (35, '23') + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in (true, 75.34) + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Left type (STRING) is not compatible with right type (type of the IN List) - query: select a, e from ta where e in (35, 75.34) + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Left type (STRING) is not compatible with right type (type of the IN List) - query: select a, e from ta where e in (35.34, 32) + - initialVersionLessThan: !current_version - error: "XX000" + - initialVersionAtLeast: !current_version + - error: "42804" - # Complex (STRUCT) left type matched against IN list - query: select a, e from ta where f in (34, 23) + - initialVersionLessThan: 4.8.2.0 + - error: "XX000" + - initialVersionAtLeast: 4.8.2.0 - error: "22000" ... From f604daa4a20188c3decad56ba415e6051416de5b Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Wed, 29 Oct 2025 19:09:29 +0000 Subject: [PATCH 07/15] Refactoring, fixes, and code cleanup. --- .../expressions/SelectExpression.java | 2 +- .../query/plan/cascades/typing/Type.java | 12 ++- .../query/plan/cascades/values/CastValue.java | 7 +- .../values/RecordConstructorValue.java | 3 +- .../relational/api/RowStruct.java | 2 +- .../relational/recordlayer/MessageTuple.java | 15 ++- .../recordlayer/query/Literals.java | 1 - .../recordlayer/query/OrderedLiteral.java | 1 - .../jdbc/RelationalArrayFacade.java | 29 +---- .../jdbc/RelationalResultSetFacade.java | 20 +--- .../jdbc/RelationalStructFacade.java | 27 ++++- .../relational/jdbc/TypeConversion.java | 24 +++++ .../server/jdbc/v1/JDBCService.java | 4 - yaml-tests/src/test/resources/vector.yamsql | 101 ++++++++++++++++++ 14 files changed, 183 insertions(+), 65 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java index 68ccce77d4..518b578d5b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java @@ -621,7 +621,7 @@ public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List 30 ? String.format(Locale.ROOT, "%02x", predicateString.hashCode()) : predicateString; + final var abbreviatedPredicateString = predicateString.length() > 3000 ? String.format(Locale.ROOT, "%02x", predicateString.hashCode()) : predicateString; return PlannerGraph.fromNodeAndChildGraphs( new PlannerGraph.LogicalOperatorNode(this, "SELECT " + resultValue, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 0e26c0e6eb..435216a2b1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -447,7 +447,7 @@ private static Type fromProtoType(@Nullable Descriptors.GenericDescriptor descri // find TypeCode of array elements final var elementField = messageDescriptor.findFieldByName(NullableArrayTypeUtils.getRepeatedFieldName()); final var elementTypeCode = TypeCode.fromProtobufFieldDescriptor(elementField.getType(), elementField.getOptions()); - return fromProtoTypeToArray(descriptor, protoType, elementTypeCode, fieldOptions, true); + return fromProtoTypeToArray(descriptor, protoType, elementTypeCode, elementField.getOptions(), true); } else if (TupleFieldsProto.UUID.getDescriptor().equals(messageDescriptor)) { return Type.uuidType(isNullable); } else { @@ -471,8 +471,14 @@ private static Array fromProtoTypeToArray(@Nullable Descriptors.GenericDescripto @Nullable DescriptorProtos.FieldOptions fieldOptions, boolean isNullable) { if (typeCode.isPrimitive()) { - final var primitiveType = primitiveType(typeCode, false); - return new Array(isNullable, primitiveType); + final Type type; + if (typeCode == TypeCode.VECTOR) { + final var vectorOptions = Objects.requireNonNull(fieldOptions).getExtension(RecordMetaDataOptionsProto.field).getVectorOptions(); + type = Type.Vector.of(false, vectorOptions.getPrecision(), vectorOptions.getDimensions()); + } else { + type = primitiveType(typeCode, false); + } + return new Array(isNullable, type); } else if (typeCode == TypeCode.ENUM) { final var enumDescriptor = (Descriptors.EnumDescriptor)Objects.requireNonNull(descriptor); final var enumType = Enum.fromProtoValues(false, enumDescriptor.getValues()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java index a5bd0b5f83..0ddfa7e9ed 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/CastValue.java @@ -563,7 +563,7 @@ private static Object castArrayToArray(Object descriptor, Object in) { SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "Target array element type cannot be null"); } - final var inputList = (java.util.List) in; + final var inputList = (List) in; // Handle empty arrays - return empty list of target type if (inputList.isEmpty()) { @@ -608,13 +608,14 @@ private static Object castArrayToVector(Object descriptor, Object in) { Verify.verify(targetArrayType.isVector()); final var targetVectorType = (Type.Vector)targetArrayType; - final var inputList = (java.util.List) in; + final var inputList = (List) in; if (sourceElementType == null) { SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "Source array element type cannot be null"); } - if (inputList.size() != targetVectorType.getDimensions()) { + int numDimensions = targetVectorType.getDimensions(); + if (inputList.size() != numDimensions) { SemanticException.fail(SemanticException.ErrorCode.INVALID_CAST, "Source array is not the same size of the vector"); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java index 70fb80495e..b7e29406d8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RecordConstructorValue.java @@ -188,7 +188,8 @@ public static Object deepCopyIfNeeded(@Nonnull TypeRepository typeRepository, final var objects = (List)field; final var elementType = Verify.verifyNotNull(((Type.Array)fieldType).getElementType()); if (elementType.isPrimitive()) { - if (elementType.getTypeCode() == Type.TypeCode.BYTES || elementType.getTypeCode() == Type.TypeCode.VERSION) { + final var elementTypeCode = elementType.getTypeCode(); + if (elementTypeCode == Type.TypeCode.VECTOR || elementTypeCode == Type.TypeCode.BYTES || elementTypeCode == Type.TypeCode.VERSION) { var resultBuilder = ImmutableList.builderWithExpectedSize(objects.size()); for (Object object : objects) { resultBuilder.add(protoObjectForPrimitive(elementType, object)); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/api/RowStruct.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/api/RowStruct.java index e2974b6c97..6edc85fe4c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/api/RowStruct.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/api/RowStruct.java @@ -258,7 +258,7 @@ public RelationalArray getArray(int oneBasedPosition) throws SQLException { final var fieldValues = (Collection) message.getField(fieldDescriptor); final var elements = new ArrayList<>(); for (var fieldValue : fieldValues) { - final var sanitizedFieldValue = MessageTuple.sanitizeField(fieldValue); + final var sanitizedFieldValue = MessageTuple.sanitizeField(fieldValue, fieldDescriptor.getOptions()); if (sanitizedFieldValue instanceof Message) { elements.add(new ImmutableRowStruct(new MessageTuple((Message) sanitizedFieldValue), arrayMetaData.getElementStructMetaData())); } else { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java index b38e06260a..1276192c5d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/MessageTuple.java @@ -31,6 +31,7 @@ import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.relational.api.exceptions.InvalidColumnReferenceException; import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -66,10 +67,10 @@ public Object getObject(int position) throws InvalidColumnReferenceException { } if (fieldDescriptor.isRepeated()) { final var list = (List) fieldValue; - return list.stream().map(MessageTuple::sanitizeField).collect(Collectors.toList()); + return list.stream().map(arrayItem -> sanitizeField(arrayItem, fieldDescriptor.getOptions())).collect(Collectors.toList()); } if (message.hasField(fieldDescriptor)) { - return sanitizeField(fieldValue); + return sanitizeField(fieldValue, fieldDescriptor.getOptions()); } else { return null; } @@ -90,7 +91,7 @@ private static RealVector getVectorFromBytes(@Nonnull final ByteString byteStrin throw new RecordCoreException("unexpected vector precision " + precision); } - public static Object sanitizeField(final Object field) { + public static Object sanitizeField(@Nonnull final Object field, @Nonnull final DescriptorProtos.FieldOptions fieldOptions) { if (field instanceof Message && ((Message) field).getDescriptorForType().equals(TupleFieldsProto.UUID.getDescriptor())) { return TupleFieldsHelper.fromProto((Message) field, TupleFieldsProto.UUID.getDescriptor()); } @@ -98,7 +99,13 @@ public static Object sanitizeField(final Object field) { return ((Descriptors.EnumValueDescriptor) field).getName(); } if (field instanceof ByteString) { - return ((ByteString) field).toByteArray(); + final var byteString = (ByteString) field; + final var fieldVectorOptionsMaybe = fieldOptions.getExtension(RecordMetaDataOptionsProto.field); + if (fieldVectorOptionsMaybe.hasVectorOptions()) { + final var precision = fieldVectorOptionsMaybe.getVectorOptions().getPrecision(); + return getVectorFromBytes(byteString, precision); + } + return byteString.toByteArray(); } return field; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java index c0a8989076..64f39bf1d6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java @@ -20,7 +20,6 @@ package com.apple.foundationdb.relational.recordlayer.query; -import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java index f170f66fa9..935671eff7 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/OrderedLiteral.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.relational.continuation.TypedQueryArgument; diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalArrayFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalArrayFacade.java index 212e069385..d26b5ce865 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalArrayFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalArrayFacade.java @@ -179,37 +179,14 @@ public RelationalResultSet getResultSet(long oneBasedIndex, int askedForCount) t int count = getCount(askedForCount, this.delegate.getElementCount(), index); var resultSetBuilder = ResultSet.newBuilder(); - final var componentType = this.delegateMetadata.getType(); - final var componentSqlType = this.delegateMetadata.getJavaSqlTypesCode(); - final var componentColumnBuilder = ColumnMetadata.newBuilder().setName("VALUE").setType(componentType).setJavaSqlTypesCode(componentSqlType); - if (componentType == Type.ARRAY) { - componentColumnBuilder.setArrayMetadata(this.delegateMetadata.getArrayMetadata()); - } else if (componentType == Type.STRUCT) { - componentColumnBuilder.setStructMetadata(this.delegateMetadata.getStructMetadata()); - } + final var columnMetadata = delegateMetadata.toBuilder().setName("VALUE").build(); resultSetBuilder.setMetadata(ResultSetMetadata.newBuilder().setColumnMetadata(ListColumnMetadata.newBuilder() .addColumnMetadata(ColumnMetadata.newBuilder().setName("INDEX").setType(Type.INTEGER).setJavaSqlTypesCode(Types.INTEGER).build()) - .addColumnMetadata(componentColumnBuilder.build()).build()).build()); + .addColumnMetadata(columnMetadata).build()).build()); for (int i = index; i < count; i++) { final var listColumnBuilder = ListColumn.newBuilder(); listColumnBuilder.addColumn(Column.newBuilder().setInteger(i + 1).build()); - final var valueColumnBuilder = Column.newBuilder(); - if (componentType == Type.STRUCT) { - valueColumnBuilder.setStruct(delegate.getElement(i).getStruct()); - } else if (componentType == Type.INTEGER) { - valueColumnBuilder.setInteger(delegate.getElement(i).getInteger()); - } else if (componentSqlType == Types.VARCHAR) { - valueColumnBuilder.setString(delegate.getElement(i).getString()); - } else if (componentSqlType == Types.DOUBLE) { - valueColumnBuilder.setDouble(delegate.getElement(i).getDouble()); - } else if (componentSqlType == Types.BOOLEAN) { - valueColumnBuilder.setBoolean(delegate.getElement(i).getBoolean()); - } else { - Assert.failUnchecked(ErrorCode.UNKNOWN_TYPE, "Type not supported: " + componentType.name()); - } - resultSetBuilder.addRow(Struct.newBuilder() - .setColumns(listColumnBuilder - .addColumn(valueColumnBuilder.build())).build()); + resultSetBuilder.addRow(Struct.newBuilder().setColumns(listColumnBuilder.addColumn(delegate.getElement(i))).build()); } return new RelationalResultSetFacade(resultSetBuilder.build()); } diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java index 712036f0f8..56415bec14 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalResultSetFacade.java @@ -20,10 +20,6 @@ package com.apple.foundationdb.relational.jdbc; -import com.apple.foundationdb.linear.DoubleRealVector; -import com.apple.foundationdb.linear.FloatRealVector; -import com.apple.foundationdb.linear.HalfRealVector; -import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalResultSet; @@ -357,7 +353,7 @@ public Object getObject(int oneBasedColumn) throws SQLException { o = getString(oneBasedColumn); break; case VECTOR: - o = parseVector(getBytes(oneBasedColumn), ((DataType.VectorType)relationalType).getPrecision()); + o = TypeConversion.parseVector(getBytes(oneBasedColumn), ((DataType.VectorType)relationalType).getPrecision()); break; default: throw new SQLException("Unsupported type " + type); @@ -404,18 +400,4 @@ public T unwrap(Class iface) throws SQLException { public boolean isWrapperFor(Class iface) throws SQLException { return iface.isInstance(this); } - - @Nonnull - public static RealVector parseVector(@Nonnull final byte[] bytes, int precision) throws SQLException { - if (precision == 16) { - return HalfRealVector.fromBytes(bytes); - } - if (precision == 32) { - return FloatRealVector.fromBytes(bytes); - } - if (precision == 64) { - return DoubleRealVector.fromBytes(bytes); - } - throw new SQLException("unexpected vector type with precision " + precision); - } } diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalStructFacade.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalStructFacade.java index 5da43330bc..c7ee6d98ab 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalStructFacade.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/RelationalStructFacade.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.jdbc; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.RelationalArray; import com.apple.foundationdb.relational.api.RelationalStruct; import com.apple.foundationdb.relational.api.RelationalStructBuilder; @@ -34,6 +35,7 @@ import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Struct; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Type; import com.apple.foundationdb.relational.jdbc.grpc.v1.column.Uuid; +import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.util.PositionalIndex; import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings; import com.google.common.annotations.VisibleForTesting; @@ -263,7 +265,18 @@ public Object getObject(int oneBasedColumn) throws SQLException { obj = getInt(oneBasedColumn); break; case Types.OTHER: - obj = getUUID(oneBasedColumn); + final var column = getMetaData().getRelationalDataType().getFields().get(oneBasedColumn - 1 + getMetaData().getLeadingPhantomColumnCount()); + final var columnRelationalType = column.getType(); + switch (columnRelationalType.getCode()) { + case UUID: + obj = getUUID(oneBasedColumn); + break; + case VECTOR: + obj = getVector(oneBasedColumn, Assert.castUnchecked(columnRelationalType, DataType.VectorType.class)); + break; + default: + throw new SQLException("Unsupported object type: " + type); + } break; default: throw new SQLException("Unsupported object type: " + type); @@ -326,6 +339,18 @@ public UUID getUUID(final int oneBasedPosition) throws SQLException { return new UUID(c.getUuid().getMostSignificantBits(), c.getUuid().getLeastSignificantBits()); } + @Nullable + private RealVector getVector(final int oneBasedPosition, @Nonnull final DataType.VectorType type) throws SQLException { + Column c = getColumnInternal(oneBasedPosition); + if (wasNull) { + return null; + } + if (!(c.hasBinary())) { + throw new SQLException("Vector", ErrorCode.CANNOT_CONVERT_TYPE.getErrorCode()); + } + return TypeConversion.parseVector(c.getBinary().toByteArray(), type.getPrecision()); + } + @Override public UUID getUUID(final String fieldName) throws SQLException { return getUUID(RelationalStruct.getOneBasedPosition(fieldName, this)); diff --git a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java index 29b7b47e4e..e408779dd0 100644 --- a/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java +++ b/fdb-relational-grpc/src/main/java/com/apple/foundationdb/relational/jdbc/TypeConversion.java @@ -21,6 +21,9 @@ package com.apple.foundationdb.relational.jdbc; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.linear.DoubleRealVector; +import com.apple.foundationdb.linear.FloatRealVector; +import com.apple.foundationdb.linear.HalfRealVector; import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.relational.api.ArrayMetaData; import com.apple.foundationdb.relational.api.Continuation; @@ -255,6 +258,7 @@ private static ColumnMetadata toColumnMetadata(@Nonnull ArrayMetaData metadata) .setJavaSqlTypesCode(metadata.getElementType()) .setType(toProtobufType(metadata.asRelationalType().getElementType())) .setNullable(metadata.isElementNullable() == DatabaseMetaData.columnNullable); + final var elementRelationalType = metadata.asRelationalType().getElementType(); // TODO phantom. // TODO: label // One-offs @@ -267,6 +271,12 @@ private static ColumnMetadata toColumnMetadata(@Nonnull ArrayMetaData metadata) var columnMetadata = toColumnMetadata(metadata.getElementArrayMetaData()); columnMetadataBuilder.setArrayMetadata(columnMetadata); break; + case Types.OTHER: + if (elementRelationalType.getCode() == DataType.Code.VECTOR) { + var vectorMetadata = toVectorMetadata((DataType.VectorType)elementRelationalType); + columnMetadataBuilder.setVectorMetadata(vectorMetadata); + } + break; default: break; } @@ -908,4 +918,18 @@ static DataType getDataType(@Nonnull Type type, @Nonnull ColumnMetadata columnMe throw new RelationalException("Not implemeneted: " + type.name(), ErrorCode.INTERNAL_ERROR).toUncheckedWrappedException(); } } + + @Nonnull + public static RealVector parseVector(@Nonnull final byte[] bytes, int precision) throws SQLException { + if (precision == 16) { + return HalfRealVector.fromBytes(bytes); + } + if (precision == 32) { + return FloatRealVector.fromBytes(bytes); + } + if (precision == 64) { + return DoubleRealVector.fromBytes(bytes); + } + throw new SQLException("unexpected vector type with precision " + precision); + } } diff --git a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java index 53a08fd756..8eaa01cd7e 100644 --- a/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java +++ b/fdb-relational-server/src/main/java/com/apple/foundationdb/relational/server/jdbc/v1/JDBCService.java @@ -100,12 +100,8 @@ public void execute(StatementRequest request, StreamObserver responseObserver.onNext(statementResponseBuilder.build()); responseObserver.onCompleted(); } catch (SQLException e) { - System.out.println("WHOOOOPS"); - e.printStackTrace(); responseObserver.onError(StatusProto.toStatusRuntimeException(GrpcSQLExceptionUtil.create(e))); } catch (RuntimeException e) { - System.out.println("WHOOOOPSY DAISY"); - e.printStackTrace(); throw handleUncaughtException(e); } } diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index d641e37a0a..5e17f144e9 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -21,9 +21,13 @@ options: supported_version: !current_version --- schema_template: + create type as struct embeddingStruct(model string, vdata vector(3, half)) + create type as struct groupedEmbeddingStruct(model string, vdatas vector(3, float) array) create table tHalfVector(a bigint, b vector(3, half), primary key(a)) create table tFloatVector(a bigint, b vector(3, float), primary key(a)) create table tDoubleVector(a bigint, b vector(3, double), primary key(a)) + create table tWithStruct(a bigint, embedding embeddingStruct, primary key(a)) + create table tWithGroupedStruct(a bigint, embeddingGrouped groupedEmbeddingStruct, primary key(a)) --- test_block: preset: single_repetition_ordered @@ -346,3 +350,100 @@ test_block: - - query: select cast([1.0, 2.0] as VECTOR(3, DOUBLE)) from tDoubleVector where a = 100 - error: "22F3H" + # Tests for tWithStruct (struct with HALF vector field) + # Insert struct with vector + - + - query: insert into tWithStruct values (100, ('model1', !! !v16 [1.1, 2.2, 3.3] !!)) + - count: 1 + - + - query: select * from tWithStruct + - result: [{100, {'model1', !v16 [1.1, 2.2, 3.3]}}] + - + - query: select embedding.vdata from tWithStruct where a = 100 + - result: [{!v16 [1.1, 2.2, 3.3]}] + - + - query: select embedding.model from tWithStruct where a = 100 + - result: [{'model1'}] + # Insert NULL struct + - + - query: insert into tWithStruct values (200, null) + - count: 1 + - + - query: select embedding.vdata is null from tWithStruct where a = 200 + - result: [{true}] + - + - query: select embedding.vdata is not null from tWithStruct where a = 200 + - result: [{false}] + # Insert struct with NULL vector field + - + - query: insert into tWithStruct values (300, ('model2', null)) + - count: 1 + - + - query: select embedding.vdata is null from tWithStruct where a = 300 + - result: [{true}] + - + - query: select embedding.model from tWithStruct where a = 300 + - result: [{'model2'}] + # Filter using struct field with vector + - + - query: select a from tWithStruct where embedding.vdata = !! !v16 [1.1, 2.2, 3.3] !! + - result: [{100}] + - + - query: select a from tWithStruct where embedding.vdata != !! !v16 [1.1, 2.2, 3.3] !! + - result: [] + #- + # https://github.com/FoundationDB/fdb-record-layer/issues/3700 + #- query: select a from tWithStruct where embedding is null + #- result: [{200}] + - + - query: select a from tWithStruct where embedding.vdata is not null + - result: [{100}] + # IS DISTINCT FROM with struct vector field + - + - query: select embedding.vdata is distinct from !! !v16 [1.1, 2.2, 3.3] !! from tWithStruct where a = 100 + - result: [{false}] + - + - query: select embedding.vdata is distinct from !! !v16 [9.9, 9.9, 9.9] !! from tWithStruct where a = 100 + - result: [{true}] + - + - query: select embedding.vdata is distinct from null from tWithStruct where a = 300 + - result: [{false}] + # IS NOT DISTINCT FROM with struct vector field + - + - query: select embedding.vdata is not distinct from !! !v16 [1.1, 2.2, 3.3] !! from tWithStruct where a = 100 + - result: [{true}] + - + - query: select embedding.vdata is not distinct from !! !v16 [9.9, 9.9, 9.9] !! from tWithStruct where a = 100 + - result: [{false}] + - + - query: select embedding.vdata is not distinct from null from tWithStruct where a = 300 + - result: [{true}] + # Tests for tWithGroupedStruct (struct with FLOAT vector array field) + # Insert struct with vector array + - + - query: insert into tWithGroupedStruct values (100, ('model1', [ !! !v32 [1.1, 2.2, 3.3] !! , !! !v32 [4.4, 5.5, 6.6] !! ] ) ) + - count: 1 + - + - query: select * from tWithGroupedStruct + - result: [{100, {'model1', [!v32 [1.1, 2.2, 3.3], !v32 [4.4, 5.5, 6.6]]}}] + - + - query: select embeddingGrouped.vdatas from tWithGroupedStruct where a = 100 + - result: [{[!v32 [1.1, 2.2, 3.3], !v32 [4.4, 5.5, 6.6]]}] + - + - query: select embeddingGrouped.model from tWithGroupedStruct where a = 100 + - result: [{'model1'}] + # Insert NULL struct + - + - query: insert into tWithGroupedStruct values (200, null) + - count: 1 + #- + # https://github.com/FoundationDB/fdb-record-layer/issues/3700 + #- query: select embeddingGrouped is null from tWithGroupedStruct where a = 200 + #- result: [{true}] + # Insert struct with single vector in array + - + - query: insert into tWithGroupedStruct values (300, ('model4', [ !! !v32 [7.7, 8.8, 9.9] !! ])) + - count: 1 + - + - query: select embeddingGrouped.vdatas from tWithGroupedStruct where a = 300 + - result: [{[!v32 [7.7, 8.8, 9.9]]}] From ab0487449baffd066523aaaabe8f22b7167b14f8 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 11:27:16 +0000 Subject: [PATCH 08/15] Fix failing tests. --- .../expressions/SelectExpression.java | 2 +- .../plan/cascades/TypeRepositoryTest.java | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java index 518b578d5b..68ccce77d4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java @@ -621,7 +621,7 @@ public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List 3000 ? String.format(Locale.ROOT, "%02x", predicateString.hashCode()) : predicateString; + final var abbreviatedPredicateString = predicateString.length() > 30 ? String.format(Locale.ROOT, "%02x", predicateString.hashCode()) : predicateString; return PlannerGraph.fromNodeAndChildGraphs( new PlannerGraph.LogicalOperatorNode(this, "SELECT " + resultValue, diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/TypeRepositoryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/TypeRepositoryTest.java index 97362e1be3..2099848626 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/TypeRepositoryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/TypeRepositoryTest.java @@ -20,6 +20,8 @@ package com.apple.foundationdb.record.query.plan.cascades; +import com.apple.foundationdb.linear.HalfRealVector; +import com.apple.foundationdb.linear.RealVector; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.TupleFieldsProto; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -29,6 +31,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.util.pair.Pair; +import com.google.protobuf.ByteString; import com.google.protobuf.DynamicMessage; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -67,6 +70,7 @@ class TypeRepositoryTest { private static final LiteralValue FLOAT_1 = new LiteralValue<>(Type.primitiveType(Type.TypeCode.FLOAT), 1.0F); private static final LiteralValue STRING_1 = new LiteralValue<>(Type.primitiveType(Type.TypeCode.STRING), "a"); private static final LiteralValue UUID_1 = new LiteralValue<>(Type.uuidType(false), UUID.fromString("eebee473-690b-48c1-beb0-07c3aca77768")); + private static final LiteralValue HALF_VECTOR_1_2_3 = new LiteralValue<>(Type.Vector.of(false, 16, 3), new HalfRealVector(new double[] {1.0d, 2.0d, 3.0d})); private static Type generateRandomType() { return generateRandomTypeInternal(0); @@ -140,6 +144,8 @@ private static Type generateType(int depth, Type.TypeCode requestedTypeCode) { return Type.Record.fromFields(fields); case UUID: return Type.uuidType(random.nextBoolean()); + case VECTOR: + return randomVectorType(); case RELATION: // fallthrough case UNKNOWN: // fallthrough case ANY: // fallthrough @@ -148,6 +154,14 @@ private static Type generateType(int depth, Type.TypeCode requestedTypeCode) { } } + @Nonnull + private static Type.Vector randomVectorType() { + final var validPrecisions = new int[] {16, 32, 64}; + final var randomPrecision = validPrecisions[random.nextInt(validPrecisions.length)]; + final var randomDimensions = random.nextInt(3000); + return Type.Vector.of(random.nextBoolean(), randomPrecision, randomDimensions); + } + @Test void addPrimitiveTypeIsNotAllowed() { TypeRepository.Builder builder = TypeRepository.newBuilder(); @@ -256,19 +270,20 @@ void createLightArrayConstructorValueWorks() { @Test void createRecordTypeConstructorWorks() { - final Typed value = new RecordConstructorValue.RecordFn().encapsulate(List.of(STRING_1, INT_2, FLOAT_1, UUID_1)); - Assertions.assertTrue(value instanceof RecordConstructorValue); + final Typed value = new RecordConstructorValue.RecordFn().encapsulate(List.of(STRING_1, INT_2, FLOAT_1, UUID_1, HALF_VECTOR_1_2_3)); + Assertions.assertInstanceOf(RecordConstructorValue.class, value); final RecordConstructorValue recordConstructorValue = (RecordConstructorValue)value; final Type resultType = recordConstructorValue.getResultType(); Assertions.assertEquals(Type.Record.fromFields(false, List.of(Type.Record.Field.of(STRING_1.getResultType(), Optional.empty()), Type.Record.Field.of(INT_2.getResultType(), Optional.empty()), Type.Record.Field.of(FLOAT_1.getResultType(), Optional.empty()), - Type.Record.Field.of(UUID_1.getResultType(), Optional.empty()) + Type.Record.Field.of(UUID_1.getResultType(), Optional.empty()), + Type.Record.Field.of(HALF_VECTOR_1_2_3.getResultType(), Optional.empty()) )), resultType); final Object result = recordConstructorValue.evalWithoutStore(EvaluationContext.forTypeRepository(TypeRepository.newBuilder().addTypeIfNeeded(recordConstructorValue.getResultType()).build())); - Assertions.assertTrue(result instanceof DynamicMessage); + Assertions.assertInstanceOf(DynamicMessage.class, result); final DynamicMessage resultMessage = (DynamicMessage)result; - Assertions.assertEquals(4, resultMessage.getAllFields().size()); + Assertions.assertEquals(5, resultMessage.getAllFields().size()); List fieldSorted = resultMessage.getAllFields().entrySet().stream() .map(kv -> Pair.of(kv.getKey().getIndex(), kv.getValue())) .sorted(Map.Entry.comparingByKey()) @@ -277,10 +292,11 @@ void createRecordTypeConstructorWorks() { Assertions.assertEquals("a", fieldSorted.get(0)); Assertions.assertEquals(2, fieldSorted.get(1)); Assertions.assertEquals(1.0F, fieldSorted.get(2)); - final var expectedUuid = UUID.fromString("eebee473-690b-48c1-beb0-07c3aca77768"); Assertions.assertEquals(TupleFieldsProto.UUID.newBuilder() - .setMostSignificantBits(expectedUuid.getMostSignificantBits()) - .setLeastSignificantBits(expectedUuid.getLeastSignificantBits()) + .setMostSignificantBits(UUID_1.getLiteralValue().getMostSignificantBits()) + .setLeastSignificantBits(UUID_1.getLiteralValue().getLeastSignificantBits()) .build(), fieldSorted.get(3)); + Assertions.assertEquals(ByteString.copyFrom(HALF_VECTOR_1_2_3.getLiteralValue().getRawData()), + fieldSorted.get(4)); } } From 2cf04d1fa794c85373b9e2cbcaa389c7a2d3c034 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 11:49:24 +0000 Subject: [PATCH 09/15] Post-merge fixes. --- .../recordlayer/query/AstNormalizer.java | 2 +- .../recordlayer/query/visitors/DdlVisitor.java | 1 + .../api/ddl/DdlStatementParsingTest.java | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java index 09796515b0..6e73e99136 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java @@ -557,7 +557,7 @@ private void processLiteral(@Nonnull final Object literal, final int tokenIndex, @Nullable final Integer unnamedParameterIndex, @Nullable final String parameterName) { if (allowLiteralAddition) { queryHasherContextBuilder.getLiteralsBuilder() - .addLiteral(Type.fromObject(literal), literal, unnamedParameterIndex, parameterName, tokenIndex); + .addLiteral(Type.any(), literal, unnamedParameterIndex, parameterName, tokenIndex); } if (allowTokenAddition) { final String canonicalName = parameterName == null ? "?" : "?" + parameterName; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 5db0608ccf..4e5145aca1 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index d0ada5e311..a6d3871aa0 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -967,7 +967,7 @@ void createViewWorks() throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V")); + final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v")); assertThat(viewMaybe).isPresent(); assertThat(Assert.optionalUnchecked(viewMaybe).getDescription()).isEqualTo("SELECT * FROM bar"); return txn -> { @@ -990,10 +990,10 @@ void createNestedViewWorks() throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var view1Maybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V1")); + final var view1Maybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v1")); assertThat(view1Maybe).isPresent(); assertThat(Assert.optionalUnchecked(view1Maybe).getDescription()).isEqualTo("SELECT * FROM bar"); - final var view2Maybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V2")); + final var view2Maybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v2")); assertThat(view2Maybe).isPresent(); assertThat(Assert.optionalUnchecked(view2Maybe).getDescription()).isEqualTo("SELECT * FROM v1"); return txn -> { @@ -1090,7 +1090,7 @@ void createViewWithCteWorks() throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V")); + final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v")); assertThat(viewMaybe).isPresent(); assertThat(Assert.optionalUnchecked(viewMaybe).getDescription()).isEqualTo("WITH C1 AS (SELECT foo_field, id, baz_field FROM bar where id > 20) SELECT * FROM C1"); return txn -> { @@ -1112,7 +1112,7 @@ void createViewWithNestedCteWorks() throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V")); + final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v")); assertThat(viewMaybe).isPresent(); assertThat(Assert.optionalUnchecked(viewMaybe).getDescription()).isEqualTo("WITH C1 AS (WITH C2 AS (SELECT foo_field, id, baz_field FROM bar where id > 20) SELECT * FROM C2) SELECT * FROM C1"); return txn -> { @@ -1135,7 +1135,7 @@ void createViewWithFunctionAndCteComplexNestingWorks() throws Exception { @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V")); + final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v")); assertThat(viewMaybe).isPresent(); assertThat(Assert.optionalUnchecked(viewMaybe).getDescription()).isEqualTo("WITH C1 AS (WITH C2 AS (SELECT foo_field, id, baz_field FROM F1(20)) SELECT * FROM C2) SELECT * FROM C1"); return txn -> { @@ -1177,7 +1177,7 @@ void createViewWithReferencesToSubsequentlyDefinedTableAndTypeWorks() throws Exc @Override public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, @Nonnull Options templateProperties) { - final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("V")); + final var viewMaybe = Assertions.assertDoesNotThrow(() -> template.findViewByName("v")); assertThat(viewMaybe).isPresent(); assertThat(Assert.optionalUnchecked(viewMaybe).getDescription()).isEqualTo("SELECT * FROM bar"); return txn -> { From f9d390b4f529012e305718090e193b78b3deec7c Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 12:20:25 +0000 Subject: [PATCH 10/15] Add documentation. --- docs/sphinx/source/reference/sql_types.rst | 172 +++++++++++++++++- .../test/java/DocumentationQueriesTests.java | 5 + .../vector-documentation-queries.yamsql | 113 ++++++++++++ 3 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql diff --git a/docs/sphinx/source/reference/sql_types.rst b/docs/sphinx/source/reference/sql_types.rst index 07dcf3944f..fb4b5456cb 100644 --- a/docs/sphinx/source/reference/sql_types.rst +++ b/docs/sphinx/source/reference/sql_types.rst @@ -72,14 +72,172 @@ In this example, `c` is an array, and each record within the array is a struct o It is possible to nest arrays within structs, and structs within arrays, to an arbitrary depth (limited by the JVM's stack size, currently). -NULL Semantics -############## +.. _vector_types: -For any unset primitive type or struct type fields, queries will return NULL for the column, unless default values are defined. +Vector Types +############ + +The Relational Layer supports *vector types* for storing fixed-size numerical vectors, commonly used in machine learning and similarity search applications. A vector type represents a fixed-dimensional array of floating-point numbers with a specific precision. + +Vector Type Declaration +======================= + +Vectors are declared using the :sql:`VECTOR(dimension, precision)` syntax, where: + +* **dimension**: The number of elements in the vector (must be a positive integer) +* **precision**: The floating-point precision, which can be: + + * :sql:`HALF` - 16-bit half-precision floating-point (2 bytes per element) + * :sql:`FLOAT` - 32-bit single-precision floating-point (4 bytes per element) + * :sql:`DOUBLE` - 64-bit double-precision floating-point (8 bytes per element) + +Examples of vector column definitions: + +.. code-block:: sql + + CREATE TABLE embeddings ( + id BIGINT, + embedding_half VECTOR(128, HALF), + embedding_float VECTOR(128, FLOAT), + embedding_double VECTOR(128, DOUBLE), + PRIMARY KEY(id) + ); + +Vectors can also be used within struct types: + +.. code-block:: sql + + CREATE TYPE AS STRUCT model_embedding ( + model_name STRING, + embedding VECTOR(512, FLOAT) + ); + CREATE TABLE documents ( + id BIGINT, + content STRING, + embedding model_embedding, + PRIMARY KEY(id) + ); + +Internal Storage Format +======================= + +Vectors are stored as byte arrays with the following format: + +* **Byte 0**: Vector type identifier (0 = HALF, 1 = FLOAT, 2 = DOUBLE) +* **Remaining bytes**: Vector components in big-endian byte order + +The storage size for each vector is: + +* :sql:`VECTOR(N, HALF)`: 1 + (2 × N) bytes +* :sql:`VECTOR(N, FLOAT)`: 1 + (4 × N) bytes +* :sql:`VECTOR(N, DOUBLE)`: 1 + (8 × N) bytes + +Working with Vectors +==================== + +Vector Literals and Prepared Statements +---------------------------------------- + +**Important**: Vector literals are not directly supported in SQL. Vectors must be inserted using **prepared statement parameters** through the JDBC API. + +In the JDBC API, you would create a prepared statement and bind vector parameters using the appropriate Java objects (e.g., :java:`HalfRealVector`, :java:`FloatRealVector`, or :java:`DoubleRealVector`): + +.. code-block:: java + + // Java JDBC example + PreparedStatement stmt = connection.prepareStatement( + "INSERT INTO embeddings VALUES (?, ?)"); + stmt.setLong(1, 1); + stmt.setObject(2, new FloatRealVector(new float[]{0.5f, 1.2f, -0.8f})); + stmt.executeUpdate(); + +For documentation and testing purposes, the examples below use a special test DSL syntax :sql:`!! !vXX [values] !!` to represent prepared statement parameters: + +* :sql:`!v16` represents a HALF precision vector parameter +* :sql:`!v32` represents a FLOAT precision vector parameter +* :sql:`!v64` represents a DOUBLE precision vector parameter + +Example insertions (using test DSL syntax for illustration): + +.. code-block:: sql + + -- Insert a HALF precision vector + INSERT INTO embeddings VALUES (1, !! !v16 [0.5, 1.2, -0.8] !!); + + -- Insert a FLOAT precision vector + INSERT INTO embeddings VALUES (2, !! !v32 [0.5, 1.2, -0.8] !!); + + -- Insert a DOUBLE precision vector + INSERT INTO embeddings VALUES (3, !! !v64 [0.5, 1.2, -0.8] !!); + +Casting Arrays to Vectors +-------------------------- + +While vector literals are not supported, you can use :sql:`CAST` to convert array expressions to vectors. The source array elements can be of any numeric type (:sql:`INTEGER`, :sql:`BIGINT`, :sql:`FLOAT`, :sql:`DOUBLE`): + +.. code-block:: sql + + -- Cast FLOAT array to FLOAT vector + SELECT CAST([1.2, 3.4, 5.6] AS VECTOR(3, FLOAT)) AS vec; + + -- Cast INTEGER array to HALF vector + SELECT CAST([1, 2, 3] AS VECTOR(3, HALF)) AS vec; + + -- Cast mixed numeric types to DOUBLE vector + SELECT CAST([1, 2.5, 3L] AS VECTOR(3, DOUBLE)) AS vec; + +The array must have exactly the same number of elements as the vector's declared dimension, or the cast will fail with error code :sql:`22F3H`. Only numeric arrays can be cast to vectors. + +Querying Vectors +---------------- + +Vectors can be selected and compared like other column types. When comparing vectors in WHERE clauses, you would typically use prepared statement parameters in your Java/JDBC code: + +.. code-block:: sql + + -- Select vectors + SELECT embedding FROM embeddings WHERE id = 1; + + -- Compare vectors for equality (using test DSL syntax; in actual code use PreparedStatement) + SELECT id FROM embeddings WHERE embedding = !! !v32 [0.5, 1.2, -0.8] !!; + + -- Compare vectors for inequality + SELECT id FROM embeddings WHERE embedding != !! !v32 [1.0, 2.0, 3.0] !!; + + -- Check for NULL vectors + SELECT id FROM embeddings WHERE embedding IS NULL; + SELECT id FROM embeddings WHERE embedding IS NOT NULL; + + -- Use IS DISTINCT FROM for NULL-safe comparisons + SELECT embedding IS DISTINCT FROM !! !v32 [0.5, 1.2, -0.8] !! FROM embeddings; + +Vectors in Struct Fields +------------------------- + +When vectors are nested within struct types, you can access them using dot notation. As with all vector comparisons, you would use prepared statement parameters in your JDBC code: + +.. code-block:: sql + + -- Access vector within a struct + SELECT embedding.embedding FROM documents WHERE id = 1; + + -- Filter by vector within struct (using test DSL syntax; in actual code use PreparedStatement) + SELECT id FROM documents + WHERE embedding.embedding = !! !v32 [0.5, 1.2, -0.8] !!; + + -- Check NULL for vector field in struct + SELECT id FROM documents WHERE embedding.embedding IS NULL; + +Supported Operations +==================== + +The following operations are supported on vector types: + +* **Equality comparison** (:sql:`=`, :sql:`!=`) +* **NULL checks** (:sql:`IS NULL`, :sql:`IS NOT NULL`) +* **NULL-safe comparison** (:sql:`IS DISTINCT FROM`, :sql:`IS NOT DISTINCT FROM`) +* **CAST from numeric arrays** to vectors -For array type fields: +Note that mathematical operations (addition, subtraction, dot product, etc.) are performed through the Java API using the :java:`RealVector` interface and its implementations (:java:`HalfRealVector`, :java:`FloatRealVector`, :java:`DoubleRealVector`), not through SQL. -* If the whole array is unset, query returns NULL. -* If the array is set to empty, query returns empty list. -* All elements in the array should be set, arrays like :sql:`[1, NULL, 2, NULL]` are not supported. diff --git a/yaml-tests/src/test/java/DocumentationQueriesTests.java b/yaml-tests/src/test/java/DocumentationQueriesTests.java index 0d4a1b0f04..e820bad3e7 100644 --- a/yaml-tests/src/test/java/DocumentationQueriesTests.java +++ b/yaml-tests/src/test/java/DocumentationQueriesTests.java @@ -38,4 +38,9 @@ void withDocumentationQueriesTests(YamlTest.Runner runner) throws Exception { void castDocumentationQueriesTests(YamlTest.Runner runner) throws Exception { runner.runYamsql(PREFIX + "/cast-documentation-queries.yamsql"); } + + @TestTemplate + void vectorDocumentationQueriesTests(YamlTest.Runner runner) throws Exception { + runner.runYamsql(PREFIX + "/vector-documentation-queries.yamsql"); + } } diff --git a/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql b/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql new file mode 100644 index 0000000000..d4d5bc5426 --- /dev/null +++ b/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql @@ -0,0 +1,113 @@ +# +# vector-documentation-queries.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +options: + supported_version: !current_version +--- +schema_template: + create type as struct model_embedding(model_name string, embedding vector(3, float)) + create table embeddings(id bigint, embedding vector(3, float), primary key(id)) + create table documents(id bigint, content string, embedding model_embedding, primary key(id)) +--- +test_block: + name: vector-documentation-tests + preset: single_repetition_ordered + options: + statement_type: prepared + tests: + # Vector Type Declaration - Insert a FLOAT precision vector (documentation example) + - + - query: INSERT INTO embeddings VALUES (1, !! !v32 [0.5, 1.2, -0.8] !!) + - count: 1 + + # Insert another row for testing + - + - query: INSERT INTO embeddings VALUES (2, !! !v32 [1.0, 2.0, 3.0] !!) + - count: 1 + + # Casting Arrays to Vectors - Cast FLOAT array to FLOAT vector (documentation example) + - + - query: SELECT CAST([1.2, 3.4, 5.6] AS VECTOR(3, FLOAT)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v32 [1.2, 3.4, 5.6]}] + + # Casting Arrays to Vectors - Cast INTEGER array to HALF vector (documentation example) + - + - query: SELECT CAST([1, 2, 3] AS VECTOR(3, HALF)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v16 [1.0, 2.0, 3.0]}] + + # Casting Arrays to Vectors - Cast mixed numeric types to DOUBLE vector (documentation example) + - + - query: SELECT CAST([1, 2.5, 3L] AS VECTOR(3, DOUBLE)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v64 [1.0, 2.5, 3.0]}] + + # Querying Vectors - Select vectors (documentation example) + - + - query: SELECT embedding FROM embeddings WHERE id = 1 + - result: [{embedding: !v32 [0.5, 1.2, -0.8]}] + + # Test CAST errors - Array must match vector dimension (documentation example) + - + - query: SELECT CAST([1.0, 2.0, 3.0, 4.0] AS VECTOR(3, FLOAT)) FROM embeddings WHERE id = 1 + - error: "22F3H" + + - + - query: SELECT CAST([1.0, 2.0] AS VECTOR(3, DOUBLE)) FROM embeddings WHERE id = 1 + - error: "22F3H" + + # Test CAST errors - Only numeric arrays can be cast to vectors (documentation example) + - + - query: SELECT CAST(['a', 'b', 'c'] AS VECTOR(3, FLOAT)) FROM embeddings WHERE id = 1 + - error: "22F3H" + + - + - query: SELECT CAST([true, false, true] AS VECTOR(3, HALF)) FROM embeddings WHERE id = 1 + - error: "22F3H" + + # Vectors in Struct Fields - Insert vector within struct (documentation example) + - + - query: INSERT INTO documents VALUES (1, 'Sample document', ('gpt-4', !! !v32 [0.5, 1.2, -0.8] !!)) + - count: 1 + + # Vectors in Struct Fields - Access vector within a struct (documentation example) + - + - query: SELECT embedding.embedding FROM documents WHERE id = 1 + - result: [{embedding: !v32 [0.5, 1.2, -0.8]}] + + # Vectors in Struct Fields - Access model name from struct (documentation example) + - + - query: SELECT embedding.model_name FROM documents WHERE id = 1 + - result: [{model_name: 'gpt-4'}] + + # Test CAST from different numeric types to different vector precisions (documentation example) + - + - query: SELECT CAST([1.0, 2.0, 3.0] AS VECTOR(3, HALF)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v16 [1.0, 2.0, 3.0]}] + + - + - query: SELECT CAST([1L, 2L, 3L] AS VECTOR(3, FLOAT)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v32 [1.0, 2.0, 3.0]}] + + - + - query: SELECT CAST([1, 2, 3] AS VECTOR(3, DOUBLE)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v64 [1.0, 2.0, 3.0]}] + + # Test negative numbers (documentation example) + - + - query: SELECT CAST([-1.5, -2.5, -3.5] AS VECTOR(3, FLOAT)) AS vec FROM embeddings WHERE id = 1 + - result: [{vec: !v32 [-1.5, -2.5, -3.5]}] From 134eda8948ea2a52f168d15c2a6444a81bc76ffa Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 15:09:50 +0000 Subject: [PATCH 11/15] Address comment. --- .../src/main/proto/grpc/relational/jdbc/v1/jdbc.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto index af4ca0dddd..8b5aa98fa9 100644 --- a/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto +++ b/fdb-relational-grpc/src/main/proto/grpc/relational/jdbc/v1/jdbc.proto @@ -111,7 +111,7 @@ message Parameter { int32 java_sql_types_code = 1; // deprecated // Reuse the column type here. It has much of what we need carrying // across parameters as well as null support, etc. - Column parameter = 2; // deprecated + Column parameter = 2; Type type = 3; // deprecated ParameterMetadata metadata = 4; } From 7783b7f57f5cf0b147263b68c21ef4979276ac44 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 15:51:13 +0000 Subject: [PATCH 12/15] Fix documentation. --- docs/sphinx/source/reference/sql_types.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/source/reference/sql_types.rst b/docs/sphinx/source/reference/sql_types.rst index fb4b5456cb..4f5d6e9c42 100644 --- a/docs/sphinx/source/reference/sql_types.rst +++ b/docs/sphinx/source/reference/sql_types.rst @@ -29,7 +29,7 @@ You can define a *struct type* (often interchangeably referred to as a *nested t .. code-block:: sql - CREATE TYPE AS STRUCT nested_type (d INT64, e STRING); + CREATE TYPE AS STRUCT nested_type (d INT64, e STRING) CREATE TABLE foo (a STRING, b DOUBLE, c nested_type, PRIMARY KEY(a)); In this example, :sql:`nested_type` is a struct within the table `foo`, and its full contents are materialized alongside the full record for each entry in the `foo` table. @@ -38,8 +38,8 @@ Struct types can have columns which are themselves struct types. Thus, this exam .. code-block:: sql - CREATE TYPE AS STRUCT nested_nested_type (f STRING, g STRING); - CREATE TYPE AS STRUCT nested_type (d INT64, e STRING, f nested_nested_type); + CREATE TYPE AS STRUCT nested_nested_type (f STRING, g STRING) + CREATE TYPE AS STRUCT nested_type (d INT64, e STRING, f nested_nested_type) CREATE TABLE foo (a STRING, b DOUBLE, c nested_type, PRIMARY KEY(a)); In this example, :sql:`nested_type` is a struct within the table :sql:`foo`, and :sql:`nested_nested_type` is a struct within the type :sql:`nested_type`. @@ -65,7 +65,7 @@ Arrays can also be created with struct columns: .. code-block:: sql - CREATE TYPE AS STRUCT nested_struct (b STRING, d STRING); + CREATE TYPE AS STRUCT nested_struct (b STRING, d STRING) CREATE TABLE structArray (a STRING, c nested_struct array); In this example, `c` is an array, and each record within the array is a struct of type :sql:`nested_struct`. You can generally treat an array as a "nested `ResultSet`"--that is to say, you can just pull up a `ResultSet` of an array type, and interrogate it as if it were the output of its own query. @@ -110,7 +110,7 @@ Vectors can also be used within struct types: CREATE TYPE AS STRUCT model_embedding ( model_name STRING, embedding VECTOR(512, FLOAT) - ); + ) CREATE TABLE documents ( id BIGINT, content STRING, From 30174125ef8358aa816dce4639e921d64c6dc61c Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 16:30:48 +0000 Subject: [PATCH 13/15] Do not use YAML constructs in doc. --- docs/sphinx/source/reference/sql_types.rst | 43 ++++++-------- .../vector-documentation-queries.yamsql | 59 +++++++++++++++++++ 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/docs/sphinx/source/reference/sql_types.rst b/docs/sphinx/source/reference/sql_types.rst index 4f5d6e9c42..7c850d38d9 100644 --- a/docs/sphinx/source/reference/sql_types.rst +++ b/docs/sphinx/source/reference/sql_types.rst @@ -138,9 +138,11 @@ Working with Vectors Vector Literals and Prepared Statements ---------------------------------------- -**Important**: Vector literals are not directly supported in SQL. Vectors must be inserted using **prepared statement parameters** through the JDBC API. +**Important**: Vector literals are not directly supported in SQL. Vectors must be inserted using **prepared statement +parameters** through the JDBC API. -In the JDBC API, you would create a prepared statement and bind vector parameters using the appropriate Java objects (e.g., :java:`HalfRealVector`, :java:`FloatRealVector`, or :java:`DoubleRealVector`): +In the JDBC API, you would create a prepared statement and bind vector parameters using the appropriate Java objects +(e.g., :java:`HalfRealVector`, :java:`FloatRealVector`, or :java:`DoubleRealVector`): .. code-block:: java @@ -151,24 +153,17 @@ In the JDBC API, you would create a prepared statement and bind vector parameter stmt.setObject(2, new FloatRealVector(new float[]{0.5f, 1.2f, -0.8f})); stmt.executeUpdate(); -For documentation and testing purposes, the examples below use a special test DSL syntax :sql:`!! !vXX [values] !!` to represent prepared statement parameters: - -* :sql:`!v16` represents a HALF precision vector parameter -* :sql:`!v32` represents a FLOAT precision vector parameter -* :sql:`!v64` represents a DOUBLE precision vector parameter - -Example insertions (using test DSL syntax for illustration): +For documentation purposes, the examples below demonstrate vector usage. While vectors can be constructed in SQL by +casting numeric arrays to vector types, **note that inserting vectors requires using prepared statement parameters +through the JDBC API** (as shown in the Java example above). CAST expressions work well for SELECT queries but have +limitations with INSERT statements in prepared statement contexts: .. code-block:: sql - -- Insert a HALF precision vector - INSERT INTO embeddings VALUES (1, !! !v16 [0.5, 1.2, -0.8] !!); - - -- Insert a FLOAT precision vector - INSERT INTO embeddings VALUES (2, !! !v32 [0.5, 1.2, -0.8] !!); - - -- Insert a DOUBLE precision vector - INSERT INTO embeddings VALUES (3, !! !v64 [0.5, 1.2, -0.8] !!); + -- Example: Constructing vectors using CAST (works in SELECT contexts) + SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, HALF)) AS half_vector; + SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) AS float_vector; + SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, DOUBLE)) AS double_vector; Casting Arrays to Vectors -------------------------- @@ -191,25 +186,25 @@ The array must have exactly the same number of elements as the vector's declared Querying Vectors ---------------- -Vectors can be selected and compared like other column types. When comparing vectors in WHERE clauses, you would typically use prepared statement parameters in your Java/JDBC code: +Vectors can be selected and compared like other column types. When comparing vectors in WHERE clauses, you would typically use prepared statement parameters in your Java/JDBC code, but for illustration purposes, vectors can also be constructed using CAST: .. code-block:: sql -- Select vectors SELECT embedding FROM embeddings WHERE id = 1; - -- Compare vectors for equality (using test DSL syntax; in actual code use PreparedStatement) - SELECT id FROM embeddings WHERE embedding = !! !v32 [0.5, 1.2, -0.8] !!; + -- Compare vectors for equality (in actual code use PreparedStatement for better performance) + SELECT id FROM embeddings WHERE embedding = CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)); -- Compare vectors for inequality - SELECT id FROM embeddings WHERE embedding != !! !v32 [1.0, 2.0, 3.0] !!; + SELECT id FROM embeddings WHERE embedding != CAST([1.0, 2.0, 3.0] AS VECTOR(3, FLOAT)); -- Check for NULL vectors SELECT id FROM embeddings WHERE embedding IS NULL; SELECT id FROM embeddings WHERE embedding IS NOT NULL; -- Use IS DISTINCT FROM for NULL-safe comparisons - SELECT embedding IS DISTINCT FROM !! !v32 [0.5, 1.2, -0.8] !! FROM embeddings; + SELECT embedding IS DISTINCT FROM CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) FROM embeddings; Vectors in Struct Fields ------------------------- @@ -221,9 +216,9 @@ When vectors are nested within struct types, you can access them using dot notat -- Access vector within a struct SELECT embedding.embedding FROM documents WHERE id = 1; - -- Filter by vector within struct (using test DSL syntax; in actual code use PreparedStatement) + -- Filter by vector within struct (in actual code use PreparedStatement for better performance) SELECT id FROM documents - WHERE embedding.embedding = !! !v32 [0.5, 1.2, -0.8] !!; + WHERE embedding.embedding = CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)); -- Check NULL for vector field in struct SELECT id FROM documents WHERE embedding.embedding IS NULL; diff --git a/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql b/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql index d4d5bc5426..9130e1f580 100644 --- a/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql +++ b/yaml-tests/src/test/resources/documentation-queries/vector-documentation-queries.yamsql @@ -32,6 +32,7 @@ test_block: statement_type: prepared tests: # Vector Type Declaration - Insert a FLOAT precision vector (documentation example) + # Note: Using YAML DSL syntax for inserts as CAST expressions don't work with prepared statements - - query: INSERT INTO embeddings VALUES (1, !! !v32 [0.5, 1.2, -0.8] !!) - count: 1 @@ -61,6 +62,47 @@ test_block: - query: SELECT embedding FROM embeddings WHERE id = 1 - result: [{embedding: !v32 [0.5, 1.2, -0.8]}] + # Vector construction examples from documentation - CAST in SELECT contexts + - + - query: SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, HALF)) AS half_vector FROM embeddings WHERE id = 1 + - result: [{half_vector: !v16 [0.5, 1.2, -0.8]}] + + - + - query: SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) AS float_vector FROM embeddings WHERE id = 1 + - result: [{float_vector: !v32 [0.5, 1.2, -0.8]}] + + - + - query: SELECT CAST([0.5, 1.2, -0.8] AS VECTOR(3, DOUBLE)) AS double_vector FROM embeddings WHERE id = 1 + - result: [{double_vector: !v64 [0.5, 1.2, -0.8]}] + + # Querying Vectors - Compare vectors for equality (documentation example) + - + - query: SELECT id FROM embeddings WHERE embedding = CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) + - result: [{1}] + + # Querying Vectors - Compare vectors for inequality (documentation example) + - + - query: SELECT id FROM embeddings WHERE embedding != CAST([1.0, 2.0, 3.0] AS VECTOR(3, FLOAT)) + - result: [{1}] + + # Querying Vectors - Check for NULL vectors (documentation example) + - + - query: INSERT INTO embeddings VALUES (3, NULL) + - count: 1 + + - + - query: SELECT id FROM embeddings WHERE embedding IS NULL + - result: [{3}] + + - + - query: SELECT id FROM embeddings WHERE embedding IS NOT NULL + - unorderedResult: [{1}, {2}] + + # Querying Vectors - IS DISTINCT FROM for NULL-safe comparisons (documentation example) + - + - query: SELECT embedding IS DISTINCT FROM CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) AS is_distinct FROM embeddings WHERE id = 1 + - result: [{is_distinct: false}] + # Test CAST errors - Array must match vector dimension (documentation example) - - query: SELECT CAST([1.0, 2.0, 3.0, 4.0] AS VECTOR(3, FLOAT)) FROM embeddings WHERE id = 1 @@ -80,6 +122,8 @@ test_block: - error: "22F3H" # Vectors in Struct Fields - Insert vector within struct (documentation example) + # Note: Using YAML DSL syntax for inserts as CAST expressions don't work with prepared statements + # https://github.com/FoundationDB/fdb-record-layer/issues/3703 - - query: INSERT INTO documents VALUES (1, 'Sample document', ('gpt-4', !! !v32 [0.5, 1.2, -0.8] !!)) - count: 1 @@ -94,6 +138,20 @@ test_block: - query: SELECT embedding.model_name FROM documents WHERE id = 1 - result: [{model_name: 'gpt-4'}] + # Vectors in Struct Fields - Filter by vector within struct (documentation example) + - + - query: SELECT id FROM documents WHERE embedding.embedding = CAST([0.5, 1.2, -0.8] AS VECTOR(3, FLOAT)) + - result: [{1}] + + # Vectors in Struct Fields - Check NULL for vector field in struct (documentation example) + - + - query: INSERT INTO documents VALUES (2, 'Document without embedding', ('gpt-4', NULL)) + - count: 1 + + - + - query: SELECT id FROM documents WHERE embedding.embedding IS NULL + - result: [{2}] + # Test CAST from different numeric types to different vector precisions (documentation example) - - query: SELECT CAST([1.0, 2.0, 3.0] AS VECTOR(3, HALF)) AS vec FROM embeddings WHERE id = 1 @@ -111,3 +169,4 @@ test_block: - - query: SELECT CAST([-1.5, -2.5, -3.5] AS VECTOR(3, FLOAT)) AS vec FROM embeddings WHERE id = 1 - result: [{vec: !v32 [-1.5, -2.5, -3.5]}] + From badd376ce3a95be2d374f8548ce83efc3736822e Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 30 Oct 2025 20:23:31 +0000 Subject: [PATCH 14/15] Address comments. --- .../record/query/expressions/Comparisons.java | 9 ++--- .../query/plan/cascades/typing/Type.java | 7 ++-- .../plan/cascades/values/RelOpValue.java | 2 ++ .../src/main/proto/record_query_plan.proto | 36 ++++++++++--------- .../src/main/resources/log4j2.xml | 6 ++-- .../yamltests/server/ExternalServer.java | 2 +- yaml-tests/src/test/resources/vector.yamsql | 13 +++++++ 7 files changed, 43 insertions(+), 32 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index c2ea478553..550632370b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -215,18 +215,13 @@ private static Comparable toComparable(@Nullable Object obj) { @Nonnull public static Object toClassWithRealEquals(@Nonnull Object obj) { - if (obj instanceof ByteString) { + if (obj instanceof ByteString || obj instanceof Comparable + || obj instanceof List || obj instanceof RealVector) { return obj; } else if (obj instanceof byte[]) { return ByteString.copyFrom((byte[])obj); } else if (obj instanceof Internal.EnumLite) { return ((Internal.EnumLite)obj).getNumber(); - } else if (obj instanceof Comparable) { - return obj; - } else if (obj instanceof List) { - return obj; - } else if (obj instanceof RealVector) { - return obj; } else { throw new RecordCoreException("Tried to compare non-comparable object " + obj.getClass()); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 435216a2b1..f273be7698 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -1215,7 +1215,7 @@ public boolean equals(final Object other) { } } - class Vector implements Type { + final class Vector implements Type { private final boolean isNullable; private final int precision; private final int dimensions; @@ -1322,8 +1322,7 @@ public PVectorType toProto(@Nonnull final PlanSerializationContext serialization @Nonnull @SuppressWarnings("PMD.ReplaceVectorWithList") - public static Vector fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PVectorType vectorTypeProto) { + public static Vector fromProto(@Nonnull final PVectorType vectorTypeProto) { Verify.verify(vectorTypeProto.hasIsNullable()); return new Vector(vectorTypeProto.getIsNullable(), vectorTypeProto.getPrecision(), vectorTypeProto.getDimensions()); } @@ -1344,7 +1343,7 @@ public Class getProtoMessageClass() { @SuppressWarnings("PMD.ReplaceVectorWithList") public Vector fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PVectorType vectorTypeProto) { - return Vector.fromProto(serializationContext, vectorTypeProto); + return Vector.fromProto(vectorTypeProto); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java index 4cb1d1cb2f..62bfe08d82 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java @@ -1043,6 +1043,8 @@ private enum BinaryPhysicalOperator { NOT_DISTINCT_FROM_NN(Comparisons.Type.NOT_DISTINCT_FROM, Type.TypeCode.NULL, Type.TypeCode.NULL, (l, r) -> true), EQ_VEC_VEC(Comparisons.Type.EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, Objects::equals), + EQ_VEC_NULL(Comparisons.Type.EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), + EQ_NULL_VEC(Comparisons.Type.EQUALS, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), NEQ_VEC_VEC(Comparisons.Type.NOT_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.VECTOR, (l, r) -> !l.equals(r)), NEQ_VEC_NULL(Comparisons.Type.NOT_EQUALS, Type.TypeCode.VECTOR, Type.TypeCode.NULL, (l, r) -> null), NEQ_NULL_VEC(Comparisons.Type.NOT_EQUALS, Type.TypeCode.NULL, Type.TypeCode.VECTOR, (l, r) -> null), diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index ffe1863912..49dc07faf9 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1174,23 +1174,25 @@ message PBinaryRelOpValue { NOT_DISTINCT_FROM_NN = 494; EQ_VEC_VEC = 495; - NEQ_VEC_VEC = 496; - NEQ_VEC_NULL = 497; - NEQ_NULL_VEC = 498; - LT_VEC_NULL = 499; - LT_NULL_VEC = 500; - LTE_VEC_NULL = 501; - LTE_NULL_VEC = 502; - GT_VEC_NULL = 503; - GT_NULL_VEC = 504; - GTE_VEC_NULL = 505; - GTE_NULL_VEC = 506; - IS_DISTINCT_FROM_NULL_VEC = 507; - IS_DISTINCT_FROM_VEC_NULL = 508; - IS_DISTINCT_FROM_VEC_VEC = 509; - NOT_DISTINCT_FROM_VEC_NULL = 510; - NOT_DISTINCT_FROM_NULL_VEC = 511; - NOT_DISTINCT_FROM_VEC_VEC = 512; + EQ_VEC_NULL = 496; + EQ_NULL_VEC = 497; + NEQ_VEC_VEC = 498; + NEQ_VEC_NULL = 499; + NEQ_NULL_VEC = 500; + LT_VEC_NULL = 501; + LT_NULL_VEC = 502; + LTE_VEC_NULL = 503; + LTE_NULL_VEC = 504; + GT_VEC_NULL = 505; + GT_NULL_VEC = 506; + GTE_VEC_NULL = 507; + GTE_NULL_VEC = 508; + IS_DISTINCT_FROM_NULL_VEC = 509; + IS_DISTINCT_FROM_VEC_NULL = 510; + IS_DISTINCT_FROM_VEC_VEC = 511; + NOT_DISTINCT_FROM_VEC_NULL = 512; + NOT_DISTINCT_FROM_NULL_VEC = 513; + NOT_DISTINCT_FROM_VEC_VEC = 514; } optional PRelOpValue super = 1; optional PBinaryPhysicalOperator operator = 2; diff --git a/fdb-relational-server/src/main/resources/log4j2.xml b/fdb-relational-server/src/main/resources/log4j2.xml index d5e5efd7e3..0b4610117f 100644 --- a/fdb-relational-server/src/main/resources/log4j2.xml +++ b/fdb-relational-server/src/main/resources/log4j2.xml @@ -19,17 +19,17 @@ limitations under the License. --> - + - + - + diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java index 316ec909f9..9e2c394136 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/server/ExternalServer.java @@ -44,7 +44,7 @@ public class ExternalServer { private static final Logger logger = LogManager.getLogger(ExternalServer.class); public static final String EXTERNAL_SERVER_PROPERTY_NAME = "yaml_testing_external_server"; - private static final boolean SAVE_SERVER_OUTPUT = true; + private static final boolean SAVE_SERVER_OUTPUT = false; @Nonnull private final File serverJar; diff --git a/yaml-tests/src/test/resources/vector.yamsql b/yaml-tests/src/test/resources/vector.yamsql index 5e17f144e9..b87c271867 100644 --- a/yaml-tests/src/test/resources/vector.yamsql +++ b/yaml-tests/src/test/resources/vector.yamsql @@ -153,6 +153,19 @@ test_block: - - query: select b is null from tDoubleVector where a = 100 - result: [{false}] + # EQUALS NULL tests for vectors + - + - query: select b = null from tHalfVector where a = 100 + - result: [{!null _}] + - + - query: select null = b from tHalfVector where a = 100 + - result: [{!null _}] + - + - query: select b != null from tHalfVector where a = 100 + - result: [{!null _}] + - + - query: select null != b from tHalfVector where a = 100 + - result: [{!null _}] # IS NOT NULL tests for vectors - - query: select b is not null from tHalfVector where a = 100 From e6dd59cab1bb3c942c23d8697863903255dbe125 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 31 Oct 2025 12:34:02 +0000 Subject: [PATCH 15/15] Revert refactoring. --- .../record/query/expressions/Comparisons.java | 9 ++++++-- .../src/test/resources/in-predicate.yamsql | 22 ------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index 550632370b..c2ea478553 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -215,13 +215,18 @@ private static Comparable toComparable(@Nullable Object obj) { @Nonnull public static Object toClassWithRealEquals(@Nonnull Object obj) { - if (obj instanceof ByteString || obj instanceof Comparable - || obj instanceof List || obj instanceof RealVector) { + if (obj instanceof ByteString) { return obj; } else if (obj instanceof byte[]) { return ByteString.copyFrom((byte[])obj); } else if (obj instanceof Internal.EnumLite) { return ((Internal.EnumLite)obj).getNumber(); + } else if (obj instanceof Comparable) { + return obj; + } else if (obj instanceof List) { + return obj; + } else if (obj instanceof RealVector) { + return obj; } else { throw new RecordCoreException("Tried to compare non-comparable object " + obj.getClass()); } diff --git a/yaml-tests/src/test/resources/in-predicate.yamsql b/yaml-tests/src/test/resources/in-predicate.yamsql index baa026e3d2..64eb0a6c5b 100644 --- a/yaml-tests/src/test/resources/in-predicate.yamsql +++ b/yaml-tests/src/test/resources/in-predicate.yamsql @@ -249,7 +249,6 @@ test_block: # "expected" as the point of error is different in both the cases. # https://github.com/FoundationDB/fdb-record-layer/issues/3583 test_block: - preset: single_repetition_ordered tests: - # LONG value matched against IN list of LONG and DOUBLE values @@ -258,50 +257,29 @@ test_block: # clear whether we want to support it as it would probably # make plan lookup in the cache much more expensive. - query: select a, b from ta where b in (1, 3.0, 5, 7.0) - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in ('foo' , 35) - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in (35, '23') - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Values of incompatible types in the IN list - query: select a, e from ta where e in (true, 75.34) - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Left type (STRING) is not compatible with right type (type of the IN List) - query: select a, e from ta where e in (35, 75.34) - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Left type (STRING) is not compatible with right type (type of the IN List) - query: select a, e from ta where e in (35.34, 32) - - initialVersionLessThan: !current_version - error: "XX000" - - initialVersionAtLeast: !current_version - - error: "42804" - # Complex (STRUCT) left type matched against IN list - query: select a, e from ta where f in (34, 23) - - initialVersionLessThan: 4.8.2.0 - - error: "XX000" - - initialVersionAtLeast: 4.8.2.0 - error: "22000" ...