diff --git a/.gitignore b/.gitignore index fe14c1529238..0cd023b42145 100644 --- a/.gitignore +++ b/.gitignore @@ -502,3 +502,5 @@ swa-cli.config.json # dapr extension files **/dapr.yaml + +*.lscache \ No newline at end of file diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 95db60cb6f98..d4e4df0a09cd 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -92,7 +92,7 @@ - + @@ -110,8 +110,8 @@ - - + + @@ -120,18 +120,18 @@ - - + + - - - - - - - - - + + + + + + + + + @@ -139,20 +139,22 @@ - + - + + + - + diff --git a/dotnet/MEVD.slnf b/dotnet/MEVD.slnf index 2a2273ba7f88..afdb2c76b516 100644 --- a/dotnet/MEVD.slnf +++ b/dotnet/MEVD.slnf @@ -18,7 +18,6 @@ "src/VectorData/SqlServer/SqlServer.csproj", "src/VectorData/Weaviate/Weaviate.csproj", - "src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj", "test/VectorData/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj", "test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj", @@ -42,9 +41,7 @@ "test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj", "test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj", "test/VectorData/Weaviate.UnitTests/Weaviate.UnitTests.csproj", - "test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj", - - "test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj" + "test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj" ] } } \ No newline at end of file diff --git a/dotnet/SK-dotnet.slnx b/dotnet/SK-dotnet.slnx index b2ff323d726c..faf9169ec443 100644 --- a/dotnet/SK-dotnet.slnx +++ b/dotnet/SK-dotnet.slnx @@ -136,7 +136,6 @@ - @@ -316,8 +315,6 @@ - - diff --git a/dotnet/SK-release.slnf b/dotnet/SK-release.slnf index 1b497da3f66e..78ee739f7407 100644 --- a/dotnet/SK-release.slnf +++ b/dotnet/SK-release.slnf @@ -45,7 +45,6 @@ "src\\VectorData\\Redis\\Redis.csproj", "src\\VectorData\\SqliteVec\\SqliteVec.csproj", "src\\VectorData\\SqlServer\\SqlServer.csproj", - "src\\VectorData\\VectorData.Abstractions\\VectorData.Abstractions.csproj", "src\\VectorData\\Weaviate\\Weaviate.csproj", "src\\Experimental\\Orchestration.Flow\\Experimental.Orchestration.Flow.csproj", diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs index 25b77f90839b..0e51c5340921 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs @@ -27,9 +27,9 @@ internal class MongoModelBuilder() : CollectionModelBuilder(s_validationOptions) UsesExternalSerializer = true, }; - protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type) + protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty) { - base.ProcessProperty(clrProperty, definitionProperty, type); + base.ProcessProperty(clrProperty, definitionProperty); if (clrProperty?.GetCustomAttribute() is { } bsonElementAttribute && this.PropertyMap.TryGetValue(clrProperty.Name, out var property)) diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj index 02b5009b6194..78d70f6bd2a7 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj +++ b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj @@ -24,7 +24,7 @@ - + diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchKernelBuilderExtensions.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchKernelBuilderExtensions.cs index 84909b8229c9..fa421d45a2bf 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchKernelBuilderExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchKernelBuilderExtensions.cs @@ -24,6 +24,7 @@ public static class TextSearchKernelBuilderExtensions ITextSearchResultMapper? resultMapper = null, VectorStoreTextSearchOptions? options = null, string? serviceId = default) + where TRecord : class { builder.Services.AddVectorStoreTextSearch(stringMapper, resultMapper, options, serviceId); return builder; diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchServiceCollectionExtensions.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchServiceCollectionExtensions.cs index 950cb46777af..5a60592cce25 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchServiceCollectionExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearch/TextSearchServiceCollectionExtensions.cs @@ -28,6 +28,7 @@ public static class TextSearchServiceCollectionExtensions ITextSearchResultMapper? resultMapper = null, VectorStoreTextSearchOptions? options = null, string? serviceId = default) + where TRecord : class { // If we are not constructing the dependent services, add the VectorStoreTextSearch as transient, since we // cannot make assumptions about how dependent services are being managed. diff --git a/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs b/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs index c83bdbcbddb9..eb26852881ce 100644 --- a/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs +++ b/dotnet/src/SemanticKernel.Core/Data/TextSearch/VectorStoreTextSearch.cs @@ -21,6 +21,7 @@ namespace Microsoft.SemanticKernel.Data; [Experimental("SKEXP0001")] #pragma warning disable CS0618 // ITextSearch is obsolete public sealed class VectorStoreTextSearch<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] TRecord> : ITextSearch, ITextSearch + where TRecord : class #pragma warning restore CS0618 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj b/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj index d43078f48cee..b1691a79cd3e 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj @@ -31,14 +31,11 @@ + - - - - diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj index 52b98d55b40e..f2c40021388b 100644 --- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj +++ b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj @@ -28,22 +28,15 @@ - - - - - + - - - - + diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj index 2849a25f118e..cbfbc8beebf4 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj @@ -25,18 +25,15 @@ - - - - - + - + + - - + + diff --git a/dotnet/src/VectorData/Directory.Build.props b/dotnet/src/VectorData/Directory.Build.props index 44a8810c0a8e..aa86f79b9a18 100644 --- a/dotnet/src/VectorData/Directory.Build.props +++ b/dotnet/src/VectorData/Directory.Build.props @@ -4,6 +4,13 @@ $(NoWarn);MEVD9000,MEVD9001 + + + true + + + + diff --git a/dotnet/src/VectorData/InMemory/InMemory.csproj b/dotnet/src/VectorData/InMemory/InMemory.csproj index aa8cbb340f55..e463411bc6ec 100644 --- a/dotnet/src/VectorData/InMemory/InMemory.csproj +++ b/dotnet/src/VectorData/InMemory/InMemory.csproj @@ -37,8 +37,4 @@ - - - - diff --git a/dotnet/src/VectorData/MongoDB/MongoDB.csproj b/dotnet/src/VectorData/MongoDB/MongoDB.csproj index fa6369f2e179..78050e3367a7 100644 --- a/dotnet/src/VectorData/MongoDB/MongoDB.csproj +++ b/dotnet/src/VectorData/MongoDB/MongoDB.csproj @@ -28,17 +28,10 @@ - - - - - - - - + diff --git a/dotnet/src/VectorData/PgVector/PgVector.csproj b/dotnet/src/VectorData/PgVector/PgVector.csproj index b171819ce395..ed07e24ce9ed 100644 --- a/dotnet/src/VectorData/PgVector/PgVector.csproj +++ b/dotnet/src/VectorData/PgVector/PgVector.csproj @@ -38,8 +38,8 @@ - - + + diff --git a/dotnet/src/VectorData/Pinecone/Pinecone.csproj b/dotnet/src/VectorData/Pinecone/Pinecone.csproj index d5c5847327a2..475285b67f79 100644 --- a/dotnet/src/VectorData/Pinecone/Pinecone.csproj +++ b/dotnet/src/VectorData/Pinecone/Pinecone.csproj @@ -31,6 +31,7 @@ + @@ -39,8 +40,4 @@ - - - - diff --git a/dotnet/src/VectorData/Qdrant/Qdrant.csproj b/dotnet/src/VectorData/Qdrant/Qdrant.csproj index 86e2503a2056..8d3480310f44 100644 --- a/dotnet/src/VectorData/Qdrant/Qdrant.csproj +++ b/dotnet/src/VectorData/Qdrant/Qdrant.csproj @@ -25,13 +25,14 @@ + - + @@ -40,8 +41,4 @@ - - - - \ No newline at end of file diff --git a/dotnet/src/VectorData/Redis/Redis.csproj b/dotnet/src/VectorData/Redis/Redis.csproj index 41e673ad61a5..590d8cc732d9 100644 --- a/dotnet/src/VectorData/Redis/Redis.csproj +++ b/dotnet/src/VectorData/Redis/Redis.csproj @@ -29,12 +29,12 @@ - - + + - + \ No newline at end of file diff --git a/dotnet/src/VectorData/SqlServer/SqlServer.csproj b/dotnet/src/VectorData/SqlServer/SqlServer.csproj index 7896ffcb03f5..b128591a19a8 100644 --- a/dotnet/src/VectorData/SqlServer/SqlServer.csproj +++ b/dotnet/src/VectorData/SqlServer/SqlServer.csproj @@ -33,8 +33,8 @@ - - + + diff --git a/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj b/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj index ab5a3f4798dd..d9e45105e395 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj +++ b/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj @@ -29,16 +29,16 @@ + + - - - - + + diff --git a/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig b/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig deleted file mode 100644 index acb2cb62caf4..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig +++ /dev/null @@ -1,3 +0,0 @@ -# Suppress missing documentation warnings for generated code (strings) -[*.Designer.cs] -dotnet_diagnostic.CS1591.severity = none diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs deleted file mode 100644 index c9419d087732..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a filter clause that filters by checking if a field consisting of a list of values contains a specific value. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public sealed class AnyTagEqualToFilterClause : FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the field with the list of values. - /// The value that the list should contain. - public AnyTagEqualToFilterClause(string fieldName, string value) - { - this.FieldName = fieldName; - this.Value = value; - } - - /// - /// Gets the name of the field with the list of values. - /// - public string FieldName { get; private set; } - - /// - /// Gets the value that the list should contain. - /// - public string Value { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs deleted file mode 100644 index 03fc678abc8b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a filter clause that filters using equality of a field value. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public sealed class EqualToFilterClause : FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - /// Field name. - /// Field value. - public EqualToFilterClause(string fieldName, object value) - { - this.FieldName = fieldName; - this.Value = value; - } - - /// - /// Gets the field name to match. - /// - public string FieldName { get; private set; } - - /// - /// Gets the field value to match. - /// - public object Value { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs deleted file mode 100644 index 97c8869a27ea..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base class for filter clauses. -/// -/// -/// A is used to request that the underlying search service should -/// filter search results based on the specified criteria. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public abstract class FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - protected FilterClause() - { - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md b/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md deleted file mode 100644 index 948b8527c6a8..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md +++ /dev/null @@ -1,51 +0,0 @@ -## About - -Contains abstractions for accessing Vector Databases and Vector Indexes. - -## Key Features - -- Base abstract classes and interfaces for Vector Database implementation. Vector Database implementations are provided separately in other packages, for example `Microsoft.SemanticKernel.Connectors.AzureAISearch`. -- Abstractions include: - - Creating, listing and deleting collections with custom schema support. - - Creating, retrieving, updating and deleting records. - - Similarty search using vector embeddings. - - Search using filters. - - Hybrid search combining vector similarity and keyword search. - - Built-in embedding generation using `Microsoft.Extensions.AI`. - -## How to Use - -This package is typically used with an implementation of the vector database abstractions such as `Microsoft.SemanticKernel.Connectors.AzureAISearch`. - -## Main Types - -The main types provided by this library are: - -- [Microsoft.Extensions.VectorData.VectorStore](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstore) -- [Microsoft.Extensions.VectorData.VectorStoreCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstorecollection-2) - -## Additional Documentation - -- [Conceptual documentation](https://learn.microsoft.com/en-us/semantic-kernel/concepts/vector-store-connectors) - -## Related Packages - -Vector Database implementations: - -- [Microsoft.SemanticKernel.Connectors.AzureAISearch](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.AzureAISearch) -- [Microsoft.SemanticKernel.Connectors.CosmosMongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosMongoDB) -- [Microsoft.SemanticKernel.Connectors.CosmosNoSQL](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosNoSQL) -- [Elastic.SemanticKernel.Connectors.Elasticsearch](https://www.nuget.org/packages/Elastic.SemanticKernel.Connectors.Elasticsearch) -- [Microsoft.SemanticKernel.Connectors.InMemory](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.InMemory) -- [Microsoft.SemanticKernel.Connectors.MongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.MongoDB) -- [Microsoft.SemanticKernel.Connectors.PgVector](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.PgVector) -- [Microsoft.SemanticKernel.Connectors.Pinecone](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Pinecone) -- [Microsoft.SemanticKernel.Connectors.Qdrant](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Qdrant) -- [Microsoft.SemanticKernel.Connectors.Redis](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Redis) -- [Microsoft.SemanticKernel.Connectors.SqliteVec](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqliteVec) -- [Microsoft.SemanticKernel.Connectors.SqlServer](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqlServer) -- [Microsoft.SemanticKernel.Connectors.Weaviate](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Weaviate) - -## Feedback & Contributing - -Microsoft.Extensions.VectorData.Abstractions is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/microsoft/semantic-kernel). diff --git a/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs b/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs deleted file mode 100644 index 09647faa37af..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -[assembly: System.Resources.NeutralResourcesLanguage("en-US")] diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs deleted file mode 100644 index 90a2db9433f3..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Reflection; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a model builder that performs logic specific to connectors that use System.Text.Json for serialization. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class CollectionJsonModelBuilder : CollectionModelBuilder -{ - private JsonSerializerOptions? _jsonSerializerOptions; - - /// - /// Constructs a new . - /// - protected CollectionJsonModelBuilder(CollectionModelBuildingOptions options) - : base(options) - { - } - - /// - /// Builds and returns a from the given and . - /// - [RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - [RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - public virtual CollectionModel Build( - Type recordType, - Type keyType, - VectorStoreCollectionDefinition? definition, - IEmbeddingGenerator? defaultEmbeddingGenerator, - JsonSerializerOptions jsonSerializerOptions) - { - this._jsonSerializerOptions = jsonSerializerOptions; - - return this.Build(recordType, keyType, definition, defaultEmbeddingGenerator); - } - - /// - /// Builds and returns a for dynamic mapping scenarios from the given . - /// - public virtual CollectionModel BuildDynamic( - VectorStoreCollectionDefinition definition, - IEmbeddingGenerator? defaultEmbeddingGenerator, - JsonSerializerOptions jsonSerializerOptions) - { - this._jsonSerializerOptions = jsonSerializerOptions; - - return this.BuildDynamic(definition, defaultEmbeddingGenerator); - } - - /// - protected override void Customize() - { - // This mimics the naming behavior of the System.Text.Json serializer, which we use for serialization/deserialization. - // The property storage names in the model must in sync with the serializer configuration, since the model is used e.g. for filtering - // even if serialization/deserialization doesn't use the model. - var namingPolicy = this._jsonSerializerOptions?.PropertyNamingPolicy; - - foreach (var property in this.Properties) - { - var keyPropertyWithReservedName = this.Options.ReservedKeyStorageName is not null && property is KeyPropertyModel; - string storageName; - - if (property.PropertyInfo?.GetCustomAttribute() is { } jsonPropertyNameAttribute) - { - if (keyPropertyWithReservedName && jsonPropertyNameAttribute.Name != this.Options.ReservedKeyStorageName) - { - throw new InvalidOperationException($"The key property for your connector must always have the reserved name '{this.Options.ReservedKeyStorageName}' and cannot be changed."); - } - - storageName = jsonPropertyNameAttribute.Name; - } - else if (namingPolicy is not null) - { - storageName = namingPolicy.ConvertName(property.ModelName); - } - else - { - storageName = property.ModelName; - } - - if (keyPropertyWithReservedName) - { - // Some providers (Weaviate, Cosmos NoSQL) have a fixed, reserved storage name for keys (id), and at the same time use an external - // JSON serializer to serialize the entire user POCO. Since the serializer is unaware of the reserved storage name, it will produce - // a storage name as usual, based on the .NET property's name, possibly with a naming policy applied to it. The connector then needs - // to look that up and replace with the reserved name. - ((KeyPropertyModel)property).SerializedKeyName = storageName; - } - else - { - property.StorageName = storageName; - } - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs deleted file mode 100644 index 7f837038dfa6..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using System.Runtime.CompilerServices; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a record in a vector store collection. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class CollectionModel -{ - private readonly Type _recordType; - private readonly Func _recordFactory; - - private KeyPropertyModel? _singleKeyProperty; - private VectorPropertyModel? _singleVectorProperty; - private DataPropertyModel? _singleFullTextSearchProperty; - - /// - /// Gets the key properties of the record. - /// - public IReadOnlyList KeyProperties { get; } - - /// - /// Gets the data properties of the record. - /// - public IReadOnlyList DataProperties { get; } - - /// - /// Gets the vector properties of the record. - /// - public IReadOnlyList VectorProperties { get; } - - /// - /// Gets all properties of the record, of all types. - /// - public IReadOnlyList Properties { get; } - - /// - /// Gets all properties of the record, of all types, indexed by their model name. - /// - public IReadOnlyDictionary PropertyMap { get; } - - /// - /// Gets a value that indicates whether any of the vector properties in the model require embedding generation. - /// - public bool EmbeddingGenerationRequired { get; } - - internal CollectionModel( - Type recordType, - Func recordFactory, - IReadOnlyList keyProperties, - IReadOnlyList dataProperties, - IReadOnlyList vectorProperties, - IReadOnlyDictionary propertyMap) - { - this._recordType = recordType; - this._recordFactory = recordFactory; - - this.KeyProperties = keyProperties; - this.DataProperties = dataProperties; - this.VectorProperties = vectorProperties; - this.PropertyMap = propertyMap; - this.Properties = propertyMap.Values.ToList(); - - this.EmbeddingGenerationRequired = vectorProperties.Any(p => p.EmbeddingType != p.Type); - } - - /// - /// Returns the single key property in the model, and throws if there are multiple key properties. - /// - public KeyPropertyModel KeyProperty => this._singleKeyProperty ??= this.KeyProperties.Single(); - - /// - /// Returns the single vector property in the model, and throws if there are multiple vector properties. - /// Suitable for connectors where validation is in place for single vectors only (). - /// - public VectorPropertyModel VectorProperty => this._singleVectorProperty ??= this.VectorProperties.Single(); - - /// - /// Instantiates a new record of the specified type. - /// - // TODO: the pattern of first instantiating via parameterless constructor and then populating the properties isn't compatible - // with read-only types, where properties have no setters. Supporting those would be problematic given the that different - // connectors have completely different representations of the data coming back from the database, and which needs to be - // populated. - public TRecord CreateRecord() - { - Debug.Assert(typeof(TRecord) == this._recordType, "Type mismatch between record type and model type."); - - return (TRecord)this._recordFactory(); - } - - /// - /// Gets the vector property with the provided name if a name is provided, and falls back - /// to a vector property in the schema if not. - /// - /// The search options, which defines the vector property name. - /// The provided property name is not a valid text data property name.ORNo name was provided and there's more than one vector property. - public VectorPropertyModel GetVectorPropertyOrSingle(VectorSearchOptions searchOptions) - { - if (searchOptions.VectorProperty is not null) - { - return this.GetMatchingProperty(searchOptions.VectorProperty, data: false); - } - - // If vector property name is not provided, check if there is a single vector property, or throw if there are no vectors or more than one. - // TODO: Make a single switch expression + coalesce from the following - dotnet format fails on it for now - if (this._singleVectorProperty is null) - { - switch (this.VectorProperties) - { - case [var singleProperty]: - this._singleVectorProperty = singleProperty; - break; - - case { Count: 0 }: - throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any vector properties."); - - default: - throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple vector properties, please specify your chosen property via options."); - } - } - - return this._singleVectorProperty; - } - - /// - /// Gets the text data property with the provided name that has full text search indexing enabled, or falls back - /// to a text data property in the schema if no name is provided. - /// - /// The full text search property selector. - /// The provided property name is not a valid text data property name.ORNo name was provided and there's more than one text data property with full text search indexing enabled. - public DataPropertyModel GetFullTextDataPropertyOrSingle(Expression>? expression) - { - if (expression is not null) - { - var property = this.GetMatchingProperty(expression, data: true); - - return property.IsFullTextIndexed - ? property - : throw new InvalidOperationException($"The property '{property.ModelName}' on '{this._recordType.Name}' must have full text search indexing enabled."); - } - - if (this._singleFullTextSearchProperty is null) - { - // If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise. - var fullTextStringProperties = this.DataProperties - .Where(l => l.Type == typeof(string) && l.IsFullTextIndexed) - .ToList(); - - // If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise. - switch (fullTextStringProperties) - { - // If there is a single property, use it. - // If there are no properties, throw. - // If there are multiple properties, throw. - case [var singleProperty]: - this._singleFullTextSearchProperty = singleProperty; - break; - - case { Count: 0 }: - throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any text data properties that have full text indexing enabled."); - - default: - throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple text data properties that have full text indexing enabled, please specify your chosen property via options."); - } - } - - return this._singleFullTextSearchProperty; - } - - /// - /// Gets the data or key property selected by the provided expression. - /// - /// The property selector. - /// The provided property name is not a valid data or key property name. - public PropertyModel GetDataOrKeyProperty(Expression> expression) - => this.GetMatchingProperty(expression, data: true); - - private TProperty GetMatchingProperty(Expression> expression, bool data) - where TProperty : PropertyModel - { - var node = expression.Body; - - // First, unwrap any object convert node: r => (object)r.PropertyName becomes r => r.PropertyName - if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } convert - && convert.Type == typeof(object)) - { - node = convert.Operand; - } - - var propertyName = node switch - { - // Simple member expression over the lambda parameter (r => r.PropertyName) - MemberExpression { Member: PropertyInfo clrProperty } member when member.Expression == expression.Parameters[0] - => clrProperty.Name, - - // Dictionary access over the lambda parameter, in dynamic mapping (r => r["PropertyName"]) - MethodCallExpression { Method.Name: "get_Item", Arguments: [var keyExpression] } methodCall - => keyExpression switch - { - ConstantExpression { Value: string text } => text, - MemberExpression field when TryGetCapturedValue(field, out object? capturedValue) && capturedValue is string text => text, - _ => throw new InvalidOperationException("Invalid dictionary key expression") - }, - - _ => throw new InvalidOperationException("Property selector lambda is invalid") - }; - - if (!this.PropertyMap.TryGetValue(propertyName, out var property)) - { - throw new InvalidOperationException($"Property '{propertyName}' could not be found."); - } - - return property is TProperty typedProperty - ? typedProperty - : throw new InvalidOperationException($"Property '{propertyName}' isn't of type '{typeof(TProperty).Name}'."); - - static bool TryGetCapturedValue(Expression expression, out object? capturedValue) - { - if (expression is MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo } - && constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate) - && Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true)) - { - capturedValue = fieldInfo.GetValue(constant.Value); - return true; - } - - capturedValue = null; - return false; - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs deleted file mode 100644 index 3b6f6ccc6b51..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs +++ /dev/null @@ -1,609 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Reflection; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a builder for a . -/// This is an internal support type meant for use by connectors only and not by applications. -/// -/// This class is single-use only, and not thread-safe. -[Experimental("MEVD9001")] -public abstract class CollectionModelBuilder -{ - /// - /// Gets the options for building the model. - /// - protected CollectionModelBuildingOptions Options { get; } - - /// - /// Gets the key properties of the record. - /// - protected List KeyProperties { get; } = []; - - /// - /// Gets the data properties of the record. - /// - protected List DataProperties { get; } = []; - - /// - /// Gets the vector properties of the record. - /// - protected List VectorProperties { get; } = []; - - /// - /// Gets all properties of the record, of all types. - /// - protected IEnumerable Properties => this.PropertyMap.Values; - - /// - /// Gets all properties of the record, of all types, indexed by their model name. - /// - protected Dictionary PropertyMap { get; } = []; - - /// - /// Gets the default embedding generator to use for vector properties, when none is specified at the property or collection level. - /// - protected IEmbeddingGenerator? DefaultEmbeddingGenerator { get; private set; } - - /// - /// Gets the collection's generic key type parameter (TKey), if provided. - /// Used by to validate that TKey corresponds to the key property type on the model. - /// - protected Type? KeyType { get; private set; } - - /// - /// Constructs a new . - /// - protected CollectionModelBuilder(CollectionModelBuildingOptions options) - => this.Options = options; - - /// - /// Builds and returns an from the given and . - /// - /// The CLR type of the record. - /// The collection's generic key type parameter (TKey), used to validate correspondence with the key property type. - /// An optional record definition that overrides attribute-based configuration. - /// An optional default embedding generator for vector properties. - [RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - [RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - public virtual CollectionModel Build(Type recordType, Type keyType, VectorStoreCollectionDefinition? definition, IEmbeddingGenerator? defaultEmbeddingGenerator) - { - this.KeyType = keyType; - - if (recordType == typeof(Dictionary)) - { - throw new ArgumentException("Dynamic mapping with Dictionary requires calling BuildDynamic()."); - } - - this.DefaultEmbeddingGenerator = definition?.EmbeddingGenerator ?? defaultEmbeddingGenerator; - - // Build a lookup of definition properties by name for matching with CLR properties. - Dictionary? definitionByName = null; - if (definition is not null) - { - definitionByName = []; - foreach (var p in definition.Properties) - { - definitionByName[p.Name] = p; - } - } - - // Process CLR properties, matching to definition properties where available. - // TODO: This traverses the CLR type's properties, making it incompatible with trimming (and NativeAOT). - // TODO: We could put [DynamicallyAccessedMembers] to preserve all properties, but that approach wouldn't - // TODO: work with hierarchical data models (#10957). - foreach (var clrProperty in recordType.GetProperties()) - { - VectorStoreProperty? definitionProperty = null; - _ = definitionByName?.TryGetValue(clrProperty.Name, out definitionProperty); - - this.ProcessProperty(clrProperty, definitionProperty, recordType); - } - - // Go over the properties, configure POCO accessors and validate type compatibility. - foreach (var property in this.Properties) - { - var clrProperty = recordType.GetProperty(property.ModelName) - ?? throw new InvalidOperationException($"Property '{property.ModelName}' not found on CLR type '{recordType.FullName}'."); - - var clrPropertyType = clrProperty.PropertyType; - if ((Nullable.GetUnderlyingType(clrPropertyType) ?? clrPropertyType) != (Nullable.GetUnderlyingType(property.Type) ?? property.Type)) - { - throw new InvalidOperationException( - $"Property '{property.ModelName}' has a different CLR type in the record definition ('{property.Type.Name}') and on the .NET property ('{clrProperty.PropertyType}')."); - } - - property.ConfigurePocoAccessors(clrProperty); - } - - this.Customize(); - this.Validate(recordType, definition); - - // Extra validation for non-dynamic mapping scenarios: ensure the type has a parameterless constructor. - if (!this.Options.UsesExternalSerializer && recordType.GetConstructor(Type.EmptyTypes) is null) - { - throw new NotSupportedException($"Type '{recordType.Name}' must have a parameterless constructor."); - } - - return new(recordType, () => Activator.CreateInstance(recordType)!, this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap); - } - - /// - /// Builds and returns an for dynamic mapping scenarios from the given . - /// - /// The record definition describing the collection's schema. - /// An optional default embedding generator for vector properties. - public virtual CollectionModel BuildDynamic(VectorStoreCollectionDefinition definition, IEmbeddingGenerator? defaultEmbeddingGenerator) - { - if (definition is null) - { - throw new ArgumentException("Vector store record definition must be provided for dynamic mapping."); - } - - this.DefaultEmbeddingGenerator = defaultEmbeddingGenerator; - - foreach (var defProp in definition.Properties) - { - this.ProcessProperty(clrProperty: null, defProp, type: null); - } - - this.Customize(); - this.Validate(type: null, definition); - - foreach (var property in this.Properties) - { - property.ConfigureDynamicAccessors(); - } - - return new(typeof(Dictionary), static () => new Dictionary(), this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap); - } - - /// - /// As part of building the model, this method processes a single property, accepting both a CLR - /// (from which attributes are read) and a from the user-provided record definition. - /// Either may be , but not both. - /// When both are provided, the record definition values override attribute-configured values. - /// - protected virtual void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type) - { - Debug.Assert(clrProperty is not null || definitionProperty is not null); - - VectorStoreKeyAttribute? keyAttribute = null; - VectorStoreDataAttribute? dataAttribute = null; - VectorStoreVectorAttribute? vectorAttribute = null; - - if (clrProperty is not null) - { - // Read attributes from CLR property. - keyAttribute = clrProperty.GetCustomAttribute(); - dataAttribute = clrProperty.GetCustomAttribute(); - vectorAttribute = clrProperty.GetCustomAttribute(); - - // Validate that at most one mapping attribute is present. - if ((keyAttribute is not null ? 1 : 0) + (dataAttribute is not null ? 1 : 0) + (vectorAttribute is not null ? 1 : 0) > 1) - { - throw new InvalidOperationException( - $"Property '{type!.Name}.{clrProperty.Name}' has multiple of {nameof(VectorStoreKeyAttribute)}, {nameof(VectorStoreDataAttribute)} or {nameof(VectorStoreVectorAttribute)}. Only one of these attributes can be specified on a property."); - } - - // If no mapping attribute and no definition, skip this property. - if (keyAttribute is null && dataAttribute is null && vectorAttribute is null && definitionProperty is null) - { - return; - } - - // Validate kind compatibility between attribute and definition. - if (definitionProperty is not null - && ((keyAttribute is not null && definitionProperty is not VectorStoreKeyProperty) - || (dataAttribute is not null && definitionProperty is not VectorStoreDataProperty) - || (vectorAttribute is not null && definitionProperty is not VectorStoreVectorProperty))) - { - string definitionKind = definitionProperty switch - { - VectorStoreKeyProperty => "key", - VectorStoreDataProperty => "data", - VectorStoreVectorProperty => "vector", - _ => throw new ArgumentException($"Unknown type '{definitionProperty.GetType().FullName}' in vector store record definition.") - }; - - throw new InvalidOperationException( - $"Property '{clrProperty.Name}' is present in the {nameof(VectorStoreCollectionDefinition)} as a {definitionKind} property, but the .NET property on type '{type?.Name}' has an incompatible attribute."); - } - } - - string propertyName = clrProperty?.Name ?? definitionProperty!.Name; - Type propertyType = clrProperty?.PropertyType - ?? definitionProperty!.Type - ?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(definitionProperty!)); - - PropertyModel property; - string? attributeStorageName = null; - - if (keyAttribute is not null || definitionProperty is VectorStoreKeyProperty) - { - var keyProperty = new KeyPropertyModel(propertyName, propertyType); - - if (keyAttribute is not null) - { - keyProperty.IsAutoGenerated = keyAttribute.IsAutoGeneratedNullable ?? this.SupportsKeyAutoGeneration(keyProperty.Type); - attributeStorageName = keyAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreKeyProperty defKey) - { - keyProperty.IsAutoGenerated = defKey.IsAutoGenerated ?? this.SupportsKeyAutoGeneration(keyProperty.Type); - } - - this.KeyProperties.Add(keyProperty); - property = keyProperty; - } - else if (dataAttribute is not null || definitionProperty is VectorStoreDataProperty) - { - var dataProperty = new DataPropertyModel(propertyName, propertyType); - - if (dataAttribute is not null) - { - dataProperty.IsIndexed = dataAttribute.IsIndexed; - dataProperty.IsFullTextIndexed = dataAttribute.IsFullTextIndexed; - attributeStorageName = dataAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreDataProperty defData) - { - dataProperty.IsIndexed = defData.IsIndexed; - dataProperty.IsFullTextIndexed = defData.IsFullTextIndexed; - } - - this.DataProperties.Add(dataProperty); - property = dataProperty; - } - else if (vectorAttribute is not null || definitionProperty is VectorStoreVectorProperty) - { - // If a definition exists, create via the definition to preserve generic type info (VectorStoreVectorProperty). - var vectorProperty = definitionProperty is VectorStoreVectorProperty defVec - ? defVec.CreatePropertyModel() - : new VectorPropertyModel(propertyName, propertyType); - - if (vectorAttribute is not null) - { - vectorProperty.Dimensions = vectorAttribute.Dimensions; - vectorProperty.IndexKind = vectorAttribute.IndexKind; - vectorProperty.DistanceFunction = vectorAttribute.DistanceFunction; - attributeStorageName = vectorAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreVectorProperty defVectorProp) - { - vectorProperty.Dimensions = defVectorProp.Dimensions; - - if (defVectorProp.IndexKind is not null) - { - vectorProperty.IndexKind = defVectorProp.IndexKind; - } - - if (defVectorProp.DistanceFunction is not null) - { - vectorProperty.DistanceFunction = defVectorProp.DistanceFunction; - } - } - - this.ConfigureVectorPropertyEmbedding( - vectorProperty, - (definitionProperty as VectorStoreVectorProperty)?.EmbeddingGenerator ?? this.DefaultEmbeddingGenerator, - (definitionProperty as VectorStoreVectorProperty)?.EmbeddingType); - - this.VectorProperties.Add(vectorProperty); - property = vectorProperty; - } - else - { - throw new UnreachableException(); - } - - // Apply storage name: attribute first, then definition (which takes precedence). - this.SetPropertyStorageName(property, attributeStorageName, type); - if (definitionProperty is not null) - { - this.SetPropertyStorageName(property, definitionProperty.StorageName, type); - } - - if (definitionProperty?.ProviderAnnotations is not null) - { - property.ProviderAnnotations = new Dictionary(definitionProperty.ProviderAnnotations); - } - - if (clrProperty is not null) - { - property.PropertyInfo = clrProperty; - } - - this.PropertyMap.Add(propertyName, property); - } - - private void SetPropertyStorageName(PropertyModel property, string? storageName, Type? type) - { - if (property is KeyPropertyModel && this.Options.ReservedKeyStorageName is not null) - { - // If we have ReservedKeyStorageName, there can only be a single key property (validated in the constructor) - property.StorageName = this.Options.ReservedKeyStorageName; - return; - } - - if (storageName is null) - { - return; - } - - // If a custom serializer is used (e.g. JsonSerializer), it would ignore our own attributes/config, and - // our model needs to be in sync with the serializer's behavior (for e.g. storage names in filters). - // So we ignore the config here as well. - // TODO: Consider throwing here instead of ignoring - if (this.Options.UsesExternalSerializer && type != null) - { - return; - } - - property.StorageName = storageName; - } - - /// - /// Gets the embedding types supported by this provider, in priority order. - /// The first type whose embedding generator is compatible with the input type will be used. - /// - /// - /// Override this property in connectors that support additional embedding types beyond of . - /// - protected virtual IReadOnlyList EmbeddingGenerationDispatchers { get; } - = [EmbeddingGenerationDispatcher.Create>()]; - - /// - /// Attempts to resolve the embedding type for the given vector property, iterating over in priority order. - /// - private (Type? EmbeddingType, EmbeddingGenerationDispatcher? Handler) ResolveEmbeddingType( - VectorPropertyModel vectorProperty, - IEmbeddingGenerator embeddingGenerator, - Type? userRequestedEmbeddingType) - { - foreach (var supported in this.EmbeddingGenerationDispatchers) - { - if (supported.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType) is { } resolved) - { - return (resolved, supported); - } - } - - return (null, null); - } - - /// - /// Resolves the embedding handler for a native vector property type, where embedding generation is only needed for search. - /// Since the property type is already a valid native type, we only check if the generator can produce the - /// embedding output type (regardless of input type, which is only known at search time). - /// - private EmbeddingGenerationDispatcher? ResolveSearchOnlyEmbeddingHandler(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator) - { - foreach (var supported in this.EmbeddingGenerationDispatchers) - { - if (supported.CanGenerateEmbedding(vectorProperty, embeddingGenerator)) - { - return supported; - } - } - - return null; - } - - /// - /// Configures embedding generation for a vector property. Sets the embedding generator, resolves the embedding type, - /// and assigns the appropriate . - /// - /// - /// If the property's type is natively supported (e.g. of ), the embedding type - /// is set to the property's type; if a generator is also configured, a search-only dispatcher is resolved so that search can convert - /// arbitrary inputs (e.g. string) to embeddings. - /// Otherwise, if a generator is configured, the embedding type is resolved from it. If resolution fails, the embedding type remains - /// and an error is deferred to the validation phase. - /// - private void ConfigureVectorPropertyEmbedding( - VectorPropertyModel vectorProperty, - IEmbeddingGenerator? embeddingGenerator, - Type? userRequestedEmbeddingType) - { - vectorProperty.EmbeddingGenerator = embeddingGenerator; - - if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out _)) - { - if (userRequestedEmbeddingType is not null && userRequestedEmbeddingType != vectorProperty.Type) - { - throw new InvalidOperationException(VectorDataStrings.DifferentEmbeddingTypeSpecifiedForNativelySupportedType(vectorProperty, userRequestedEmbeddingType)); - } - - vectorProperty.EmbeddingType = vectorProperty.Type; - - // Even for native types, if an embedding generator is configured, resolve the dispatcher - // so that search can convert arbitrary inputs (e.g. string) to embeddings. - if (embeddingGenerator is not null) - { - vectorProperty.EmbeddingGenerationDispatcher = this.ResolveSearchOnlyEmbeddingHandler(vectorProperty, embeddingGenerator); - } - } - else if (embeddingGenerator is not null) - { - // The property type isn't a valid embedding type, but an embedding generator is configured. - // Try to resolve the embedding type from it: if the configured generator supports translating the input type (e.g. string) to - // an output type supported by the provider, we set that as the embedding type. - // If this fails, EmbeddingType remains null and we defer the error to the validation phase. - var (embeddingType, handler) = this.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType); - vectorProperty.EmbeddingType = embeddingType; - vectorProperty.EmbeddingGenerationDispatcher = handler; - } - - // If the property type isn't valid and there's no embedding generator, that's an error. - // But we throw later, in validation, to allow for provider customization to correct this invalid state after this step. - } - - /// - /// Extension hook for connectors to be able to customize the model. - /// - protected virtual void Customize() - { - } - - /// - /// Validates the model after all properties have been processed. - /// - protected virtual void Validate(Type? type, VectorStoreCollectionDefinition? definition) - { - if (this.KeyProperties.Count > 1) - { - throw new NotSupportedException($"Multiple key properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported."); - } - - if (this.KeyProperties.Count == 0) - { - throw new NotSupportedException($"No key property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required."); - } - - if (this.Options.RequiresAtLeastOneVector && this.VectorProperties.Count == 0) - { - throw new NotSupportedException($"No vector property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required."); - } - - if (!this.Options.SupportsMultipleVectors && this.VectorProperties.Count > 1) - { - throw new NotSupportedException($"Multiple vector properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported."); - } - - var storageNameMap = new Dictionary(); - - foreach (var property in this.PropertyMap.Values) - { - this.ValidateProperty(property, definition); - - if (storageNameMap.TryGetValue(property.StorageName, out var otherproperty)) - { - throw new InvalidOperationException($"Property '{property.ModelName}' is being mapped to storage name '{property.StorageName}', but property '{otherproperty.ModelName}' is already mapped to the same storage name."); - } - - storageNameMap[property.StorageName] = property; - } - - string TypeMessage() => type is null ? "" : $"type '{type.Name}' or "; - } - - /// - /// Validates a single property, performing validation on it. - /// - protected virtual void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) - { - var type = propertyModel.Type; - - Debug.Assert(propertyModel.Type is not null); - - switch (propertyModel) - { - case KeyPropertyModel keyProperty: - if (keyProperty.IsAutoGenerated && !this.SupportsKeyAutoGeneration(keyProperty.Type)) - { - throw new NotSupportedException( - $"Property '{keyProperty.ModelName}' is configured for auto-generation, but key properties of type '{keyProperty.Type.Name}' do not support auto-generation."); - } - - this.ValidateKeyProperty(keyProperty); - break; - - case DataPropertyModel dataProperty: - if (!this.IsDataPropertyTypeValid(dataProperty.Type, out var supportedTypes)) - { - throw new NotSupportedException( - $"Property '{dataProperty.ModelName}' has unsupported type '{type.Name}'. Data properties must be one of the supported types: {supportedTypes}."); - } - break; - - case VectorPropertyModel vectorProperty: - if (vectorProperty.EmbeddingType is null) - { - if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out string? supportedVectorTypes)) - { - throw new UnreachableException("EmbeddingType cannot be null when the property type is supported."); - } - - if (vectorProperty.EmbeddingGenerator is null) - { - throw new InvalidOperationException(VectorDataStrings.UnsupportedVectorPropertyWithoutEmbeddingGenerator(vectorProperty)); - } - - // If the user has configured a desired embedding type (done to use en embedding type other than the provider's default one), throw errors tailored to that. - // Throw errors related to that. - var userRequestedEmbeddingType = definition?.Properties.OfType().SingleOrDefault(p => p.Name == vectorProperty.ModelName)?.EmbeddingType; - if (userRequestedEmbeddingType is not null) - { - throw new InvalidOperationException(this.IsVectorPropertyTypeValid(userRequestedEmbeddingType, out _) - ? VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes) - : VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes)); - } - - throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGenerator(vectorProperty, vectorProperty.EmbeddingGenerator, supportedVectorTypes)); - } - - if (!this.IsVectorPropertyTypeValid(vectorProperty.EmbeddingType, out string? supportedVectorTypes2)) - { - // Should in principle never happen, only with incorrect provider customization. - throw new InvalidOperationException($"Property '{vectorProperty.ModelName}' has unsupported embedding type '{vectorProperty.EmbeddingType.Name}'. Vector properties must be one of the supported types: {supportedVectorTypes2}."); - } - - if (vectorProperty.Dimensions <= 0) - { - throw new InvalidOperationException($"Vector property '{propertyModel.ModelName}' must have a positive number of dimensions."); - } - - break; - - default: - throw new UnreachableException(); - } - } - - /// - /// Configures auto-generation for the given key property. - /// Defaults to configuring key properties as auto-generated, and throwing if auto-generation is requested for - /// any other type. - /// - protected virtual bool SupportsKeyAutoGeneration(Type keyPropertyType) - => keyPropertyType == typeof(Guid); - - /// - /// Validates the key property. The default implementation validates that the collection's generic key type () - /// corresponds to the key property type on the model, if was provided. - /// Provider overrides should call the base implementation. - /// - protected virtual void ValidateKeyProperty(KeyPropertyModel keyProperty) - { - if (this.KeyType is not null && this.KeyType != typeof(object) && this.KeyType != keyProperty.Type) - { - throw new InvalidOperationException( - $"The collection's generic key type is '{this.KeyType.Name}', but the key property '{keyProperty.ModelName}' has type '{keyProperty.Type.Name}'. The generic key type must match the key property type."); - } - } - - /// - /// Validates that the .NET type for a data property is supported by the provider. - /// - protected abstract bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes); - - /// - /// Validates that the .NET type for a vector property is supported by the provider. - /// - protected abstract bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs deleted file mode 100644 index 28ae1f026af0..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Contains options affecting model building; passed to . -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class CollectionModelBuildingOptions -{ - /// - /// Gets a value that indicates whether multiple vector properties are supported. - /// - public required bool SupportsMultipleVectors { get; init; } - - /// - /// Gets a value that indicates whether at least one vector property is required. - /// - public required bool RequiresAtLeastOneVector { get; init; } - - /// - /// Gets a value that indicates whether an external serializer will be used (for example, System.Text.Json). - /// - public bool UsesExternalSerializer { get; init; } - - /// - /// Gets the special, reserved name for the key property of the database. - /// When set, the model builder manages the key storage name, and users cannot customize it. - /// - public string? ReservedKeyStorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs deleted file mode 100644 index b2d800245054..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a data property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class DataPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; set; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; set; } - - /// - public override string ToString() - => $"{this.ModelName} (Data, {this.Type.Name})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs deleted file mode 100644 index b1825cda295a..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a supported embedding type for a vector store provider. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -/// -/// Each instance encapsulates both build-time embedding type resolution and runtime embedding generation -/// for a specific subtype. -/// -[Experimental("MEVD9001")] -public abstract class EmbeddingGenerationDispatcher -{ - /// - /// Gets the type that this instance supports. - /// - public abstract Type EmbeddingType { get; } - - /// - /// Attempts to resolve the embedding type for the given , using the given . - /// - /// The resolved embedding type, or if the generator does not support this embedding type. - public abstract Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType); - - /// - /// Generates embeddings of this type from the given , using the embedding generator configured on the . - /// - public abstract Task> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable values, CancellationToken cancellationToken); - - /// - /// Generates a single embedding of this type from the given , using the embedding generator configured on the . - /// - public abstract Task GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken); - - /// - /// Checks whether the given can produce embeddings of this type for any of the input types - /// supported by the given . - /// This is used for native vector property types (e.g., of ), where embedding generation - /// is only needed for search and the input type is not known at model-build time. - /// - public abstract bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator); - - /// - /// Creates a new for the given type. - /// - public static EmbeddingGenerationDispatcher Create() - where TEmbedding : Embedding - => new EmbeddingGenerationDispatcher(); -} - -/// -/// A implementation for a specific type. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class EmbeddingGenerationDispatcher : EmbeddingGenerationDispatcher - where TEmbedding : Embedding -{ - /// - public override Type EmbeddingType => typeof(TEmbedding); - - /// - public override Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - => vectorProperty.ResolveEmbeddingType(embeddingGenerator, userRequestedEmbeddingType); - - /// - public override bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator) - => vectorProperty.CanGenerateEmbedding(embeddingGenerator); - - /// - public override Task> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable values, CancellationToken cancellationToken) - => vectorProperty.GenerateEmbeddingsCoreAsync(values, cancellationToken); - - /// - public override Task GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken) - => vectorProperty.GenerateEmbeddingCoreAsync(value, cancellationToken); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs deleted file mode 100644 index 4b0cc8f07e2b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// Options for filter expression preprocessing. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class FilterPreprocessingOptions -{ - /// - /// Whether the connector supports parameterization. - /// - /// - /// If , the visitor will inline captured variables and constant member accesses as simple constant nodes. - /// If , these will instead be replaced with nodes. - /// - public bool SupportsParameterization { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs deleted file mode 100644 index ad847b34b1d1..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// Base class for filter translators used by vector data connectors. -/// Provides common functionality for preprocessing filter expressions and matching common patterns. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class FilterTranslatorBase -{ - /// - /// The collection model for the current translation operation. - /// - protected CollectionModel Model { get; private set; } = null!; - - /// - /// The parameter expression representing the record in the filter lambda. - /// - protected ParameterExpression RecordParameter { get; private set; } = null!; - - /// - /// Preprocesses the filter expression before translation. - /// Sets and , runs the preprocessing visitor, - /// and returns the preprocessed expression. - /// - /// The filter lambda expression to preprocess. - /// The collection model containing property information. - /// Options controlling the preprocessing behavior. - /// The preprocessed expression ready for translation. - protected Expression PreprocessFilter(LambdaExpression lambdaExpression, CollectionModel model, FilterPreprocessingOptions options) - { - this.Model = model; - this.RecordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor(options.SupportsParameterization); - return preprocessor.Preprocess(lambdaExpression.Body); - } - - /// - /// Tries to match a Contains method call expression and extract the source collection and item expressions. - /// - /// The method call expression to match. - /// When successful, the source collection expression. - /// When successful, the item expression being searched for. - /// if the expression is a recognized Contains pattern; otherwise, . - protected static bool TryMatchContains( - MethodCallExpression methodCall, - [NotNullWhen(true)] out Expression? source, - [NotNullWhen(true)] out Expression? item) - { - switch (methodCall) - { - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var itm] } - when methodCall.Method.DeclaringType == typeof(Enumerable): - source = src; - item = itm; - return true; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression src, - Arguments: [var itm] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - source = src; - item = itm; - return true; - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var itm, ..] } - when methodCall.Method.DeclaringType == typeof(MemoryExtensions) - && (methodCall.Arguments.Count is 2 - || (methodCall.Arguments.Count is 3 && methodCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var src): - source = src; - item = itm; - return true; - - default: - source = null; - item = null; - return false; - } - } - - /// - /// Tries to bind an expression to a property in the collection model. - /// - /// The expression to bind. - /// When successful, the property model that was bound. - /// if the expression was successfully bound to a property; otherwise, . - protected virtual bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? propertyModel) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this.RecordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this.RecordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - propertyModel = null; - return false; - } - - if (!this.Model.PropertyMap.TryGetValue(modelName, out propertyModel)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{propertyModel.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{propertyModel.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - /// - /// Tries to unwrap an implicit cast to Span or ReadOnlySpan that may be present in expressions - /// when C# 14's first-class span support causes MemoryExtensions methods to be resolved. - /// - /// The expression to unwrap. - /// When successful, the unwrapped expression. - /// if a span implicit cast was unwrapped; otherwise, . - protected static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } - - #region FilterTranslationPreprocessor - - /// - /// A processor for user-provided filter expressions which performs various common transformations before actual translation takes place. - /// - private sealed class FilterTranslationPreprocessor : ExpressionVisitor - { - private readonly bool _supportsParameterization; - private List? _parameterNames; - - internal FilterTranslationPreprocessor(bool supportsParameterization) - { - this._supportsParameterization = supportsParameterization; - } - - internal Expression Preprocess(Expression node) - { - if (this._supportsParameterization) - { - this._parameterNames = []; - } - - return this.Visit(node); - } - - /// - protected override Expression VisitMember(MemberExpression node) - { - var visited = (MemberExpression)base.VisitMember(node); - - // This identifies field and property access over constants, which can be evaluated immediately. - // This covers captured variables, since those are actually member accesses over compiled-generated closure types: - // var x = 8; - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x }); - // - // This also covers member variables: - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x }); - // ... as "this" here is represented by a ConstantExpression node in the tree. - // - // Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders - // from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those - // connectors, and centralizes the pattern matching in a single centralized place). - // For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't - // even need to be aware of it. - - // Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for - // static members (expression is null). - object? baseValue; - switch (visited.Expression) - { - // Member access over constant (i.e. instance members) - case ConstantExpression { Value: var v }: - baseValue = v; - break; - - // Member constant over null (i.e. static members) - case null: - baseValue = null; - break; - - // Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something) - case QueryParameterExpression p: - baseValue = p.Value; - - // The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names. - this._parameterNames!.Remove(p.Name); - break; - - default: - return visited; - } - - object? evaluatedValue; - - var memberInfo = visited.Member; - - switch (memberInfo) - { - case FieldInfo fieldInfo: - evaluatedValue = fieldInfo.GetValue(baseValue); - break; - - case PropertyInfo { GetMethod.IsStatic: false } propertyInfo when baseValue is null: - throw new InvalidOperationException($"Cannot access member '{propertyInfo.Name}' on null object."); - - case PropertyInfo propertyInfo: - evaluatedValue = propertyInfo.GetValue(baseValue); - break; - default: - return visited; - } - - // Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly), - if (!this._supportsParameterization) - { - return Expression.Constant(evaluatedValue, visited.Type); - } - - // Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter). - - // TODO: Share the same parameter when it references the same captured value - - // Make sure parameter names are unique. - var origName = memberInfo.Name; - var name = origName; - for (var i = 0; this._parameterNames!.Contains(name); i++) - { - name = $"{origName}_{i}"; - } - this._parameterNames.Add(name); - - return new QueryParameterExpression(name, evaluatedValue, visited.Type); - } - - /// - protected override Expression VisitNew(NewExpression node) - { - var visited = (NewExpression)base.VisitNew(node); - - // Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression. - // This is particularly useful for converting inline instantiation of DateTime, DateTimeOffset, DateOnly, and TimeOnly to constants, which can then be easily translated. - switch (visited.Constructor) - { - case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime) -#if NET - || constructor.DeclaringType == typeof(DateOnly) || constructor.DeclaringType == typeof(TimeOnly) -#endif - : - var constantArguments = new object?[visited.Arguments.Count]; - - // We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1). - // If an argument isn't a constant (e.g. new DateTimeOffset(..., TimeSpan.FromHours(2))), we fall back to trying the LINQ interpreter - // as a general-purpose expression evaluator - but note that this is considerably slower. - for (var i = 0; i < visited.Arguments.Count; i++) - { - if (visited.Arguments[i] is ConstantExpression constantArgument) - { - constantArguments[i] = constantArgument.Value; - } - else - { - // There's a non-constant argument - try the LINQ interpreter. -#pragma warning disable CA1031 // Do not catch general exception types - try - { - var evaluated = Expression.Lambda>(Expression.Convert(visited, typeof(object))) -#if NET - .Compile(preferInterpretation: true) -#else - .Compile() -#endif - .Invoke(); - - return Expression.Constant(evaluated, constructor.DeclaringType); - } - catch - { - return visited; - } -#pragma warning restore CA1031 - } - } - - var constantValue = constructor.Invoke(constantArguments); - return Expression.Constant(constantValue, constructor.DeclaringType); - } - - return visited; - } - } - - #endregion FilterTranslationPreprocessor -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs deleted file mode 100644 index 0c37722a3bcc..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// An expression representation a query parameter (captured variable) in the filter expression. -/// -[Experimental("MEVD9001")] -public class QueryParameterExpression(string name, object? value, Type type) : Expression -{ - /// - /// The name of the parameter. - /// - public string Name { get; } = name; - - /// - /// The value of the parameter. - /// - public object? Value { get; } = value; - - /// - public override ExpressionType NodeType => ExpressionType.Extension; - - /// - public override Type Type => type; - - /// - protected override Expression VisitChildren(ExpressionVisitor visitor) => this; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs deleted file mode 100644 index 079659ce5bcb..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -internal interface IRecordCreator -{ - TRecord Create(); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs deleted file mode 100644 index 2b31e56a3d9d..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a key property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class KeyPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - public bool IsAutoGenerated { get; set; } - - /// - /// Gets or sets the name that the JSON serializer will produce for this key property. - /// This is needed for connectors that use an external JSON serializer combined with a reserved key storage name - /// (e.g. CosmosDB NoSQL uses "id"): the serializer produces a JSON object with the policy-transformed name, and - /// the connector needs to find and replace it with the reserved storage name. - /// - public string? SerializedKeyName { get; set; } - - /// - public override string ToString() - => $"{this.ModelName} (Key, {this.Type.Name}{(this.IsAutoGenerated ? ", auto-generated" : "")})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs deleted file mode 100644 index 7b3635911e95..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Reflection; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class PropertyModel(string modelName, Type type) -{ - private string? _storageName; - private Func? _getter; - private Action? _setter; - - /// - /// Gets or sets the model name of the property. If the property corresponds to a .NET property, this name is the name of that property. - /// - public string ModelName { get; set; } = modelName; - - /// - /// Gets or sets the storage name of the property. This is the name to which the property is mapped in the vector store. - /// - public string StorageName - { - get => this._storageName ?? this.ModelName; - set => this._storageName = value; - } - - /// - /// Gets or sets the CLR type of the property. - /// - public Type Type { get; set; } = type; - - /// - /// Gets or sets the reflection for the .NET property. - /// - /// - /// The reflection for the .NET property. - /// when using dynamic mapping. - /// - public PropertyInfo? PropertyInfo { get; set; } - - /// - /// Gets or sets a dictionary of provider-specific annotations for this property. - /// - /// - /// This allows setting database-specific configuration options that aren't universal across all vector stores. - /// - public Dictionary? ProviderAnnotations { get; set; } - - /// - /// Gets whether the property type is nullable. For value types, this is when the type is - /// . For reference types on .NET 6+, this uses NRT annotations via - /// NullabilityInfoContext when a is available - /// (i.e., POCO mapping); otherwise, reference types are assumed nullable. - /// - public bool IsNullable - { - get - { - // Value types: nullable only if Nullable - if (this.Type.IsValueType) - { - return Nullable.GetUnderlyingType(this.Type) is not null; - } - - // Reference types: check NRT annotation via NullabilityInfoContext when available -#if NET - if (this.PropertyInfo is { } propertyInfo) - { - var nullabilityInfo = new NullabilityInfoContext().Create(propertyInfo); - return nullabilityInfo.ReadState != NullabilityState.NotNull; - } -#endif - - // Dynamic mapping or old framework: assume nullable for reference types - return true; - } - } - - /// - /// Configures the property accessors using a CLR for POCO mapping. - /// - // TODO: Implement compiled delegates for better performance, #11122 - // TODO: Implement source-generated accessors for NativeAOT, #10256 - internal void ConfigurePocoAccessors(PropertyInfo propertyInfo) - { - this.PropertyInfo = propertyInfo; - this._getter = propertyInfo.GetValue; - this._setter = (record, value) => - { - // If the value is null, no need to set the property (it's the CLR default) - if (value is not null) - { - propertyInfo.SetValue(record, value); - } - }; - } - - /// - /// Configures the property accessors for dynamic mapping using . - /// - internal void ConfigureDynamicAccessors() - { - var modelName = this.ModelName; - var propertyType = this.Type; - - this._getter = record => - { - var dictionary = (Dictionary)record; - var value = dictionary.TryGetValue(modelName, out var tempValue) ? tempValue : null; - - if (value is not null && value.GetType() != (Nullable.GetUnderlyingType(propertyType) ?? propertyType)) - { - throw new InvalidCastException($"Property '{modelName}' has a value of type '{value.GetType().Name}', but its configured type is '{propertyType.Name}'."); - } - - return value; - }; - - this._setter = (record, value) => ((Dictionary)record)[modelName] = value; - } - - /// - /// Reads the property from the given , returning the value as an . - /// - public object? GetValueAsObject(object record) - { - Debug.Assert(this._getter is not null, "Property accessors have not been configured."); - return this._getter!(record); - } - - /// - /// Writes the property from the given , accepting the value to write as an . - /// - public void SetValueAsObject(object record, object? value) - { - Debug.Assert(this._setter is not null, "Property accessors have not been configured."); - this._setter!(record, value); - } - - /// - /// Reads the property from the given . - /// - // TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors - public T GetValue(object record) - => (T)(object)this.GetValueAsObject(record)!; - - /// - /// Writes the property from the given . - /// - // TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors - public void SetValue(object record, T value) - => this.SetValueAsObject(record, value); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs deleted file mode 100644 index 22a65a732f8b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - -/// -/// Exposes methods for constructing strings that should be used by providers when throwing exceptions. -/// -[Experimental("MEVD9001")] -public static class VectorDataStrings -{ - public static string ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes) - => $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your embedding generator."; - - public static string ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes) - => $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your provider. Supported types are {supportedVectorTypes}."; - - public static string EmbeddingGeneratorWithInvalidEmbeddingType(VectorPropertyModel vectorProperty) - => $"An embedding generator was configured on property '{vectorProperty.ModelName}', but output embedding type '{vectorProperty.EmbeddingType.Name}' isn't supported by the connector."; - - public static string EmbeddingPropertyTypeIncompatibleWithEmbeddingGenerator(VectorPropertyModel vectorProperty) - => $"Property '{vectorProperty.ModelName}' has embedding type '{TypeName(vectorProperty.Type)}', but an embedding generator is configured on the property. Remove the embedding generator or change the property's .NET type to a non-embedding input type to the generator (e.g. string)."; - - public static string DifferentEmbeddingTypeSpecifiedForNativelySupportedType(VectorPropertyModel vectorProperty, Type embeddingType) - => $"Property '{vectorProperty.ModelName}' has {nameof(VectorStoreVectorProperty.EmbeddingType)} configured to '{TypeName(embeddingType)}', but the property already has natively supported '{TypeName(vectorProperty.Type)}'. {nameof(VectorStoreVectorProperty.EmbeddingType)} only needs to be specified for properties that require embedding generation."; - - public static string GetCollectionWithDictionaryNotSupported - => "Dynamic mapping via Dictionary is not supported via this method, call GetDynamicCollection() instead."; - - public static string IncludeVectorsNotSupportedWithEmbeddingGeneration - => "When an embedding generator is configured, `Include Vectors` cannot be enabled."; - - public static string IncompatibleEmbeddingGenerator(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, string supportedOutputTypes) - => $"Embedding generator '{TypeName(embeddingGenerator.GetType())}' on vector property '{vectorProperty.ModelName}' cannot convert the input type '{TypeName(vectorProperty.Type)}' to a supported vector type (one of: {supportedOutputTypes})."; - - public static string IncompatibleEmbeddingGeneratorWasConfiguredForInputType(Type inputType, Type embeddingGeneratorType) - => $"An input of type '{TypeName(inputType)}' was provided, but an incompatible embedding generator of type '{TypeName(embeddingGeneratorType)}' was configured."; - - public static string InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(Type inputType, string supportedVectorTypes) - => $"A value of type '{TypeName(inputType)}' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are: {supportedVectorTypes}."; - - public static string MissingTypeOnPropertyDefinition(VectorStoreProperty property) - => $"Property '{property.Name}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition."; - - public static string UnsupportedVectorPropertyWithoutEmbeddingGenerator(VectorPropertyModel vectorProperty) - => $"Vector property '{vectorProperty.ModelName}' has type '{TypeName(vectorProperty.Type)}' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting '{TypeName(vectorProperty.Type)}' to vector type supported by your provider."; - - public static string NonDynamicCollectionWithDictionaryNotSupported(Type dynamicCollectionType) - => $"Dynamic mapping via Dictionary is not supported via this class, use '{TypeName(dynamicCollectionType)}' instead."; - - private static string TypeName(this Type type) - { - var i = type.Name.IndexOf('`'); - if (i == -1) - { - return type.Name switch - { - "Int32" => "int", - "Int64" => "long", - "Boolean" => "bool", - "Double" => "double", - "Single" => "float", - "String" => "string", - - _ => type.Name - }; - } - - var genericTypeName = type.Name.Substring(0, i); - var genericArgs = string.Join(", ", type.GetGenericArguments().Select(t => t.TypeName())); - return $"{genericTypeName}<{genericArgs}>"; - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs deleted file mode 100644 index 75d3a1057ff9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a vector property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class VectorPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - private int _dimensions; - - /// - /// Gets or sets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions - { - get => this._dimensions; - - set - { - if (value <= 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero."); - } - - this._dimensions = value; - } - } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default varies by database type. For more information, see the documentation of your chosen database connector. - /// - /// - public string? IndexKind { get; set; } - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default varies by database type. For more information, see the documentation of your chosen database connector. - /// - /// - public string? DistanceFunction { get; set; } - - /// - /// Gets or sets the type representing the embedding stored in the database if is set. - /// Otherwise, this property is identical to . - /// - /// - /// This property may be during model building while the embedding type is being resolved, - /// but is guaranteed to be non-null after building completes (validation ensures this). - /// - [AllowNull] - public Type EmbeddingType { get; set; } = null!; - - /// - /// Gets or sets the embedding generator to use for this property. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } - - /// - /// Gets or sets the that was resolved for this property during model building. - /// This handler is used for runtime embedding generation dispatch. - /// - /// - /// This is for vector properties whose type is natively supported by the provider - /// (e.g., of , [], ), - /// since no embedding generation is needed. - /// - public EmbeddingGenerationDispatcher? EmbeddingGenerationDispatcher { get; set; } - - /// - /// Checks whether the given can produce embeddings of type - /// for any input type known to this property model. The base implementation checks for and ; - /// also checks for TInput. - /// - /// This is used for native vector property types, where the input type isn't known at model-build time. - public virtual bool CanGenerateEmbedding(IEmbeddingGenerator embeddingGenerator) - where TEmbedding : Embedding - => embeddingGenerator is IEmbeddingGenerator - || embeddingGenerator is IEmbeddingGenerator; - - /// - /// Checks whether the configured on this property supports the given embedding type. - /// The implementation on this non-generic checks for - /// and as input types for . - /// - public virtual Type? ResolveEmbeddingType(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - where TEmbedding : Embedding - => embeddingGenerator switch - { - // On the TInput side, this out-of-the-box/simple implementation supports string and DataContent only - // (users who want arbitrary TInput types need to use the generic subclass of this type). - // The TEmbedding side is provided by the connector via the generic type parameter to this method, as the connector controls/knows which embedding types are supported. - // Note that if the user has manually specified an embedding type (e.g. to choose Embedding rather than the default Embedding), - // that's provided via the userRequestedEmbeddingType argument; we use that as a filter. - IEmbeddingGenerator when this.Type == typeof(string) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - IEmbeddingGenerator when this.Type == typeof(DataContent) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - - null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."), - _ => null - }; - - /// - /// Generates embeddings for the given , using the configured . - /// - /// Thrown if no is configured on this property. - public Task> GenerateEmbeddingsAsync(IEnumerable values, CancellationToken cancellationToken) - => this.EmbeddingGenerationDispatcher is not { } dispatcher - ? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.") - : dispatcher.GenerateEmbeddingsAsync(this, values, cancellationToken); - - /// - /// Generates a single embedding for the given , using the configured . - /// - /// Thrown if no is configured on this property. - public Task GenerateEmbeddingAsync(object? value, CancellationToken cancellationToken) - => this.EmbeddingGenerationDispatcher is not { } dispatcher - ? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.") - : dispatcher.GenerateEmbeddingAsync(this, value, cancellationToken); - - /// - /// Core method to generate a batch of embeddings. Called by with the correct type parameter. - /// - internal virtual async Task> GenerateEmbeddingsCoreAsync(IEnumerable values, CancellationToken cancellationToken) - where TEmbedding : Embedding - => this.EmbeddingGenerator switch - { - IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding) - => await generator.GenerateAsync( - values.Select(v => v is string s - ? s - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a string, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false), - - IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding) - => await generator.GenerateAsync( - values.Select(v => v is DataContent c - ? c - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {nameof(DataContent)}, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false), - - null => throw new UnreachableException("This method should only be called when an embedding generator is configured."), - - _ => throw new InvalidOperationException( - $"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type."), - }; - - /// - /// Core method to generate a single embedding. Called by with the correct type parameter. - /// - internal virtual async Task GenerateEmbeddingCoreAsync(object? value, CancellationToken cancellationToken) - where TEmbedding : Embedding - => this.EmbeddingGenerator switch - { - IEmbeddingGenerator generator when value is string s - => await generator.GenerateAsync(s, options: null, cancellationToken).ConfigureAwait(false), - - IEmbeddingGenerator generator when value is DataContent c - => await generator.GenerateAsync(c, options: null, cancellationToken).ConfigureAwait(false), - - null => throw new UnreachableException("This method should only be called when an embedding generator is configured."), - - _ => throw new InvalidOperationException( - VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(value?.GetType() ?? typeof(object), this.EmbeddingGenerator!.GetType())), - }; - - /// - /// Returns the types of input that this property model supports. - /// - public virtual Type[] GetSupportedInputTypes() => [typeof(string), typeof(DataContent)]; - - /// - public override string ToString() - => $"{this.ModelName} (Vector, {this.Type.Name})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs deleted file mode 100644 index e7f51d2c58e1..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -[Experimental("MEVD9001")] -public sealed class VectorPropertyModel(string modelName) : VectorPropertyModel(modelName, typeof(TInput)) -{ - /// - public override bool CanGenerateEmbedding(IEmbeddingGenerator embeddingGenerator) - => embeddingGenerator is IEmbeddingGenerator - || base.CanGenerateEmbedding(embeddingGenerator); - - /// - public override Type? ResolveEmbeddingType(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - => embeddingGenerator switch - { - IEmbeddingGenerator when this.Type == typeof(TInput) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - - null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."), - _ => null - }; - - /// - internal override async Task> GenerateEmbeddingsCoreAsync(IEnumerable values, CancellationToken cancellationToken) - { - switch (this.EmbeddingGenerator) - { - case IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding): - return await generator.GenerateAsync( - values.Select(v => v is TInput s - ? s - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {typeof(TInput).Name}, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false); - - case null: - throw new UnreachableException("This method should only be called when an embedding generator is configured."); - - default: - throw new InvalidOperationException( - $"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type."); - } - } - - /// - internal override async Task GenerateEmbeddingCoreAsync(object? value, CancellationToken cancellationToken) - { - if (this.EmbeddingGenerator is IEmbeddingGenerator generator && value is TInput t) - { - return await generator.GenerateAsync(t, options: null, cancellationToken).ConfigureAwait(false); - } - - // Fall through to base class which checks for string and DataContent input types. - return await base.GenerateEmbeddingCoreAsync(value, cancellationToken).ConfigureAwait(false); - } - - /// - public override Type[] GetSupportedInputTypes() => [typeof(TInput)]; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs deleted file mode 100644 index d326bb068760..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as 'data'. -/// -/// -/// Marking a property as 'data' means that the property is not a key and not a vector. But optionally, -/// this property can have an associated vector field containing an embedding for this data. -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreDataAttribute : Attribute -{ - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; init; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; init; } - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs deleted file mode 100644 index 5e4001a447d3..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as the key under which the record is stored in a vector store. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreKeyAttribute : Attribute -{ - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } - - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - /// - /// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations. - /// The getter returns when the value has not been explicitly set; use to distinguish - /// between "explicitly set to false" and "not set". - /// The getter does not throw, even though it cannot distinguish "explicitly set to false" and "not set"; this is a workaround for a C# compiler - /// limitation that does not allow to be used as an attribute argument. - /// - public bool IsAutoGenerated - { - // The getter returns GetValueOrDefault() rather than throwing, as a workaround for a C# compiler limitation: - // Nullable cannot be used as a compile-time attribute argument, so the public property must be bool. - get => this.IsAutoGeneratedNullable.GetValueOrDefault(); - set => this.IsAutoGeneratedNullable = value; - } - - /// - /// Gets whether this key property's value is auto-generated or not, or if not set. - /// - internal bool? IsAutoGeneratedNullable { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs deleted file mode 100644 index 4f898116c69f..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as a vector. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreVectorAttribute : Attribute -{ - /// - /// Initializes a new instance of the class. - /// - /// The number of dimensions that the vector has. - public VectorStoreVectorAttribute(int Dimensions) - { - if (Dimensions <= 0) - { - throw new ArgumentOutOfRangeException(nameof(Dimensions), "Dimensions must be greater than zero."); - } - - this.Dimensions = Dimensions; - } - - /// - /// Gets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions { get; private set; } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default value varies by database type. See the documentation of your chosen database connector for more information. - /// - /// -#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete. - public string? IndexKind { get; init; } -#pragma warning restore CA1019 - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default value varies by database type. See the documentation of your chosen database connector for more information. - /// - /// -#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete. - public string? DistanceFunction { get; init; } -#pragma warning restore CA1019 - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs deleted file mode 100644 index 068caed55807..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a list of well-known distance functions that can be used to compare vectors. -/// -/// -/// Not all Vector Store connectors support all distance functions, and some connectors might -/// support additional distance functions that aren't defined here. -/// For more information on what's supported, see the documentation for each connector. -/// -public static class DistanceFunction -{ - /// - /// Specifies the function that measures the cosine (angular) similarity between two vectors. - /// - /// - /// Cosine similarity measures only the angle between the two vectors, without taking into account the length of the vectors. - /// ConsineSimilarity = 1 - CosineDistance. - /// -1 means vectors are opposite. - /// 0 means vectors are orthogonal. - /// 1 means vectors are identical. - /// - public const string CosineSimilarity = nameof(CosineSimilarity); - - /// - /// Specifies the function that measures the cosine (angular) distance between two vectors. - /// - /// - /// CosineDistance = 1 - CosineSimilarity. - /// 2 means vectors are opposite. - /// 1 means vectors are orthogonal. - /// 0 means vectors are identical. - /// - public const string CosineDistance = nameof(CosineDistance); - - /// - /// Specifies the dot product similarity function, which measures both the length and angle between two vectors. - /// - /// - /// The higher the value, the more similar the vectors. - /// - public const string DotProductSimilarity = nameof(DotProductSimilarity); - - /// - /// Specifies the negative dot product similarity function, which measures both the length and angle between two vectors. - /// - /// - /// The value of NegativeDotProduct = -1 * DotProductSimilarity. - /// The higher the value, the greater the distance between the vectors and the less similar the vectors. - /// - public const string NegativeDotProductSimilarity = nameof(NegativeDotProductSimilarity); - - /// - /// Specifies the function that measures the Euclidean distance between two vectors. - /// - /// - /// Also known as l2-norm. - /// - public const string EuclideanDistance = nameof(EuclideanDistance); - - /// - /// Specifies the function that measures the Euclidean squared distance between two vectors. - /// - /// - /// Also known as l2-squared. - /// - public const string EuclideanSquaredDistance = nameof(EuclideanSquaredDistance); - - /// - /// Specifies the function that measures the number of differences between vectors at each dimension. - /// - public const string HammingDistance = nameof(HammingDistance); - - /// - /// Specifies the function that measures the Manhattan distance between two vectors. - /// - public const string ManhattanDistance = nameof(ManhattanDistance); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs deleted file mode 100644 index 372f451e4c05..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a list of well-known index types that can be used to index vectors. -/// -/// -/// Not all Vector Store connectors support all index types, and some connectors might -/// support additional index types that aren't defined here. For more information on what's -/// supported, see the documentation for each connector. -/// -public static class IndexKind -{ - /// - /// Specifies the Hierarchical Navigable Small World, which performs an approximate nearest neighbor (ANN) search. - /// - /// - /// This search has lower accuracy than exhaustive k nearest neighbor, but is faster and more efficient. - /// - public const string Hnsw = nameof(Hnsw); - - /// - /// Specifies the brute force search to find the nearest neighbors. - /// - /// - /// This search calculates the distances between all pairs of data points, so it has a linear time complexity that grows directly proportional to the number of points. - /// It's also referred to as "exhaustive k nearest neighbor" in some databases. - /// This search has high recall accuracy, but is slower and more expensive than HNSW. - /// It works better with smaller datasets. - /// - public const string Flat = nameof(Flat); - - /// - /// Specifies an Inverted File with Flat Compression. - /// - /// - /// This search is designed to enhance search efficiency by narrowing the search area through the use of neighbor partitions or clusters. - /// Also referred to as approximate nearest neighbor (ANN) search. - /// - public const string IvfFlat = nameof(IvfFlat); - - /// - /// Specifies the Disk-based Approximate Nearest Neighbor algorithm, which is designed for efficiently searching for approximate nearest neighbors (ANN) in high-dimensional spaces. - /// - /// - /// The primary focus of DiskANN is to handle large-scale datasets that can't fit entirely into memory, leveraging disk storage to store the data while maintaining fast search times. - /// - public const string DiskAnn = nameof(DiskAnn); - - /// - /// Specifies an index that compresses vectors using DiskANN-based quantization methods for better efficiency in the kNN search. - /// - public const string QuantizedFlat = nameof(QuantizedFlat); - - /// - /// Specifies a dynamic index that switches automatically from to indexes. - /// - public const string Dynamic = nameof(Dynamic); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs deleted file mode 100644 index b7ba514b0adc..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Describes the properties of a record in a vector store collection. -/// -/// -/// Each property contains additional information about how the property will be treated by the vector store. -/// -public sealed class VectorStoreCollectionDefinition -{ - private IList? _properties; - - /// - /// Gets or sets the list of properties that are stored in the record. - /// - [AllowNull] - public IList Properties - { - get => this._properties ??= []; - set => this._properties = value; - } - - /// - /// Gets or sets the default embedding generator for vector properties in this collection. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs deleted file mode 100644 index 0f1a7d764bde..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a data property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public sealed class VectorStoreDataProperty : VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. Required when using a record type of Dictionary<string, object?> (dynamic mapping), but can be omitted when mapping any other .NET type. - public VectorStoreDataProperty(string name, Type? type = null) - : base(name, type) - { - } - - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; set; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs deleted file mode 100644 index 2c2a6f6262f0..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a key property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public sealed class VectorStoreKeyProperty : VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. Required when using a record type of Dictionary<string, object?> (dynamic mapping), but can be omitted when mapping any other .NET type. - public VectorStoreKeyProperty(string name, Type? type = null) - : base(name, type) - { - } - - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - /// - /// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations. - /// - public bool? IsAutoGenerated { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs deleted file mode 100644 index d3f2b8faced9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base property class for properties on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public abstract class VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. - private protected VectorStoreProperty(string name, Type? type) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); - } - - this.Name = name; - this.Type = type; - } - - private protected VectorStoreProperty(VectorStoreProperty source) - { - this.Name = source.Name; - this.StorageName = source.StorageName; - this.Type = source.Type; - this.ProviderAnnotations = source.ProviderAnnotations is not null - ? new Dictionary(source.ProviderAnnotations) - : null; - } - - /// - /// Gets or sets the name of the property on the data model. - /// - public string Name { get; set; } - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// This property is only respected by implementations that don't support a well-known - /// serialization mechanism like JSON, in which case the attributes used by that serialization system will - /// be used. - /// - public string? StorageName { get; set; } - - /// - /// Gets or sets the type of the property. - /// - public Type? Type { get; set; } - - /// - /// Gets or sets a dictionary of provider-specific annotations for this property. - /// - /// - /// This allows setting database-specific configuration options that aren't universal across all vector stores. - /// Use provider-specific extension methods to set and get values in a strongly-typed manner. - /// - public Dictionary? ProviderAnnotations { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs deleted file mode 100644 index 54762c7ebe81..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData.ProviderServices; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a vector property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public class VectorStoreVectorProperty : VectorStoreProperty -{ - private int _dimensions; - - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The number of dimensions that the vector has. - public VectorStoreVectorProperty(string name, int dimensions) - : base(name, type: null) - { - this.Dimensions = dimensions; - } - - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. - /// The number of dimensions that the vector has. - public VectorStoreVectorProperty(string name, Type type, int dimensions) - : base(name, type) - { - this.Dimensions = dimensions; - } - - /// - /// Gets or sets the default embedding generator to use for this property. - /// - /// - /// If not set, embedding generation will be performed in the database, if supported by your connector. - /// Otherwise, if your database does not support embedding generation, only pregenerated embeddings can be used (for example, ReadOnlyMemory<float>). - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } - - /// - /// Gets or sets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions - { - get => this._dimensions; - - set - { - if (value <= 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero."); - } - - this._dimensions = value; - } - } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default varies by database type. See the documentation of your chosen database connector for more information. - /// - /// - public string? IndexKind { get; set; } - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default varies by database type. See the documentation of your chosen database connector for more information. - /// - /// - public string? DistanceFunction { get; set; } - - /// - /// Gets or sets the desired embedding type (for example, Embedding<Half>) for cases where the default (typically Embedding<float>) isn't suitable. - /// - public Type? EmbeddingType { get; set; } - - internal virtual VectorPropertyModel CreatePropertyModel() - => new(this.Name, this.Type ?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(this))) - { - Dimensions = this.Dimensions, - IndexKind = this.IndexKind, - DistanceFunction = this.DistanceFunction, - EmbeddingGenerator = this.EmbeddingGenerator, - EmbeddingType = this.EmbeddingType! - }; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs deleted file mode 100644 index 23a63b27604c..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData.ProviderServices; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a vector property on a vector store record. -/// -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -/// -/// This generic version of only needs to be used when an is -/// configured on the property, and a custom .NET type is used as input (any type other than or ). -/// -/// -public class VectorStoreVectorProperty : VectorStoreVectorProperty -{ - /// - public VectorStoreVectorProperty(string propertyName, int dimensions) - : base(propertyName, typeof(TInput), dimensions) - { - } - - internal override VectorPropertyModel CreatePropertyModel() - => new VectorPropertyModel(this.Name) - { - Dimensions = this.Dimensions, - IndexKind = this.IndexKind, - DistanceFunction = this.DistanceFunction, - EmbeddingGenerator = this.EmbeddingGenerator - }; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs deleted file mode 100644 index 0ef217b8b4de..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq.Expressions; -using System.Threading; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for calling . -/// -/// The type of the record. -public sealed class FilteredRecordRetrievalOptions -{ - private int _skip = 0; - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets the data property to order by. - /// - /// - /// If not provided, the order of returned results is non-deterministic. - /// - public Func? OrderBy { get; set; } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Represents a builder for sorting. - /// - // This type does not derive any collection in order to avoid Intellisense suggesting LINQ methods. - public sealed class OrderByDefinition - { - private readonly List _values = []; - - /// - /// Gets the expressions to sort by. - /// - /// This property is intended to be consumed by the connectors to retrieve the configuration. - public IReadOnlyList Values => this._values; - - /// - /// Creates an ascending sort. - /// - public OrderByDefinition Ascending(Expression> propertySelector) - { - if (propertySelector is null) - { - throw new ArgumentNullException(nameof(propertySelector)); - } - - this._values.Add(new(propertySelector, true)); - return this; - } - - /// - /// Creates a descending sort. - /// - public OrderByDefinition Descending(Expression> propertySelector) - { - if (propertySelector is null) - { - throw new ArgumentNullException(nameof(propertySelector)); - } - - this._values.Add(new(propertySelector, false)); - return this; - } - - /// - /// Provides a way to define property ordering. - /// - /// This class is intended to be consumed by the connectors to retrieve the configuration. - public sealed class SortInfo - { - internal SortInfo(Expression> propertySelector, bool isAscending) - { - this.PropertySelector = propertySelector; - this.Ascending = isAscending; - } - - /// - /// Gets the expression to select the property to sort by. - /// - public Expression> PropertySelector { get; } - - /// - /// Gets a value that indicates whether the sort is ascending; otherwise, false. - /// - /// - /// if the sort is ascending; otherwise, . - /// - public bool Ascending { get; } - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs deleted file mode 100644 index ac17b697136e..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for calling -/// or . -/// -public class RecordRetrievalOptions -{ - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs b/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs deleted file mode 100644 index 42682c708155..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -internal static class Throw -{ - /// Throws an exception indicating that a required service is not available. - public static InvalidOperationException CreateMissingServiceException(Type serviceType, object? serviceKey) => - new(serviceKey is null ? - $"No service of type '{serviceType}' is available." : - $"No service of type '{serviceType}' for the key '{serviceKey}' is available."); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj b/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj deleted file mode 100644 index ea63dd6f4065..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj +++ /dev/null @@ -1,71 +0,0 @@ - - - - Microsoft.Extensions.VectorData.Abstractions - Microsoft.Extensions.VectorData - net10.0;net8.0;netstandard2.0;net462 - true - - false - - - - - - 10.1.0 - 10.0.0.0 - - 10.0.1 - Microsoft.Extensions.VectorData.Abstractions - $(AssemblyName) - Abstractions for vector database access. - -Commonly Used Types: -Microsoft.Extensions.VectorData.IVectorStore -Microsoft.Extensions.VectorData.IVectorStoreRecordCollection<TKey, TRecord> - neticon.png - neticon.png - PACKAGE.md - - Vector, Database, SDK - $(PackageDescription) - https://dot.net/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs deleted file mode 100644 index e3ca2cc0b430..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for hybrid search when using a dense vector and string keywords to do the search. -/// -public class HybridSearchOptions -{ - private int _skip = 0; - - /// - /// Gets or sets a search filter to use before doing the hybrid search. - /// - public Expression>? Filter { get; set; } - - /// - /// Gets or sets the target dense vector property to search on. - /// Only needs to be set when the collection has multiple vector properties. - /// - /// - /// If this property isn't set, checks if there is a vector property to use by default, and - /// throws if either none or multiple exist. - /// - public Expression>? VectorProperty { get; set; } - - /// - /// Gets or sets the additional target property to do the text or keyword search on. - /// The property must have full text indexing enabled. - /// - /// - /// If this property isn't set, checks if there is a text property with full text indexing enabled, and - /// throws an exception if either none or multiple exist. - /// - public Expression>? AdditionalProperty { get; set; } - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Gets or sets the score threshold to filter results. - /// - /// - /// - /// The meaning of the score is a combination of the distance function configured for and the text - /// relevance score for the full-text search on . - /// - /// - /// The range of scores also depends on the distance function; for example, cosine similarity/distance scores - /// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases. - /// - /// - public double? ScoreThreshold { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs deleted file mode 100644 index d0bb9ee95c00..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Threading; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Contains a method for performing a hybrid search using a vector and keywords. -/// -/// The record data model to use for retrieving data from the store. -public interface IKeywordHybridSearchable -{ - /// - /// Performs a hybrid search for records that match the given embedding and keywords, after applying the provided filters. - /// - /// The type of the input value on which to perform the vector similarity search. - /// The value on which to perform the similarity search. See the remarks section for more details. - /// A collection of keywords to search the store with. - /// The maximum number of results to return. - /// The options that control the behavior of the search. - /// The to monitor for cancellation requests. The default is . - /// The records found by the hybrid search, including their result scores. - /// - /// The types supported for the vary based on the provider being used and the embedding generation configured: - /// - /// - /// - /// A or (for images, sound...) if an appropriate has been configured that accepts that type as input. - /// For example, register an that accepts as input in your dependency injection container, and then pass in a - /// argument to this method; the argument will be automatically passed to the to generate the embedding and perform the search. - /// Some databases support generating embeddings at the database side. In this case, you can pass in a or without configuring an - /// with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation. - /// - /// - /// Arbitrary .NET types can also be passed in as long as an appropriate has been configured; for example, you can create your own - /// that accepts your own custom types as input, and uses another to generate embedding from multiple properties. For .NET types beyond and - /// , you must use the generic in your record definition. - /// - /// - /// To work with embeddings directly, pass in a or a .NET array of the appropriate type. Most providers support at least ReadOnlyMemory<float> and float[], - /// but some support other types (for example, ReadOnlyMemory<Half>, ). Some providers might also support their own custom types as well, for example, to represent sparse embeddings. - /// Consult your provider's documentation for supported types. - /// - /// - /// If you're using directly in your code, that type returns an (for example, Embedding{float}), - /// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your with the provider - /// instead and pass in the input type (for example, ). - /// - /// - /// - IAsyncEnumerable> HybridSearchAsync( - TInput searchValue, - ICollection keywords, - int top, - HybridSearchOptions? options = default, - CancellationToken cancellationToken = default) - where TInput : notnull; - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - object? GetService(Type serviceType, object? serviceKey = null); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs deleted file mode 100644 index 8d6dc76e8daf..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Threading; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an interface for performing vector searches on a vector store. -/// -/// The record data model to use for retrieving data from the store. -public interface IVectorSearchable -{ - /// - /// Searches the vector store for records that are similar to the given value. - /// - /// The type of the input value on which to perform the similarity search. - /// The value on which to perform the similarity search. See the remarks section for more details. - /// The maximum number of results to return. - /// The options that control the behavior of the search. - /// The to monitor for cancellation requests. The default is . - /// The records found by the vector search, including their result scores. - /// - /// The types supported for the vary based on the provider being used and the embedding generation configured: - /// - /// - /// - /// A or (for images, sound...) if an appropriate has been configured that accepts that type as input. - /// For example, register an that accepts as input in your dependency injection container, and then pass in a - /// argument to this method; the argument will be automatically passed to the to generate the embedding and perform the search. - /// Some databases support generating embeddings at the database side. In this case, you can pass in a or without configuring an - /// with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation. - /// - /// - /// Arbitrary .NET types can also be passed in as long as an appropriate has been configured; for example, you can create your own - /// that accepts your own custom types as input, and uses another to generate embedding from multiple properties. For .NET types beyond and - /// , you must use the generic in your record definition. - /// - /// - /// To work with embeddings directly, pass in a or a .NET array of the appropriate type. Most providers support at least ReadOnlyMemory<float> and float[], - /// but some support other types (for example, ReadOnlyMemory<Half>, ). Some providers might also support their own custom types as well, for example, to represent sparse embeddings. - /// Consult your provider's documentation for supported types. - /// - /// - /// If you're using directly in your code, that type returns an (for example, Embedding{float}), - /// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your with the provider - /// instead and pass in the input type (for example, ). - /// - /// - /// - IAsyncEnumerable> SearchAsync( - TInput searchValue, - int top, - VectorSearchOptions? options = default, - CancellationToken cancellationToken = default) - where TInput : notnull; - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - object? GetService(Type serviceType, object? serviceKey = null); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs deleted file mode 100644 index dea184745fb9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class KeywordHybridSearchExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throw an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The keyword hybrid search. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this IKeywordHybridSearchable keywordHybridSearch, Type serviceType, object? serviceKey = null) - { - if (keywordHybridSearch is null) { throw new ArgumentNullException(nameof(keywordHybridSearch)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - keywordHybridSearch.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs deleted file mode 100644 index 207034bd95c9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for vector search via . -/// -public class VectorSearchOptions -{ - private int _skip = 0; - - /// - /// Gets or sets a search filter to use before doing the vector search. - /// - public Expression>? Filter { get; set; } - - /// - /// Gets or sets the vector property to search on. - /// Only needs to be set when the collection has multiple vector properties. - /// - /// - /// If this property isn't set provided, checks if there is a vector property to use by default, and - /// throws an exception if either none or multiple exist. - /// - public Expression>? VectorProperty { get; set; } - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Gets or sets the score threshold to filter results. - /// - /// - /// - /// The meaning of the score depends on the distance function configured for the vector property. - /// For similarity functions (e.g. , ), - /// higher scores indicate more similar results, and results with scores lower than the threshold will be filtered out. - /// For distance functions (e.g. , ), - /// lower scores indicate more similar results, and results with scores higher than the threshold will be filtered out. - /// - /// - /// The range of scores also depends on the distance function; for example, cosine similarity/distance scores - /// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases. - /// - /// - public double? ScoreThreshold { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs deleted file mode 100644 index 65e931c99d4e..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class VectorSearchExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throws an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The vector search. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this IVectorSearchable vectorSearch, Type serviceType, object? serviceKey = null) - { - if (vectorSearch is null) { throw new ArgumentNullException(nameof(vectorSearch)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - vectorSearch.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs deleted file mode 100644 index f5793844d674..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a single search result from a vector search. -/// -/// The record data model to use for retrieving data from the store. -public sealed class VectorSearchResult -{ - /// - /// Initializes a new instance of the class. - /// - /// The record that was found by the search. - /// The score of this result in relation to the search query. - public VectorSearchResult(TRecord record, double? score) - { - this.Record = record; - this.Score = score; - } - - /// - /// Gets the record that was found by the search. - /// - public TRecord Record { get; } - - /// - /// Gets the score of this result in relation to the search query. - /// - public double? Score { get; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs deleted file mode 100644 index ab284796fd6b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a vector store that contains collections of records. -/// -/// -/// This type can be used with collections of any schema type, but requires you to provide schema information when getting a collection. -/// Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads. -/// -public abstract class VectorStore : IDisposable -{ - /// - /// Gets a collection from the vector store. - /// - /// The data type of the record key. - /// The record data model to use for adding, updating, and retrieving data from the collection. - /// The name of the collection. - /// The schema of the record type. - /// A new instance for managing the records in the collection. - /// - /// To successfully request a collection, either must be annotated with attributes that define the schema of - /// the record type, or must be provided. - /// - /// - /// - /// - [RequiresDynamicCode("This API is not compatible with NativeAOT. For dynamic mapping via Dictionary, use GetCollectionDynamic() instead.")] - [RequiresUnreferencedCode("This API is not compatible with trimming. For dynamic mapping via Dictionary, use GetCollectionDynamic() instead.")] - public abstract VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) - where TKey : notnull - where TRecord : class; - - /// - /// Gets a collection from the vector store, using dynamic mapping; the record type is represented as a . - /// - /// The name of the collection. - /// The schema of the record type. - /// A new instance for managing the records in the collection. - public abstract VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition); - - /// - /// Retrieves the names of all the collections in the vector store. - /// - /// The to monitor for cancellation requests. The default is . - /// The list of names of all the collections in the vector store. - public abstract IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default); - - /// - /// Checks if the collection exists in the vector store. - /// - /// The name of the collection. - /// The to monitor for cancellation requests. The default is . - /// if the collection exists, otherwise. - public abstract Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default); - - /// - /// Deletes the collection from the vector store. - /// - /// The name of the collection to delete. - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been deleted. - public abstract Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default); - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - public abstract object? GetService(Type serviceType, object? serviceKey = null); - - /// - /// Disposes the and releases any resources it holds. - /// - /// if called from ; if called from a finalizer. - protected virtual void Dispose(bool disposing) - { - } - - /// - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs deleted file mode 100644 index cf5636215712..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq.Expressions; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a named collection of records in a vector store, and can be used to search and manage records, and to create or delete the collection itself. -/// -/// The data type of the record key. -/// The record data model to use for adding, updating, and retrieving data from the store. -/// -/// Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads. -/// -#pragma warning disable CA1711 // Identifiers should not have incorrect suffix (Collection) -public abstract class VectorStoreCollection : IVectorSearchable, IDisposable -#pragma warning restore CA1711 - where TKey : notnull - where TRecord : class -{ - /// - /// Gets the name of the collection. - /// - public abstract string Name { get; } - - /// - /// Checks if the collection exists in the vector store. - /// - /// The to monitor for cancellation requests. The default is . - /// if the collection exists, otherwise. - public abstract Task CollectionExistsAsync(CancellationToken cancellationToken = default); - - /// - /// Creates this collection in the vector store if it doesn't already exist. - /// - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been created. - public abstract Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default); - - /// - /// Deletes the collection from the vector store if it exists. - /// - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been deleted. - public abstract Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default); - - /// - /// Gets a record from the vector store. Does not guarantee that the collection exists. - /// Returns null if the record is not found. - /// - /// The unique ID associated with the record to get. - /// Optional options for retrieving the record. - /// The to monitor for cancellation requests. The default is . - /// The record if found, otherwise null. - /// The command fails to execute for any reason. - public abstract Task GetAsync(TKey key, RecordRetrievalOptions? options = default, CancellationToken cancellationToken = default); - - /// - /// Gets a batch of records from the vector store. Does not guarantee that the collection exists. - /// - /// The unique IDs associated with the record to get. - /// Optional options for retrieving the records. - /// The to monitor for cancellation requests. The default is . - /// The records associated with the specified unique keys. - /// - /// - /// The exact method of retrieval is implementation-specific and can vary based on database support. - /// The default implementation of this method retrieves the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation. - /// - /// - /// Only found records are returned, so the result set might be smaller than the requested keys. - /// - /// - /// This method throws for any issues other than records not being found. - /// - /// - /// The command fails to execute for any reason. - public virtual async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - Verify.NotNull(keys); - - foreach (var key in keys) - { - var record = await this.GetAsync(key, options, cancellationToken).ConfigureAwait(false); - - if (record is not null) - { - yield return record; - } - } - } - - /// - /// Deletes a record from the vector store. Does not guarantee that the collection exists. - /// - /// The unique ID associated with the record to remove. - /// The to monitor for cancellation requests. The default is . - /// The unique identifier for the record. - /// The command fails to execute for any reason other than that the record does not exist. - public abstract Task DeleteAsync(TKey key, CancellationToken cancellationToken = default); - - /// - /// Deletes a batch of records from the vector store. Does not guarantee that the collection exists. - /// - /// The unique IDs associated with the records to remove. - /// The to monitor for cancellation requests. The default is . - /// A that completes when the records have been deleted. - /// - /// - /// The exact method of deleting is implementation-specific and can vary based on database support. - /// The default implementation of this method deletes the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation. - /// - /// - /// If a record isn't found, it is ignored and the batch succeeds. - /// If any record can't be deleted for any other reason, the operation throws. Some records might have already been deleted while others might not have, so the entire operation should be retried. - /// - /// - /// The command fails to execute for any reason other than that a record does not exist. - public virtual async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) - { - Verify.NotNull(keys); - - foreach (var key in keys) - { - await this.DeleteAsync(key, cancellationToken).ConfigureAwait(false); - } - } - - /// - /// Upserts a record into the vector store. Does not guarantee that the collection exists. - /// If the record already exists, it is updated. - /// If the record does not exist, it is created. - /// - /// The record to upsert. - /// The to monitor for cancellation requests. The default is . - /// The command fails to execute for any reason. - public abstract Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default); - - /// - /// Upserts a batch of records into the vector store. Does not guarantee that the collection exists. - /// If the record already exists, it is updated. - /// If the record does not exist, it is created. - /// - /// The records to upsert. - /// The to monitor for cancellation requests. The default is . - /// - /// - /// The exact method of upserting the batch is implementation-specific and can vary based on database support. - /// - /// - /// Similarly, the error behavior can vary across databases: where possible, the batch should be upserted atomically, so that any errors cause the entire batch to be rolled - /// back. Where not supported, some records might be upserted while others are not. If key properties are set by the user, then the entire upsert operation is idempotent, - /// and can simply be retried again if an error occurs. However, if store-generated keys are in use, the upsert operation is no longer idempotent; in that case, if the - /// database doesn't guarantee atomicity, retrying could cause duplicate records to be created. - /// - /// - /// Implementations of should implement this method in a way which performs embedding generation once for the batch, rather than - /// generating an embedding for each record separately. This is why a default implementation that calls is not provided. - /// - /// - /// The command fails to execute for any reason. - public abstract Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default); - - /// - /// Gets matching records from the vector store. Does not guarantee that the collection exists. - /// - /// The predicate to filter the records. - /// The maximum number of results to return. - /// Options for retrieving the records. - /// The to monitor for cancellation requests. The default is . - /// The records that match the given predicate. - /// The command fails to execute for any reason. - public abstract IAsyncEnumerable GetAsync(Expression> filter, int top, FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default); - - /// - public abstract IAsyncEnumerable> SearchAsync(TInput searchValue, int top, VectorSearchOptions? options = null, CancellationToken cancellationToken = default) - where TInput : notnull; - - /// - public abstract object? GetService(Type serviceType, object? serviceKey = null); - - /// - /// Disposes the and releases any resources it holds. - /// - /// if called from ; if called from a finalizer. - protected virtual void Dispose(bool disposing) - { - } - - /// - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs deleted file mode 100644 index f36fe294ce63..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -public class VectorStoreCollectionMetadata -{ - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } - - /// - /// Gets or sets the name of a collection (table, container) within the vector store (database). - /// - public string? CollectionName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs deleted file mode 100644 index 04f67aae662b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// Defines an abstract base class for options passed to a collection. -public abstract class VectorStoreCollectionOptions -{ - /// - /// Initializes a new instance of the class. - /// - protected VectorStoreCollectionOptions() - { - } - - /// - /// Initializes a new instance of the class. - /// - protected VectorStoreCollectionOptions(VectorStoreCollectionOptions? source) - { - this.Definition = source?.Definition; - this.EmbeddingGenerator = source?.EmbeddingGenerator; - } - - /// - /// Gets or sets an optional record definition that defines the schema of the record type. - /// - /// - /// If not provided, the schema will be inferred from the record model class using reflection. - /// In this case, the record model properties must be annotated with the appropriate attributes to indicate their usage. - /// See , , and . - /// - public VectorStoreCollectionDefinition? Definition { get; set; } - - /// - /// Gets or sets the default embedding generator to use when generating vectors embeddings with this collection. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs deleted file mode 100644 index a90599b34142..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base exception type for any type of failure when using vector stores. -/// -public class VectorStoreException : Exception -{ - /// - /// Initializes a new instance of the class. - /// - public VectorStoreException() - { - } - - /// - /// Initializes a new instance of the class with a specified error message. - /// - /// The error message that explains the reason for the exception. - public VectorStoreException(string? message) : base(message) - { - } - - /// - /// Initializes a new instance of the class with a specified error message and a reference to the inner exception that's the cause of this exception. - /// - /// The error message that explains the reason for the exception. - /// The exception that's the cause of the current exception, or a null reference if no inner exception is specified. - public VectorStoreException(string? message, Exception? innerException) : base(message, innerException) - { - } - - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } - - /// - /// Gets or sets the name of the vector store collection that the failing operation was performed on. - /// - public string? CollectionName { get; init; } - - /// - /// Gets or sets the name of the vector store operation that failed. - /// - public string? OperationName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs deleted file mode 100644 index ceeecb8166bb..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class VectorStoreExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throws an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The vector store. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this VectorStore vectorStore, Type serviceType, object? serviceKey = null) - { - if (vectorStore is null) { throw new ArgumentNullException(nameof(vectorStore)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - vectorStore.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs deleted file mode 100644 index 97571a0691bf..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -public class VectorStoreMetadata -{ - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/neticon.png b/dotnet/src/VectorData/VectorData.Abstractions/neticon.png deleted file mode 100644 index a0f1fdbf4d5e..000000000000 Binary files a/dotnet/src/VectorData/VectorData.Abstractions/neticon.png and /dev/null differ diff --git a/dotnet/src/VectorData/Weaviate/Weaviate.csproj b/dotnet/src/VectorData/Weaviate/Weaviate.csproj index 26b59e293cb9..0107d11a4d01 100644 --- a/dotnet/src/VectorData/Weaviate/Weaviate.csproj +++ b/dotnet/src/VectorData/Weaviate/Weaviate.csproj @@ -25,19 +25,16 @@ + - + - - - - diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj index 7f642f209a84..a9ced5e1c21a 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj @@ -28,7 +28,6 @@ - diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs index a69879faa3c5..da487571a509 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchAllSupportedTypesTests.cs @@ -2,14 +2,13 @@ using AzureAISearch.ConformanceTests.Support; using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace AzureAISearch.ConformanceTests; public class AzureAISearchAllSupportedTypesTests(AzureAISearchFixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task AllTypesBatchGetAsync() { var collection = fixture.TestStore.DefaultVectorStore.GetCollection("all-types", AzureAISearchAllTypes.GetRecordDefinition()); diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs index 844e905999f3..57c9e13650a4 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchIndexKindTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.VectorData; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace AzureAISearch.ConformanceTests; @@ -12,7 +11,7 @@ namespace AzureAISearch.ConformanceTests; public class AzureAISearchIndexKindTests(AzureAISearchIndexKindTests.Fixture fixture) : IndexKindTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Hnsw() => this.Test(IndexKind.Hnsw); diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs index 7458ef02a07b..cbb67c1c8afd 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Properties/AssemblyAttributes.cs @@ -1,3 +1 @@ // Copyright (c) Microsoft. All rights reserved. - -[assembly: AzureAISearch.ConformanceTests.Support.AzureAISearchUrlRequiredAttribute] diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs index 1c099bc7f4d1..26437a5e5c9e 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchAllTypes.cs @@ -70,7 +70,7 @@ public class AzureAISearchAllTypes [VectorStoreData] public List DateTimeOffsetList { get; set; } - [VectorStoreVector(Dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] + [VectorStoreVector(dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] public ReadOnlyMemory? Embedding { get; set; } internal void AssertEqual(AzureAISearchAllTypes other) diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs index dc63ea196fae..600136e9b1c5 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchTestEnvironment.cs @@ -23,7 +23,7 @@ static AzureAISearchTestEnvironment() .AddJsonFile(path: "testsettings.json", optional: true) .AddJsonFile(path: "testsettings.development.json", optional: true) .AddEnvironmentVariables() - .AddUserSecrets() + .AddUserSecrets() .Build(); var azureAISearchSection = configuration.GetSection("AzureAISearch"); diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchUrlRequiredAttribute.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchUrlRequiredAttribute.cs deleted file mode 100644 index c04c71595c5a..000000000000 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/Support/AzureAISearchUrlRequiredAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -namespace AzureAISearch.ConformanceTests.Support; - -/// -/// Checks whether the sqlite_vec extension is properly installed, and skips the test(s) otherwise. -/// -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class AzureAISearchUrlRequiredAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() => new(AzureAISearchTestEnvironment.IsConnectionInfoDefined); - - public string Skip { get; set; } = "Service URL is not configured, set AzureAISearch:ServiceUrl (and AzureAISearch:ApiKey if you don't use managed identity)."; - - public string SkipReason - => this.Skip; -} diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs index 535f51632b29..b0d577ef481f 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.VectorData; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace AzureAISearch.ConformanceTests.TypeTests; @@ -13,7 +12,7 @@ public class AzureAISearchDataTypeTests(AzureAISearchDataTypeTests.Fixture fixtu : DataTypeTests(fixture), IClassFixture { - [ConditionalFact(Skip = "Issues around empty collection initialization")] + [Fact(Skip = "Issues around empty collection initialization")] public override Task String_array() => Task.CompletedTask; protected override object? GenerateEmptyProperty(VectorStoreProperty property) diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs index 2a9266654964..378bbf67e45b 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchKeyTypeTests.cs @@ -3,7 +3,6 @@ using AzureAISearch.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace AzureAISearch.ConformanceTests.TypeTests; @@ -11,7 +10,7 @@ namespace AzureAISearch.ConformanceTests.TypeTests; public class AzureAISearchKeyTypeTests(AzureAISearchKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs index 44b99c00d98d..549eb724ad46 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoBsonMappingTests.cs @@ -5,16 +5,14 @@ using Microsoft.SemanticKernel.Connectors.CosmosMongoDB; using MongoDB.Bson.Serialization.Attributes; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosMongoDB.ConformanceTests; -[CosmosConnectionStringRequired] public sealed class CosmosMongoBsonMappingTests(CosmosMongoBsonMappingTests.Fixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_model_works() { var store = (CosmosMongoTestStore)fixture.TestStore; @@ -53,7 +51,7 @@ public async Task Upsert_with_bson_model_works() } } - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_vector_store_model_works() { var store = (CosmosMongoTestStore)fixture.TestStore; @@ -80,7 +78,7 @@ public async Task Upsert_with_bson_vector_store_model_works() } } - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_vector_store_with_name_model_works() { var store = (CosmosMongoTestStore)fixture.TestStore; diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj index 8c0a1d9b37b7..a302e8425ec1 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj @@ -25,7 +25,6 @@ - diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs index a644f1f1b00d..1b857a4d3383 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoFilterTests.cs @@ -3,7 +3,6 @@ using CosmosMongoDB.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosMongoDB.ConformanceTests; @@ -12,7 +11,7 @@ public class CosmosMongoFilterTests(CosmosMongoFilterTests.Fixture fixture) : FilterTests(fixture), IClassFixture { // Specialized MongoDB syntax for NOT over Contains ($nin) - [ConditionalFact] + [Fact] public virtual Task Not_over_Contains() => this.TestFilterAsync( r => !new[] { 8, 10 }.Contains(r.Int), diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs index af6c091c4dab..77fcccfcaa72 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoIndexKindTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.VectorData; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosMongoDB.ConformanceTests; @@ -13,11 +12,11 @@ public class CosmosMongoIndexKindTests(CosmosMongoIndexKindTests.Fixture fixture : IndexKindTests(fixture), IClassFixture { // Note: Cosmos Mongo support HNSW, but only in a specific tier. - // [ConditionalFact] + // [Fact] // public virtual Task Hnsw() // => this.Test(IndexKind.Hnsw); - [ConditionalFact] + [Fact] public virtual Task IvfFlat() => this.Test(IndexKind.IvfFlat); diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs index 4889cb87e6b6..cbb67c1c8afd 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Properties/AssemblyAttributes.cs @@ -1,3 +1 @@ // Copyright (c) Microsoft. All rights reserved. - -[assembly: CosmosMongoDB.ConformanceTests.Support.CosmosConnectionStringRequired] diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs deleted file mode 100644 index 5eae7f790892..000000000000 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -namespace CosmosMongoDB.ConformanceTests.Support; - -/// -/// Checks whether the sqlite_vec extension is properly installed, and skips the test(s) otherwise. -/// -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class CosmosConnectionStringRequiredAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() => new(CosmosMongoTestEnvironment.IsConnectionStringDefined); - - public string Skip { get; set; } = "The Cosmos connection string hasn't been configured (CosmosMongo:ConnectionString)."; - - public string SkipReason - => this.Skip; -} diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs index e86d0d13b3b3..7c6581121cee 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/Support/CosmosMongoTestEnvironment.cs @@ -18,7 +18,7 @@ static CosmosMongoTestEnvironment() .AddJsonFile(path: "testsettings.json", optional: true) .AddJsonFile(path: "testsettings.development.json", optional: true) .AddEnvironmentVariables() - .AddUserSecrets() + .AddUserSecrets() .Build(); ConnectionString = configuration["CosmosMongo:ConnectionString"]; diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs index 62533ade31e3..ebfe0ff8a0f9 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoKeyTypeTests.cs @@ -4,7 +4,6 @@ using MongoDB.Bson; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosMongoDB.ConformanceTests.TypeTests; @@ -12,16 +11,16 @@ namespace CosmosMongoDB.ConformanceTests.TypeTests; public class CosmosMongoKeyTypeTests(CosmosMongoKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ObjectId() => this.Test(new("652f8c3e8f9b2c1a4d3e6a7b"), supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs b/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs index cdb1dc794918..793075e47cc0 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoCollectionTests.cs @@ -700,11 +700,11 @@ private sealed class VectorSearchModel [VectorStoreData] public string? HotelName { get; set; } - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat, StorageName = "test_embedding_1")] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat, StorageName = "test_embedding_1")] public ReadOnlyMemory TestEmbedding1 { get; set; } [BsonElement("test_embedding_2")] - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] public ReadOnlyMemory TestEmbedding2 { get; set; } } #pragma warning restore CA1812 diff --git a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs b/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs index ec83f23ad9f1..fc434d094989 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoHotelModel.cs @@ -39,6 +39,6 @@ public class CosmosMongoHotelModel(string hotelId) public string? Description { get; set; } /// A vector field. - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj index 838c0f16e0f8..d8b012a849d2 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj @@ -25,7 +25,6 @@ - diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionOptionsTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionOptionsTests.cs index b7fae39008be..fbbe3b8f696c 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionOptionsTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionOptionsTests.cs @@ -5,16 +5,14 @@ using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel.Connectors.CosmosNoSql; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosNoSql.ConformanceTests; -[CosmosConnectionStringRequired] public sealed class CosmosNoSqlCollectionOptionsTests(CosmosNoSqlCollectionOptionsTests.Fixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task Collection_supports_partition_key_composite_key() { var store = (CosmosNoSqlTestStore)fixture.TestStore; @@ -53,7 +51,7 @@ public async Task Collection_supports_partition_key_composite_key() } } - [ConditionalTheory] + [Theory] [InlineData(IndexingMode.Consistent)] [InlineData(IndexingMode.Lazy)] [InlineData(IndexingMode.None)] @@ -108,7 +106,7 @@ private sealed class PartitionedHotel [VectorStoreData] public string? Description { get; set; } - [VectorStoreVector(Dimensions: 3)] + [VectorStoreVector(dimensions: 3)] public ReadOnlyMemory Embedding { get; set; } } @@ -120,7 +118,7 @@ private sealed class IndexingModeHotel [VectorStoreData] public string HotelName { get; set; } = string.Empty; - [VectorStoreVector(Dimensions: 3)] + [VectorStoreVector(dimensions: 3)] public ReadOnlyMemory Embedding { get; set; } } } diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs index e95cc67d1d54..b6ee0432fc3e 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.VectorData; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosNoSql.ConformanceTests; @@ -12,7 +11,7 @@ namespace CosmosNoSql.ConformanceTests; public class CosmosNoSqlIndexKindTests(CosmosNoSqlIndexKindTests.Fixture fixture) : IndexKindTests(fixture), IClassFixture { - [ConditionalFact(Skip = "DiskANN is supported by Cosmos NoSQL, but is not supported on the emulator and needs to be explicitly enabled")] + [Fact(Skip = "DiskANN is supported by Cosmos NoSQL, but is not supported on the emulator and needs to be explicitly enabled")] public virtual Task DiskANN() => this.Test(IndexKind.DiskAnn); diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Properties/AssemblyAttributes.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Properties/AssemblyAttributes.cs index 55e7e46eb05b..cbb67c1c8afd 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Properties/AssemblyAttributes.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Properties/AssemblyAttributes.cs @@ -1,3 +1 @@ // Copyright (c) Microsoft. All rights reserved. - -[assembly: CosmosNoSql.ConformanceTests.Support.CosmosConnectionStringRequired] diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs deleted file mode 100644 index cb84c96d4ea7..000000000000 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosConnectionStringRequiredAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -namespace CosmosNoSql.ConformanceTests.Support; - -/// -/// Checks whether the sqlite_vec extension is properly installed, and skips the test(s) otherwise. -/// -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class CosmosConnectionStringRequiredAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() => new(CosmosNoSqlTestEnvironment.IsConnectionStringDefined); - - public string Skip { get; set; } = "The Cosmos connection string hasn't been configured (CosmosNoSql:ConnectionString)."; - - public string SkipReason - => this.Skip; -} diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs index 4b50c98a64a3..ebb9b6357712 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs @@ -18,7 +18,7 @@ static CosmosNoSqlTestEnvironment() .AddJsonFile(path: "testsettings.json", optional: true) .AddJsonFile(path: "testsettings.development.json", optional: true) .AddEnvironmentVariables() - .AddUserSecrets() + .AddUserSecrets() .Build(); ConnectionString = configuration["CosmosNoSql:ConnectionString"]; diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs index cfd228cdac17..f41f9465a1a8 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs @@ -3,7 +3,6 @@ using CosmosNoSql.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosNoSql.ConformanceTests.TypeTests; @@ -13,7 +12,7 @@ public class CosmosNoSqlDataTypeTests(CosmosNoSqlDataTypeTests.Fixture fixture) { // Cosmos doesn't support DateTimeOffset with non-zero offset, so we convert it to UTC. // See https://github.com/dotnet/efcore/issues/35310 - [ConditionalFact(Skip = "Need to convert DateTimeOffset to UTC before sending to Cosmos")] + [Fact(Skip = "Need to convert DateTimeOffset to UTC before sending to Cosmos")] public override Task DateTimeOffset() => this.Test( "DateTimeOffset", diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlEmbeddingTypeTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlEmbeddingTypeTests.cs index 1bb0e73d20fa..4e80e9b76dc9 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlEmbeddingTypeTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlEmbeddingTypeTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.AI; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; #pragma warning disable CA2000 // Dispose objects before losing scope @@ -14,41 +13,41 @@ namespace CosmosNoSql.ConformanceTests.TypeTests; public class CosmosNoSqlEmbeddingTypeTests(CosmosNoSqlEmbeddingTypeTests.Fixture fixture) : EmbeddingTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ReadOnlyMemory_of_byte() => this.Test>( new ReadOnlyMemory([1, 2, 3]), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Embedding_of_byte() => this.Test>( new Embedding(new ReadOnlyMemory([1, 2, 3])), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Array_of_byte() => this.Test( [1, 2, 3], new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3])); - [ConditionalFact] + [Fact] public virtual Task ReadOnlyMemory_of_sbyte() => this.Test>( new ReadOnlyMemory([1, 2, 3]), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Embedding_of_sbyte() => this.Test>( new Embedding(new ReadOnlyMemory([1, 2, 3])), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Array_of_sbyte() => this.Test( [1, 2, 3], diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlKeyTypeTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlKeyTypeTests.cs index 0e64d33e9e3b..f70db5a30787 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlKeyTypeTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlKeyTypeTests.cs @@ -3,7 +3,6 @@ using CosmosNoSql.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace CosmosNoSql.ConformanceTests.TypeTests; @@ -11,7 +10,7 @@ namespace CosmosNoSql.ConformanceTests.TypeTests; public class CosmosNoSqlKeyTypeTests(CosmosNoSqlKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionTests.cs b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionTests.cs index 7efb13bab864..ce7e8867d5bc 100644 --- a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlCollectionTests.cs @@ -632,13 +632,13 @@ private sealed class TestIndexingModel [VectorStoreKey] public string? Id { get; set; } - [VectorStoreVector(Dimensions: 2, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Flat)] + [VectorStoreVector(dimensions: 2, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding2 { get; set; } - [VectorStoreVector(Dimensions: 3, DistanceFunction = DistanceFunction.DotProductSimilarity, IndexKind = IndexKind.QuantizedFlat)] + [VectorStoreVector(dimensions: 3, DistanceFunction = DistanceFunction.DotProductSimilarity, IndexKind = IndexKind.QuantizedFlat)] public ReadOnlyMemory? DescriptionEmbedding3 { get; set; } - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.EuclideanDistance, IndexKind = IndexKind.DiskAnn)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.EuclideanDistance, IndexKind = IndexKind.DiskAnn)] public ReadOnlyMemory? DescriptionEmbedding4 { get; set; } [VectorStoreData(IsIndexed = true)] diff --git a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlHotel.cs b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlHotel.cs index 9967675b5214..7913a0baa5db 100644 --- a/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlHotel.cs +++ b/dotnet/test/VectorData/CosmosNoSql.UnitTests/CosmosNoSqlHotel.cs @@ -39,6 +39,6 @@ public class CosmosNoSqlHotel(string hotelId) /// A vector field. [JsonPropertyName("description_embedding")] - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Flat)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/test/VectorData/Directory.Build.props b/dotnet/test/VectorData/Directory.Build.props index 8c60dd1d8b50..4154d5929a88 100644 --- a/dotnet/test/VectorData/Directory.Build.props +++ b/dotnet/test/VectorData/Directory.Build.props @@ -19,6 +19,13 @@ $(NoWarn);IDE0340 + + + true + + + + diff --git a/dotnet/test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj b/dotnet/test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj index 7c7e170881b2..678d14ed5bf1 100644 --- a/dotnet/test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj +++ b/dotnet/test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj @@ -21,7 +21,6 @@ - diff --git a/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs b/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs index d0d8f4e3bf8c..1de57166871d 100644 --- a/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs +++ b/dotnet/test/VectorData/InMemory.ConformanceTests/TypeTests/InMemoryKeyTypeTests.cs @@ -3,7 +3,6 @@ using InMemory.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace InMemory.ConformanceTests.TypeTests; @@ -13,13 +12,13 @@ public class InMemoryKeyTypeTests(InMemoryKeyTypeTests.Fixture fixture) { // The InMemory provider supports all .NET types as keys; below are just a few basic tests. - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8, 9); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L, 9L); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); protected override async Task Test(TKey key1, TKey key2, bool supportsAutoGeneration = false) diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoBsonMappingTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoBsonMappingTests.cs index f17caa806825..a09202fd79e3 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoBsonMappingTests.cs +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoBsonMappingTests.cs @@ -5,7 +5,6 @@ using MongoDB.Bson.Serialization.Attributes; using MongoDB.ConformanceTests.Support; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace MongoDB.ConformanceTests; @@ -13,7 +12,7 @@ namespace MongoDB.ConformanceTests; public sealed class MongoBsonMappingTests(MongoBsonMappingTests.Fixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_model_works() { var store = (MongoTestStore)fixture.TestStore; @@ -52,7 +51,7 @@ public async Task Upsert_with_bson_model_works() } } - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_vector_store_model_works() { var store = (MongoTestStore)fixture.TestStore; @@ -79,7 +78,7 @@ public async Task Upsert_with_bson_vector_store_model_works() } } - [ConditionalFact] + [Fact] public async Task Upsert_with_bson_vector_store_with_name_model_works() { var store = (MongoTestStore)fixture.TestStore; diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj index f6aa797d164e..b42c59f8a352 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj @@ -24,7 +24,6 @@ - diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs index d1541b604db6..7dff736a2fba 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoFilterTests.cs @@ -3,7 +3,6 @@ using MongoDB.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace MongoDB.ConformanceTests; @@ -12,7 +11,7 @@ public class MongoFilterTests(MongoFilterTests.Fixture fixture) : FilterTests(fixture), IClassFixture { // Specialized MongoDB syntax for NOT over Contains ($nin) - [ConditionalFact] + [Fact] public virtual Task Not_over_Contains() => this.TestFilterAsync( r => !new[] { 8, 10 }.Contains(r.Int), @@ -65,7 +64,7 @@ public override Task Contains_with_MemoryExtensions_Contains() #endif #if NET10_0_OR_GREATER - [ConditionalFact] + [Fact] public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer() => Assert.ThrowsAsync(base.Contains_with_MemoryExtensions_Contains_with_null_comparer); #endif diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/Properties/AssemblyInfo.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/Properties/AssemblyInfo.cs deleted file mode 100644 index 2e8748461853..000000000000 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -[assembly: DisableTests(Skip = "The MongoDB container is intermittently timing out at startup time blocking prs, so these test should be run manually.")] diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoKeyTypeTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoKeyTypeTests.cs index 89989a9e128c..09299942fcec 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoKeyTypeTests.cs +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoKeyTypeTests.cs @@ -4,7 +4,6 @@ using MongoDB.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace MongoDB.ConformanceTests.TypeTests; @@ -12,16 +11,16 @@ namespace MongoDB.ConformanceTests.TypeTests; public class MongoKeyTypeTests(MongoKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ObjectId() => this.Test(new("652f8c3e8f9b2c1a4d3e6a7b"), supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionTests.cs b/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionTests.cs index 7faaab6e7190..ad91553e0f64 100644 --- a/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionTests.cs +++ b/dotnet/test/VectorData/MongoDB.UnitTests/MongoCollectionTests.cs @@ -696,11 +696,11 @@ private sealed class VectorSearchModel [VectorStoreData] public string? HotelName { get; set; } - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat, StorageName = "test_embedding_1")] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat, StorageName = "test_embedding_1")] public ReadOnlyMemory TestEmbedding1 { get; set; } [BsonElement("test_embedding_2")] - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.IvfFlat)] public ReadOnlyMemory TestEmbedding2 { get; set; } } #pragma warning restore CA1812 diff --git a/dotnet/test/VectorData/MongoDB.UnitTests/MongoHotelModel.cs b/dotnet/test/VectorData/MongoDB.UnitTests/MongoHotelModel.cs index 053fe8a3dbbb..f9b0e7bd1797 100644 --- a/dotnet/test/VectorData/MongoDB.UnitTests/MongoHotelModel.cs +++ b/dotnet/test/VectorData/MongoDB.UnitTests/MongoHotelModel.cs @@ -39,6 +39,6 @@ public class MongoHotelModel(string hotelId) public string? Description { get; set; } /// A vector field. - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj b/dotnet/test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj index 1dac770bbb2f..a855a1fc1c68 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj @@ -26,7 +26,6 @@ - diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresIndexKindTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresIndexKindTests.cs index 021cd96a911f..d6e91a258638 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresIndexKindTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/PostgresIndexKindTests.cs @@ -4,7 +4,6 @@ using PgVector.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace PgVector.ConformanceTests; @@ -12,7 +11,7 @@ namespace PgVector.ConformanceTests; public class PostgresIndexKindTests(PostgresIndexKindTests.Fixture fixture) : IndexKindTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Hnsw() => this.Test(IndexKind.Hnsw); diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs index 8a015c95ce94..c840831a1dc5 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresEmbeddingTypeTests.cs @@ -7,7 +7,6 @@ using PgVector.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; #pragma warning disable CA2000 // Dispose objects before losing scope @@ -18,35 +17,35 @@ public class PostgresEmbeddingTypeTests(PostgresEmbeddingTypeTests.Fixture fixtu : EmbeddingTypeTests(fixture), IClassFixture { #if NET - [ConditionalFact] + [Fact] public virtual Task ReadOnlyMemory_of_Half() => this.Test>( new ReadOnlyMemory([(byte)1, (byte)2, (byte)3]), new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Embedding_of_Half() => this.Test>( new Embedding(new ReadOnlyMemory([(byte)1, (byte)2, (byte)3])), new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Array_of_Half() => this.Test( [(byte)1, (byte)2, (byte)3], new ReadOnlyMemoryEmbeddingGenerator([(byte)1, (byte)2, (byte)3])); #endif - [ConditionalFact] + [Fact] public virtual Task BitArray() => this.Test( new BitArray([true, false, true]), new BinaryEmbeddingGenerator(new BitArray([true, false, true])), distanceFunction: DistanceFunction.HammingDistance); - [ConditionalFact] + [Fact] public virtual Task BinaryEmbedding() => this.Test( new BinaryEmbedding(new([true, false, true])), @@ -54,7 +53,7 @@ public virtual Task BinaryEmbedding() distanceFunction: DistanceFunction.HammingDistance, vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector, a.Vector)); - [ConditionalFact] + [Fact] public virtual Task SparseVector() => this.Test(new SparseVector(new ReadOnlyMemory([1, 2, 3])), embeddingGenerator: null); diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs index 659b4d8bdcb6..4efdb86ead54 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresKeyTypeTests.cs @@ -3,7 +3,6 @@ using PgVector.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace PgVector.ConformanceTests.TypeTests; @@ -11,13 +10,13 @@ namespace PgVector.ConformanceTests.TypeTests; public class PostgresKeyTypeTests(PostgresKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj b/dotnet/test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj index f6e06ddf3779..3785e8013cdc 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj @@ -37,7 +37,6 @@ - \ No newline at end of file diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs index 849425659895..297649f65391 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeAllSupportedTypesTests.cs @@ -2,14 +2,13 @@ using Microsoft.Extensions.VectorData; using Pinecone.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Pinecone.ConformanceTests; public class PineconeAllSupportedTypesTests(PineconeFixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task AllTypesBatchGetAsync() { var collection = fixture.TestStore.DefaultVectorStore.GetCollection("all-types", PineconeAllTypes.GetRecordDefinition()); diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs index b20a4d4a5b1d..59d55776af2b 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeFilterTests.cs @@ -3,7 +3,6 @@ using Pinecone.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Pinecone.ConformanceTests; @@ -14,7 +13,7 @@ public class PineconeFilterTests(PineconeFilterTests.Fixture fixture) : FilterTests(fixture), IClassFixture { // Specialized Pinecone syntax for NOT over Contains ($nin) - [ConditionalFact] + [Fact] public virtual Task Not_over_Contains() => this.TestFilterAsync( r => !new[] { 8, 10 }.Contains(r.Int), diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeIndexKindTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeIndexKindTests.cs index 7bc83979b511..175a74e4ceea 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeIndexKindTests.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeIndexKindTests.cs @@ -3,7 +3,6 @@ using Pinecone.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Pinecone.ConformanceTests; @@ -14,7 +13,7 @@ public class PineconeIndexKindTests(PineconeIndexKindTests.Fixture fixture) // Pinecone does not support index-less searching public override Task Flat() => Assert.ThrowsAsync(base.Flat); - [ConditionalFact] + [Fact] public virtual Task PGA() => this.Test("PGA"); diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs index 81ed102a6d05..34939433a070 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/Support/PineconeAllTypes.cs @@ -48,7 +48,7 @@ public record PineconeAllTypes() [VectorStoreData] public List? NullableStringList { get; set; } - [VectorStoreVector(Dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] + [VectorStoreVector(dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)] public ReadOnlyMemory? Embedding { get; set; } internal void AssertEqual(PineconeAllTypes other) diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs index 6dce9486b2b1..77737371cf4e 100644 --- a/dotnet/test/VectorData/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs +++ b/dotnet/test/VectorData/Pinecone.ConformanceTests/TypeTests/PineconeKeyTypeTests.cs @@ -3,7 +3,6 @@ using Pinecone.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Pinecone.ConformanceTests.TypeTests; @@ -11,7 +10,7 @@ namespace Pinecone.ConformanceTests.TypeTests; public class PineconeKeyTypeTests(PineconeKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj b/dotnet/test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj index f664f696500a..d96862e05318 100644 --- a/dotnet/test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj +++ b/dotnet/test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj @@ -22,7 +22,6 @@ - diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantIndexKindTests.cs b/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantIndexKindTests.cs index 398ceb69ec9c..f5a5a93da743 100644 --- a/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantIndexKindTests.cs +++ b/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantIndexKindTests.cs @@ -4,7 +4,6 @@ using Qdrant.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Qdrant.ConformanceTests; @@ -15,7 +14,7 @@ public class QdrantIndexKindTests(QdrantIndexKindTests.Fixture fixture) // Qdrant does not support index-less searching public override Task Flat() => Assert.ThrowsAsync(base.Flat); - [ConditionalFact] + [Fact] public virtual Task Hnsw() => this.Test(IndexKind.Hnsw); diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs index fcd8f55566ef..65d3caa82f70 100644 --- a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs +++ b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs @@ -3,7 +3,6 @@ using Qdrant.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Qdrant.ConformanceTests.TypeTests; @@ -13,15 +12,15 @@ public class QdrantDataTypeTests(QdrantDataTypeTests.Fixture fixture) { // Qdrant doesn't seem to support filtering on float/double or string ararys, // https://qdrant.tech/documentation/concepts/filtering/#match - [ConditionalFact] + [Fact] public override Task Float() => this.Test("Float", 8.5f, 9.5f, isFilterable: false); - [ConditionalFact] + [Fact] public override Task Double() => this.Test("Double", 8.5d, 9.5d, isFilterable: false); - [ConditionalFact] + [Fact] public override Task String_array() => this.Test( "StringArray", diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs index 6594bddb3e35..45746f2874e6 100644 --- a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs +++ b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantKeyTypeTests.cs @@ -3,7 +3,6 @@ using Qdrant.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Qdrant.ConformanceTests.TypeTests; @@ -11,7 +10,7 @@ namespace Qdrant.ConformanceTests.TypeTests; public class QdrantKeyTypeTests(QdrantKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ULong() => this.Test(8UL); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj b/dotnet/test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj index 81fd4c52d7df..18c4c492dac4 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj +++ b/dotnet/test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj @@ -21,7 +21,6 @@ - diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/RedisJsonOptionsTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/RedisJsonOptionsTests.cs index 4743183b1389..ed36c841354e 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/RedisJsonOptionsTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/RedisJsonOptionsTests.cs @@ -6,7 +6,6 @@ using Redis.ConformanceTests.Support; using StackExchange.Redis; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Redis.ConformanceTests; @@ -14,7 +13,7 @@ namespace Redis.ConformanceTests; public sealed class RedisJsonOptionsTests(RedisJsonOptionsTests.Fixture fixture) : IClassFixture { - [ConditionalFact] + [Fact] public async Task Json_collection_with_prefix_and_nested_address_roundtrips() { var store = (RedisTestStore)fixture.TestStore; @@ -54,7 +53,7 @@ public async Task Json_collection_with_prefix_and_nested_address_roundtrips() } } - [ConditionalFact] + [Fact] public async Task Json_collection_get_throws_for_invalid_schema() { var store = (RedisTestStore)fixture.TestStore; diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs index dfbc028c0dc9..92c28cf50d10 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetEmbeddingTypeTests.cs @@ -4,7 +4,6 @@ using Redis.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; #pragma warning disable CA2000 // Dispose objects before losing scope @@ -14,21 +13,21 @@ namespace Redis.ConformanceTests.TypeTests; public class RedisHashSetEmbeddingTypeTests(RedisHashSetEmbeddingTypeTests.Fixture fixture) : EmbeddingTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ReadOnlyMemory_of_double() => this.Test>( new ReadOnlyMemory([1d, 2d, 3d]), new ReadOnlyMemoryEmbeddingGenerator([1d, 2d, 3d]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Embedding_of_double() => this.Test>( new Embedding(new ReadOnlyMemory([1, 2, 3])), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Array_of_double() => this.Test( [1, 2, 3], diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs index bbf40144fea9..876924ff11c4 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisHashSetKeyTypeTests.cs @@ -4,7 +4,6 @@ using Redis.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Redis.ConformanceTests.TypeTests; @@ -12,7 +11,7 @@ namespace Redis.ConformanceTests.TypeTests; public class RedisHashSetKeyTypeTests(RedisHashSetKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs index 090b94c8e88c..169ae0c59a32 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonEmbeddingTypeTests.cs @@ -4,7 +4,6 @@ using Redis.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; #pragma warning disable CA2000 // Dispose objects before losing scope @@ -14,21 +13,21 @@ namespace Redis.ConformanceTests.TypeTests; public class RedisJsonEmbeddingTypeTests(RedisJsonEmbeddingTypeTests.Fixture fixture) : EmbeddingTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task ReadOnlyMemory_of_double() => this.Test>( new ReadOnlyMemory([1d, 2d, 3d]), new ReadOnlyMemoryEmbeddingGenerator([1d, 2d, 3d]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Span.ToArray(), a.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Embedding_of_double() => this.Test>( new Embedding(new ReadOnlyMemory([1, 2, 3])), new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.Span.ToArray(), a.Vector.Span.ToArray())); - [ConditionalFact] + [Fact] public virtual Task Array_of_double() => this.Test( [1, 2, 3], diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs index ecc2030a65e2..7831e785108c 100644 --- a/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs +++ b/dotnet/test/VectorData/Redis.ConformanceTests/TypeTests/RedisJsonKeyTypeTests.cs @@ -4,7 +4,6 @@ using Redis.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace Redis.ConformanceTests.TypeTests; @@ -12,7 +11,7 @@ namespace Redis.ConformanceTests.TypeTests; public class RedisJsonKeyTypeTests(RedisJsonKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/ModelTests/SqlServerBasicModelTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/ModelTests/SqlServerBasicModelTests.cs index 4202035a649d..4118de03dd02 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/ModelTests/SqlServerBasicModelTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/ModelTests/SqlServerBasicModelTests.cs @@ -4,7 +4,6 @@ using SqlServer.ConformanceTests.Support; using VectorData.ConformanceTests.ModelTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace SqlServer.ConformanceTests.ModelTests; @@ -14,7 +13,7 @@ public class SqlServerBasicModelTests(SqlServerBasicModelTests.Fixture fixture) { private const int SqlServerMaxParameters = 2_100; - [ConditionalFact] + [Fact] private async Task Split_batches_to_account_for_max_parameter_limit() { var collection = fixture.Collection; @@ -43,7 +42,7 @@ private async Task Split_batches_to_account_for_max_parameter_limit() Assert.Empty(await collection.GetAsync(keys).ToArrayAsync()); } - [ConditionalFact] + [Fact] public async Task Upsert_batch_is_atomic() { var collection = fixture.Collection; diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj index af2ed087924f..77c39df13afa 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj @@ -34,7 +34,6 @@ - diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerDiskAnnVectorSearchTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerDiskAnnVectorSearchTests.cs index bf661df40b87..a669087055a7 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerDiskAnnVectorSearchTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerDiskAnnVectorSearchTests.cs @@ -14,7 +14,6 @@ using Microsoft.Extensions.VectorData; using SqlServer.ConformanceTests.Support; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace SqlServer.ConformanceTests; @@ -29,7 +28,7 @@ public class SqlServerDiskAnnVectorSearchTests( /// Tests that approximate vector search via VECTOR_SEARCH() returns correct results /// when a DiskANN index exists on the table. /// - [ConditionalFact] + [Fact] public async Task VectorSearch_WithDiskAnnIndex() { using var collection = this.CreateDiskAnnCollection(); @@ -44,7 +43,7 @@ public async Task VectorSearch_WithDiskAnnIndex() /// /// Tests that VECTOR_SEARCH() correctly returns multiple results ordered by distance. /// - [ConditionalFact] + [Fact] public async Task VectorSearch_WithDiskAnnIndex_TopN() { using var collection = this.CreateDiskAnnCollection(); @@ -61,7 +60,7 @@ public async Task VectorSearch_WithDiskAnnIndex_TopN() /// Tests that VECTOR_SEARCH() throws when a LINQ filter is specified, /// since SQL Server's VECTOR_SEARCH only supports post-filtering. /// - [ConditionalFact] + [Fact] public async Task VectorSearch_WithDiskAnnIndex_WithFilter_Throws() { using var collection = this.CreateDiskAnnCollection(); diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerIndexKindTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerIndexKindTests.cs index e17de1f54393..6865dc92e178 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerIndexKindTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerIndexKindTests.cs @@ -4,7 +4,6 @@ using SqlServer.ConformanceTests.Support; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace SqlServer.ConformanceTests; @@ -15,7 +14,7 @@ public class SqlServerIndexKindTests(SqlServerIndexKindTests.Fixture fixture) // SQL Server 2025 currently makes tables with vector indexes read-only, so data must be inserted before // the index is created. See SqlServerDiskAnnVectorSearchTests for a temporary workaround test that inserts // data first and then creates the index. Remove the Skip and delete that class once this limitation is lifted. - [ConditionalFact(Skip = "SQL Server 2025 read-only vector index limitation; see SqlServerDiskAnnVectorSearchTests")] + [Fact(Skip = "SQL Server 2025 read-only vector index limitation; see SqlServerDiskAnnVectorSearchTests")] public virtual Task DiskAnn() => this.Test(IndexKind.DiskAnn); diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerEmbeddingTypeTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerEmbeddingTypeTests.cs index 4a939691da9d..ca33bf98df1b 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerEmbeddingTypeTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerEmbeddingTypeTests.cs @@ -4,7 +4,6 @@ using SqlServer.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; #pragma warning disable CA2000 // Dispose objects before losing scope @@ -14,7 +13,7 @@ namespace SqlServer.ConformanceTests.TypeTests; public class SqlServerEmbeddingTypeTests(SqlServerEmbeddingTypeTests.Fixture fixture) : EmbeddingTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task SqlVector_of_float() => this.Test>( new SqlVector(new float[] { 1, 2, 3 }), diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerKeyTypeTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerKeyTypeTests.cs index 27d5d2019eb2..5cc312e36afc 100644 --- a/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerKeyTypeTests.cs +++ b/dotnet/test/VectorData/SqlServer.ConformanceTests/TypeTests/SqlServerKeyTypeTests.cs @@ -3,7 +3,6 @@ using SqlServer.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace SqlServer.ConformanceTests.TypeTests; @@ -11,13 +10,13 @@ namespace SqlServer.ConformanceTests.TypeTests; public class SqlServerKeyTypeTests(SqlServerKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs index ffcc58e906c3..d26685815cb0 100644 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs +++ b/dotnet/test/VectorData/SqliteVec.ConformanceTests/Properties/AssemblyAttributes.cs @@ -2,6 +2,5 @@ using Xunit; -[assembly: SqliteVec.ConformanceTests.Support.SqliteVecRequired] // Disable test parallelization in order to prevent from "database is locked" errors [assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj index ab89ee34cb9e..341a3eca871c 100644 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj +++ b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj @@ -20,7 +20,6 @@ - diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/Support/SqliteVecRequiredAttribute.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/Support/SqliteVecRequiredAttribute.cs deleted file mode 100644 index 107a1e306f02..000000000000 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/Support/SqliteVecRequiredAttribute.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -namespace SqliteVec.ConformanceTests.Support; - -/// -/// Checks whether the sqlite_vec extension is properly installed, and skips the test(s) otherwise. -/// -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class SqliteVecRequiredAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() => new(SqliteTestEnvironment.CanUseSqlite); - - public string Skip { get; set; } = "Some native Sqlite dependencies are missing."; - - public string SkipReason - => this.Skip; -} diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs index 8561d6eaba9e..7d1914e452d9 100644 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs +++ b/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteKeyTypeTests.cs @@ -3,7 +3,6 @@ using SqliteVec.ConformanceTests.Support; using VectorData.ConformanceTests.Support; using VectorData.ConformanceTests.TypeTests; -using VectorData.ConformanceTests.Xunit; using Xunit; namespace SqliteVec.ConformanceTests.TypeTests; @@ -11,13 +10,13 @@ namespace SqliteVec.ConformanceTests.TypeTests; public class SqliteKeyTypeTests(SqliteKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Int() => this.Test(8, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task Long() => this.Test(8L, supportsAutoGeneration: true); - [ConditionalFact] + [Fact] public virtual Task String() => this.Test("foo", "bar"); public new class Fixture : KeyTypeTests.Fixture diff --git a/dotnet/test/VectorData/SqliteVec.UnitTests/SqliteHotel.cs b/dotnet/test/VectorData/SqliteVec.UnitTests/SqliteHotel.cs index 1df5691074b1..cad3e15ac695 100644 --- a/dotnet/test/VectorData/SqliteVec.UnitTests/SqliteHotel.cs +++ b/dotnet/test/VectorData/SqliteVec.UnitTests/SqliteHotel.cs @@ -32,6 +32,6 @@ public class SqliteHotel() public string? Description { get; set; } /// A vector field. - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.EuclideanDistance)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.EuclideanDistance)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/CollectionManagementTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/CollectionManagementTests.cs deleted file mode 100644 index 1e32e41f37a6..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/CollectionManagementTests.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests; - -public abstract class CollectionManagementTests(VectorStoreFixture fixture) : IAsyncLifetime - where TKey : notnull -{ - public Task InitializeAsync() - => fixture.VectorStore.EnsureCollectionDeletedAsync(this.CollectionName); - - [ConditionalFact] - public async Task Collection_Ensure_Exists_Delete() - { - var collection = this.GetCollection(); - - Assert.False(await collection.CollectionExistsAsync()); - await collection.EnsureCollectionExistsAsync(); - Assert.True(await collection.CollectionExistsAsync()); - await collection.EnsureCollectionDeletedAsync(); - Assert.False(await collection.CollectionExistsAsync()); - - // Deleting a non-existing collection does not throw - await fixture.TestStore.DefaultVectorStore.EnsureCollectionDeletedAsync(collection.Name); - } - - [ConditionalFact] - public async Task EnsureCollectionExists_twice_does_not_throw() - { - var collection = this.GetCollection(); - - await collection.EnsureCollectionExistsAsync(); - await collection.EnsureCollectionExistsAsync(); - Assert.True(await collection.CollectionExistsAsync()); - } - - [ConditionalFact] - public async Task Store_CollectionExists() - { - var store = fixture.VectorStore; - var collection = this.GetCollection(); - - Assert.False(await store.CollectionExistsAsync(collection.Name)); - await collection.EnsureCollectionExistsAsync(); - Assert.True(await store.CollectionExistsAsync(collection.Name)); - } - - [ConditionalFact] - public async Task Store_DeleteCollection() - { - var store = fixture.VectorStore; - var collection = this.GetCollection(); - - await collection.EnsureCollectionExistsAsync(); - await fixture.TestStore.DefaultVectorStore.EnsureCollectionDeletedAsync(collection.Name); - Assert.False(await collection.CollectionExistsAsync()); - } - - [ConditionalFact] - public async Task Store_ListCollections() - { - var store = fixture.VectorStore; - var collection = this.GetCollection(); - - Assert.Empty(await store.ListCollectionNamesAsync().Where(n => n == collection.Name).ToListAsync()); - - await collection.EnsureCollectionExistsAsync(); - - var name = Assert.Single(await store.ListCollectionNamesAsync().Where(n => n == collection.Name).ToListAsync()); - Assert.Equal(collection.Name, name); - } - - [ConditionalFact] - public void Collection_metadata() - { - var collection = this.GetCollection(); - - var collectionMetadata = (VectorStoreCollectionMetadata?)collection.GetService(typeof(VectorStoreCollectionMetadata)); - - Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreSystemName); - Assert.NotNull(collectionMetadata.CollectionName); - } - - protected virtual string CollectionNameBase => nameof(CollectionManagementTests); - public virtual string CollectionName => fixture.TestStore.AdjustCollectionName(this.CollectionNameBase); - - public sealed class Record : TestRecord - { - public string? Text { get; set; } - public int Number { get; set; } - public ReadOnlyMemory Floats { get; set; } - } - - public virtual VectorStoreCollection GetCollection() - => fixture.TestStore.CreateCollection(this.CollectionName, this.CreateRecordDefinition()); - - public virtual VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)) { StorageName = "key" }, - new VectorStoreDataProperty(nameof(Record.Text), typeof(string)) { StorageName = "text" }, - new VectorStoreDataProperty(nameof(Record.Number), typeof(int)) { StorageName = "number" }, - new VectorStoreVectorProperty(nameof(Record.Floats), typeof(ReadOnlyMemory), 10) { IndexKind = fixture.TestStore.DefaultIndexKind } - ] - }; - - public Task DisposeAsync() => Task.CompletedTask; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/DependencyInjectionTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/DependencyInjectionTests.cs deleted file mode 100644 index 1a0bb1493785..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/DependencyInjectionTests.cs +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Linq.Expressions; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using Xunit; - -namespace VectorData.ConformanceTests; - -// The type argument object? might not be serializable, which may cause Test Explorer to not enumerate individual data rows. Consider using a type that is known to be serializable. -#pragma warning disable xUnit1045 - -public abstract class DependencyInjectionTests : DependencyInjectionTests - where TVectorStore : VectorStore - where TCollection : VectorStoreCollection - where TKey : notnull - where TRecord : class -{ - protected virtual string CollectionName => "collectionName"; - - protected abstract void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null); - - public abstract IEnumerable> StoreDelegates { get; } - - public abstract IEnumerable> CollectionDelegates { get; } - - [Fact] - public void ServiceCollectionCantBeNull() - { - foreach (var registrationDelegate in this.StoreDelegates) - { - Assert.Throws(() => registrationDelegate(null!, null, ServiceLifetime.Singleton)); - Assert.Throws(() => registrationDelegate(null!, "serviceKey", ServiceLifetime.Singleton)); - } - - foreach (var registrationDelegate in this.CollectionDelegates) - { - Assert.Throws(() => registrationDelegate(null!, null, this.CollectionName, ServiceLifetime.Singleton)); - Assert.Throws(() => registrationDelegate(null!, "serviceKey", this.CollectionName, ServiceLifetime.Singleton)); - } - } - - [Fact] - public void CollectionNameCantBeNullOrEmpty() - { - const string EmptyCollectionName = ""; - - foreach (var registrationDelegate in this.CollectionDelegates) - { - IServiceCollection services = this.CreateServices(); - - Assert.Throws(() => registrationDelegate(services, null, null!, ServiceLifetime.Singleton)); - Assert.Throws(() => registrationDelegate(services, "serviceKey", null!, ServiceLifetime.Singleton)); - Assert.Throws(() => registrationDelegate(services, null, EmptyCollectionName, ServiceLifetime.Singleton)); - Assert.Throws(() => registrationDelegate(services, "serviceKey", EmptyCollectionName, ServiceLifetime.Singleton)); - } - } - - [Theory, MemberData(nameof(LifetimesAndServiceKeys))] - public virtual void CanRegisterVectorStore(ServiceLifetime lifetime, object? serviceKey) - { - foreach (var registrationDelegate in this.StoreDelegates) - { - IServiceCollection services = this.CreateServices(serviceKey); - - registrationDelegate(services, serviceKey, lifetime); - - using ServiceProvider serviceProvider = services.BuildServiceProvider(); - // let's ensure that concrete types are registered - Verify(serviceProvider, lifetime, serviceKey); - // and the abstraction too - Verify(serviceProvider, lifetime, serviceKey); - } - } - - [Theory, MemberData(nameof(LifetimesAndServiceKeys))] - public void CanRegisterCollections(ServiceLifetime lifetime, object? serviceKey) - { - foreach (var registrationDelegate in this.CollectionDelegates) - { - IServiceCollection services = this.CreateServices(serviceKey); - - registrationDelegate(services, serviceKey, this.CollectionName, lifetime); - - using ServiceProvider serviceProvider = services.BuildServiceProvider(); - // Let's ensure that concrete types are registered. - Verify(serviceProvider, lifetime, serviceKey); - // And the VectorStoreCollection abstraction. - Verify>(serviceProvider, lifetime, serviceKey); - // And the IVectorSearchable abstraction. - Verify>(serviceProvider, lifetime, serviceKey); - - if (typeof(IKeywordHybridSearchable).IsAssignableFrom(typeof(TCollection))) - { - Verify>(serviceProvider, lifetime, serviceKey); - } - else - { - Assert.Null(serviceProvider.GetService>()); - } - } - } - - [Theory, MemberData(nameof(LifetimesAndServiceKeys))] - public virtual void CanRegisterConcreteTypeVectorStoreAfterSomeAbstractionHasBeenRegistered(ServiceLifetime lifetime, object? serviceKey) - { - foreach (var registrationDelegate in this.StoreDelegates) - { - IServiceCollection services = this.CreateServices(serviceKey); - - // Users may be willing to register more than one IVectorStore implementation. - services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, (sp, key) => new FakeVectorStore(), lifetime)); - - registrationDelegate(services, serviceKey, lifetime); - - using ServiceProvider serviceProvider = services.BuildServiceProvider(); - // let's ensure that concrete types are registered - Verify(serviceProvider, lifetime, serviceKey); - } - } - - [Theory, MemberData(nameof(LifetimesAndServiceKeys))] - public void CanRegisterConcreteTypeCollectionsAfterSomeAbstractionHasBeenRegistered(ServiceLifetime lifetime, object? serviceKey) - { - foreach (var registrationDelegate in this.CollectionDelegates) - { - IServiceCollection services = this.CreateServices(serviceKey); - - // Users may be willing to register more than one VectorStoreCollection implementation. - services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, (sp, key) => new FakeVectorStoreRecordCollection(), lifetime)); - - registrationDelegate(services, serviceKey, this.CollectionName, lifetime); - - using ServiceProvider serviceProvider = services.BuildServiceProvider(); - // let's ensure that concrete types are registered - Verify(serviceProvider, lifetime, serviceKey); - } - } - - [Theory, MemberData(nameof(LifetimesAndServiceKeys))] - public void EmbeddingGeneratorIsResolved(ServiceLifetime lifetime, object? serviceKey) - { - foreach (var registrationDelegate in this.CollectionDelegates) - { - IServiceCollection services = this.CreateServices(serviceKey); - - bool wasResolved = false; - services.AddSingleton(sp => - { - wasResolved = true; - return null!; - }); - - registrationDelegate(services, serviceKey, this.CollectionName, lifetime); - - Assert.False(wasResolved); // it's lazy - - using ServiceProvider serviceProvider = services.BuildServiceProvider(); - using var collection = serviceKey is null - ? serviceProvider.GetRequiredService() - : serviceProvider.GetRequiredKeyedService(serviceKey); - - Assert.True(wasResolved); - } - } - -#pragma warning disable CA1000 // Do not declare static members on generic types - public static TheoryData LifetimesAndServiceKeys { get; } = GetLifetimesAndServiceKeys(); -#pragma warning restore CA1000 // Do not declare static members on generic types - - private static TheoryData GetLifetimesAndServiceKeys() - { - TheoryData result = []; - - foreach (ServiceLifetime lifetime in new ServiceLifetime[] { ServiceLifetime.Scoped, ServiceLifetime.Singleton, ServiceLifetime.Transient }) - { - result.Add(lifetime, null); - result.Add(lifetime, "key"); - result.Add(lifetime, 8); - } - - return result; - } - - protected IServiceCollection CreateServices(object? serviceKey = null) - { - IServiceCollection services = new ServiceCollection(); -#pragma warning disable CA2000 // Dispose objects before losing scope - ConfigurationManager configuration = new(); -#pragma warning restore CA2000 // Dispose objects before losing scope - services.AddSingleton(configuration); - - this.PopulateConfiguration(configuration, serviceKey); - - return services; - } - - private static void Verify(ServiceProvider serviceProvider, ServiceLifetime lifetime, object? serviceKey) - where TService : class - { - TService? serviceFromFirstScope, serviceFromSecondScope, secondServiceFromSecondScope; - - using (IServiceScope scope1 = serviceProvider.CreateScope()) - { - serviceFromFirstScope = Resolve(scope1.ServiceProvider, serviceKey); - } - - using (IServiceScope scope2 = serviceProvider.CreateScope()) - { - serviceFromSecondScope = Resolve(scope2.ServiceProvider, serviceKey); - - secondServiceFromSecondScope = Resolve(scope2.ServiceProvider, serviceKey); - } - - Assert.NotNull(serviceFromFirstScope); - Assert.NotNull(serviceFromSecondScope); - Assert.NotNull(secondServiceFromSecondScope); - - switch (lifetime) - { - case ServiceLifetime.Singleton: - Assert.Same(serviceFromFirstScope, serviceFromSecondScope); - Assert.Same(serviceFromSecondScope, secondServiceFromSecondScope); - break; - case ServiceLifetime.Scoped: - Assert.NotSame(serviceFromFirstScope, serviceFromSecondScope); - Assert.Same(serviceFromSecondScope, secondServiceFromSecondScope); - break; - case ServiceLifetime.Transient: - Assert.NotSame(serviceFromFirstScope, serviceFromSecondScope); - Assert.NotSame(serviceFromSecondScope, secondServiceFromSecondScope); - break; - } - } - - protected static string CreateConfigKey(string prefix, object? serviceKey, string suffix) - => serviceKey is null ? $"{prefix}:{suffix}" : $"{prefix}:{serviceKey}:{suffix}"; - - private static TService Resolve(IServiceProvider serviceProvider, object? serviceKey = null) where TService : notnull - => serviceKey is null - ? serviceProvider.GetRequiredService() - : serviceProvider.GetRequiredKeyedService(serviceKey); - - private sealed class FakeVectorStore : VectorStore - { - public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) => throw new NotImplementedException(); - public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null) => throw new NotImplementedException(); - public override object? GetService(Type serviceType, object? serviceKey = null) => throw new NotImplementedException(); - public override IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - } - - private sealed class FakeVectorStoreRecordCollection : VectorStoreCollection - { - public override string Name => throw new NotImplementedException(); - public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override IAsyncEnumerable GetAsync(Expression> filter, int top, FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override object? GetService(Type serviceType, object? serviceKey = null) => throw new NotImplementedException(); - public override IAsyncEnumerable> SearchAsync(TInput searchValue, int top, VectorSearchOptions? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public override Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - } -} - -public abstract class DependencyInjectionTests -{ - public sealed class Record : TestRecord - { - [VectorStoreData(StorageName = "number")] - public int Number { get; set; } - - [VectorStoreVector(Dimensions: 3, StorageName = "embedding")] - public ReadOnlyMemory Floats { get; set; } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/DistanceFunctionTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/DistanceFunctionTests.cs deleted file mode 100644 index 182954d9d985..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/DistanceFunctionTests.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests; - -public abstract class DistanceFunctionTests(DistanceFunctionTests.Fixture fixture) - where TKey : notnull -{ - [ConditionalFact] - public virtual Task CosineDistance() - => this.Test(DistanceFunction.CosineDistance, 0, 2, 1, [0, 2, 1]); - - [ConditionalFact] - public virtual Task CosineSimilarity() - => this.Test(DistanceFunction.CosineSimilarity, 1, -1, 0, [0, 2, 1]); - - [ConditionalFact] - public virtual Task DotProductSimilarity() - => this.Test(DistanceFunction.DotProductSimilarity, 1, -1, 0, [0, 2, 1]); - - [ConditionalFact] - public virtual Task NegativeDotProductSimilarity() - => this.Test(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]); - - [ConditionalFact] - public virtual Task EuclideanDistance() - => this.Test(DistanceFunction.EuclideanDistance, 0, 2, 1.73, [0, 2, 1]); - - [ConditionalFact] - public virtual Task EuclideanSquaredDistance() - => this.Test(DistanceFunction.EuclideanSquaredDistance, 0, 4, 3, [0, 2, 1]); - - [ConditionalFact] - public virtual Task HammingDistance() - => this.Test(DistanceFunction.HammingDistance, 0, 1, 3, [0, 1, 2]); - - [ConditionalFact] - public virtual Task ManhattanDistance() - => this.Test(DistanceFunction.ManhattanDistance, 0, 2, 3, [0, 1, 2]); - - protected virtual async Task Test( - string distanceFunction, - double expectedExactMatchScore, - double expectedOppositeScore, - double expectedOrthogonalScore, - int[] resultOrder) - { - using var collection = fixture.CreateCollection(distanceFunction); - await collection.EnsureCollectionDeletedAsync(); - await collection.EnsureCollectionExistsAsync(); - - ReadOnlyMemory baseVector = new([1, 0, 0, 0]); - ReadOnlyMemory oppositeVector = new([-1, 0, 0, 0]); - ReadOnlyMemory orthogonalVector = new([0f, -1f, -1f, 0f]); - - double[] scoreDictionary = - [ - expectedExactMatchScore, - expectedOppositeScore, - expectedOrthogonalScore - ]; - - double[] expectedScores = - [ - scoreDictionary[resultOrder[0]], - scoreDictionary[resultOrder[1]], - scoreDictionary[resultOrder[2]] - ]; - - List insertedRecords = - [ - new() - { - Key = fixture.GenerateNextKey(), - Int = 1, - Vector = baseVector, - }, - new() - { - Key = fixture.GenerateNextKey(), - Int = 2, - Vector = oppositeVector, - }, - new() - { - Key = fixture.GenerateNextKey(), - Int = 3, - Vector = orthogonalVector, - } - ]; - SearchRecord[] expectedRecords = - [ - insertedRecords[resultOrder[0]], - insertedRecords[resultOrder[1]], - insertedRecords[resultOrder[2]] - ]; - - await collection.UpsertAsync(insertedRecords); - - await fixture.TestStore.WaitForDataAsync(collection, insertedRecords.Count, vectorSize: 4); - - var results = await collection.SearchAsync(baseVector, top: 3).ToListAsync(); - - Assert.Equal(expectedRecords.Length, results.Count); - for (int i = 0; i < results.Count; i++) - { - Assert.Equal(expectedRecords[i].Key, results[i].Record.Key); - Assert.Equal(expectedRecords[i].Int, results[i].Record.Int); - if (fixture.AssertScores) - { - Assert.Equal(Math.Round(expectedScores[i], 2), Math.Round(results[i].Score!.Value, 2)); - } - } - - await this.TestScoreThreshold(collection); - } - - protected virtual async Task TestScoreThreshold(VectorStoreCollection collection) - { - if (!fixture.TestStore.SupportsScoreThreshold) - { - await Assert.ThrowsAsync(async () => - { - await collection - .SearchAsync( - new ReadOnlyMemory([1, 0, 0, 0]), - top: 3, - new() { ScoreThreshold = 0.9 }) - .ToListAsync(); - }); - - return; - } - - // Fetch the top three records, then use the second's returned score as the threshold. - var results = await collection - .SearchAsync(new ReadOnlyMemory([1, 0, 0, 0]), top: 3) - .ToListAsync(); - - var threshold = results[1].Score; - - var filteredResults = await collection - .SearchAsync( - new ReadOnlyMemory([1, 0, 0, 0]), - top: 3, - new() { ScoreThreshold = threshold }) - .ToListAsync(); - - // Some providers use inclusive thresholds (>=), returning 2 results (first and second), - // while others use exclusive thresholds (>), returning only 1 result (first). - Assert.True(filteredResults.Count is 1 or 2); - Assert.Equal(results[0].Record.Key, filteredResults[0].Record.Key); - if (filteredResults.Count == 2) - { - Assert.Equal(results[1].Record.Key, filteredResults[1].Record.Key); - } - } - - public abstract class Fixture : VectorStoreFixture - { - protected virtual string CollectionNameBase => nameof(DistanceFunctionTests); - public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase); - - protected virtual string? IndexKind => null; - - public virtual bool AssertScores { get; } = true; - - public virtual VectorStoreCollection CreateCollection(string distanceFunction) - { - VectorStoreCollectionDefinition definition = new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(SearchRecord.Key), typeof(TKey)), - new VectorStoreDataProperty(nameof(SearchRecord.Int), typeof(int)), - new VectorStoreVectorProperty(nameof(SearchRecord.Vector), typeof(ReadOnlyMemory), dimensions: 4) - { - DistanceFunction = distanceFunction, - IndexKind = this.IndexKind ?? this.DefaultIndexKind - } - ] - }; - - return this.TestStore.CreateCollection(this.CollectionName, definition); - } - } - - public class SearchRecord - { - public TKey Key { get; set; } = default!; - public int Int { get; set; } - public ReadOnlyMemory Vector { get; set; } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/EmbeddingGenerationTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/EmbeddingGenerationTests.cs deleted file mode 100644 index c3b13dc8fde5..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/EmbeddingGenerationTests.cs +++ /dev/null @@ -1,623 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.AI; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests; - -#pragma warning disable CA2000 // Don't actually need to dispose FakeEmbeddingGenerator -#pragma warning disable CS8605 // Unboxing a possibly null value. - -public abstract class EmbeddingGenerationTests(EmbeddingGenerationTests.StringVectorFixture stringVectorFixture, EmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) - where TKey : notnull -{ - #region Search - - [ConditionalFact] - public virtual async Task SearchAsync_with_property_generator() - { - // Property level: embedding generators are defined at all levels. The property generator should take precedence. - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Property ([1, 1, 3])", result.Record.Text); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_property_generator_dynamic() - { - // Property level: embedding generators are defined at all levels. The property generator should take precedence. - var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Property ([1, 1, 3])", result.Record[nameof(Record.Text)]); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_collection_generator() - { - // Collection level: embedding generators are defined at the collection and store level - the collection generator should take precedence. - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: false); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Collection ([1, 1, 2])", result.Record.Text); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_store_generator() - { - // Store level: an embedding generator is defined at the store level only. - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: false, propertyGenerator: false); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Store ([1, 1, 1])", result.Record.Text); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_store_dependency_injection() - { - foreach (var registrationDelegate in stringVectorFixture.DependencyInjectionStoreRegistrationDelegates) - { - IServiceCollection serviceCollection = new ServiceCollection(); - - serviceCollection.AddSingleton(new FakeEmbeddingGenerator(replaceLast: 1)); - registrationDelegate(serviceCollection); - - await using var serviceProvider = serviceCollection.BuildServiceProvider(); - - var vectorStore = serviceProvider.GetRequiredService(); - var collection = vectorStore.GetCollection(stringVectorFixture.CollectionName, stringVectorFixture.CreateRecordDefinition()); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Store ([1, 1, 1])", result.Record.Text); - } - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_collection_dependency_injection() - { - foreach (var registrationDelegate in stringVectorFixture.DependencyInjectionCollectionRegistrationDelegates) - { - IServiceCollection serviceCollection = new ServiceCollection(); - - serviceCollection.AddSingleton(new FakeEmbeddingGenerator(replaceLast: 1)); - registrationDelegate(serviceCollection); - - await using var serviceProvider = serviceCollection.BuildServiceProvider(); - - var collection = serviceProvider.GetRequiredService>(); - - var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync(); - - Assert.Equal("Store ([1, 1, 1])", result.Record.Text); - } - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_custom_input_type() - { - var recordDefinition = new VectorStoreCollectionDefinition() - { - Properties = stringVectorFixture.CreateRecordDefinition().Properties - .Select(p => p is VectorStoreVectorProperty vectorProperty - ? new VectorStoreVectorProperty(nameof(Record.Embedding), dimensions: 3) - { - DistanceFunction = stringVectorFixture.DefaultDistanceFunction, - IndexKind = stringVectorFixture.DefaultIndexKind - } - : p) - .ToList() - }; - - var collection = stringVectorFixture.GetCollection( - stringVectorFixture.CreateVectorStore(new FakeCustomerEmbeddingGenerator([1, 1, 1])), - stringVectorFixture.CollectionName, - recordDefinition); - - var result = await collection.SearchAsync(new Customer(), top: 1).SingleAsync(); - - Assert.Equal("Store ([1, 1, 1])", result.Record.Text); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_search_only_embedding_generation() - { - var collection = romOfFloatVectorFixture.GetCollection( - romOfFloatVectorFixture.CreateVectorStore(new FakeEmbeddingGenerator()), - romOfFloatVectorFixture.CollectionName, - romOfFloatVectorFixture.CreateRecordDefinition()); - - var result = await collection.SearchAsync("[1, 1, 1]", top: 1).SingleAsync(); - - Assert.Equal("Store ([1, 1, 1])", result.Record.Text); - } - - [ConditionalFact] - public virtual async Task SearchAsync_string_without_generator_throws() - { - // The database doesn't support embedding generation, and no client-side generator has been configured at any level, - // so SearchAsync should throw for string. - var collection = stringVectorFixture.GetCollection(stringVectorFixture.TestStore.DefaultVectorStore, stringVectorFixture.CollectionName + "withoutgenerator"); - - var exception = await Assert.ThrowsAsync(() => collection.SearchAsync("foo", top: 1).ToListAsync().AsTask()); - - Assert.StartsWith( - "A value of type 'string' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are:", - exception.Message); - } - - public class RawRecord - { - [VectorStoreKey] - public TKey Key { get; set; } = default!; - [VectorStoreVector(Dimensions: 3)] - public ReadOnlyMemory Embedding { get; set; } - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_incompatible_generator_throws() - { - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - // We have a generator configured for string, not int. - var exception = await Assert.ThrowsAsync(() => collection.SearchAsync(8, top: 1).ToListAsync().AsTask()); - - Assert.Equal($"An input of type 'int' was provided, but an incompatible embedding generator of type '{nameof(FakeEmbeddingGenerator)}' was configured.", exception.Message); - } - - #endregion Search - - #region Upsert - - [ConditionalFact] - public virtual async Task UpsertAsync() - { - var counter = stringVectorFixture.GenerateNextCounter(); - - var record = new Record - { - Key = stringVectorFixture.GenerateNextKey(), - Embedding = "[100, 1, 0]", - Counter = counter, - Text = nameof(UpsertAsync) - }; - - // Property level: embedding generators are defined at all levels. The property generator should take precedence. - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - await collection.UpsertAsync(record).ConfigureAwait(false); - - await stringVectorFixture.TestStore.WaitForDataAsync(collection, 1, filter: r => r.Counter == counter); - - var result = await collection.SearchAsync(new ReadOnlyMemory([100, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter, result.Record.Counter); - } - - [ConditionalFact] - public virtual async Task UpsertAsync_dynamic() - { - var counter = stringVectorFixture.GenerateNextCounter(); - - var record = new Dictionary - { - [nameof(Record.Key)] = stringVectorFixture.GenerateNextKey(), - [nameof(Record.Embedding)] = "[200, 1, 0]", - [nameof(Record.Counter)] = counter, - [nameof(Record.Text)] = nameof(UpsertAsync_dynamic) - }; - - // Property level: embedding generators are defined at all levels. The property generator should take precedence. - var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - await collection.UpsertAsync(record).ConfigureAwait(false); - - await stringVectorFixture.TestStore.WaitForDataAsync(collection, 1, filter: r => (int)r[nameof(Record.Counter)] == counter); - - var result = await collection.SearchAsync(new ReadOnlyMemory([200, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter, result.Record[nameof(Record.Counter)]); - } - - [ConditionalFact] - public virtual async Task UpsertAsync_batch() - { - var (counter1, counter2) = (stringVectorFixture.GenerateNextCounter(), stringVectorFixture.GenerateNextCounter()); - - Record[] records = - [ - new() - { - Key = stringVectorFixture.GenerateNextKey(), - Embedding = "[300, 1, 0]", - Counter = counter1, - Text = nameof(UpsertAsync_batch) + "1" - }, - new() - { - Key = stringVectorFixture.GenerateNextKey(), - Embedding = "[400, 1, 0]", - Counter = counter2, - Text = nameof(UpsertAsync_batch) + "2" - } - ]; - - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - await collection.UpsertAsync(records).ConfigureAwait(false); - - await stringVectorFixture.TestStore.WaitForDataAsync(collection, 2, filter: r => (int)r.Counter == counter1 || (int)r.Counter == counter2); - - var result = await collection.SearchAsync(new ReadOnlyMemory([300, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter1, result.Record.Counter); - - result = await collection.SearchAsync(new ReadOnlyMemory([400, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter2, result.Record.Counter); - } - - [ConditionalFact] - public virtual async Task UpsertAsync_batch_dynamic() - { - var (counter1, counter2) = (stringVectorFixture.GenerateNextCounter(), stringVectorFixture.GenerateNextCounter()); - - Dictionary[] records = - [ - new() - { - [nameof(Record.Key)] = stringVectorFixture.GenerateNextKey(), - [nameof(Record.Embedding)] = "[500, 1, 0]", - [nameof(Record.Counter)] = counter1, - [nameof(Record.Text)] = nameof(UpsertAsync_batch_dynamic) + "1" - }, - new() - { - [nameof(Record.Key)] = stringVectorFixture.GenerateNextKey(), - [nameof(Record.Embedding)] = "[600, 1, 0]", - [nameof(Record.Counter)] = counter2, - [nameof(Record.Text)] = nameof(UpsertAsync_batch_dynamic) + "2" - } - ]; - - var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - await collection.UpsertAsync(records).ConfigureAwait(false); - - await stringVectorFixture.TestStore.WaitForDataAsync(collection, 2, filter: r => (int)r[nameof(Record.Counter)] == counter1 || (int)r[nameof(Record.Counter)] == counter2); - - var result = await collection.SearchAsync(new ReadOnlyMemory([500, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter1, result.Record[nameof(Record.Counter)]); - - result = await collection.SearchAsync(new ReadOnlyMemory([600, 1, 3]), top: 1).SingleAsync(); - Assert.Equal(counter2, result.Record[nameof(Record.Counter)]); - } - - #endregion Upsert - - #region IncludeVectors - - [ConditionalFact] - public virtual async Task SearchAsync_with_IncludeVectors_throws() - { - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - var exception = await Assert.ThrowsAsync(() => collection.SearchAsync("[1, 0, 0]", top: 1, new() { IncludeVectors = true }).ToListAsync().AsTask()); - - Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_IncludeVectors_throws() - { - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - var exception = await Assert.ThrowsAsync(() => collection.GetAsync(stringVectorFixture.TestData[0].Key, new() { IncludeVectors = true })); - - Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message); - } - - [ConditionalFact] - public virtual async Task GetAsync_enumerable_with_IncludeVectors_throws() - { - var collection = this.GetCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true); - - var exception = await Assert.ThrowsAsync(() => - collection.GetAsync( - [stringVectorFixture.TestData[0].Key, stringVectorFixture.TestData[1].Key], - new() { IncludeVectors = true }) - .ToListAsync().AsTask()); - - Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message); - } - - #endregion IncludeVectors - - #region Support - - public class Record : TestRecord - { - public string? Embedding { get; set; } - - public int Counter { get; set; } - public string? Text { get; set; } - } - - public class RecordWithAttributes - { - [VectorStoreKey] - public TKey Key { get; set; } = default!; - - [VectorStoreVector(Dimensions: 3)] - public string? Embedding { get; set; } - - [VectorStoreData(IsIndexed = true)] - public int Counter { get; set; } - - [VectorStoreData] - public string? Text { get; set; } - } - - public class RecordWithCustomerVectorProperty - { - public TKey Key { get; set; } = default!; - public Customer? Embedding { get; set; } - - public int Counter { get; set; } - public string? Text { get; set; } - } - - public class RecordWithRomOfFloatVectorProperty : TestRecord - { - public ReadOnlyMemory Embedding { get; set; } - - public int Counter { get; set; } - public string? Text { get; set; } - } - - public class Customer - { - public string? FirstName { get; set; } - public string? LastName { get; set; } - } - - private VectorStoreCollection GetCollection( - bool storeGenerator = false, - bool collectionGenerator = false, - bool propertyGenerator = false) - where TRecord : class - { - var (vectorStore, recordDefinition) = this.GetStoreAndRecordDefinition(storeGenerator, collectionGenerator, propertyGenerator); - - return stringVectorFixture.GetCollection(vectorStore, stringVectorFixture.CollectionName, recordDefinition); - } - - private VectorStoreCollection> GetDynamicCollection( - bool storeGenerator = false, - bool collectionGenerator = false, - bool propertyGenerator = false) - { - var (vectorStore, recordDefinition) = this.GetStoreAndRecordDefinition(storeGenerator, collectionGenerator, propertyGenerator); - - return stringVectorFixture.GetDynamicCollection(vectorStore, stringVectorFixture.CollectionName, recordDefinition); - } - - private (VectorStore, VectorStoreCollectionDefinition) GetStoreAndRecordDefinition( - bool storeGenerator = false, - bool collectionGenerator = false, - bool propertyGenerator = false) - { - var recordDefinition = stringVectorFixture.CreateRecordDefinition(); - - if (propertyGenerator) - { - foreach (var vectorProperty in recordDefinition.Properties.OfType()) - { - vectorProperty.EmbeddingGenerator = new FakeEmbeddingGenerator(replaceLast: 3); - } - } - - if (collectionGenerator) - { - recordDefinition.EmbeddingGenerator = new FakeEmbeddingGenerator(replaceLast: 2); - } - - var vectorStore = stringVectorFixture.CreateVectorStore(storeGenerator ? new FakeEmbeddingGenerator(replaceLast: 1) : null); - - return (vectorStore, recordDefinition); - } - - public abstract class StringVectorFixture : VectorStoreCollectionFixture - { - private int _counter; - - protected override string CollectionNameBase => nameof(EmbeddingGenerationTests); - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(Record.Embedding), typeof(string), dimensions: 3) - { - DistanceFunction = this.DefaultDistanceFunction, - IndexKind = this.DefaultIndexKind - }, - - new VectorStoreDataProperty(nameof(Record.Counter), typeof(int)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(Record.Text), typeof(string)) - ], - EmbeddingGenerator = new FakeEmbeddingGenerator() - }; - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Embedding = "[1, 1, 1]", - Counter = this.GenerateNextCounter(), - Text = "Store ([1, 1, 1])" - }, - new() - { - Key = this.GenerateNextKey(), - Embedding = "[1, 1, 2]", - Counter = this.GenerateNextCounter(), - Text = "Collection ([1, 1, 2])" - }, - new() - { - Key = this.GenerateNextKey(), - Embedding = "[1, 1, 3]", - Counter = this.GenerateNextCounter(), - Text = "Property ([1, 1, 3])" - } - ]; - - public virtual VectorStoreCollection GetCollection( - VectorStore vectorStore, - string collectionName, - VectorStoreCollectionDefinition? recordDefinition = null) - where TRecord : class - => vectorStore.GetCollection(collectionName, recordDefinition); - - public virtual VectorStoreCollection> GetDynamicCollection( - VectorStore vectorStore, - string collectionName, - VectorStoreCollectionDefinition recordDefinition) - => vectorStore.GetDynamicCollection(collectionName, recordDefinition); - - public abstract VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null); - - public abstract Func[] DependencyInjectionStoreRegistrationDelegates { get; } - public abstract Func[] DependencyInjectionCollectionRegistrationDelegates { get; } - - public virtual int GenerateNextCounter() - => Interlocked.Increment(ref this._counter); - } - - public abstract class RomOfFloatVectorFixture : VectorStoreCollectionFixture - { - private int _counter; - - protected override string CollectionNameBase => "SearchOnlyEmbeddingGenerationTests"; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithRomOfFloatVectorProperty.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(RecordWithRomOfFloatVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - { - DistanceFunction = this.DefaultDistanceFunction, - IndexKind = this.DefaultIndexKind - }, - - new VectorStoreDataProperty(nameof(RecordWithRomOfFloatVectorProperty.Counter), typeof(int)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(RecordWithRomOfFloatVectorProperty.Text), typeof(string)) - ], - EmbeddingGenerator = new FakeEmbeddingGenerator() - }; - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Embedding = new ReadOnlyMemory([1, 1, 1]), - Counter = this.GenerateNextCounter(), - Text = "Store ([1, 1, 1])", - }, - new() - { - Key = this.GenerateNextKey(), - Embedding = new ReadOnlyMemory([1, 1, 2]), - Counter = this.GenerateNextCounter(), - Text = "Collection ([1, 1, 2])" - }, - new() - { - Key = this.GenerateNextKey(), - Embedding = new ReadOnlyMemory([1, 1, 3]), - Counter = this.GenerateNextCounter(), - Text = "Property ([1, 1, 3])" - } - ]; - - public virtual VectorStoreCollection GetCollection( - VectorStore vectorStore, - string collectionName, - VectorStoreCollectionDefinition? recordDefinition = null) - where TRecord : class - => vectorStore.GetCollection(collectionName, recordDefinition); - - public virtual VectorStoreCollection> GetDynamicCollection( - VectorStore vectorStore, - string collectionName, - VectorStoreCollectionDefinition recordDefinition) - => vectorStore.GetDynamicCollection(collectionName, recordDefinition); - - public abstract VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null); - - public abstract Func[] DependencyInjectionStoreRegistrationDelegates { get; } - public abstract Func[] DependencyInjectionCollectionRegistrationDelegates { get; } - - public virtual int GenerateNextCounter() - => Interlocked.Increment(ref this._counter); - } - - protected sealed class FakeEmbeddingGenerator(int? replaceLast = null) : IEmbeddingGenerator> - { - public Task>> GenerateAsync( - IEnumerable values, - EmbeddingGenerationOptions? options = null, - CancellationToken cancellationToken = default) - { - var results = new GeneratedEmbeddings>(); - - foreach (var value in values) - { - var vector = value.TrimStart('[').TrimEnd(']').Split(',').Select(s => float.Parse(s.Trim())).ToArray(); - - if (replaceLast is not null) - { - vector[vector.Length - 1] = replaceLast.Value; - } - - results.Add(new Embedding(vector)); - } - - return Task.FromResult(results); - } - - public object? GetService(Type serviceType, object? serviceKey = null) - => null; - - public void Dispose() - { - } - } - - private sealed class FakeCustomerEmbeddingGenerator(float[] embedding) : IEmbeddingGenerator> - { - public Task>> GenerateAsync(IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) - => Task.FromResult(new GeneratedEmbeddings> { new(embedding) }); - - public object? GetService(Type serviceType, object? serviceKey = null) - => null; - - public void Dispose() - { - } - } - - #endregion Support -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs deleted file mode 100644 index cdb714eca080..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/FilterTests.cs +++ /dev/null @@ -1,722 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Linq.Expressions; -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -#pragma warning disable CS8605 // Unboxing a possibly null value. -#pragma warning disable CS0252 // Possible unintended reference comparison; left hand side needs cast -#pragma warning disable RCS1098 // Constant values should be placed on right side of comparisons - -namespace VectorData.ConformanceTests; - -public abstract class FilterTests(FilterTests.Fixture fixture) - where TKey : notnull -{ - #region Equality - - [ConditionalFact] - public virtual Task Equal_with_int() - => this.TestFilterAsync( - r => r.Int == 8, - r => (int)r["Int"] == 8); - - [ConditionalFact] - public virtual Task Equal_with_string() - => this.TestFilterAsync( - r => r.String == "foo", - r => r["String"] == "foo"); - - [ConditionalFact] - public virtual Task Equal_with_string_sql_injection_in_value() - { - string sqlInjection = $"foo; DROP TABLE {fixture.Collection.Name};"; - - return this.TestFilterAsync( - r => r.String == sqlInjection, - r => r["String"] == sqlInjection, - expectZeroResults: true); - } - - [ConditionalFact] - public virtual async Task Equal_with_string_sql_injection_in_name() - { - if (fixture.TestDynamic) - { - await Assert.ThrowsAsync( - async () => await fixture.DynamicCollection.SearchAsync( - new ReadOnlyMemory([1, 2, 3]), - top: 1, - new() { Filter = r => r["String = \"not\"; DROP TABLE FilterTests;"] == "" }).ToListAsync()); - } - } - - [ConditionalFact] - public virtual Task Equal_with_string_containing_special_characters() - => this.TestFilterAsync( - r => r.String == fixture.SpecialCharactersText, - r => r["String"] == fixture.SpecialCharactersText); - - [ConditionalFact] - public virtual Task Equal_with_string_is_not_Contains() - => this.TestFilterAsync( - r => r.String == "some", - r => r["String"] == "some", - expectZeroResults: true); - - [ConditionalFact] - public virtual Task Equal_reversed() - => this.TestFilterAsync( - r => 8 == r.Int, - r => 8 == (int)r["Int"]); - - [ConditionalFact] - public virtual Task Equal_with_null_reference_type() - => this.TestFilterAsync( - r => r.String == null, - r => r["String"] == null); - - [ConditionalFact] - public virtual Task Equal_with_null_captured() - { - string? s = null; - - return this.TestFilterAsync( - r => r.String == s, - r => r["String"] == s); - } - - [ConditionalFact] - public virtual Task Equal_int_property_with_nonnull_nullable_int() - { - int? i = 8; - - return this.TestFilterAsync( - r => r.Int == i, - r => (int)r["Int"] == i); - } - - [ConditionalFact] - public virtual Task Equal_int_property_with_null_nullable_int() - { - int? i = null; - - return this.TestFilterAsync( - r => r.Int == i, - r => (int)r["Int"] == i, - expectZeroResults: true); - } - - [ConditionalFact] - public virtual Task Equal_int_property_with_nonnull_nullable_int_Value() - { - int? i = 8; - - return this.TestFilterAsync( - r => r.Int == i.Value, - r => (int)r["Int"] == i.Value); - } - -#pragma warning disable CS8629 // Nullable value type may be null. - [ConditionalFact] - public virtual async Task Equal_int_property_with_null_nullable_int_Value() - { - int? i = null; - - // TODO: Some connectors wrap filter translation exceptions in a VectorStoreException (#11766) - var exception = await Assert.ThrowsAnyAsync(() => this.TestFilterAsync( - r => r.Int == i.Value, - r => (int)r["Int"] == i.Value, - expectZeroResults: true)); - - if (exception is not InvalidOperationException and not VectorStoreException { InnerException: InvalidOperationException }) - { - Assert.Fail($"Expected {nameof(InvalidOperationException)} or {nameof(VectorStoreException)} but got {exception.GetType()}"); - } - } -#pragma warning restore CS8629 - - [ConditionalFact] - public virtual Task NotEqual_with_int() - => this.TestFilterAsync( - r => r.Int != 8, - r => (int)r["Int"] != 8); - - [ConditionalFact] - public virtual Task NotEqual_with_string() - => this.TestFilterAsync( - r => r.String != "foo", - r => r["String"] != "foo"); - - [ConditionalFact] - public virtual Task NotEqual_reversed() - => this.TestFilterAsync( - r => r.Int != 8, - r => (int)r["Int"] != 8); - - [ConditionalFact] - public virtual Task NotEqual_with_null_reference_type() - => this.TestFilterAsync( - r => r.String != null, - r => r["String"] != null); - - [ConditionalFact] - public virtual Task NotEqual_with_null_captured() - { - string? s = null; - - return this.TestFilterAsync( - r => r.String != s, - r => r["String"] != s); - } - - [ConditionalFact] - public virtual Task Bool() - => this.TestFilterAsync( - r => r.Bool, - r => (bool)r["Bool"]); - - [ConditionalFact] - public virtual Task Bool_And_Bool() - => this.TestFilterAsync( - r => r.Bool && r.Bool, - r => (bool)r["Bool"] && (bool)r["Bool"]); - - [ConditionalFact] - public virtual Task Bool_Or_Not_Bool() - => this.TestFilterAsync( - r => r.Bool || !r.Bool, - r => (bool)r["Bool"] || !(bool)r["Bool"], - expectAllResults: true); - - #endregion Equality - - #region Comparison - - [ConditionalFact] - public virtual Task GreaterThan_with_int() - => this.TestFilterAsync( - r => r.Int > 9, - r => (int)r["Int"] > 9); - - [ConditionalFact] - public virtual Task GreaterThanOrEqual_with_int() - => this.TestFilterAsync( - r => r.Int >= 9, - r => (int)r["Int"] >= 9); - - [ConditionalFact] - public virtual Task LessThan_with_int() - => this.TestFilterAsync( - r => r.Int < 10, - r => (int)r["Int"] < 10); - - [ConditionalFact] - public virtual Task LessThanOrEqual_with_int() - => this.TestFilterAsync( - r => r.Int <= 10, - r => (int)r["Int"] <= 10); - - #endregion Comparison - - #region Logical operators - - [ConditionalFact] - public virtual Task And() - => this.TestFilterAsync( - r => r.Int == 8 && r.String == "foo", - r => (int)r["Int"] == 8 && r["String"] == "foo"); - - [ConditionalFact] - public virtual Task Or() - => this.TestFilterAsync( - r => r.Int == 8 || r.String == "foo", - r => (int)r["Int"] == 8 || r["String"] == "foo"); - - [ConditionalFact] - public virtual Task And_within_And() - => this.TestFilterAsync( - r => (r.Int == 8 && r.String == "foo") && r.Int2 == 80, - r => ((int)r["Int"] == 8 && r["String"] == "foo") && (int)r["Int2"] == 80); - - [ConditionalFact] - public virtual Task And_within_Or() - => this.TestFilterAsync( - r => (r.Int == 8 && r.String == "foo") || r.Int2 == 100, - r => ((int)r["Int"] == 8 && r["String"] == "foo") || (int)r["Int2"] == 100); - - [ConditionalFact] - public virtual Task Or_within_And() - => this.TestFilterAsync( - r => (r.Int == 8 || r.Int == 9) && r.String == "foo", - r => ((int)r["Int"] == 8 || (int)r["Int"] == 9) && r["String"] == "foo"); - - [ConditionalFact] - public virtual Task Not_over_Equal() - // ReSharper disable once NegativeEqualityExpression - => this.TestFilterAsync( - r => !(r.Int == 8), - r => !((int)r["Int"] == 8)); - - [ConditionalFact] - public virtual Task Not_over_NotEqual() - // ReSharper disable once NegativeEqualityExpression - => this.TestFilterAsync( - r => !(r.Int != 8), - r => !((int)r["Int"] != 8)); - - [ConditionalFact] - public virtual Task Not_over_And() - => this.TestFilterAsync( - r => !(r.Int == 8 && r.String == "foo"), - r => !((int)r["Int"] == 8 && r["String"] == "foo")); - - [ConditionalFact] - public virtual Task Not_over_Or() - => this.TestFilterAsync( - r => !(r.Int == 8 || r.String == "foo"), - r => !((int)r["Int"] == 8 || r["String"] == "foo")); - - [ConditionalFact] - public virtual Task Not_over_bool() - => this.TestFilterAsync( - r => !r.Bool, - r => !(bool)r["Bool"]); - - [ConditionalFact] - public virtual Task Not_over_bool_And_Comparison() - => this.TestFilterAsync( - r => !r.Bool && r.Int != int.MaxValue, - r => !(bool)r["Bool"] && (int)r["Int"] != int.MaxValue); - - #endregion Logical operators - - #region Contains - - [ConditionalFact] - public virtual Task Contains_over_field_string_array() - => this.TestFilterAsync( - r => r.StringArray.Contains("x"), - r => ((string[])r["StringArray"]!).Contains("x")); - - [ConditionalFact] - public virtual Task Contains_over_field_string_List() - => this.TestFilterAsync( - r => r.StringList.Contains("x"), - r => ((List)r["StringList"]!).Contains("x")); - - [ConditionalFact] - public virtual Task Contains_over_inline_int_array() - => this.TestFilterAsync( - r => new[] { 8, 10 }.Contains(r.Int), - r => new[] { 8, 10 }.Contains((int)r["Int"])); - - [ConditionalFact] - public virtual Task Contains_over_inline_string_array() - => this.TestFilterAsync( - r => new[] { "foo", "baz", "unknown" }.Contains(r.String), - r => new[] { "foo", "baz", "unknown" }.Contains(r["String"])); - - [ConditionalFact] - public virtual Task Contains_over_inline_string_array_with_weird_chars() - => this.TestFilterAsync( - r => new[] { "foo", "baz", "un , ' \"" }.Contains(r.String), - r => new[] { "foo", "baz", "un , ' \"" }.Contains(r["String"])); - - [ConditionalFact] - public virtual Task Contains_over_captured_string_array() - { - var array = new[] { "foo", "baz", "unknown" }; - - return this.TestFilterAsync( - r => array.Contains(r.String), - r => array.Contains(r["String"])); - } - -#pragma warning disable RCS1196 // Call extension method as instance method - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // The following tests the various Contains variants directly, without using extension syntax, to ensure everything's - // properly supported. - [ConditionalFact] - public virtual Task Contains_with_Enumerable_Contains() - => this.TestFilterAsync( - r => Enumerable.Contains(r.StringArray, "x"), - r => ((string[])r["StringArray"]!).Contains("x")); - -#if !NETFRAMEWORK - [ConditionalFact] - public virtual Task Contains_with_MemoryExtensions_Contains() - => this.TestFilterAsync( - r => MemoryExtensions.Contains(r.StringArray, "x"), - r => ((string[])r["StringArray"]!).Contains("x")); -#endif - -#if NET10_0_OR_GREATER - [ConditionalFact] - public virtual Task Contains_with_MemoryExtensions_Contains_with_null_comparer() - => this.TestFilterAsync( - r => MemoryExtensions.Contains(r.StringArray, "x", comparer: null), - r => ((string[])r["StringArray"]!).Contains("x")); -#endif - -#pragma warning restore RCS1196 // Call extension method as instance method - - [ConditionalFact] - public virtual Task Any_with_Contains_over_inline_string_array() - => this.TestFilterAsync( - r => r.StringArray.Any(s => new[] { "x", "z", "nonexistent" }.Contains(s)), - r => ((string[])r["StringArray"]!).Any(s => new[] { "x", "z", "nonexistent" }.Contains(s))); - - [ConditionalFact] - public virtual Task Any_with_Contains_over_captured_string_array() - { - string[] tagsToFind = ["x", "z", "nonexistent"]; - - return this.TestFilterAsync( - r => r.StringArray.Any(s => tagsToFind.Contains(s)), - r => ((string[])r["StringArray"]!).Any(s => tagsToFind.Contains(s))); - } - - [ConditionalFact] - public virtual Task Any_with_Contains_over_captured_string_list() - { - List tagsToFind = ["x", "z", "nonexistent"]; - - return this.TestFilterAsync( - r => r.StringArray.Any(s => tagsToFind.Contains(s)), - r => ((string[])r["StringArray"]!).Any(s => tagsToFind.Contains(s))); - } - - [ConditionalFact] - public virtual Task Any_over_List_with_Contains_over_captured_string_array() - { - string[] tagsToFind = ["x", "z", "nonexistent"]; - - return this.TestFilterAsync( - r => r.StringList.Any(s => tagsToFind.Contains(s)), - r => ((List)r["StringList"]!).Any(s => tagsToFind.Contains(s))); - } - - #endregion Contains - - #region Variable types - - [ConditionalFact] - public virtual Task Captured_local_variable() - { - // ReSharper disable once ConvertToConstant.Local - var i = 8; - - return this.TestFilterAsync( - r => r.Int == i, - r => (int)r["Int"] == i); - } - - [ConditionalFact] - public virtual Task Member_field() - => this.TestFilterAsync( - r => r.Int == this._memberField, - r => (int)r["Int"] == this._memberField); - - [ConditionalFact] - public virtual Task Member_readonly_field() - => this.TestFilterAsync( - r => r.Int == this._memberReadOnlyField, - r => (int)r["Int"] == this._memberReadOnlyField); - - [ConditionalFact] - public virtual Task Member_static_field() - => this.TestFilterAsync( - r => r.Int == _staticMemberField, - r => (int)r["Int"] == _staticMemberField); - - [ConditionalFact] - public virtual Task Member_static_readonly_field() - => this.TestFilterAsync( - r => r.Int == _staticMemberReadOnlyField, - r => (int)r["Int"] == _staticMemberReadOnlyField); - - [ConditionalFact] - public virtual Task Member_nested_access() - => this.TestFilterAsync( - r => r.Int == this._someWrapper.SomeWrappedValue, - r => (int)r["Int"] == this._someWrapper.SomeWrappedValue); - -#pragma warning disable RCS1169 // Make field read-only -#pragma warning disable IDE0044 // Make field read-only -#pragma warning disable RCS1187 // Use constant instead of field -#pragma warning disable CA1802 // Use literals where appropriate - private int _memberField = 8; - private readonly int _memberReadOnlyField = 8; - - private static int _staticMemberField = 8; - private static readonly int _staticMemberReadOnlyField = 8; - - private SomeWrapper _someWrapper = new(); -#pragma warning restore CA1802 -#pragma warning restore RCS1187 -#pragma warning restore RCS1169 -#pragma warning restore IDE0044 - - private sealed class SomeWrapper - { - public int SomeWrappedValue = 8; - } - - #endregion Variable types - - #region Miscellaneous - - [ConditionalFact] - public virtual Task True() - => this.TestFilterAsync(r => true, r => true, expectAllResults: true); - - #endregion Miscellaneous - - protected virtual async Task TestFilterAsync( - Expression> filter, - Expression, bool>> dynamicFilter, - bool expectZeroResults = false, - bool expectAllResults = false) - { - var expected = fixture.TestData.AsQueryable().Where(filter).OrderBy(r => r.Key).ToList(); - - if (expected.Count == 0 && !expectZeroResults) - { - Assert.Fail("The test returns zero results, and so may be unreliable"); - } - else if (expectZeroResults && expected.Count != 0) - { - Assert.Fail($"{nameof(expectZeroResults)} was true, but the test returned {expected.Count} results."); - } - - if (expected.Count == fixture.TestData.Count && !expectAllResults) - { - Assert.Fail("The test returns all results, and so may be unreliable"); - } - else if (expectAllResults && expected.Count != fixture.TestData.Count) - { - Assert.Fail($"{nameof(expectAllResults)} was true, but the test returned {expected.Count} results instead of the expected {fixture.TestData.Count}."); - } - - // Execute the query against the vector store, once using the strongly typed filter - // and once using the dynamic filter - var actual = await fixture.Collection.SearchAsync( - new ReadOnlyMemory([1, 2, 3]), - top: fixture.TestData.Count, - new() { Filter = filter }) - .Select(r => r.Record) - .OrderBy(r => r.Key) - .ToListAsync(); - - if (actual.Count != expected.Count) - { - Assert.Fail($"Expected {expected.Count} results, but got {actual.Count}"); - } - - foreach (var (e, a) in expected.Zip(actual, (e, a) => (e, a))) - { - fixture.AssertEqualFilterRecord(e, a); - } - - if (fixture.TestDynamic) - { - var dynamicActual = await fixture.DynamicCollection.SearchAsync( - new ReadOnlyMemory([1, 2, 3]), - top: fixture.TestData.Count, - new() { Filter = dynamicFilter }) - .Select(r => r.Record) - .OrderBy(r => r[nameof(FilterRecord.Key)]) - .ToListAsync(); - - if (dynamicActual.Count != expected.Count) - { - Assert.Fail($"Expected {expected.Count} results, but got {actual.Count}"); - } - - foreach (var (e, a) in expected.Zip(dynamicActual, (e, a) => (e, a))) - { - fixture.AssertEqualDynamic(e, a); - } - } - } - - public class FilterRecord : TestRecord - { - public ReadOnlyMemory? Vector { get; set; } - - public int Int { get; set; } - public string? String { get; set; } - public bool Bool { get; set; } - public int Int2 { get; set; } - public string[] StringArray { get; set; } = null!; - public List StringList { get; set; } = null!; - } - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(FilterTests); - - public virtual string SpecialCharactersText => """>with $om[ specia]"chara> DynamicCollection { get; protected set; } = null!; - - public virtual bool TestDynamic => true; - - public override async Task InitializeAsync() - { - await base.InitializeAsync(); - - if (this.TestDynamic) - { - this.DynamicCollection = this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition()); - } - } - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(FilterRecord.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(FilterRecord.Vector), typeof(ReadOnlyMemory?), 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - }, - - new VectorStoreDataProperty(nameof(FilterRecord.Int), typeof(int)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(FilterRecord.String), typeof(string)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(FilterRecord.Bool), typeof(bool)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(FilterRecord.Int2), typeof(int)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(FilterRecord.StringArray), typeof(string[])) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(FilterRecord.StringList), typeof(List)) { IsIndexed = true } - ] - }; - - protected override List BuildTestData() - { - // All records have the same vector - this fixture is about testing criteria filtering only - var vector = new ReadOnlyMemory([1, 2, 3]); - - return - [ - new() - { - Key = this.GenerateNextKey(), - Int = 8, - String = "foo", - Bool = true, - Int2 = 80, - StringArray = ["x", "y"], - StringList = ["x", "y"], - Vector = vector - }, - new() - { - Key = this.GenerateNextKey(), - Int = 9, - String = "bar", - Bool = false, - Int2 = 90, - StringArray = ["a", "b"], - StringList = ["a", "b"], - Vector = vector - }, - new() - { - Key = this.GenerateNextKey(), - Int = 9, - String = "foo", - Bool = true, - Int2 = 9, - StringArray = ["x"], - StringList = ["x"], - Vector = vector - }, - new() - { - Key = this.GenerateNextKey(), - Int = 10, - String = null, - Bool = false, - Int2 = 100, - StringArray = ["x", "y", "z"], - StringList = ["x", "y", "z"], - Vector = vector - }, - new() - { - Key = this.GenerateNextKey(), - Int = 11, - Bool = true, - String = this.SpecialCharactersText, - Int2 = 101, - StringArray = ["y", "z"], - StringList = ["y", "z"], - Vector = vector - } - ]; - } - - public virtual void AssertEqualFilterRecord(FilterRecord x, FilterRecord y) - { - var definitionProperties = this.CreateRecordDefinition().Properties; - - Assert.Equal(x.Key, y.Key); - Assert.Equal(x.Int, y.Int); - Assert.Equal(x.String, y.String); - Assert.Equal(x.Int2, y.Int2); - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.Bool))) - { - Assert.Equal(x.Bool, y.Bool); - } - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringArray))) - { - Assert.Equivalent(x.StringArray, y.StringArray); - } - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringList))) - { - Assert.Equivalent(x.StringList, y.StringList); - } - } - - public virtual void AssertEqualDynamic(FilterRecord x, Dictionary y) - { - var definitionProperties = this.CreateRecordDefinition().Properties; - - Assert.Equal(x.Key, y["Key"]); - Assert.Equal(x.Int, y["Int"]); - Assert.Equal(x.String, y["String"]); - Assert.Equal(x.Int2, y["Int2"]); - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.Bool))) - { - Assert.Equal(x.Bool, y["Bool"]); - } - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringArray))) - { - Assert.Equivalent(x.StringArray, y["StringArray"]); - } - - if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringList))) - { - Assert.Equivalent(x.StringList, y["StringList"]); - } - } - - // In some databases (Azure AI Search), the data shows up but the filtering index isn't yet updated, - // so filtered searches show empty results. Add a filter to the seed data check below. - protected override Task WaitForDataAsync() - => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, r => r.Int > 0); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/HybridSearchTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/HybridSearchTests.cs deleted file mode 100644 index f63059ac6b8b..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/HybridSearchTests.cs +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests; - -/// -/// Base class for common integration tests that should pass for any . -/// -/// The type of key to use with the record collection. -public abstract class HybridSearchTests( - HybridSearchTests.VectorAndStringFixture vectorAndStringFixture, - HybridSearchTests.MultiTextFixture multiTextFixture) - where TKey : notnull -{ - [ConditionalFact] - public async Task HybridSearchAsync() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - // All records have the same vector, but the third contains Grapes, so searching for - // Grapes should return the third record first. - var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["Grapes"], top: 3).ToListAsync(); - - // Assert - Assert.Equal(3, results.Count); - Assert.Equal(3, results[0].Record.Code); - } - - [ConditionalFact] - public async Task HybridSearchAsync_with_filter() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - // All records have the same vector, but the second contains Oranges, however - // adding the filter should limit the results to only the first. - var results = await vectorAndStringFixture.HybridSearchable - .HybridSearchAsync( - vector, - ["Oranges"], - top: 3, - new() { Filter = r => r.Code == 1 }) - .ToListAsync(); - - // Assert - Assert.Equal(1, Assert.Single(results).Record.Code); - } - - [ConditionalFact] - public async Task HybridSearchAsync_with_top() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - // All records have the same vector, but the second contains Oranges, so the - // second should be returned first. - var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["Oranges"], top: 1).ToListAsync(); - - // Assert - Assert.Single(results); - - Assert.Equal(2, results[0].Record.Code); - } - - [ConditionalFact] - public async Task HybridSearchAsync_with_Skip() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - // All records have the same vector, but the first and third contain healthy, - // so when skipping the first two results, we should get the second record. - var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["healthy"], top: 3, new() { Skip = 2 }).ToListAsync(); - - // Assert - Assert.Single(results); - - Assert.Equal(2, results[0].Record.Code); - } - - [ConditionalFact] - public async Task HybridSearchAsync_with_multiple_keywords_ranks_matched_keywords_higher() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["tangy", "nourishing"], top: 3).ToListAsync(); - - // Assert - Assert.Equal(3, results.Count); - - Assert.True(results[0].Record.Code.Equals(1) || results[0].Record.Code.Equals(2)); - Assert.True(results[1].Record.Code.Equals(1) || results[1].Record.Code.Equals(2)); - Assert.Equal(3, results[2].Record.Code); - } - - [ConditionalFact] - public async Task HybridSearchAsync_with_multiple_text_properties() - { - // Arrange - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - // Act - var results1 = await multiTextFixture.HybridSearchable - .HybridSearchAsync(vector, ["Apples"], top: 4, new() { AdditionalProperty = r => r.Text2 }) - .ToListAsync(); - var results2 = await multiTextFixture.HybridSearchable - .HybridSearchAsync(vector, ["Oranges"], top: 4, new() { AdditionalProperty = r => r.Text2 }) - .ToListAsync(); - - // Assert - Assert.Equal(2, results1.Count); - - Assert.Equal(2, results1[0].Record.Code); - Assert.Equal(1, results1[1].Record.Code); - - Assert.Equal(2, results2.Count); - - Assert.Equal(1, results2[0].Record.Code); - Assert.Equal(2, results2[1].Record.Code); - } - - [ConditionalFact] - public Task HybridSearchAsync_without_explicitly_specified_property_fails() - => Assert.ThrowsAsync(async () => - await multiTextFixture.HybridSearchable - .HybridSearchAsync(new ReadOnlyMemory([1, 0, 0, 0]), ["Apples"], top: 3) - .ToListAsync()); - - public sealed class VectorAndStringRecord : TestRecord - { - public string Text { get; set; } = string.Empty; - public int Code { get; set; } - public ReadOnlyMemory Vector { get; set; } - } - - public sealed class MultiTextStringRecord : TestRecord - { - public string Text1 { get; set; } = string.Empty; - public string Text2 { get; set; } = string.Empty; - public int Code { get; set; } - public ReadOnlyMemory Vector { get; set; } - } - - public abstract class VectorAndStringFixture : VectorStoreCollectionFixture> - { - protected override string CollectionNameBase => "HybridSearchTests"; - - public IKeywordHybridSearchable> HybridSearchable - => (IKeywordHybridSearchable>)this.Collection; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty("Key", typeof(TKey)), - new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, - new VectorStoreDataProperty("Code", typeof(int)) { IsIndexed = true }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4) { IndexKind = this.IndexKind }, - ] - }; - - protected override List> BuildTestData() - { - // All records have the same vector - this fixture is about testing the full text search portion of hybrid search - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - return - [ - new() - { - Key = this.GenerateNextKey(), - Text = "Apples are a healthy and nourishing snack", - Vector = vector, - Code = 1 - }, - new() - { - Key = this.GenerateNextKey(), - Text = "Oranges are tangy and contain vitamin c", - Vector = vector, - Code = 2 - }, - new() - { - Key = this.GenerateNextKey(), - Text = "Grapes are healthy, sweet and juicy", - Vector = vector, - Code = 3 - } - ]; - } - - protected override Task WaitForDataAsync() - => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorSize: 4); - } - - public abstract class MultiTextFixture : VectorStoreCollectionFixture> - { - protected override string CollectionNameBase => "MultiTextHybridSearchTests"; - - public IKeywordHybridSearchable> HybridSearchable - => (IKeywordHybridSearchable>)this.Collection; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty("Key", typeof(TKey)), - new VectorStoreDataProperty("Text1", typeof(string)) { IsFullTextIndexed = true }, - new VectorStoreDataProperty("Text2", typeof(string)) { IsFullTextIndexed = true }, - new VectorStoreDataProperty("Code", typeof(int)) { IsIndexed = true }, - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4) { IndexKind = this.IndexKind }, - ] - }; - - protected override List> BuildTestData() - { - // All records have the same vector - this fixture is about testing the full text search portion of hybrid search - var vector = new ReadOnlyMemory([1, 0, 0, 0]); - - return - [ - new() - { - Key = this.GenerateNextKey(), - Text1 = "Apples", - Text2 = "Oranges", - Code = 1, - Vector = vector - }, - new() - { - Key = this.GenerateNextKey(), - Text1 = "Oranges", - Text2 = "Apples", - Code = 2, - Vector = vector - } - ]; - } - - protected override Task WaitForDataAsync() - => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorSize: 4); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/IndexKindTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/IndexKindTests.cs deleted file mode 100644 index 26738dbeba66..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/IndexKindTests.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests; - -public abstract class IndexKindTests(IndexKindTests.Fixture fixture) - where TKey : notnull -{ - [ConditionalFact] - public virtual Task Flat() - => this.Test(IndexKind.Flat); - - protected virtual async Task Test(string indexKind) - { - using var collection = fixture.CreateCollection(indexKind); - await collection.EnsureCollectionDeletedAsync(); - await collection.EnsureCollectionExistsAsync(); - - SearchRecord[] records = - [ - new() - { - Key = fixture.GenerateNextKey(), - Int = 1, - Vector = new([1, 2, 3]), - }, - new() - { - Key = fixture.GenerateNextKey(), - Int = 2, - Vector = new([10, 30, 50]), - }, - new() - { - Key = fixture.GenerateNextKey(), - Int = 3, - Vector = new([100, 40, 70]), - } - ]; - - await collection.UpsertAsync(records); - - await fixture.TestStore.WaitForDataAsync(collection, records.Length); - - var result = await collection.SearchAsync(new ReadOnlyMemory([10, 30, 50]), top: 1).SingleAsync(); - - Assert.NotNull(result); - Assert.Equal(2, result.Record.Int); - } - - public abstract class Fixture : VectorStoreFixture - { - protected virtual string CollectionNameBase => nameof(IndexKindTests); - public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase); - - protected virtual string? DistanceFunction => null; - - public virtual VectorStoreCollection CreateCollection(string indexKind) - { - VectorStoreCollectionDefinition definition = new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(SearchRecord.Key), typeof(TKey)), - new VectorStoreDataProperty(nameof(SearchRecord.Int), typeof(int)), - new VectorStoreVectorProperty(nameof(SearchRecord.Vector), typeof(ReadOnlyMemory), dimensions: 3) - { - IndexKind = indexKind, - DistanceFunction = this.DistanceFunction ?? this.DefaultDistanceFunction - } - ] - }; - - return this.TestStore.CreateCollection(this.CollectionName, definition); - } - } - - public class SearchRecord - { - public TKey Key { get; set; } = default!; - public int Int { get; set; } - public ReadOnlyMemory Vector { get; set; } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/BasicModelTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/BasicModelTests.cs deleted file mode 100644 index 3acc39b0d780..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/BasicModelTests.cs +++ /dev/null @@ -1,552 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.ModelTests; - -public abstract class BasicModelTests(BasicModelTests.Fixture fixture) : IAsyncLifetime - where TKey : notnull -{ - #region Get - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_single_record(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); - - expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_multiple_records(bool includeVectors) - { - var expectedRecords = fixture.TestData.Take(2); - var ids = expectedRecords.Select(record => record.Key); - - var received = await this.Collection.GetAsync(ids, new() { IncludeVectors = includeVectors }).ToArrayAsync(); - - foreach (var record in expectedRecords) - { - record.AssertEqual( - received.Single(r => r.Key.Equals(record.Key)), - includeVectors, - fixture.TestStore.VectorsComparable); - } - } - - [ConditionalFact] - public virtual async Task GetAsync_throws_for_null_key() - { - // Skip this test for value type keys - if (default(TKey) is not null) - { - return; - } - - ArgumentNullException ex = await Assert.ThrowsAsync(() => this.Collection.GetAsync((TKey)default!)); - Assert.Equal("key", ex.ParamName); - } - - [ConditionalFact] - public virtual async Task GetAsync_throws_for_null_keys() - { - ArgumentNullException ex = await Assert.ThrowsAsync(() => this.Collection.GetAsync(keys: null!).ToArrayAsync().AsTask()); - Assert.Equal("keys", ex.ParamName); - } - - [ConditionalFact] - public virtual async Task GetAsync_returns_null_for_missing_key() - { - TKey key = fixture.GenerateNextKey(); - - Assert.Null(await this.Collection.GetAsync(key)); - } - - [ConditionalFact] - public virtual async Task GetAsync_multiple_records_with_missing_keys_returns_only_existing() - { - var expectedRecords = fixture.TestData.Take(2).ToArray(); - var missingKey = fixture.GenerateNextKey(); - var ids = expectedRecords.Select(record => record.Key).Append(missingKey).ToArray(); - - var received = await this.Collection.GetAsync(ids).ToListAsync(); - - Assert.Equal(2, received.Count); - - foreach (var record in expectedRecords) - { - record.AssertEqual( - received.Single(r => r.Key.Equals(record.Key)), - includeVectors: false, - fixture.TestStore.VectorsComparable); - } - } - - [ConditionalFact] - public virtual async Task GetAsync_returns_empty_for_empty_keys() - { - Assert.Empty(await this.Collection.GetAsync([]).ToArrayAsync()); - } - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_with_filter(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var results = await this.Collection.GetAsync( - r => r.Number == 1, - top: 2, - new() { IncludeVectors = includeVectors }) - .ToListAsync(); - - var receivedRecord = Assert.Single(results); - expectedRecord.AssertEqual(receivedRecord, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_by_true() - { - Assert.True(fixture.TestData.Count < 100); - - var count = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - - Assert.Equal(fixture.TestData.Count, count); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_OrderBy() - { - var ascendingNumbers = fixture.TestData.Where(r => r.Number > 1).OrderBy(r => r.Number).Take(2).Select(r => r.Number).ToList(); - var descendingNumbers = fixture.TestData.Where(r => r.Number > 1).OrderByDescending(r => r.Number).Take(2).Select(r => r.Number).ToList(); - - // Make sure the actual results are different for ascending/descending, otherwise the test is meaningless - Assert.NotEqual(ascendingNumbers, descendingNumbers); - - var results = await this.Collection.GetAsync( - r => r.Number > 1, - top: 2, - new() { OrderBy = o => o.Ascending(r => r.Number) }) - .Select(r => r.Number) - .ToListAsync(); - - Assert.Equal(ascendingNumbers, results); - - results = await this.Collection.GetAsync( - r => r.Number > 1, - top: 2, - new() { OrderBy = o => o.Descending(r => r.Number) }) - .Select(r => r.Number) - .ToListAsync(); - - Assert.Equal(descendingNumbers, results); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_multiple_OrderBys() - { - var ascendingNumbers = fixture.TestData - .OrderByDescending(r => r.Text) - .ThenBy(r => r.Number) - .Take(2).Select(r => r.Number).ToList(); - var descendingNumbers = fixture.TestData - .OrderByDescending(r => r.Text) - .ThenByDescending(r => r.Number) - .Take(2) - .Select(r => r.Number) - .ToList(); - - // Make sure the actual results are different for ascending/descending, otherwise the test is meaningless - Assert.NotEqual(ascendingNumbers, descendingNumbers); - - var results = await this.Collection.GetAsync( - r => true, - top: 2, - new() { OrderBy = o => o.Descending(r => r.Text).Ascending(r => r.Number) }) - .Select(r => r.Number) - .ToListAsync(); - - Assert.Equal(ascendingNumbers, results); - - results = await this.Collection.GetAsync( - r => true, - top: 2, - new() { OrderBy = o => o.Descending(r => r.Text).Descending(r => r.Number) }) - .Select(r => r.Number) - .ToListAsync(); - - Assert.Equal(descendingNumbers, results); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_OrderBy_and_Skip() - { - var results = await this.Collection.GetAsync( - r => r.Number > 1, - top: 2, - new() { OrderBy = o => o.Ascending(r => r.Number), Skip = 1 }) - .Select(r => r.Number) - .ToListAsync(); - - Assert.Equal( - fixture.TestData.Where(r => r.Number > 1).OrderBy(r => r.Number).Skip(1).Take(2).Select(r => r.Number), - results); - } - - #endregion Get - - #region Upsert - - [ConditionalFact] - public virtual async Task Insert_single_record() - { - TKey expectedKey = fixture.GenerateNextKey(); - Record inserted = new() - { - Key = expectedKey, - Text = "New record", - Number = 123, - Vector = new([10, 0, 0]) - }; - - Assert.Null(await this.Collection.GetAsync(expectedKey)); - await this.Collection.UpsertAsync(inserted); - - var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true }); - inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable); - - await fixture.TestStore.WaitForDataAsync(this.Collection, recordCount: fixture.TestData.Count + 1); - } - - [ConditionalFact] - public virtual async Task Update_single_record() - { - var existingRecord = fixture.TestData[1]; - Record updated = new() - { - Key = existingRecord.Key, - Text = "Updated record", - Number = 456, - Vector = new([10, 0, 0]) - }; - - Assert.NotNull(await this.Collection.GetAsync(existingRecord.Key)); - await this.Collection.UpsertAsync(updated); - - var received = await this.Collection.GetAsync(existingRecord.Key, new() { IncludeVectors = true }); - updated.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Insert_multiple_records() - { - Record[] newRecords = - [ - new() - { - Key = fixture.GenerateNextKey(), - Number = 100, - Text = "New record 1", - Vector = new([10, 0, 1]) - }, - new() - { - Key = fixture.GenerateNextKey(), - Number = 101, - Text = "New record 2", - Vector = new([10, 0, 2]) - }, - ]; - - var keys = newRecords.Select(record => record.Key).ToArray(); - Assert.Empty(await this.Collection.GetAsync(keys).ToArrayAsync()); - - await this.Collection.UpsertAsync(newRecords); - - var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync(); - - Assert.Collection( - received.OrderBy(r => r.Number), - r => newRecords[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable), - r => newRecords[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable)); - } - - [ConditionalFact] - public virtual async Task Update_multiple_records() - { - Record[] existingRecords = - [ - new() - { - Key = fixture.TestData[0].Key, - Number = 101, - Text = "Updated record 1", - Vector = new([10, 0, 1]) - }, - new() - { - Key = fixture.TestData[1].Key, - Number = 102, - Text = "Updated record 2", - Vector = new([10, 0, 2]) - } - ]; - - await this.Collection.UpsertAsync(existingRecords); - - var keys = existingRecords.Select(record => record.Key).ToArray(); - var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync(); - - Assert.Collection( - received.OrderBy(r => r.Number), - r => existingRecords[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable), - r => existingRecords[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable)); - } - - [ConditionalFact] - public virtual async Task Insert_and_update_in_same_batch() - { - Record[] records = - [ - new() - { - Key = fixture.GenerateNextKey(), - Number = 101, - Text = "New record", - Vector = new([10, 0, 1]) - }, - new() - { - Key = fixture.TestData[0].Key, - Number = 102, - Text = "Updated record", - Vector = new([10, 0, 2]) - }, - ]; - - await this.Collection.UpsertAsync(records); - - var keys = records.Select(record => record.Key).ToArray(); - var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync(); - - Assert.Collection( - received.OrderBy(r => r.Number), - r => records[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable), - r => records[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable)); - } - - [ConditionalFact] - public virtual async Task UpsertAsync_throws_for_null_batch() - { - ArgumentNullException ex = await Assert.ThrowsAsync(() => this.Collection.UpsertAsync(records: null!)); - Assert.Equal("records", ex.ParamName); - } - - [ConditionalFact] - public virtual async Task UpsertAsync_does_nothing_for_empty_batch() - { - Assert.True(fixture.TestData.Count < 100); - - var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - await this.Collection.UpsertAsync([]); - var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - - Assert.Equal(afterCount, beforeCount); - } - - #endregion Upsert - - #region Delete - - [ConditionalFact] - public virtual async Task Delete_single_record() - { - var keyToRemove = fixture.TestData[0].Key; - - await this.Collection.DeleteAsync(keyToRemove); - Assert.Null(await this.Collection.GetAsync(keyToRemove)); - } - - [ConditionalFact] - public virtual async Task Delete_multiple_records() - { - TKey[] keysToRemove = [fixture.TestData[0].Key, fixture.TestData[1].Key]; - - await this.Collection.DeleteAsync(keysToRemove); - Assert.Empty(await this.Collection.GetAsync(keysToRemove).ToArrayAsync()); - } - - [ConditionalFact] - public virtual async Task DeleteAsync_does_nothing_for_non_existing_key() - { - var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - await this.Collection.DeleteAsync(fixture.GenerateNextKey()); - var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - - Assert.Equal(afterCount, beforeCount); - } - - [ConditionalFact] - public virtual async Task DeleteAsync_does_nothing_for_empty_batch() - { - var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - await this.Collection.DeleteAsync([]); - var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync(); - - Assert.Equal(afterCount, beforeCount); - } - - [ConditionalFact] - public virtual async Task DeleteAsync_throws_for_null_keys() - { - ArgumentNullException ex = await Assert.ThrowsAsync(() => this.Collection.DeleteAsync(keys: null!)); - Assert.Equal("keys", ex.ParamName); - } - - #endregion Delete - - #region Search - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task SearchAsync(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var result = await this.Collection - .SearchAsync( - expectedRecord.Vector, - top: 1, - new() { IncludeVectors = includeVectors }) - .SingleAsync(); - - expectedRecord.AssertEqual(result.Record, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_Skip() - { - var result = await this.Collection - .SearchAsync( - fixture.TestData[0].Vector, - top: 1, - new() { Skip = 1 }) - .SingleAsync(); - - fixture.TestData[1].AssertEqual(result.Record, includeVectors: false, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_Filter() - { - var result = await this.Collection - .SearchAsync( - fixture.TestData[0].Vector, - top: 1, - new() { Filter = r => r.Number == 2 }) - .SingleAsync(); - - fixture.TestData[1].AssertEqual(result.Record, includeVectors: false, fixture.TestStore.VectorsComparable); - } - - // For ScoreThreshold, see DistanceFunctionTests (to ensure we tests thresholds for each and every function) - - #endregion Search - - protected VectorStoreCollection Collection => fixture.Collection; - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(BasicModelTests); - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Number = 1, - Text = "foo", - Vector = new([1, 2, 3]) - }, - new() - { - Key = this.GenerateNextKey(), - Number = 2, - Text = "bar", - Vector = new([1, 2, 4]) - }, - new() - { - Key = this.GenerateNextKey(), - Number = 3, - Text = "foo", // identical text as above - Vector = new([1, 2, 5]) - } - ]; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(Record.Vector), typeof(ReadOnlyMemory), 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - }, - - new VectorStoreDataProperty(nameof(Record.Number), typeof(int)) { IsIndexed = true }, - new VectorStoreDataProperty(nameof(Record.Text), typeof(string)) { IsIndexed = true }, - ] - }; - } - - public sealed class Record : TestRecord - { - [VectorStoreData(StorageName = "text")] - public string? Text { get; set; } - - [VectorStoreData(StorageName = "number")] - public int Number { get; set; } - - [VectorStoreVector(Dimensions: 3, StorageName = "vector")] - public ReadOnlyMemory Vector { get; set; } - - public void AssertEqual(Record? other, bool includeVectors, bool compareVectors) - { - Assert.NotNull(other); - Assert.Equal(this.Key, other.Key); - Assert.Equal(this.Text, other.Text); - Assert.Equal(this.Number, other.Number); - - if (includeVectors) - { - Assert.Equal(this.Vector.Span.Length, other.Vector.Span.Length); - - if (compareVectors) - { - Assert.Equal(this.Vector.ToArray(), other.Vector.ToArray()); - } - } - else - { - Assert.Equal(0, other.Vector.Length); - } - } - - public override string ToString() - => $"Key: {this.Key}, Text: {this.Text}"; - } - - public Task InitializeAsync() - => fixture.ReseedAsync(); - - public Task DisposeAsync() - => Task.CompletedTask; - - public static readonly TheoryData IncludeVectorsData = [false, true]; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/DynamicModelTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/DynamicModelTests.cs deleted file mode 100644 index 387f3ad05fe3..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/DynamicModelTests.cs +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.ModelTests; - -public abstract class DynamicModelTests(DynamicModelTests.Fixture fixture) : IAsyncLifetime - where TKey : notnull -{ - #region Get - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_single_record(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var received = await fixture.Collection.GetAsync( - (TKey)expectedRecord[KeyPropertyName]!, - new() { IncludeVectors = includeVectors }); - - AssertEquivalent(expectedRecord, received, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_multiple_records(bool includeVectors) - { - var expectedRecords = fixture.TestData.Take(2); - var ids = expectedRecords.Select(record => record[KeyPropertyName]!); - - var received = await fixture.Collection.GetAsync(ids, new() { IncludeVectors = includeVectors }).ToArrayAsync(); - - foreach (var record in expectedRecords) - { - AssertEquivalent( - record, - received.Single(r => r[KeyPropertyName]!.Equals(record[KeyPropertyName])), - includeVectors, - fixture.TestStore.VectorsComparable); - } - } - - [ConditionalFact] - public virtual async Task GetAsync_throws_for_null_key() - { - // Skip this test for value type keys - if (default(TKey) is not null) - { - return; - } - - ArgumentNullException ex = await Assert.ThrowsAsync(() => fixture.Collection.GetAsync((TKey)default!)); - Assert.Equal("key", ex.ParamName); - } - - [ConditionalFact] - public virtual async Task GetAsync_throws_for_null_keys() - { - ArgumentNullException ex = await Assert.ThrowsAsync(() => fixture.Collection.GetAsync(keys: null!).ToArrayAsync().AsTask()); - Assert.Equal("keys", ex.ParamName); - } - - [ConditionalFact] - public virtual async Task GetAsync_returns_null_for_missing_key() - { - TKey key = fixture.GenerateNextKey(); - - Assert.Null(await fixture.Collection.GetAsync(key)); - } - - [ConditionalFact] - public virtual async Task GetAsync_returns_empty_for_empty_keys() - { - Assert.Empty(await fixture.Collection.GetAsync([]).ToArrayAsync()); - } - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_with_filter(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var results = await fixture.Collection.GetAsync( - r => (int)r[IntegerPropertyName]! == 1, - top: 2, - new() { IncludeVectors = includeVectors }) - .ToListAsync(); - - var receivedRecord = Assert.Single(results); - AssertEquivalent(expectedRecord, receivedRecord, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_by_true() - { - var count = await fixture.Collection.GetAsync(r => true, top: 100).CountAsync(); - Assert.Equal(fixture.TestData.Count, count); - Assert.True(count < 100); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_OrderBy() - { - var ascendingNumbers = fixture.TestData - .Where(r => (int)r[IntegerPropertyName]! > 1) - .OrderBy(r => r[IntegerPropertyName]) - .Take(2) - .Select(r => (int)r[IntegerPropertyName]!) - .ToList(); - - var descendingNumbers = fixture.TestData - .Where(r => (int)r[IntegerPropertyName]! > 1) - .OrderByDescending(r => r[IntegerPropertyName]) - .Take(2) - .Select(r => (int)r[IntegerPropertyName]!) - .ToList(); - - // Make sure the actual results are different for ascending/descending, otherwise the test is meaningless - Assert.NotEqual(ascendingNumbers, descendingNumbers); - - // Finally, query once with ascending and once with descending, comparing against the expected results above. - var results = await fixture.Collection.GetAsync( - r => (int)r[IntegerPropertyName]! > 1, - top: 2, - new() { OrderBy = o => o.Ascending(r => r[IntegerPropertyName]) }) - .Select(r => (int)r[IntegerPropertyName]!) - .ToListAsync(); - - Assert.Equal(ascendingNumbers, results); - - results = await fixture.Collection.GetAsync( - r => (int)r[IntegerPropertyName]! > 1, - top: 2, - new() { OrderBy = o => o.Descending(r => r[IntegerPropertyName]) }) - .Select(r => (int)r[IntegerPropertyName]!) - .ToListAsync(); - - Assert.Equal(descendingNumbers, results); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_multiple_OrderBys() - { - var ascendingNumbers = fixture.TestData - .OrderByDescending(r => r[StringPropertyName]) - .ThenBy(r => r[IntegerPropertyName]) - .Take(2) - .Select(r => (int)r[IntegerPropertyName]!) - .ToList(); - - var descendingNumbers = fixture.TestData - .OrderByDescending(r => r[StringPropertyName]) - .ThenByDescending(r => r[IntegerPropertyName]) - .Take(2) - .Select(r => (int)r[IntegerPropertyName]!) - .ToList(); - - // Make sure the actual results are different for ascending/descending, otherwise the test is meaningless - Assert.NotEqual(ascendingNumbers, descendingNumbers); - - var results = await fixture.Collection.GetAsync( - r => true, - top: 2, - new() { OrderBy = o => o.Descending(r => r[StringPropertyName]).Ascending(r => r[IntegerPropertyName]) }) - .Select(r => (int)r[IntegerPropertyName]!) - .ToListAsync(); - - Assert.Equal(ascendingNumbers, results); - - results = await fixture.Collection.GetAsync( - r => true, - top: 2, - new() { OrderBy = o => o.Descending(r => r[StringPropertyName]).Descending(r => r[IntegerPropertyName]) }) - .Select(r => (int)r[IntegerPropertyName]!) - .ToListAsync(); - - Assert.Equal(descendingNumbers, results); - } - - [ConditionalFact] - public virtual async Task GetAsync_with_filter_and_OrderBy_and_Skip() - { - var results = await fixture.Collection.GetAsync( - r => (int)r[IntegerPropertyName]! > 1, - top: 2, - new() { OrderBy = o => o.Ascending(r => r[IntegerPropertyName]), Skip = 1 }) - .Select(r => (int)r[IntegerPropertyName]!) - .ToListAsync(); - - Assert.Equal( - fixture.TestData - .Where(r => (int)r[IntegerPropertyName]! > 1) - .OrderBy(r => r[IntegerPropertyName]) - .Skip(1) - .Take(2) - .Select(r => (int)r[IntegerPropertyName]!), - results); - } - - #endregion Get - - #region Upsert - - [ConditionalFact] - public virtual async Task Insert_single_record() - { - TKey expectedKey = fixture.GenerateNextKey(); - var inserted = new Dictionary - { - [KeyPropertyName] = expectedKey, - [StringPropertyName] = "some", - [IntegerPropertyName] = 123, - [VectorPropertyName] = new ReadOnlyMemory([10, 0, 0]) - }; - - Assert.Null(await this.Collection.GetAsync(expectedKey)); - await this.Collection.UpsertAsync(inserted); - - var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true }); - AssertEquivalent(inserted, received, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Update_single_record() - { - var existingRecord = fixture.TestData[1]; - var updated = new Dictionary - { - [KeyPropertyName] = existingRecord[KeyPropertyName], - [StringPropertyName] = "different", - [IntegerPropertyName] = 456, - [VectorPropertyName] = new ReadOnlyMemory(Enumerable.Repeat(0.7f, 3).ToArray()) - }; - - Assert.NotNull(await this.Collection.GetAsync((TKey)existingRecord[KeyPropertyName]!)); - await this.Collection.UpsertAsync(updated); - - var received = await this.Collection.GetAsync((TKey)existingRecord[KeyPropertyName]!, new() { IncludeVectors = true }); - AssertEquivalent(updated, received, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Insert_multiple_records() - { - Dictionary[] newRecords = - [ - new() - { - [KeyPropertyName] = fixture.GenerateNextKey(), - [IntegerPropertyName] = 100, - [StringPropertyName] = "New record 1", - [VectorPropertyName] = new ReadOnlyMemory([10, 0, 1]) - }, - new() - { - [KeyPropertyName] = fixture.GenerateNextKey(), - [IntegerPropertyName] = 101, - [StringPropertyName] = "New record 2", - [VectorPropertyName] = new ReadOnlyMemory([10, 0, 2]) - }, - ]; - - var keys = newRecords.Select(record => record[KeyPropertyName]!).ToArray(); - Assert.Empty(await this.Collection.GetAsync(keys).ToArrayAsync()); - - await this.Collection.UpsertAsync(newRecords); - - var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync(); - - Assert.Collection( - received.OrderBy(r => r[IntegerPropertyName]), - r => AssertEquivalent(newRecords[0], r, includeVectors: true, fixture.TestStore.VectorsComparable), - r => AssertEquivalent(newRecords[1], r, includeVectors: true, fixture.TestStore.VectorsComparable)); - } - - #endregion Upsert - - #region Delete - - [ConditionalFact] - public async Task Delete_single_record() - { - var recordToRemove = fixture.TestData[2]; - - Assert.NotNull(await fixture.Collection.GetAsync((TKey)recordToRemove[KeyPropertyName]!)); - await fixture.Collection.DeleteAsync((TKey)recordToRemove[KeyPropertyName]!); - Assert.Null(await fixture.Collection.GetAsync((TKey)recordToRemove[KeyPropertyName]!)); - } - - // TODO: https://github.com/microsoft/semantic-kernel/issues/13303 - // [ConditionalFact] - // public virtual async Task Delete_multiple_records() - // { - // TKey[] keysToRemove = [(TKey)fixture.TestData[0][KeyPropertyName]!, (TKey)fixture.TestData[1][KeyPropertyName]!]; - - // await this.Collection.DeleteAsync(keysToRemove); - // Assert.Empty(await this.Collection.GetAsync(keysToRemove).ToArrayAsync()); - - // Assert.Equal(fixture.TestData.Count - 2, await this.GetRecordCount()); - // } - - [ConditionalFact] - public virtual async Task DeleteAsync_does_nothing_for_non_existing_key() - { - TKey key = fixture.GenerateNextKey(); - - await fixture.Collection.DeleteAsync(key); - } - - #endregion Delete - - #region Search - - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task SearchAsync(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var result = await this.Collection - .SearchAsync( - expectedRecord[VectorPropertyName]!, - top: 1, - new() { IncludeVectors = includeVectors }) - .SingleAsync(); - - AssertEquivalent(expectedRecord, result.Record, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_Skip() - { - var result = await this.Collection - .SearchAsync( - fixture.TestData[0][VectorPropertyName]!, - top: 1, - new() { Skip = 1 }) - .SingleAsync(); - - AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: false, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_Filter() - { - var result = await this.Collection - .SearchAsync( - fixture.TestData[0][VectorPropertyName]!, - top: 1, - new() { Filter = r => (int)r[IntegerPropertyName]! == 2 }) - .SingleAsync(); - - AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: false, fixture.TestStore.VectorsComparable); - } - - #endregion Search - - protected static void AssertEquivalent(Dictionary expected, Dictionary? actual, bool includeVectors, bool compareVectors) - { - Assert.NotNull(actual); - Assert.Equal(expected[KeyPropertyName], actual[KeyPropertyName]); - - Assert.Equal(expected[StringPropertyName], actual[StringPropertyName]); - Assert.Equal(expected[IntegerPropertyName], actual[IntegerPropertyName]); - - if (includeVectors) - { - Assert.Equal( - ((ReadOnlyMemory)expected[VectorPropertyName]!).Length, - ((ReadOnlyMemory)actual[VectorPropertyName]!).Length); - - if (compareVectors) - { - Assert.Equal( - ((ReadOnlyMemory)expected[VectorPropertyName]!).ToArray(), - ((ReadOnlyMemory)actual[VectorPropertyName]!).ToArray()); - } - } - else - { - Assert.False(actual.ContainsKey(VectorPropertyName)); - } - } - - public const string KeyPropertyName = "key"; - public const string StringPropertyName = "text"; - public const string IntegerPropertyName = "integer"; - public const string VectorPropertyName = "vector"; - - protected VectorStoreCollection> Collection => fixture.Collection; - - public abstract class Fixture : DynamicVectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(DynamicModelTests); - - protected override string KeyPropertyName => DynamicModelTests.KeyPropertyName; - - protected override VectorStoreCollection> GetCollection() - => this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition()); - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(this.KeyPropertyName, typeof(TKey)), - new VectorStoreDataProperty(StringPropertyName, typeof(string)) { IsIndexed = true}, - new VectorStoreDataProperty(IntegerPropertyName, typeof(int)) { IsIndexed = true }, - new VectorStoreVectorProperty(VectorPropertyName, typeof(ReadOnlyMemory), dimensions: 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - } - ] - }; - - protected override List> BuildTestData() => - [ - new() - { - [this.KeyPropertyName] = this.GenerateNextKey(), - [StringPropertyName] = "foo", - [IntegerPropertyName] = 1, - [VectorPropertyName] = new ReadOnlyMemory([1, 2, 3]) - }, - new() - { - [this.KeyPropertyName] = this.GenerateNextKey(), - [StringPropertyName] = "bar", - [IntegerPropertyName] = 2, - [VectorPropertyName] = new ReadOnlyMemory([1, 2, 4]) - }, - new() - { - [this.KeyPropertyName] = this.GenerateNextKey(), - [StringPropertyName] = "foo", // identical text as above - [IntegerPropertyName] = 3, - [VectorPropertyName] = new ReadOnlyMemory([1, 2, 5]) - } - ]; - } - - public Task InitializeAsync() - => fixture.ReseedAsync(); - - public Task DisposeAsync() - => Task.CompletedTask; - - public static readonly TheoryData IncludeVectorsData = [false, true]; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/MultiVectorConformanceTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/MultiVectorConformanceTests.cs deleted file mode 100644 index 6061d432a123..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/MultiVectorConformanceTests.cs +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.ModelTests; - -/// -/// Tests using a model with multiple vectors. -/// -public class MultiVectorModelTests(MultiVectorModelTests.Fixture fixture) : IAsyncLifetime - where TKey : notnull -{ - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_single_record(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); - - expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Insert_single_record() - { - TKey expectedKey = fixture.GenerateNextKey(); - MultiVectorRecord inserted = new() - { - Key = expectedKey, - Number = 10, - Vector1 = new([10, 0, 0]), - Vector2 = new([10, 0, 0]), - }; - - Assert.Null(await this.Collection.GetAsync(expectedKey)); - await this.Collection.UpsertAsync(inserted); - - var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true }); - inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Delete_single_record() - { - var keyToRemove = fixture.TestData[0].Key; - - await this.Collection.DeleteAsync(keyToRemove); - Assert.Null(await this.Collection.GetAsync(keyToRemove)); - } - - [ConditionalFact] - public virtual async Task SearchAsync_with_multiple_vector_properties() - { - var result = await this.Collection - .SearchAsync(new ReadOnlyMemory([1, 2, 3]), top: 1, new() { VectorProperty = r => r.Vector1, IncludeVectors = true }) - .SingleAsync(); - fixture.TestData[0].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); - - result = await this.Collection - .SearchAsync(new ReadOnlyMemory([10, 2, 6]), top: 1, new() { VectorProperty = r => r.Vector2, IncludeVectors = true }) - .SingleAsync(); - fixture.TestData[1].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Search_without_explicitly_specified_vector_property_fails() - { - var exception = await Assert.ThrowsAsync(async () => - await this.Collection.SearchAsync(new ReadOnlyMemory([1, 2, 3]), top: 1).ToListAsync()); - - Assert.Equal($"The '{nameof(MultiVectorRecord)}' type has multiple vector properties, please specify your chosen property via options.", exception.Message); - } - - protected VectorStoreCollection Collection => fixture.Collection; - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => "MultiVectorModelTests"; - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Number = 1, - Vector1 = new([1, 2, 3]), - Vector2 = new([10, 2, 4]) - }, - new() - { - Key = this.GenerateNextKey(), - Number = 2, - Vector1 = new([1, 2, 5]), - Vector2 = new([10, 2, 6]) - } - ]; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(MultiVectorRecord.Key), typeof(TKey)), - new VectorStoreDataProperty(nameof(MultiVectorRecord.Number), typeof(int)), - - new VectorStoreVectorProperty(nameof(MultiVectorRecord.Vector1), typeof(ReadOnlyMemory), 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - }, - - new VectorStoreVectorProperty(nameof(MultiVectorRecord.Vector2), typeof(ReadOnlyMemory), 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - } - ] - }; - - protected override Task WaitForDataAsync() - => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorProperty: r => r.Vector1); - } - - public sealed class MultiVectorRecord : TestRecord - { - public int Number { get; set; } - - public ReadOnlyMemory Vector1 { get; set; } - public ReadOnlyMemory Vector2 { get; set; } - - public void AssertEqual(MultiVectorRecord? other, bool includeVectors, bool compareVectors) - { - Assert.NotNull(other); - - Assert.Equal(this.Key, other.Key); - Assert.Equal(this.Number, other.Number); - - if (includeVectors) - { - Assert.Equal(this.Vector1.Span.Length, other.Vector1.Span.Length); - Assert.Equal(this.Vector2.Span.Length, other.Vector2.Span.Length); - - if (compareVectors) - { - Assert.True(this.Vector1.Span.SequenceEqual(other.Vector1.Span)); - Assert.True(this.Vector2.Span.SequenceEqual(other.Vector2.Span)); - } - } - } - } - - public Task InitializeAsync() - => fixture.ReseedAsync(); - - public Task DisposeAsync() - => Task.CompletedTask; - - public static readonly TheoryData IncludeVectorsData = [false, true]; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoDataModelTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoDataModelTests.cs deleted file mode 100644 index c3b0888724e4..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoDataModelTests.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.ModelTests; - -/// -/// Tests using a model without data fields, only a key and an embedding. -/// -public class NoDataModelTests(NoDataModelTests.Fixture fixture) : IAsyncLifetime - where TKey : notnull -{ - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_single_record(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); - - expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Insert_single_record() - { - TKey expectedKey = fixture.GenerateNextKey(); - NoDataRecord inserted = new() - { - Key = expectedKey, - Floats = new([10, 0, 0]) - }; - - Assert.Null(await this.Collection.GetAsync(expectedKey)); - await this.Collection.UpsertAsync(inserted); - - var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true }); - inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable); - } - - [ConditionalFact] - public virtual async Task Delete_single_record() - { - var keyToRemove = fixture.TestData[0].Key; - - await this.Collection.DeleteAsync(keyToRemove); - Assert.Null(await this.Collection.GetAsync(keyToRemove)); - } - - protected VectorStoreCollection Collection => fixture.Collection; - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(NoDataModelTests); - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Floats = new([1, 2, 3]) - }, - new() - { - Key = this.GenerateNextKey(), - Floats = new([1, 2, 4]) - } - ]; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(NoDataRecord.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(NoDataRecord.Floats), typeof(ReadOnlyMemory), 3) - { - IndexKind = this.IndexKind - } - ] - }; - } - - public sealed class NoDataRecord : TestRecord - { - [VectorStoreVector(Dimensions: 3, StorageName = "embedding")] - public ReadOnlyMemory Floats { get; set; } - - public void AssertEqual(NoDataRecord? other, bool includeVectors, bool compareVectors) - { - Assert.NotNull(other); - Assert.Equal(this.Key, other.Key); - - if (includeVectors) - { - Assert.Equal(this.Floats.Span.Length, other.Floats.Span.Length); - - if (compareVectors) - { - Assert.True(this.Floats.Span.SequenceEqual(other.Floats.Span)); - } - } - } - } - - public Task InitializeAsync() - => fixture.ReseedAsync(); - - public Task DisposeAsync() - => Task.CompletedTask; - - public static readonly TheoryData IncludeVectorsData = [false, true]; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoVectorConformanceTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoVectorConformanceTests.cs deleted file mode 100644 index 91e0b4a88fd0..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/ModelTests/NoVectorConformanceTests.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.ModelTests; - -/// -/// Tests operations using a model without a vector. -/// This is only supported by a subset of databases so only extend if applicable for your database. -/// -public class NoVectorModelTests(NoVectorModelTests.Fixture fixture) : IAsyncLifetime - where TKey : notnull -{ - [ConditionalTheory, MemberData(nameof(IncludeVectorsData))] - public virtual async Task GetAsync_single_record(bool includeVectors) - { - var expectedRecord = fixture.TestData[0]; - - var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors }); - - expectedRecord.AssertEqual(received); - } - - [ConditionalFact] - public virtual async Task Insert_single_record() - { - TKey expectedKey = fixture.GenerateNextKey(); - NoVectorRecord inserted = new() - { - Key = expectedKey, - Text = "New record" - }; - - Assert.Null(await this.Collection.GetAsync(expectedKey)); - await this.Collection.UpsertAsync(inserted); - - var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true }); - inserted.AssertEqual(received); - } - - [ConditionalFact] - public virtual async Task Delete_single_record() - { - var keyToRemove = fixture.TestData[0].Key; - - await this.Collection.DeleteAsync(keyToRemove); - Assert.Null(await this.Collection.GetAsync(keyToRemove)); - } - - protected VectorStoreCollection Collection => fixture.Collection; - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(NoVectorModelTests); - - protected override List BuildTestData() => - [ - new() - { - Key = this.GenerateNextKey(), - Text = "foo", - }, - new() - { - Key = this.GenerateNextKey(), - Text = "bar", - } - ]; - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(NoVectorRecord.Key), typeof(TKey)), - new VectorStoreDataProperty(nameof(NoVectorRecord.Text), typeof(string)) { IsIndexed = true } - ] - }; - - // The default implementation of WaitForDataAsync uses SearchAsync, but our model has no vectors. - protected override async Task WaitForDataAsync() - { - for (var i = 0; i < 200; i++) - { - var results = await this.Collection.GetAsync([this.TestData[0].Key, this.TestData[1].Key]).ToArrayAsync(); - if (results.Length == this.TestData.Count && results.All(r => r != null)) - { - return; - } - - await Task.Delay(TimeSpan.FromMilliseconds(100)); - } - - throw new InvalidOperationException("Data did not appear in the collection within the expected time."); - } - } - - public sealed class NoVectorRecord : TestRecord - { - [VectorStoreData(StorageName = "text")] - public string? Text { get; set; } - - public void AssertEqual(NoVectorRecord? other) - { - Assert.NotNull(other); - Assert.Equal(this.Key, other.Key); - Assert.Equal(this.Text, other.Text); - } - } - - public Task InitializeAsync() - => fixture.ReseedAsync(); - - public Task DisposeAsync() - => Task.CompletedTask; - - public static readonly TheoryData IncludeVectorsData = [false, true]; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/DynamicVectorStoreCollectionFixture.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/DynamicVectorStoreCollectionFixture.cs deleted file mode 100644 index 6e33a54adf98..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/DynamicVectorStoreCollectionFixture.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace VectorData.ConformanceTests.Support; - -public abstract class DynamicVectorStoreCollectionFixture : VectorStoreCollectionFixtureBase> - where TKey : notnull -{ - protected abstract string KeyPropertyName { get; } - - public virtual async Task ReseedAsync() - { - // TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830 - - const int BatchSize = 100; - - List keys = []; - do - { - await foreach (var record in this.Collection.GetAsync(r => true, top: BatchSize)) - { - // TODO: We don't use batching delete because of https://github.com/microsoft/semantic-kernel/issues/13303 - await this.Collection.DeleteAsync((TKey)record[this.KeyPropertyName]!); - } - } while (keys.Count == BatchSize); - - await this.SeedAsync(); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestRecord.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestRecord.cs deleted file mode 100644 index 2df6b952a83a..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestRecord.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; - -namespace VectorData.ConformanceTests.Support; - -public abstract class TestRecord -{ - [VectorStoreKey] - public TKey Key { get; set; } = default!; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestStore.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestStore.cs deleted file mode 100644 index ced2b255d15a..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestStore.cs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Globalization; -using System.Linq.Expressions; -using Microsoft.Extensions.VectorData; - -namespace VectorData.ConformanceTests.Support; - -#pragma warning disable CA1001 // Type owns disposable fields but is not disposable - -public abstract class TestStore -{ - private readonly SemaphoreSlim _lock = new(1, 1); - private int _referenceCount; - private VectorStore? _defaultVectorStore; - - /// - /// Some databases modify vectors on upsert, e.g. normalizing them, so vectors - /// returned cannot be compared with the original ones. - /// - public virtual bool VectorsComparable => true; - - /// - /// Whether the database supports filtering by score threshold in vector search. - /// - public virtual bool SupportsScoreThreshold => true; - - public virtual string DefaultDistanceFunction => DistanceFunction.CosineSimilarity; - public virtual string DefaultIndexKind => IndexKind.Flat; - - protected abstract Task StartAsync(); - - protected virtual Task StopAsync() - => Task.CompletedTask; - - public VectorStore DefaultVectorStore - { - get => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); - set => this._defaultVectorStore = value; - } - - public virtual async Task ReferenceCountingStartAsync() - { - await this._lock.WaitAsync(); - try - { - if (this._referenceCount++ == 0) - { - await this.StartAsync(); - } - } - finally - { - this._lock.Release(); - } - } - - public virtual async Task ReferenceCountingStopAsync() - { - await this._lock.WaitAsync(); - try - { - if (--this._referenceCount == 0) - { - await this.StopAsync(); - this._defaultVectorStore?.Dispose(); - } - } - finally - { - this._lock.Release(); - } - } - - public virtual TKey GenerateKey(int value) - => typeof(TKey) switch - { - _ when typeof(TKey) == typeof(int) => (TKey)(object)value, - _ when typeof(TKey) == typeof(long) => (TKey)(object)(long)value, - _ when typeof(TKey) == typeof(ulong) => (TKey)(object)(ulong)value, - _ when typeof(TKey) == typeof(string) => (TKey)(object)value.ToString(CultureInfo.InvariantCulture), - _ when typeof(TKey) == typeof(Guid) => (TKey)(object)new Guid($"00000000-0000-0000-0000-00{value:0000000000}"), - - _ => throw new NotSupportedException($"Unsupported key of type '{typeof(TKey).Name}', override {nameof(TestStore)}.{nameof(this.GenerateKey)}") - }; - - /// - /// Applies any provider-specific rules to collection names (e.g. all-lowercase). - /// - /// - /// - public virtual string AdjustCollectionName(string baseName) - => baseName; - - /// - /// Creates a collection for the given name and definition. - /// Override this to provide provider-specific collection options (e.g., partition key configuration). - /// - public virtual VectorStoreCollection CreateCollection( - string name, - VectorStoreCollectionDefinition definition) - where TKey : notnull - where TRecord : class - => this.DefaultVectorStore.GetCollection(name, definition); - - /// - /// Creates a dynamic collection for the given name and definition. - /// Override this to provide provider-specific collection options (e.g., partition key configuration). - /// - public virtual VectorStoreCollection> CreateDynamicCollection( - string name, - VectorStoreCollectionDefinition definition) - => this.DefaultVectorStore.GetDynamicCollection(name, definition); - - /// Loops until the expected number of records is visible in the given collection. - /// Some databases upsert asynchronously, meaning that our seed data may not be visible immediately to tests. - public virtual async Task WaitForDataAsync( - VectorStoreCollection collection, - int recordCount, - Expression>? filter = null, - Expression>? vectorProperty = null, - int? vectorSize = null, - object? dummyVector = null) - where TKey : notnull - where TRecord : class - { - if (vectorSize is not null && dummyVector is not null) - { - throw new ArgumentException("vectorSize or dummyVector can't both be set"); - } - - var vector = dummyVector ?? new ReadOnlyMemory(Enumerable.Range(0, vectorSize ?? 3).Select(i => (float)i).ToArray()); - - for (var i = 0; i < 200; i++) - { - // Note that we very intentionally use SearchAsync and not filtering GetAsync, as we want to wait until the data is visible - // specifically via vector search (some databases may show data via filtering before they are indexed for vector search). - var results = collection.SearchAsync( - vector, - top: recordCount is 0 ? 1 : recordCount, - new() - { - Filter = filter, - VectorProperty = vectorProperty - }); - var count = await results.CountAsync(); - if (count == recordCount) - { - return; - } - - await Task.Delay(TimeSpan.FromMilliseconds(100)); - } - - throw new InvalidOperationException("Data did not appear in the collection within the expected time."); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixture.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixture.cs deleted file mode 100644 index 9ceafd703e4b..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixture.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace VectorData.ConformanceTests.Support; - -public abstract class VectorStoreCollectionFixture : VectorStoreCollectionFixtureBase - where TKey : notnull - where TRecord : TestRecord -{ - public virtual async Task ReseedAsync() - { - // TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830 - - const int BatchSize = 100; - - TKey[] keys; - do - { - keys = await this.Collection.GetAsync(r => true, top: BatchSize).Select(r => r.Key).ToArrayAsync(); - await this.Collection.DeleteAsync(keys); - } while (keys.Length == BatchSize); - - await this.SeedAsync(); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixtureBase.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixtureBase.cs deleted file mode 100644 index 10ae77bc8a7d..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixtureBase.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; - -namespace VectorData.ConformanceTests.Support; - -#pragma warning disable CA1721 // Property names should not match get methods - -/// -/// A test fixture that sets up a single collection in the test vector store, with a specific record definition -/// and test data. -/// -public abstract class VectorStoreCollectionFixtureBase : VectorStoreFixture - where TKey : notnull - where TRecord : class -{ - private List? _testData; - - public abstract VectorStoreCollectionDefinition CreateRecordDefinition(); - protected virtual List BuildTestData() => []; - - /// - /// The base name for the test collection used in tests, before any provider-specific collection naming rules have been applied. - /// - // TODO: Make protected - protected abstract string CollectionNameBase { get; } - - /// - /// The actual name of the test collection, after any provider-specific collection naming rules have been applied. - /// - public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase); - - protected virtual string DistanceFunction => this.TestStore.DefaultDistanceFunction; - protected virtual string IndexKind => this.TestStore.DefaultIndexKind; - - protected virtual VectorStoreCollection GetCollection() - => this.TestStore.CreateCollection(this.CollectionName, this.CreateRecordDefinition()); - - public override async Task InitializeAsync() - { - await base.InitializeAsync(); - - this.Collection = this.GetCollection(); - - if (await this.Collection.CollectionExistsAsync()) - { - await this.Collection.EnsureCollectionDeletedAsync(); - } - - await this.Collection.EnsureCollectionExistsAsync(); - await this.SeedAsync(); - } - - public virtual VectorStoreCollection Collection { get; private set; } = null!; - - public List TestData => this._testData ??= this.BuildTestData(); - - protected virtual async Task SeedAsync() - { - if (this.TestData.Count > 0) - { - await this.Collection.UpsertAsync(this.TestData); - await this.WaitForDataAsync(); - } - } - - protected virtual Task WaitForDataAsync() - => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count); -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreFixture.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreFixture.cs deleted file mode 100644 index 091de0bcee69..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreFixture.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using Xunit; - -namespace VectorData.ConformanceTests.Support; - -public abstract class VectorStoreFixture : IAsyncLifetime -{ - private int _nextKeyValue = 1; - - public abstract TestStore TestStore { get; } - public virtual VectorStore VectorStore => this.TestStore.DefaultVectorStore; - - public virtual string DefaultDistanceFunction => this.TestStore.DefaultDistanceFunction; - public virtual string DefaultIndexKind => this.TestStore.DefaultIndexKind; - - public virtual Task InitializeAsync() - => this.TestStore.ReferenceCountingStartAsync(); - - public virtual Task DisposeAsync() - => this.TestStore.ReferenceCountingStopAsync(); - - public virtual TKey GenerateNextKey() - => this.TestStore.GenerateKey(Interlocked.Increment(ref this._nextKeyValue)); - - /// - /// Creates a collection for the given name and definition. - /// Delegates to which can be overridden for provider-specific options. - /// - public virtual VectorStoreCollection CreateCollection(string name, VectorStoreCollectionDefinition definition) - where TKey : notnull - where TRecord : class - => this.TestStore.CreateCollection(name, definition); - - /// - /// Creates a dynamic collection for the given name and definition. - /// Delegates to which can be overridden for provider-specific options. - /// - public virtual VectorStoreCollection> CreateDynamicCollection(string name, VectorStoreCollectionDefinition definition) - => this.TestStore.CreateDynamicCollection(name, definition); -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TestSuiteImplementationTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TestSuiteImplementationTests.cs deleted file mode 100644 index eb29903bbf90..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/TestSuiteImplementationTests.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Reflection; -using System.Text.RegularExpressions; -using Xunit; - -namespace VectorData.ConformanceTests; - -/// -/// A test that ensures that all base test suites are implemented (or explicitly ignored) in provider implementations. -/// Used to make sure that test coverage is complete. -/// -public abstract class TestSuiteImplementationTests -{ - protected virtual ICollection IgnoredTestBases { get; } = []; - - [Fact] - public virtual void All_test_bases_must_be_implemented() - { - var concreteTests - = this.GetType().Assembly.GetTypes() - .Where(c => c.BaseType != typeof(object) && !c.IsAbstract && (c.IsPublic || c.IsNestedPublic)) - .ToList(); - - var nonImplementedBases - = this.GetBaseTestClasses() - .Where(t => !this.IgnoredTestBases.Contains(t) && !concreteTests.Any(c => Implements(c, t))) - .Select(t => t.FullName) - .ToList(); - - Assert.False( - nonImplementedBases.Count > 0, - "\r\n-- Missing derived classes for --\r\n" + string.Join(Environment.NewLine, nonImplementedBases)); - } - - // Filter for abstract base types which end with Tests and possibly generic arity (e.g. FooTests`2) - protected virtual IEnumerable GetBaseTestClasses() - => typeof(TestSuiteImplementationTests).Assembly.ExportedTypes - .Where(t => Regex.IsMatch(t.Name, """Tests(`\d+)?$""") && t.IsAbstract && !t.IsSealed && !t.IsInterface); - - private static bool Implements(Type type, Type interfaceOrBaseType) - => (type.IsPublic || type.IsNestedPublic) && interfaceOrBaseType.IsGenericTypeDefinition - ? GetGenericTypeImplementations(type, interfaceOrBaseType).Any() - : interfaceOrBaseType.IsAssignableFrom(type); - - private static IEnumerable GetGenericTypeImplementations(Type type, Type interfaceOrBaseType) - { - var typeInfo = type.GetTypeInfo(); - - if (!typeInfo.IsGenericTypeDefinition) - { - var baseTypes = interfaceOrBaseType.IsInterface - ? typeInfo.ImplementedInterfaces - : GetBaseTypes(type); - foreach (var baseType in baseTypes) - { - if (baseType.IsGenericType - && baseType.GetGenericTypeDefinition() == interfaceOrBaseType) - { - yield return baseType; - } - } - - if (type.IsGenericType - && type.GetGenericTypeDefinition() == interfaceOrBaseType) - { - yield return type; - } - } - } - - private static IEnumerable GetBaseTypes(Type type) - { - var t = type.BaseType; - - while (t != null) - { - yield return t; - - t = t.BaseType; - } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/DataTypeTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/DataTypeTests.cs deleted file mode 100644 index ddc400f56c07..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/DataTypeTests.cs +++ /dev/null @@ -1,602 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Linq.Expressions; -using System.Reflection; -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.TypeTests; - -public abstract class DataTypeTests(DataTypeTests.Fixture fixture) : DataTypeTests() - where TKey : notnull - where TRecord : DataTypeTests.RecordBase, new() -{ - // Note: nullable value types are tested automatically within TestTypeStructAsync - - [ConditionalFact] - public virtual Task Byte() - => fixture.UnsupportedDefaultTypes.Contains(typeof(byte)) - ? Task.CompletedTask - : this.Test("Byte", 8, 9); - - [ConditionalFact] - public virtual Task Short() - => fixture.UnsupportedDefaultTypes.Contains(typeof(short)) - ? Task.CompletedTask - : this.Test("Short", 8, 9); - - [ConditionalFact] - public virtual Task Int() - => fixture.UnsupportedDefaultTypes.Contains(typeof(int)) - ? Task.CompletedTask - : this.Test("Int", 8, 9); - - [ConditionalFact] - public virtual Task Long() - => fixture.UnsupportedDefaultTypes.Contains(typeof(long)) - ? Task.CompletedTask - : this.Test("Long", 8L, 9L); - - [ConditionalFact] - public virtual Task Float() - => fixture.UnsupportedDefaultTypes.Contains(typeof(float)) - ? Task.CompletedTask - : this.Test("Float", 8.5f, 9.5f); - - [ConditionalFact] - public virtual Task Double() - => fixture.UnsupportedDefaultTypes.Contains(typeof(double)) - ? Task.CompletedTask - : this.Test("Double", 8.5d, 9.5d); - - [ConditionalFact] - public virtual Task Decimal() - => fixture.UnsupportedDefaultTypes.Contains(typeof(decimal)) - ? Task.CompletedTask - : this.Test("Decimal", 8.5m, 9.5m); - - [ConditionalFact] - public virtual Task String() - => fixture.UnsupportedDefaultTypes.Contains(typeof(string)) - ? Task.CompletedTask - : this.Test("String", "foo", "bar"); - - [ConditionalFact] - public virtual Task Bool() - => fixture.UnsupportedDefaultTypes.Contains(typeof(bool)) - ? Task.CompletedTask - : this.Test("Bool", true, false); - - [ConditionalFact] - public virtual Task Guid() - => fixture.UnsupportedDefaultTypes.Contains(typeof(Guid)) - ? Task.CompletedTask - : this.Test( - "Guid", - new Guid("603840bf-cf91-4521-8b8e-8b6a2e75910a"), - new Guid("e9a97807-8cf0-4741-8ce3-82df676ca0f0")); - - [ConditionalFact] - public virtual Task DateTime() - => fixture.UnsupportedDefaultTypes.Contains(typeof(DateTime)) - ? Task.CompletedTask - : this.Test( - "DateTime", - new DateTime(2020, 1, 1, 12, 30, 45), - new DateTime(2021, 2, 3, 13, 40, 55), - instantiationExpression: () => new DateTime(2020, 1, 1, 12, 30, 45)); - - [ConditionalFact] - public virtual Task DateTimeOffset() - => fixture.UnsupportedDefaultTypes.Contains(typeof(DateTimeOffset)) - ? Task.CompletedTask - : this.Test( - "DateTimeOffset", - new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(2)), - new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.FromHours(3)), - instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(2))); - - [ConditionalFact] - public virtual Task DateOnly() - { -#if NET - return fixture.UnsupportedDefaultTypes.Contains(typeof(DateOnly)) - ? Task.CompletedTask - : this.Test( - "DateOnly", - new DateOnly(2020, 1, 1), - new DateOnly(2021, 2, 3)); -#else - return Task.CompletedTask; -#endif - } - - [ConditionalFact] - public virtual Task TimeOnly() - { -#if NET - return fixture.UnsupportedDefaultTypes.Contains(typeof(TimeOnly)) - ? Task.CompletedTask - : this.Test( - "TimeOnly", - new TimeOnly(12, 30, 45), - new TimeOnly(13, 40, 55)); -#else - return Task.CompletedTask; -#endif - } - - [ConditionalFact] - public virtual Task String_array() - => fixture.UnsupportedDefaultTypes.Contains(typeof(string[])) - ? Task.CompletedTask - : this.Test( - "StringArray", - ["foo", "bar"], - ["foo", "baz"]); - - [ConditionalFact] - public virtual Task Nullable_value_type() - => fixture.UnsupportedDefaultTypes.Contains(typeof(int?)) - ? Task.CompletedTask - : this.Test("NullableInt", 8, 9); - - protected virtual async Task Test( - string propertyName, - TTestType mainValue, - TTestType otherValue, - bool isFilterable = true, - Action? comparisonAction = null, - Expression>? instantiationExpression = null) - { - if (propertyName is "Key" or "Vector") - { - throw new ArgumentException($"The property name '{propertyName}' is reserved and cannot be used for testing.", nameof(propertyName)); - } - - var property = typeof(TRecord).GetProperty(propertyName) - ?? throw new ArgumentException($"The type '{typeof(TRecord).Name}' does not have a property named '{propertyName}'.", nameof(propertyName)); - comparisonAction ??= (a, b) => Assert.Equal(a, b); - var instantiationExpressionBody = instantiationExpression is null - ? Expression.Constant(mainValue, typeof(TTestType)) - : instantiationExpression.Body; - - await fixture.Collection.DeleteAsync([fixture.MainRecordKey, fixture.OtherRecordKey, fixture.NullRecordKey]); - await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: 0); - - // Step 1: Insert data - await this.InsertData(property, mainValue, otherValue); - - // Step 2: Read the values back via GetAsync - TRecord result = await fixture.Collection.GetAsync(fixture.MainRecordKey) ?? throw new InvalidOperationException($"Record with key '{fixture.MainRecordKey}' was not found."); - comparisonAction(mainValue, (TTestType)property.GetValue(result)!); - - // Step 3: Exercise filtering by the value, using a constant in the filter expression - if (isFilterable) - { - await this.TestFiltering(fixture.Collection, property, mainValue, comparisonAction, instantiationExpressionBody); - } - - /////////////////////// - // Test dynamic mapping - /////////////////////// - if (fixture.RecreateCollection) - { - await fixture.Collection.EnsureCollectionDeletedAsync(); - } - else - { - await fixture.Collection.DeleteAsync([fixture.MainRecordKey, fixture.OtherRecordKey, fixture.NullRecordKey]); - await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: 0); - } - - var dynamicCollection = fixture.CreateDynamicCollection(fixture.CollectionName, fixture.CreateRecordDefinition()); - - if (fixture.RecreateCollection) - { - await dynamicCollection.EnsureCollectionExistsAsync(); - } - - // Step 1: Insert data - await this.InsertDynamicData(dynamicCollection, propertyName, mainValue, otherValue); - - // Step 2: Read the values back via GetAsync - var dynamicResult = await dynamicCollection.GetAsync(fixture.MainRecordKey) ?? throw new InvalidOperationException($"Record with key '{fixture.MainRecordKey}' was not found."); - comparisonAction(mainValue, (TTestType)dynamicResult[propertyName]!); - - // Step 3: Exercise dynamic filtering by the value, using a constant in the filter expression - if (isFilterable) - { - await this.TestDynamicFiltering(dynamicCollection, propertyName, mainValue, comparisonAction, instantiationExpressionBody); - } - } - - private async Task InsertData(PropertyInfo property, TTestType mainValue, TTestType otherValue) - { - // Note that all records have the same vector - var mainRecord = GenerateEmptyRecord(); - mainRecord.Key = fixture.MainRecordKey; - mainRecord.Vector = fixture.Vector; - property.SetValue(mainRecord, mainValue); - - var otherRecord = GenerateEmptyRecord(); - otherRecord.Key = fixture.OtherRecordKey; - otherRecord.Vector = fixture.Vector; - property.SetValue(otherRecord, otherValue); - - List testData = [mainRecord, otherRecord]; - - if (default(TTestType) == null && fixture.IsNullSupported && IsPropertyNullable(property)) - { - var nullRecord = GenerateEmptyRecord(); - nullRecord.Key = fixture.NullRecordKey; - nullRecord.Vector = fixture.Vector; - property.SetValue(nullRecord, null); - testData.Add(nullRecord); - } - - await fixture.Collection.UpsertAsync(testData); - await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: testData.Count); - - TRecord GenerateEmptyRecord() - { - var record = new TRecord(); - - foreach (var property in fixture.CreateRecordDefinition().Properties) - { - var propertyInfo = typeof(TRecord).GetProperty(property.Name) - ?? throw new InvalidOperationException($"Property '{property.Name}' not found on record type '{typeof(TRecord).Name}'."); - propertyInfo.SetValue(record, this.GenerateEmptyProperty(property)); - } - - return record; - } - } - - private async Task InsertDynamicData( - VectorStoreCollection> dynamicCollection, - string propertyName, - TTestType mainValue, - TTestType otherValue) - { - // Note that all records have the same vector - var mainRecord = GenerateEmptyRecord(); - mainRecord[nameof(RecordBase.Key)] = fixture.MainRecordKey; - mainRecord[nameof(RecordBase.Vector)] = fixture.Vector; - mainRecord[propertyName] = mainValue; - - var otherRecord = GenerateEmptyRecord(); - otherRecord[nameof(RecordBase.Key)] = fixture.OtherRecordKey; - otherRecord[nameof(RecordBase.Vector)] = fixture.Vector; - otherRecord[propertyName] = otherValue; - - List> testData = [mainRecord, otherRecord]; - - var pocoProperty = typeof(TRecord).GetProperty(propertyName); - if (default(TTestType) == null && fixture.IsNullSupported && (pocoProperty is null || IsPropertyNullable(pocoProperty))) - { - var nullRecord = GenerateEmptyRecord(); - nullRecord[nameof(RecordBase.Key)] = fixture.NullRecordKey; - nullRecord[nameof(RecordBase.Vector)] = fixture.Vector; - nullRecord[propertyName] = null; - testData.Add(nullRecord); - } - - await dynamicCollection.UpsertAsync(testData); - await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: testData.Count); - - Dictionary GenerateEmptyRecord() - { - var record = new Dictionary(); - - foreach (var property in fixture.CreateRecordDefinition().Properties) - { - record[property.Name] = this.GenerateEmptyProperty(property); - } - - return record; - } - } - - /// - /// Checks whether a property is nullable, taking into account NRT annotations on .NET 6+. - /// - private static bool IsPropertyNullable(PropertyInfo property) - { - if (property.PropertyType.IsValueType) - { - return Nullable.GetUnderlyingType(property.PropertyType) is not null; - } - -#if NET - return new NullabilityInfoContext().Create(property).ReadState != NullabilityState.NotNull; -#else - return true; // Without NRT support, assume reference types are nullable -#endif - } - - protected virtual object? GenerateEmptyProperty(VectorStoreProperty property) - => property.Type switch - { - null => throw new InvalidOperationException($"Property '{property.Name}' has no type defined."), - - // For value types, we create an instance with the default value. - // This is necessary for relational providers where non-nullable columns are created. - var t when t.IsValueType => Activator.CreateInstance(t), - - // In some cases (Azure AI Search), array fields must be non-null - var t when t.IsArray => Array.CreateInstance(t.GetElementType()!, 0), - - _ => null - }; - - private async Task TestFiltering( - VectorStoreCollection collection, - PropertyInfo property, - TTestType mainValue, - Action comparisonAction, - Expression instantiationExpression) - { - // Note: we need to manually build the expression tree since the equality operator can't be used over - // unconstrained generic types. - var lambdaParameter = Expression.Parameter(typeof(TRecord), "r"); - var filter = Expression.Lambda>( - Expression.Equal( - Expression.Property(lambdaParameter, property), - instantiationExpression), - lambdaParameter); - - // Some databases (Mongo) update the filter index asynchronously, so we wait until the record appears under the filter, - // and then do the main search to make sure only the main record is returned. - await fixture.TestStore.WaitForDataAsync(collection, filter: filter, recordCount: 1); - var result = (await collection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record; - - Assert.Equal(fixture.MainRecordKey, result.Key); - comparisonAction(mainValue, (TTestType)property.GetValue(result)!); - - // Exercise filtering by a null value - if (default(TTestType) == null && fixture.IsNullFilteringSupported && IsPropertyNullable(property)) - { - lambdaParameter = Expression.Parameter(typeof(TRecord), "r"); - filter = Expression.Lambda>( - Expression.Equal( - Expression.Property(lambdaParameter, property), - Expression.Constant(null, typeof(TTestType))), - lambdaParameter); - - result = (await collection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record; - - Assert.Equal(fixture.NullRecordKey, result.Key); - } - } - - private async Task TestDynamicFiltering( - VectorStoreCollection> dynamicCollection, - string propertyName, - TTestType mainValue, - Action comparisonAction, - Expression instantiationExpression) - { - // Note: we need to manually build the expression tree since we want the property name to be a constant - var lambdaParameter = Expression.Parameter(typeof(Dictionary), "r"); - var filter = Expression.Lambda, bool>>( - Expression.Equal( - Expression.Convert( - Expression.Call(lambdaParameter, DynamicDictionaryIndexer, Expression.Constant(propertyName)), - typeof(TTestType)), - instantiationExpression), - lambdaParameter); - - // Some databases (Mongo) update the filter index asynchronously, so we wait until the record appears under the filter, - // and then do the main search to make sure only the main record is returned. - await fixture.TestStore.WaitForDataAsync(dynamicCollection, filter: filter, recordCount: 1); - var result = (await dynamicCollection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record; - Assert.Equal(fixture.MainRecordKey, result[nameof(RecordBase.Key)]); - comparisonAction(mainValue, (TTestType)result[propertyName]!); - - // Exercise filtering by a null value - var pocoProperty = typeof(TRecord).GetProperty(propertyName); - if (default(TTestType) == null && fixture.IsNullFilteringSupported && (pocoProperty is null || IsPropertyNullable(pocoProperty))) - { - lambdaParameter = Expression.Parameter(typeof(Dictionary), "r"); - filter = Expression.Lambda, bool>>( - Expression.Equal( - Expression.Convert( - Expression.Call(lambdaParameter, DynamicDictionaryIndexer, Expression.Constant(propertyName)), - typeof(TTestType)), - Expression.Constant(null, typeof(TTestType))), - lambdaParameter); - - result = (await dynamicCollection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record; - - Assert.Equal(fixture.NullRecordKey, result[nameof(RecordBase.Key)]); - } - } - - private static readonly MethodInfo DynamicDictionaryIndexer = typeof(Dictionary).GetMethod("get_Item")!; - - public abstract class Fixture : VectorStoreCollectionFixture - { - protected override string CollectionNameBase => nameof(DataTypeTests); - - public virtual bool IsNullSupported => true; - public virtual bool IsNullFilteringSupported => true; - - public virtual Type[] UnsupportedDefaultTypes { get; } = []; - - public virtual TKey MainRecordKey { get; protected set; } = default!; - public virtual TKey OtherRecordKey { get; protected set; } = default!; - public virtual TKey NullRecordKey { get; protected set; } = default!; - - public virtual float[] Vector { get; } = [1, 2, 3]; - - private readonly IList _defaultDataProperties = null!; - - /// - /// Whether the recreate the collection while testing, as opposed to deleting the records. - /// Necessary for InMemory, where the .NET mapped on the collection cannot be changed. - /// - public virtual bool RecreateCollection => false; - -#pragma warning disable CA2214 // Do not call overridable methods in constructors - protected Fixture() - { - this._defaultDataProperties = this.GetDataProperties(); - } -#pragma warning restore CA2214 - - public override async Task InitializeAsync() - { - await base.InitializeAsync(); - - this.MainRecordKey = this.GenerateNextKey(); - this.OtherRecordKey = this.GenerateNextKey(); - this.NullRecordKey = this.GenerateNextKey(); - } - - public override VectorStoreCollectionDefinition CreateRecordDefinition() - => new() - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordBase.Key), typeof(TKey)), - new VectorStoreVectorProperty(nameof(RecordBase.Vector), typeof(float[]), 3) - { - DistanceFunction = this.DistanceFunction, - IndexKind = this.IndexKind - }, - - .. this._defaultDataProperties - ] - }; - - public virtual IList GetDataProperties() - { - var properties = new List(); - - if (!this.UnsupportedDefaultTypes.Contains(typeof(byte))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Byte), typeof(byte)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(short))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Short), typeof(short)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(int))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Int), typeof(int)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(long))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Long), typeof(long)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(float))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Float), typeof(float)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(double))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Double), typeof(double)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(decimal))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Decimal), typeof(decimal)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(string))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.String), typeof(string)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(bool))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Bool), typeof(bool)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(Guid))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Guid), typeof(Guid)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(DateTime))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateTime), typeof(DateTime)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(DateTimeOffset))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateTimeOffset), typeof(DateTimeOffset)) { IsIndexed = true }); - } - -#if NET - if (!this.UnsupportedDefaultTypes.Contains(typeof(DateOnly))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateOnly), typeof(DateOnly)) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(TimeOnly))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.TimeOnly), typeof(TimeOnly)) { IsIndexed = true }); - } -#endif - if (!this.UnsupportedDefaultTypes.Contains(typeof(string[]))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.StringArray), typeof(string[])) { IsIndexed = true }); - } - - if (!this.UnsupportedDefaultTypes.Contains(typeof(int?))) - { - properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.NullableInt), typeof(int?)) { IsIndexed = true }); - } - - return properties; - } - } -} - -// We have this base class so the Record type can be referenced in subtypes (the main TypeTests class -// is generic over the record type as well). -public abstract class DataTypeTests() - where TKey : notnull -{ - public class RecordBase : TestRecord - { - public float[] Vector { get; set; } = default!; - } - - public class DefaultRecord : RecordBase - { - public byte Byte { get; set; } - public short Short { get; set; } - public int Int { get; set; } - public long Long { get; set; } - - public float Float { get; set; } - public double Double { get; set; } - public decimal Decimal { get; set; } - - public string? String { get; set; } - public bool Bool { get; set; } - public Guid Guid { get; set; } - - public DateTime DateTime { get; set; } - public DateTimeOffset DateTimeOffset { get; set; } - -#if NET - public DateOnly DateOnly { get; set; } - public TimeOnly TimeOnly { get; set; } -#endif - - public string[] StringArray { get; set; } = null!; - - public int? NullableInt { get; set; } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/EmbeddingTypeTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/EmbeddingTypeTests.cs deleted file mode 100644 index 5fafde527ced..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/EmbeddingTypeTests.cs +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. -#pragma warning disable CA2000 // Dispose objects before losing scope - -namespace VectorData.ConformanceTests.TypeTests; - -/// -/// Tests that the various embedding types natively supported by the provider (ReadOnlyMemory<float>, ReadOnlyMemory<Half>...) work correctly. -/// -public abstract class EmbeddingTypeTests(EmbeddingTypeTests.Fixture fixture) - where TKey : notnull -{ - [ConditionalFact] - public virtual Task ReadOnlyMemory_of_float() - => this.Test>( - new ReadOnlyMemory([1, 2, 3]), - new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), - vectorEqualityAsserter: (e, a) => Assert.Equal(e.ToArray(), a.ToArray())); - - [ConditionalFact] - public virtual Task Embedding_of_float() - => this.Test>( - new Embedding(new ReadOnlyMemory([1, 2, 3])), - new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3]), - vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.ToArray(), a.Vector.ToArray())); - - [ConditionalFact] - public virtual Task Array_of_float() - => this.Test( - [1, 2, 3], - new ReadOnlyMemoryEmbeddingGenerator([1, 2, 3])); - - protected virtual async Task Test( - TVector value, - IEmbeddingGenerator? embeddingGenerator = null, - Action? vectorEqualityAsserter = null, - string? distanceFunction = null, - int dimensions = 3) - where TVector : notnull - { - vectorEqualityAsserter ??= (e, a) => Assert.Equal(e, a); - - await fixture.VectorStore.EnsureCollectionDeletedAsync(fixture.CollectionName); - - var collection = fixture.CreateCollection>(fixture.CollectionName, fixture.CreateRecordDefinition(embeddingGenerator: null, distanceFunction, dimensions)); - await collection.EnsureCollectionExistsAsync(); - - var key = fixture.GenerateNextKey(); - var record = new Record - { - Key = key, - Vector = value, - Int = 42 - }; - - await collection.UpsertAsync(record); - - await fixture.TestStore.WaitForDataAsync(collection, recordCount: 1, filter: r => r.Int == 42, dummyVector: value); - - var result1 = await collection.GetAsync(key, new() { IncludeVectors = true }); - Assert.Equal(42, result1!.Int); - if (fixture.TestStore.VectorsComparable) - { - vectorEqualityAsserter(value, result1.Vector); - } - - // Note that some databases leak indexing information across deletion/recreation of the same collection name (e.g. Pinecone) - so we also - // filter by Int here. - var result2 = await collection.SearchAsync(value, top: 1, new() { IncludeVectors = true, Filter = r => r.Int == 42 }).SingleAsync(); - Assert.Equal(42, result2.Record.Int); - if (fixture.TestStore.VectorsComparable) - { - vectorEqualityAsserter(value, result2.Record.Vector); - } - - /////////////////////// - // Test dynamic mapping - /////////////////////// - if (fixture.RecreateCollection) - { - await collection.EnsureCollectionDeletedAsync(); - } - else - { - await collection.DeleteAsync(key); - } - - var dynamicCollection = fixture.CreateDynamicCollection(fixture.CollectionName, fixture.CreateRecordDefinition(embeddingGenerator, distanceFunction, dimensions)); - - if (fixture.RecreateCollection) - { - await dynamicCollection.EnsureCollectionExistsAsync(); - } - - key = fixture.GenerateNextKey(); - var dynamicRecord = new Dictionary - { - ["Key"] = key, - ["Vector"] = value, - ["Int"] = 43 - }; - - await dynamicCollection.UpsertAsync(dynamicRecord); - - await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: 1, filter: r => (int)r["Int"]! == 43, dummyVector: value); - - var dynamicResult1 = await dynamicCollection.GetAsync(key, new() { IncludeVectors = true }); - Assert.Equal(43, dynamicResult1!["Int"]); - if (fixture.TestStore.VectorsComparable) - { - vectorEqualityAsserter(value, (TVector)dynamicResult1["Vector"]!); - } - - // Note that some databases leak indexing information across deletion/recreation of the same collection name (e.g. Pinecone) - so we also - // filter by Int here. - var dynamicResult2 = await dynamicCollection.SearchAsync(value, top: 1, new() { IncludeVectors = true, Filter = r => (int)r["Int"]! == 43 }).SingleAsync(); - Assert.Equal(43, dynamicResult2.Record["Int"]); - if (fixture.TestStore.VectorsComparable) - { - vectorEqualityAsserter(value, (TVector)dynamicResult2.Record["Vector"]!); - } - - //////////////////////////// - // Test embedding generation - //////////////////////////// - if (embeddingGenerator is not null) - { - if (fixture.RecreateCollection) - { - await collection.EnsureCollectionDeletedAsync(); - } - else - { - await collection.DeleteAsync(key); - } - - var collection2 = fixture.CreateCollection(fixture.CollectionName, fixture.CreateRecordDefinition(embeddingGenerator, distanceFunction, dimensions)); - - if (fixture.RecreateCollection) - { - await collection2.EnsureCollectionExistsAsync(); - } - - key = fixture.GenerateNextKey(); - var record2 = new RecordWithString - { - Key = key, - Vector = "does not matter", - Int = 44 - }; - - await collection2.UpsertAsync(record2); - - await fixture.TestStore.WaitForDataAsync(collection2, recordCount: 1, filter: r => r.Int == 44, dummyVector: value); - - var result3 = await collection2.GetAsync(key); - Assert.Equal(44, result3!.Int); - if (fixture.AssertNoVectorsLoadedWithEmbeddingGeneration) - { - Assert.Null(result3.Vector); - } - - var result4 = await collection2.SearchAsync("does not matter", top: 1, new() { Filter = r => r.Int == 44 }).SingleAsync(); - Assert.Equal(44, result4.Record.Int); - if (fixture.AssertNoVectorsLoadedWithEmbeddingGeneration) - { - Assert.Null(result4.Record.Vector); - } - } - } - - protected sealed class ReadOnlyMemoryEmbeddingGenerator(T[] data) : IEmbeddingGenerator> - { - public Task>> GenerateAsync( - IEnumerable values, - EmbeddingGenerationOptions? options = null, - CancellationToken cancellationToken = default) - => Task.FromResult(new GeneratedEmbeddings>([new(data)])); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - public void Dispose() { } - } - - protected sealed class BinaryEmbeddingGenerator(BitArray data) : IEmbeddingGenerator - { - public Task> GenerateAsync( - IEnumerable values, - EmbeddingGenerationOptions? options = null, - CancellationToken cancellationToken = default) - => Task.FromResult(new GeneratedEmbeddings([new(data)])); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - public void Dispose() { } - } - - public class RecordWithString - { - public TKey Key { get; set; } - public string Vector { get; set; } - public int Int { get; set; } - } - - public class Record - { - public TKey Key { get; set; } - public TVector Vector { get; set; } - public int Int { get; set; } - } - - public abstract class Fixture : VectorStoreFixture - { - protected virtual string CollectionNameBase => nameof(EmbeddingTypeTests<>); - public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase); - - public virtual VectorStoreCollectionDefinition CreateRecordDefinition(IEmbeddingGenerator? embeddingGenerator, string? distanceFunction, int dimensions) - => new() - { - Properties = - [ - new VectorStoreKeyProperty("Key", typeof(TKey)), - new VectorStoreVectorProperty("Vector", typeof(TVectorProperty), dimensions) - { - DistanceFunction = distanceFunction ?? this.DefaultDistanceFunction, - IndexKind = this.DefaultIndexKind - }, - new VectorStoreDataProperty("Int", typeof(int)) { IsIndexed = true } - ], - EmbeddingGenerator = embeddingGenerator - }; - - /// - /// Whether the recreate the collection while testing, as opposed to deleting the records. - /// Necessary for InMemory, where the .NET mapped on the collection cannot be changed. - /// - public virtual bool RecreateCollection => false; - - /// - /// Whether to assert that no vectors were loaded when embedding generation is used. - /// Necessary for InMemory which returns the same object which was inserted, and therefore contains - /// the original input value. - /// - public virtual bool AssertNoVectorsLoadedWithEmbeddingGeneration => true; - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs deleted file mode 100644 index 124c5e705869..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/TypeTests/KeyTypeTests.cs +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.VectorData; -using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; -using Xunit; - -namespace VectorData.ConformanceTests.TypeTests; - -public abstract class KeyTypeTests(KeyTypeTests.Fixture fixture) -{ - // All MEVD providers are expected to support Guid keys (possibly by storing them as strings), including - // auto-generation. - // This allows upper layers such as Microsoft.Extensions.DataIngestion to use Guid keys consistently. - [ConditionalFact] - public virtual Task Guid() - => this.Test( - new Guid("603840bf-cf91-4521-8b8e-8b6a2e75910a"), - supportsAutoGeneration: true); - - /// - /// Verifies that creating a collection with a TKey that doesn't match the key property type on the model throws. - /// - [ConditionalFact] - public virtual void MismatchedKeyTypeThrows() - { - // The definition says the key property is string (matching Record.Key), - // but TKey is Guid - this mismatch should be detected during model building. - Assert.Throws(() => - fixture.TestStore.CreateCollection>( - fixture.CollectionName, fixture.CreateRecordDefinition(withAutoGeneration: false))); - } - - protected virtual Task Test(TKey key, bool supportsAutoGeneration = false) - where TKey : struct - => this.Test(key, default!, supportsAutoGeneration: supportsAutoGeneration); - - // Note that we do not currently support testing auto generation for reference types, since - // no such case currently exists in a known provider. As a result we require a second key - // value. - protected virtual Task Test(TKey key1, TKey key2) - where TKey : class - => this.Test(key1, key2, supportsAutoGeneration: false); - - protected virtual async Task Test( - TKey key1, - TKey key2, - bool supportsAutoGeneration = false) - where TKey : notnull - { - Assert.NotEqual(key1, key2); - - using var collection = fixture.CreateCollection(withAutoGeneration: false); - - { - await collection.EnsureCollectionDeletedAsync(); - await collection.EnsureCollectionExistsAsync(); - - var record = new Record - { - Key = key1, - Int = 8, - Vector = new ReadOnlyMemory([1, 2, 3]) - }; - - var nextRecord = new Record - { - Key = key2, - Int = 9, - Vector = new ReadOnlyMemory([3, 2, 1]) - }; - - await collection.UpsertAsync(record); - // Exercise multi-record plus updating existing record - await collection.UpsertAsync([record, nextRecord]); - await fixture.TestStore.WaitForDataAsync(collection, recordCount: 2); - - // Single record get - var result = await collection.GetAsync(key1); - Assert.NotNull(result); - Assert.Equal(key1, result.Key); - Assert.Equal(8, result.Int); - - // Multiple record get - // Also ensures that the second record - with the default key value - got properly inserted and did not trigger auto-generation - // (as we haven't configured it). - var results = await collection.GetAsync([key1, key2]).ToListAsync(); - Assert.Equal(2, results.Count); - var firstRecord = Assert.Single(results, r => r.Key.Equals(key1)); - Assert.Equal(8, firstRecord.Int); - var secondRecord = Assert.Single(results, r => r.Key.Equals(key2)); - Assert.Equal(9, secondRecord.Int); - } - - /////////////////////// - // Test dynamic mapping - /////////////////////// - await collection.DeleteAsync(key1); - await collection.DeleteAsync([key1, key2]); - await fixture.TestStore.WaitForDataAsync(collection, recordCount: 0); - - using (var dynamicCollection = fixture.CreateDynamicCollection(withAutoGeneration: false)) - { - await dynamicCollection.EnsureCollectionExistsAsync(); - - var dynamicRecord = new Dictionary - { - [nameof(Record.Key)] = key1, - [nameof(Record.Int)] = 8, - [nameof(Record.Vector)] = new ReadOnlyMemory([1, 2, 3]) - }; - var nextDynamicRecord = new Dictionary - { - [nameof(Record.Key)] = key2, - [nameof(Record.Int)] = 9, - [nameof(Record.Vector)] = new ReadOnlyMemory([3, 2, 1]) - }; - - await dynamicCollection.UpsertAsync(dynamicRecord); - // Exercise multi-record plus updating existing record - await dynamicCollection.UpsertAsync([dynamicRecord, nextDynamicRecord]); - await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: 2); - - // Single record get - var dynamicResult = await dynamicCollection.GetAsync(key1); - Assert.NotNull(dynamicResult); - Assert.IsType(dynamicResult[nameof(Record.Key)]); - Assert.Equal(key1, (TKey)dynamicResult[nameof(Record.Key)]!); - Assert.Equal(8, dynamicResult[nameof(Record.Int)]); - - // Multiple record get - // Also ensures that the second record - with the default key value - got properly inserted and did not trigger auto-generation - // (as we haven't configured it). - var dynamicResults = await dynamicCollection.GetAsync([key1, key2]).ToListAsync(); - Assert.Equal(2, dynamicResults.Count); - var firstDynamicRecord = Assert.Single(dynamicResults, r => r[nameof(Record.Key)]!.Equals(key1)); - Assert.IsType(firstDynamicRecord[nameof(Record.Key)]); - Assert.Equal(8, firstDynamicRecord[nameof(Record.Int)]); - var secondDynamicRecord = Assert.Single(dynamicResults, r => r[nameof(Record.Key)]!.Equals(key2)); - Assert.IsType(secondDynamicRecord[nameof(Record.Key)]); - Assert.Equal(9, secondDynamicRecord[nameof(Record.Int)]); - } - - if (supportsAutoGeneration) - { - // Above we tested with a collection where auto-generation isn't enabled - including with the default key value, - // which would have triggered auto-generation if it was enabled. - // Now, drop and recreate the collection with auto-generation enabled, and test that it works. - await collection.EnsureCollectionDeletedAsync(); - - // Pass null to test the provider's default behavior, which should be to enable auto-generation. - using var collectionWithAutoGeneration = fixture.CreateCollection(withAutoGeneration: null); - await collectionWithAutoGeneration.EnsureCollectionExistsAsync(); - - var record = new Record - { - Key = key1, - Int = 8, - Vector = new ReadOnlyMemory([1, 2, 3]) - }; - - var recordWithDefaultValueKey1 = new Record - { - Key = key2, - Int = 9, - Vector = new ReadOnlyMemory([3, 2, 1]) - }; - - var recordWithDefaultValueKey2 = new Record - { - Key = key2, - Int = 10, - Vector = new ReadOnlyMemory([3, 2, 1]) - }; - - var recordWithDefaultValueKey3 = new Record - { - Key = key2, - Int = 11, - Vector = new ReadOnlyMemory([3, 2, 1]) - }; - - // recordWithDefaultValueKey1 gets inserted alone, exercising single-record upsert with auto-generation. - await collectionWithAutoGeneration.UpsertAsync(recordWithDefaultValueKey1); - Assert.NotEqual(recordWithDefaultValueKey1.Key, key2); - var preUpdateGeneratedKey = recordWithDefaultValueKey1.Key; - recordWithDefaultValueKey1.Int = 99; - - // recordWithDefaultValueKey1 gets upserted, exercising update instead of insert. - // recordWithDefaultValueKey2 and 3 get inserted, exercising multi-record upsert with auto-generation; we insert two records to make - // sure the correct key gets injected back into each record. - // Finally, record gets inserted with a non-generated key, to make sure auto-generation doesn't kick in for non-CLR-default keys. - await collectionWithAutoGeneration.UpsertAsync([recordWithDefaultValueKey1, recordWithDefaultValueKey2, recordWithDefaultValueKey3, record]); - await fixture.TestStore.WaitForDataAsync(collectionWithAutoGeneration, recordCount: 4); - - Assert.Equal(recordWithDefaultValueKey1.Key, preUpdateGeneratedKey); - Assert.Equal(99, recordWithDefaultValueKey1.Int); - Assert.NotEqual(recordWithDefaultValueKey2.Key, key2); - Assert.NotEqual(recordWithDefaultValueKey3.Key, key2); - Assert.NotEqual(recordWithDefaultValueKey2.Key, recordWithDefaultValueKey1.Key!); - Assert.NotEqual(recordWithDefaultValueKey3.Key, recordWithDefaultValueKey1.Key!); - Assert.NotEqual(recordWithDefaultValueKey3.Key, recordWithDefaultValueKey2.Key!); - Assert.Equal(record.Key, key1); - - var results = await collectionWithAutoGeneration.GetAsync([key1, recordWithDefaultValueKey1.Key, recordWithDefaultValueKey2.Key, recordWithDefaultValueKey3.Key]).ToListAsync(); - Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey1.Key)); - Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey2.Key)); - Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey3.Key)); - Assert.Single(results, r => r.Key.Equals(key1)); - } - else - { - // Auto-generation is not supported for this type; ensure that model validation throws. - Assert.Throws(() => fixture.CreateCollection(withAutoGeneration: true)); - } - } - - public abstract class Fixture : VectorStoreFixture - { - protected virtual string CollectionNameBase => nameof(KeyTypeTests); - public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase); - - public virtual VectorStoreCollection> CreateCollection(bool? withAutoGeneration) - where TKey : notnull - => this.TestStore.CreateCollection>(this.CollectionName, this.CreateRecordDefinition(withAutoGeneration)); - - public virtual VectorStoreCollection> CreateDynamicCollection(bool withAutoGeneration) - where TKey : notnull - => this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition(withAutoGeneration)); - - public virtual VectorStoreCollectionDefinition CreateRecordDefinition(bool? withAutoGeneration) - where TKey : notnull - => new() - { - Properties = - [ - new VectorStoreKeyProperty("Key", typeof(TKey)) { IsAutoGenerated = withAutoGeneration }, - new VectorStoreDataProperty("Int", typeof(int)), - new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), dimensions: 3) - { - DistanceFunction = this.DefaultDistanceFunction, - IndexKind = this.DefaultIndexKind - } - ] - }; - } - - public class Record - { - public TKey Key { get; set; } = default!; - public int Int { get; set; } - public ReadOnlyMemory Vector { get; set; } - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj b/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj deleted file mode 100644 index dfdf5e4c0da0..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - net10.0;net472 - enable - enable - false - VectorData.ConformanceTests - $(NoWarn);MEVD9000 - $(NoWarn);MEVD9001 - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactAttribute.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactAttribute.cs deleted file mode 100644 index ecd4c815608c..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactAttribute.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -[AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("VectorData.ConformanceTests.Xunit.ConditionalFactDiscoverer", "VectorData.ConformanceTests")] -public sealed class ConditionalFactAttribute : FactAttribute; diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactDiscoverer.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactDiscoverer.cs deleted file mode 100644 index 3d80911fb11a..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactDiscoverer.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -/// -/// Used dynamically from . -/// Make sure to update that class if you move this type. -/// -public class ConditionalFactDiscoverer(IMessageSink messageSink) : FactDiscoverer(messageSink) -{ - protected override IXunitTestCase CreateTestCase( - ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo factAttribute) - => new ConditionalFactTestCase( - this.DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod); -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactTestCase.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactTestCase.cs deleted file mode 100644 index 8c31832a7bf2..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalFactTestCase.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -public sealed class ConditionalFactTestCase : XunitTestCase -{ - [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] - public ConditionalFactTestCase() - { - } - - public ConditionalFactTestCase( - IMessageSink diagnosticMessageSink, - TestMethodDisplay defaultMethodDisplay, - TestMethodDisplayOptions defaultMethodDisplayOptions, - ITestMethod testMethod, - object[]? testMethodArguments = null) - : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) - { - } - - public override async Task RunAsync( - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - object[] constructorArguments, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - => await XunitTestCaseExtensions.TrySkipAsync(this, messageBus) - ? new RunSummary { Total = 1, Skipped = 1 } - : await base.RunAsync( - diagnosticMessageSink, - messageBus, - constructorArguments, - aggregator, - cancellationTokenSource); -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryAttribute.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryAttribute.cs deleted file mode 100644 index a11859575eaa..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryAttribute.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -[AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("VectorData.ConformanceTests.Xunit.ConditionalTheoryDiscoverer", "VectorData.ConformanceTests")] -public sealed class ConditionalTheoryAttribute : TheoryAttribute; diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryDiscoverer.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryDiscoverer.cs deleted file mode 100644 index 986da24ab2da..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryDiscoverer.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -/// -/// Used dynamically from . -/// Make sure to update that class if you move this type. -/// -public class ConditionalTheoryDiscoverer(IMessageSink messageSink) : TheoryDiscoverer(messageSink) -{ - protected override IEnumerable CreateTestCasesForTheory( - ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute) - { - yield return new ConditionalTheoryTestCase( - this.DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod); - } - - protected override IEnumerable CreateTestCasesForDataRow( - ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, - IAttributeInfo theoryAttribute, - object[] dataRow) - { - yield return new ConditionalFactTestCase( - this.DiagnosticMessageSink, - discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), - testMethod, - dataRow); - } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryTestCase.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryTestCase.cs deleted file mode 100644 index 377c2ceaecb1..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ConditionalTheoryTestCase.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -public sealed class ConditionalTheoryTestCase : XunitTheoryTestCase -{ - [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] - public ConditionalTheoryTestCase() - { - } - - public ConditionalTheoryTestCase( - IMessageSink diagnosticMessageSink, - TestMethodDisplay defaultMethodDisplay, - TestMethodDisplayOptions defaultMethodDisplayOptions, - ITestMethod testMethod) - : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) - { - } - - public override async Task RunAsync( - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - object[] constructorArguments, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - => await XunitTestCaseExtensions.TrySkipAsync(this, messageBus) - ? new RunSummary { Total = 1, Skipped = 1 } - : await base.RunAsync( - diagnosticMessageSink, - messageBus, - constructorArguments, - aggregator, - cancellationTokenSource); -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/DisableTestsAttribute.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/DisableTestsAttribute.cs deleted file mode 100644 index 88e011f51ed9..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/DisableTestsAttribute.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace VectorData.ConformanceTests.Xunit; - -/// -/// Disable the tests in the decorated scope. -/// -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)] -public sealed class DisableTestsAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() - { - return new(false); - } - - public string Skip { get; set; } = "Test disabled due to usage of DisableTestsAttribute"; - - public string SkipReason - => this.Skip; -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ITestCondition.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ITestCondition.cs deleted file mode 100644 index 04cc5fd97847..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/ITestCondition.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace VectorData.ConformanceTests.Xunit; - -public interface ITestCondition -{ - ValueTask IsMetAsync(); - - string SkipReason { get; } -} diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/XunitTestCaseExtensions.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/XunitTestCaseExtensions.cs deleted file mode 100644 index 0a41f70507fb..000000000000 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/Xunit/XunitTestCaseExtensions.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Concurrent; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace VectorData.ConformanceTests.Xunit; - -public static class XunitTestCaseExtensions -{ - private static readonly ConcurrentDictionary> s_typeAttributes = new(); - private static readonly ConcurrentDictionary> s_assemblyAttributes = new(); - - public static async ValueTask TrySkipAsync(XunitTestCase testCase, IMessageBus messageBus) - { - var method = testCase.Method; - var type = testCase.TestMethod.TestClass.Class; - var assembly = type.Assembly; - - var skipReasons = new List(); - var attributes = - s_assemblyAttributes.GetOrAdd( - assembly.Name, - a => assembly.GetCustomAttributes(typeof(ITestCondition)).ToList()) - .Concat( - s_typeAttributes.GetOrAdd( - type.Name, - t => type.GetCustomAttributes(typeof(ITestCondition)).ToList())) - .Concat(method.GetCustomAttributes(typeof(ITestCondition))) - .OfType() - .Select(attributeInfo => (ITestCondition)attributeInfo.Attribute); - - foreach (var attribute in attributes) - { - if (!await attribute.IsMetAsync()) - { - skipReasons.Add(attribute.SkipReason); - } - } - - if (skipReasons.Count > 0) - { - messageBus.QueueMessage( - new TestSkipped(new XunitTest(testCase, testCase.DisplayName), string.Join(Environment.NewLine, skipReasons))); - - return true; - } - - return false; - } -} diff --git a/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs b/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs deleted file mode 100644 index fe588e1100d1..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs +++ /dev/null @@ -1,525 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; -using Xunit; - -namespace VectorData.UnitTests; - -#pragma warning disable CA2000 // Dispose objects before losing scope - -public class CollectionModelBuilderTests -{ - [Fact] - public void Default_embedding_generator_without_record_definition() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_clr_type_and_record_definition() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - // The following configures the property to be Embedding (non-default embedding type for this connector) - EmbeddingType = typeof(Embedding) - } - ] - }; - - var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_dynamic() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_dynamic_and_non_default_EmbeddingType() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) - } - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator); - - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Property_embedding_generator_takes_precedence_over_default_generator() - { - using var propertyEmbeddingGenerator = new FakeEmbeddingGenerator>(); - using var defaultEmbeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingGenerator = propertyEmbeddingGenerator - } - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator); - - Assert.Same(propertyEmbeddingGenerator, model.VectorProperty.EmbeddingGenerator); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public void Embedding_property_type_with_default_embedding_generator(bool dynamic) - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var model = dynamic - ? new CustomModelBuilder().BuildDynamic( - new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - ] - }, - embeddingGenerator) - : new CustomModelBuilder().Build(typeof(RecordWithEmbeddingVectorProperty), typeof(int), definition: null, embeddingGenerator); - - var vectorProperty = model.VectorProperty; - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(ReadOnlyMemory), vectorProperty.Type); - } - - [Fact] - public void Embedding_property_type_with_property_embedding_generator() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var model = new CustomModelBuilder().Build( - typeof(RecordWithEmbeddingVectorProperty), - typeof(int), - new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - { - EmbeddingGenerator = embeddingGenerator - } - ] - }, - embeddingGenerator); - - var vectorProperty = model.VectorProperty; - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(ReadOnlyMemory), vectorProperty.EmbeddingType); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public void Custom_input_type(bool dynamic) - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - // TODO: Allow custom input type without a record definition (i.e. generic attribute) - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), dimensions: 3) - ] - }; - - var model = dynamic - ? new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator) - : new CustomModelBuilder().Build(typeof(RecordWithCustomerVectorProperty), typeof(int), recordDefinition, embeddingGenerator); - - var vectorProperty = model.VectorProperty; - - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(Customer), vectorProperty.Type); - Assert.Same(typeof(Embedding), vectorProperty.EmbeddingType); - } - - [Fact] - public void Incompatible_embedding_on_embedding_generator_throws() - { - // Embedding is not a supported embedding type by the connector - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator)); - - Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]).", exception.Message); - } - - [Fact] - public void Incompatible_input_on_embedding_generator_throws() - { - // int is not a supported input type for the embedding generator - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator)); - - Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]).", exception.Message); - } - - [Fact] - public void Non_embedding_vector_property_without_embedding_generator_throws() - { - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, defaultEmbeddingGenerator: null)); - - Assert.Equal($"Vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' has type 'string' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting 'string' to vector type supported by your provider.", exception.Message); - } - - [Fact] - public void EmbeddingType_not_supported_by_provider() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) // The provider supports float/Half only, not byte - } - ] - }; - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator)); - - Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding' configured, but that type isn't supported by your provider. Supported types are ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[].", exception.Message); - } - - [Fact] - public void EmbeddingType_not_supported_by_generator() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) // The generator (instantiated above) supports only Embedding - } - ] - }; - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator)); - - Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding' configured, but that type isn't supported by your embedding generator.", exception.Message); - } - - [Fact] - public void Missing_Type_on_property_definition() - { - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - ] - }; - - // Key - recordDefinition.Properties[0].Type = null; - var exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Id)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - - // Data - recordDefinition.Properties[0].Type = typeof(int); - recordDefinition.Properties[1].Type = null; - exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Name)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - - // Vector - recordDefinition.Properties[1].Type = typeof(string); - recordDefinition.Properties[2].Type = null; - exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Embedding)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - } - - public class RecordWithStringVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public string Embedding { get; set; } - } - - public class RecordWithEmbeddingVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public ReadOnlyMemory Embedding { get; set; } - } - - public class RecordWithCustomerVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public Customer Embedding { get; set; } - } - - public class Customer - { - public string FirstName { get; set; } - public string LastName { get; set; } - } - - private sealed class CustomModelBuilder(CollectionModelBuildingOptions? options = null) - : CollectionModelBuilder(options ?? s_defaultOptions) - { - private static readonly CollectionModelBuildingOptions s_defaultOptions = new() - { - SupportsMultipleVectors = true, - RequiresAtLeastOneVector = false - }; - - protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) - { - var type = keyProperty.Type; - - if (type != typeof(string) && type != typeof(int)) - { - throw new NotSupportedException( - $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, int."); - } - } - - protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = "string, int"; - - if (Nullable.GetUnderlyingType(type) is Type underlyingType) - { - type = underlyingType; - } - - return type == typeof(string) || type == typeof(int); - } - - protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - => IsVectorPropertyTypeValidCore(type, out supportedTypes); - - internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = "ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]"; - - if (Nullable.GetUnderlyingType(type) is Type underlyingType) - { - type = underlyingType; - } - - return type == typeof(ReadOnlyMemory) - || type == typeof(Embedding) - || type == typeof(float[]) - || type == typeof(ReadOnlyMemory) - || type == typeof(Embedding) - || type == typeof(Half[]); - } - - protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = - [ - EmbeddingGenerationDispatcher.Create>(), - EmbeddingGenerationDispatcher.Create>() - ]; - } - - [Fact] - public void IsAutoGenerated_attribute_true_overrides_default() - { - // Guid key with explicit IsAutoGenerated = true; the attribute should override SupportsKeyAutoGeneration. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyAutoGeneratedTrue), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.True(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_attribute_false_overrides_default() - { - // Guid key with explicit IsAutoGenerated = false; the attribute should override SupportsKeyAutoGeneration, - // which would otherwise return true for Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyAutoGeneratedFalse), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.False(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_true() - { - // Guid key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns true for Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyNoIsAutoGenerated), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.True(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_false() - { - // int key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns false for non-Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithIntKeyNoIsAutoGenerated), typeof(int), definition: null, defaultEmbeddingGenerator: null); - - Assert.False(model.KeyProperty.IsAutoGenerated); - } - - public class RecordWithGuidKeyAutoGeneratedTrue - { - [VectorStoreKey(IsAutoGenerated = true)] - public Guid Id { get; set; } - } - - public class RecordWithGuidKeyAutoGeneratedFalse - { - [VectorStoreKey(IsAutoGenerated = false)] - public Guid Id { get; set; } - } - - public class RecordWithGuidKeyNoIsAutoGenerated - { - [VectorStoreKey] - public Guid Id { get; set; } - } - - public class RecordWithIntKeyNoIsAutoGenerated - { - [VectorStoreKey] - public int Id { get; set; } - } - - private sealed class GuidKeyModelBuilder() - : CollectionModelBuilder(new CollectionModelBuildingOptions - { - SupportsMultipleVectors = true, - RequiresAtLeastOneVector = false - }) - { - protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = null; - return true; - } - - protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = null; - return true; - } - } - - private sealed class FakeEmbeddingGenerator : IEmbeddingGenerator - where TEmbedding : Embedding - { - public Task> GenerateAsync( - IEnumerable values, - EmbeddingGenerationOptions? options = null, - CancellationToken cancellationToken = default) - => throw new UnreachableException(); - - public object? GetService(Type serviceType, object? serviceKey = null) - => throw new UnreachableException(); - - public void Dispose() { } - } -} diff --git a/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs b/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs deleted file mode 100644 index aad7b64beee3..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData.ProviderServices; -using Xunit; - -namespace VectorData.UnitTests; - -public class PropertyModelTests -{ - #region Value type nullability - - [Fact] - public void IsNullable_NonNullableValueType_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(int)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableValueType_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(int?)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_Guid_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(Guid)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableGuid_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(Guid?)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ReadOnlyMemoryFloat_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(ReadOnlyMemory)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableReadOnlyMemoryFloat_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(ReadOnlyMemory?)); - Assert.True(property.IsNullable); - } - - #endregion - - #region Reference type nullability (dynamic mapping, no PropertyInfo) - - [Fact] - public void IsNullable_String_WithoutPropertyInfo_ReturnsTrue() - { - // Without PropertyInfo (dynamic mapping), reference types are assumed nullable - var property = new DataPropertyModel("test", typeof(string)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ByteArray_WithoutPropertyInfo_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(byte[])); - Assert.True(property.IsNullable); - } - - #endregion - -#if NET - #region NRT detection via NullabilityInfoContext (POCO mapping with PropertyInfo) - - [Fact] - public void IsNullable_NonNullableString_WithPropertyInfo_ReturnsFalse() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableString))!; - var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableString_WithPropertyInfo_ReturnsTrue() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableString))!; - var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo }; - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_NonNullableByteArray_WithPropertyInfo_ReturnsFalse() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableByteArray))!; - var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableByteArray_WithPropertyInfo_ReturnsTrue() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableByteArray))!; - var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo }; - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ValueType_WithPropertyInfo_StillUsesTypeCheck() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableInt))!; - var property = new DataPropertyModel("test", typeof(int)) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - - var nullablePropertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableInt))!; - var nullableProperty = new DataPropertyModel("test", typeof(int?)) { PropertyInfo = nullablePropertyInfo }; - Assert.True(nullableProperty.IsNullable); - } - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor -#pragma warning disable CA1812 // Class is used via reflection - private sealed class NrtTestRecord - { - public string NonNullableString { get; set; } - public string? NullableString { get; set; } - public byte[] NonNullableByteArray { get; set; } - public byte[]? NullableByteArray { get; set; } - public int NonNullableInt { get; set; } - public int? NullableInt { get; set; } - } -#pragma warning restore CA1812 -#pragma warning restore CS8618 - - #endregion -#endif -} diff --git a/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj b/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj deleted file mode 100644 index cc7bf6d4e2bc..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj +++ /dev/null @@ -1,43 +0,0 @@ - - - - VectorData.UnitTests - VectorData.UnitTests - net10.0 - true - enable - disable - false - $(NoWarn);VSTHRD111,CA2007,CS8618 - $(NoWarn);MEVD9001 - $(NoWarn);CA1515 - $(NoWarn);CA1707 - $(NoWarn);CA1716 - $(NoWarn);CA1720 - $(NoWarn);CA1721 - $(NoWarn);CA1861 - $(NoWarn);CA1863 - $(NoWarn);CA2007;VSTHRD111 - $(NoWarn);CS1591 - $(NoWarn);IDE1006 - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - diff --git a/dotnet/test/VectorData/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs b/dotnet/test/VectorData/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs deleted file mode 100644 index e0b8afb13cc5..000000000000 --- a/dotnet/test/VectorData/Weaviate.ConformanceTests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorData.ConformanceTests.Xunit; - -[assembly: DisableTests(Skip = "Weaviate tests are failing on the build server with connection reset errors, but passing locally.")] diff --git a/dotnet/test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj b/dotnet/test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj index d78de074348f..06239b432672 100644 --- a/dotnet/test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj +++ b/dotnet/test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj @@ -21,7 +21,6 @@ - diff --git a/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs b/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs index 4fb39d5548bf..813da61a041d 100644 --- a/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs +++ b/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateIndexKindTests.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.VectorData; using VectorData.ConformanceTests; using VectorData.ConformanceTests.Support; -using VectorData.ConformanceTests.Xunit; using Weaviate.ConformanceTests.Support; using Xunit; @@ -12,11 +11,11 @@ namespace Weaviate.ConformanceTests; public class WeaviateIndexKindTests(WeaviateIndexKindTests.Fixture fixture) : IndexKindTests(fixture), IClassFixture { - [ConditionalFact] + [Fact] public virtual Task Hnsw() => this.Test(IndexKind.Hnsw); - [ConditionalFact] + [Fact] public virtual Task Dynamic() => this.Test(IndexKind.Dynamic); diff --git a/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateHotel.cs b/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateHotel.cs index 4e24f8095f50..a6b07f275bf3 100644 --- a/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateHotel.cs +++ b/dotnet/test/VectorData/Weaviate.UnitTests/WeaviateHotel.cs @@ -44,6 +44,6 @@ public sealed record WeaviateHotel public DateTimeOffset Timestamp { get; set; } /// A vector field. - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw)] + [VectorStoreVector(dimensions: 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } }