diff --git a/CommunityToolkit.AI.slnx b/CommunityToolkit.AI.slnx index 3dbd050..0267de3 100644 --- a/CommunityToolkit.AI.slnx +++ b/CommunityToolkit.AI.slnx @@ -9,6 +9,7 @@ + @@ -22,6 +23,8 @@ + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 3bea2a0..1ba608a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,6 +12,7 @@ + @@ -45,6 +46,7 @@ + diff --git a/MEVD/MEVD.slnf b/MEVD/MEVD.slnf index 324331c..3d753ac 100644 --- a/MEVD/MEVD.slnf +++ b/MEVD/MEVD.slnf @@ -4,6 +4,7 @@ "projects": [ "MEVD/src/AzureAISearch/AzureAISearch.csproj", + "MEVD/src/CosmosNoSql/CosmosNoSql.csproj", "MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj", "MEVD/src/InMemory/InMemory.csproj", "MEVD/src/Pinecone/Pinecone.csproj", @@ -16,6 +17,8 @@ "MEVD/test/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj", "MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj", + "MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj", + "MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj", "MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj", "MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj", "MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj", diff --git a/MEVD/src/CosmosNoSql/AssemblyInfo.cs b/MEVD/src/CosmosNoSql/AssemblyInfo.cs new file mode 100644 index 0000000..53616c2 --- /dev/null +++ b/MEVD/src/CosmosNoSql/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/CosmosNoSql/ByteArrayJsonConverter.cs b/MEVD/src/CosmosNoSql/ByteArrayJsonConverter.cs new file mode 100644 index 0000000..15aba28 --- /dev/null +++ b/MEVD/src/CosmosNoSql/ByteArrayJsonConverter.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// A JSON converter for byte arrays that serializes them as JSON arrays of numbers +/// instead of base64-encoded strings. +/// +/// +/// This is needed because Cosmos DB's VectorDistance function requires vectors to be arrays of numbers, +/// not base64-encoded strings. +/// +internal sealed class ByteArrayJsonConverter : JsonConverter +{ + /// + public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Support reading both base64 strings (for backward compatibility) and number arrays + if (reader.TokenType == JsonTokenType.String) + { + return reader.GetBytesFromBase64(); + } + + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException($"Expected StartArray or String token, got {reader.TokenType}"); + } + + var list = new List(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + list.Add(reader.GetByte()); + } + return list.ToArray(); + } + + /// + public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) + { + writer.WriteStartArray(); + foreach (var b in value) + { + writer.WriteNumberValue(b); + } + writer.WriteEndArray(); + } +} + +/// +/// A JSON converter for of byte that serializes as JSON arrays of numbers +/// instead of base64-encoded strings. +/// +/// +/// This is needed because Cosmos DB's VectorDistance function requires vectors to be arrays of numbers, +/// not base64-encoded strings. +/// +internal sealed class ReadOnlyMemoryByteJsonConverter : JsonConverter> +{ + /// + public override ReadOnlyMemory Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Support reading both base64 strings (for backward compatibility) and number arrays + if (reader.TokenType == JsonTokenType.String) + { + return new ReadOnlyMemory(reader.GetBytesFromBase64()); + } + + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException($"Expected StartArray or String token, got {reader.TokenType}"); + } + + var list = new List(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + list.Add(reader.GetByte()); + } + return new ReadOnlyMemory(list.ToArray()); + } + + /// + public override void Write(Utf8JsonWriter writer, ReadOnlyMemory value, JsonSerializerOptions options) + { + writer.WriteStartArray(); + foreach (var b in value.Span) + { + writer.WriteNumberValue(b); + } + writer.WriteEndArray(); + } +} diff --git a/MEVD/src/CosmosNoSql/ClientWrapper.cs b/MEVD/src/CosmosNoSql/ClientWrapper.cs new file mode 100644 index 0000000..53eec7a --- /dev/null +++ b/MEVD/src/CosmosNoSql/ClientWrapper.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using Microsoft.Azure.Cosmos; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +internal sealed class ClientWrapper : IDisposable +{ + private readonly bool _ownsClient; + private int _referenceCount = 1; + + internal ClientWrapper(CosmosClient cosmosClient, bool ownsClient) + { + this.Client = cosmosClient; + this._ownsClient = ownsClient; + } + + internal CosmosClient Client { get; } + + internal ClientWrapper Share() + { + if (this._ownsClient) + { + Interlocked.Increment(ref this._referenceCount); + } + + return this; + } + + public void Dispose() + { + if (this._ownsClient) + { + if (Interlocked.Decrement(ref this._referenceCount) == 0) + { + this.Client.Dispose(); + } + } + } +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSql.csproj b/MEVD/src/CosmosNoSql/CosmosNoSql.csproj new file mode 100644 index 0000000..64a7173 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSql.csproj @@ -0,0 +1,27 @@ + + + 1.0.0-preview.1 + CommunityToolkit.VectorData.CosmosNoSql + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + Azure Cosmos DB for NoSQL provider for Microsoft.Extensions.VectorData + Azure Cosmos DB for NoSQL provider for Microsoft.Extensions.VectorData by the .NET Community Toolkit + + true + + + + + + + + + + + + + + + + diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlCollection.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlCollection.cs new file mode 100644 index 0000000..8cd8988 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlCollection.cs @@ -0,0 +1,940 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Linq.Expressions; +using System.Net; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using DistanceFunction = Microsoft.Azure.Cosmos.DistanceFunction; +using IndexKind = Microsoft.Extensions.VectorData.IndexKind; +using MEAI = Microsoft.Extensions.AI; +using SKDistanceFunction = Microsoft.Extensions.VectorData.DistanceFunction; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Service for storing and retrieving vector records, that uses Azure CosmosDB NoSQL as the underlying storage. +/// +/// +/// The data type of the record key. Supported types are and (in which case the partition key +/// is the document ID), and (for other partition key configurations). +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class CosmosNoSqlCollection : VectorStoreCollection, IKeywordHybridSearchable + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + /// The default options for hybrid vector search. + private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); + + private readonly ClientWrapper _clientWrapper; + + /// that can be used to manage the collections in Azure CosmosDB NoSQL. + private readonly Database _database; + + /// The model for this collection. + private readonly CollectionModel _model; + + // TODO: Refactor this into the model + /// The properties to use as partition key (supports hierarchical partition keys up to 3 levels). + private readonly List _partitionKeyProperties; + + /// The mapper to use when mapping between the consumer data model and the Azure CosmosDB NoSQL record. + private readonly ICosmosNoSqlMapper _mapper; + + /// + public override string Name { get; } + + /// The indexing mode in the Azure Cosmos DB service. + private readonly IndexingMode _indexingMode; + + /// Whether automatic indexing is enabled for a collection in the Azure Cosmos DB service. + private readonly bool _automatic; + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB NoSQL. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] + public CosmosNoSqlCollection(Database database, string name, CosmosNoSqlCollectionOptions? options = default) + : this(new(database.Client, ownsClient: false), _ => database, name, options) + { + Throw.IfNull(database); + Throw.IfNullOrWhitespace(name); + } + + /// + /// Initializes a new instance of the class. + /// + /// Connection string required to connect to Azure CosmosDB NoSQL. + /// Database name for Azure CosmosDB NoSQL. + /// The name of the collection that this will access. + /// Optional configuration options for . + /// Optional configuration options for . + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] + public CosmosNoSqlCollection( + string connectionString, + string databaseName, + string name, + CosmosClientOptions? clientOptions = null, + CosmosNoSqlCollectionOptions? options = null) + : this( + new ClientWrapper(new CosmosClient(connectionString, clientOptions), ownsClient: true), + client => client.GetDatabase(databaseName), + name, + options) + { + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + Throw.IfNullOrWhitespace(name); + } + + internal CosmosNoSqlCollection( + ClientWrapper clientWrapper, + Func databaseProvider, + string name, + CosmosNoSqlCollectionOptions? options) + : this( + clientWrapper, + databaseProvider, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(CosmosNoSqlDynamicCollection))) + : new CosmosNoSqlModelBuilder() + .Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator, options.JsonSerializerOptions ?? JsonSerializerOptions.Default), + options) + { + } + + internal CosmosNoSqlCollection( + ClientWrapper clientWrapper, + Func databaseProvider, + string name, + Func modelFactory, + CosmosNoSqlCollectionOptions? options) + { + // Validate TKey: must be string or Guid (partition key = document ID), CosmosNoSqlKey (other partition key cases), or object (used by CosmosNoSqlDynamicCollection). + if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(CosmosNoSqlKey) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException($"The key type must be string, Guid, or CosmosNoSqlKey. Received: {typeof(TKey).Name}"); + } + + try + { + this._database = databaseProvider(clientWrapper.Client); + + if (clientWrapper.Client.ClientOptions?.UseSystemTextJsonSerializerWithOptions is null) + { + throw new ArgumentException( + $"Property {nameof(CosmosClientOptions.UseSystemTextJsonSerializerWithOptions)} in CosmosClient.ClientOptions " + + $"is required to be configured for {this.GetType().Name}."); + } + + options ??= CosmosNoSqlCollectionOptions.Default; + + // Assign. + this.Name = name; + this._model = modelFactory(options); + this._indexingMode = options.IndexingMode; + this._automatic = options.Automatic; + var jsonSerializerOptions = options.JsonSerializerOptions ?? JsonSerializerOptions.Default; + + // Assign mapper. + this._mapper = typeof(TRecord) == typeof(Dictionary) + ? (new CosmosNoSqlDynamicMapper(this._model, jsonSerializerOptions) as ICosmosNoSqlMapper)! + : new CosmosNoSqlMapper(this._model, options.JsonSerializerOptions); + + // Setup partition key properties (supports hierarchical partition keys up to 3 levels) + if (options.PartitionKeyProperties is null) + { + // No partition key property names configured: default to using the key property as the partition key. + // This is the most common Cosmos DB strategy where the document ID and partition key are the same value. + this._partitionKeyProperties = [this._model.KeyProperty]; + } + else + { + if (options.PartitionKeyProperties.Count > 3) + { + throw new ArgumentException("Cosmos DB supports at most 3 levels of hierarchical partition keys."); + } + + this._partitionKeyProperties = new List(options.PartitionKeyProperties.Count); + + foreach (var propertyName in options.PartitionKeyProperties) + { + if (!this._model.PropertyMap.TryGetValue(propertyName, out var property)) + { + throw new ArgumentException($"Partition key property '{propertyName}' is not part of the record definition."); + } + + if (property.Type != typeof(string) && property.Type != typeof(bool) && property.Type != typeof(double) && property.Type != typeof(Guid)) + { + throw new ArgumentException($"Partition key property '{propertyName}' must be string, bool, double, or Guid."); + } + + this._partitionKeyProperties.Add(property); + } + } + + // When using simple key types (string/Guid), the partition key must be the key property only, + // since the partition key value cannot be independently specified via a simple key. + // Users who need hierarchical partition keys, a non-key partition key, or no partition key at all + // must use CosmosNoSqlKey as the key type. + if ((typeof(TKey) == typeof(string) || typeof(TKey) == typeof(Guid)) + && (this._partitionKeyProperties is not [var singlePkProperty] || singlePkProperty != this._model.KeyProperty)) + { + throw new ArgumentException( + $"When using {typeof(TKey).Name} as the key type, the partition key must be the key property " + + $"('{this._model.KeyProperty.ModelName}'). To use a different partition key configuration, use CosmosNoSqlKey as the key type."); + } + + this._collectionMetadata = new() + { + VectorStoreSystemName = CosmosNoSqlConstants.VectorStoreSystemName, + VectorStoreName = this._database.Id, + CollectionName = name + }; + } + catch (Exception) + { + // Something went wrong, we dispose the client and don't store a reference. + clientWrapper.Dispose(); + + throw; + } + + this._clientWrapper = clientWrapper; + } + + /// + protected override void Dispose(bool disposing) + { + this._clientWrapper.Dispose(); + base.Dispose(disposing); + } + + /// + public override async Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "ListCollectionNamesAsync"; + const string Query = "SELECT c.id FROM c WHERE c.id = @collectionName"; + var queryDefinition = new QueryDefinition(Query).WithParameter("@collectionName", this.Name); + + using var feedIterator = VectorStoreErrorHandler.RunOperation, CosmosException>( + this._collectionMetadata, + OperationName, + () => this._database.GetContainerQueryIterator(queryDefinition)); + + while (feedIterator.HasMoreResults) + { + var next = await feedIterator.ReadNextAsync(cancellationToken).ConfigureAwait(false); + + foreach (var result in next.Resource) + { + return true; + } + } + + return false; + } + + private sealed class ContainerIdResult + { + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + } + + /// + public override async Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + { + // Don't even try to create if the collection already exists. + if (await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + await this._database.CreateContainerAsync(this.GetContainerProperties(), cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.Conflict) + { + // Do nothing, since the container is already created. + } + catch (CosmosException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = CosmosNoSqlConstants.VectorStoreSystemName, + VectorStoreName = this._collectionMetadata.VectorStoreName, + CollectionName = this.Name, + OperationName = "CreateContainer" + }; + } + } + + /// + public override async Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + { + try + { + await this._database + .GetContainer(this.Name) + .DeleteContainerAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + // Do nothing, since the container is already deleted. + } + catch (CosmosException ex) + { + throw new VectorStoreException("Call to vector store failed.", ex) + { + VectorStoreSystemName = CosmosNoSqlConstants.VectorStoreSystemName, + VectorStoreName = this._collectionMetadata.VectorStoreName, + CollectionName = this.Name, + OperationName = "DeleteContainer" + }; + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + var (documentId, partitionKey) = this.GetDocumentIdAndPartitionKey(key); + + return this.RunOperationAsync("DeleteItem", async () => + { + try + { + await this._database + .GetContainer(this.Name) + .DeleteItemAsync(documentId, partitionKey, cancellationToken: cancellationToken) + .ConfigureAwait(false); + return 0; + } + catch (CosmosException e) when (e.StatusCode == System.Net.HttpStatusCode.NotFound) + { + // Ignore not found errors + return 0; + } + }); + } + + // TODO: Implement bulk delete, #11350 + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + const string OperationName = "ReadItem"; + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && this._model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var (documentId, partitionKey) = this.GetDocumentIdAndPartitionKey(key); + + var jsonObject = await this.RunOperationAsync(OperationName, async () => + { + try + { + var response = await this._database + .GetContainer(this.Name) + .ReadItemAsync(documentId, partitionKey, cancellationToken: cancellationToken) + .ConfigureAwait(false); + return response.Resource; + } + catch (CosmosException e) when (e.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + }).ConfigureAwait(false); + + return jsonObject is null ? default : this._mapper.MapFromStorageToDataModel(jsonObject, includeVectors); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + const string OperationName = "ReadManyItems"; + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && this._model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var items = keys.Select(this.GetDocumentIdAndPartitionKey).ToList(); + + if (items.Count == 0) + { + yield break; + } + + var response = await this.RunOperationAsync(OperationName, () => + this._database + .GetContainer(this.Name) + .ReadManyItemsAsync(items, cancellationToken: cancellationToken)) + .ConfigureAwait(false); + + foreach (var jsonObject in response.Resource) + { + var record = this._mapper.MapFromStorageToDataModel(jsonObject, includeVectors); + + if (record is not null) + { + yield return record; + } + } + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + (_, var generatedEmbeddings) = await ProcessEmbeddingsAsync(this._model, [record], cancellationToken).ConfigureAwait(false); + + await this.UpsertCoreAsync(record, recordIndex: 0, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + (records, var generatedEmbeddings) = await ProcessEmbeddingsAsync(this._model, records, cancellationToken).ConfigureAwait(false); + + // TODO: Do proper bulk upsert rather than single inserts, #11350 + var i = 0; + + foreach (var record in records) + { + await this.UpsertCoreAsync(record, i++, generatedEmbeddings, cancellationToken).ConfigureAwait(false); + } + } + + private async Task UpsertCoreAsync(TRecord record, int recordIndex, IReadOnlyList?[]? generatedEmbeddings, CancellationToken cancellationToken = default) + { + const string OperationName = "UpsertItem"; + + // Handle auto-generated keys (client-side for Cosmos DB NoSQL, which doesn't support server-side auto-generation) + var keyProperty = this._model.KeyProperty; + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + + var partitionKey = this.BuildPartitionKey(record); + + var jsonObject = this._mapper.MapFromDataToStorageModel(record, recordIndex, generatedEmbeddings); + + var keyValue = jsonObject.TryGetPropertyValue(this._model.KeyProperty.StorageName!, out var jsonKey) ? jsonKey?.ToString() : null; + + if (string.IsNullOrWhiteSpace(keyValue)) + { + throw new ArgumentException($"Key property {this._model.KeyProperty.ModelName} is not initialized."); + } + + await this.RunOperationAsync(OperationName, () => + this._database + .GetContainer(this.Name) + .UpsertItemAsync(jsonObject, partitionKey, cancellationToken: cancellationToken)) + .ConfigureAwait(false); + } + + private static async ValueTask<(IEnumerable records, IReadOnlyList?[]?)> ProcessEmbeddingsAsync( + CollectionModel model, + IEnumerable records, + CancellationToken cancellationToken) + { + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + IReadOnlyList?[]? generatedEmbeddings = null; + + var vectorPropertyCount = model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = model.VectorProperties[i]; + + if (CosmosNoSqlModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + // We have a property with embedding generation; materialize the records' enumerable if needed, to + // prevent multiple enumeration. + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return (records, null); + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new IReadOnlyList?[vectorPropertyCount]; + generatedEmbeddings[i] = await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + return (records, generatedEmbeddings); + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + const string OperationName = "VectorizedSearch"; + const string ScorePropertyName = "SimilarityScore"; + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && this._model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = this._model.GetVectorPropertyOrSingle(options); + object vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + + var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( + vector, + null, + this._model, + vectorProperty.StorageName, + vectorProperty.DistanceFunction, + textPropertyName: null, + ScorePropertyName, + options.Filter, + options.ScoreThreshold, + top, + options.Skip, + options.IncludeVectors); + + var searchResults = this.GetItemsAsync(queryDefinition, OperationName, cancellationToken); + + await foreach (var record in this.MapSearchResultsAsync(searchResults, ScorePropertyName, OperationName, options.IncludeVectors, cancellationToken).ConfigureAwait(false)) + { + yield return record; + } + } + + private static async ValueTask GetSearchVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + => searchValue switch + { + // float32 + ReadOnlyMemory m => m, + float[] a => new ReadOnlyMemory(a), + Embedding e => e.Vector, + + // int8 + ReadOnlyMemory m => m, + sbyte[] a => new ReadOnlyMemory(a), + Embedding e => e.Vector, + + // uint8 + ReadOnlyMemory m => m, + byte[] a => new ReadOnlyMemory(a), + Embedding e => e.Vector, + + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false), + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), CosmosNoSqlModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + const string OperationName = "GetAsync"; + + options ??= new(); + + var (whereClause, filterParameters) = new CosmosNoSqlFilterTranslator().Translate(filter, this._model); + + var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( + this._model, + whereClause, + filterParameters, + options, + top); + + var searchResults = this.GetItemsAsync(queryDefinition, OperationName, cancellationToken); + + await foreach (var jsonObject in searchResults.ConfigureAwait(false)) + { + var record = this._mapper.MapFromStorageToDataModel(jsonObject, options.IncludeVectors); + + yield return record; + } + } + + /// + public async IAsyncEnumerable> HybridSearchAsync( + TInput searchValue, + ICollection keywords, + int top, + HybridSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + const string OperationName = "VectorizedSearch"; + const string ScorePropertyName = "SimilarityScore"; + + this.VerifyVectorType(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultKeywordVectorizedHybridSearchOptions; + var vectorProperty = this._model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + object vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + var textProperty = this._model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + + var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery( + vector, + keywords, + this._model, + vectorProperty.StorageName, + vectorProperty.DistanceFunction, + textProperty.StorageName, + ScorePropertyName, + options.Filter, + options.ScoreThreshold, + top, + options.Skip, + options.IncludeVectors); + + var searchResults = this.GetItemsAsync(queryDefinition, OperationName, cancellationToken); + + await foreach (var record in this.MapSearchResultsAsync(searchResults, ScorePropertyName, OperationName, options.IncludeVectors, cancellationToken).ConfigureAwait(false)) + { + yield return record; + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + #region private + + private void VerifyVectorType(TVector? vector) + { + Throw.IfNull(vector); + + var vectorType = vector!.GetType(); + + if (!CosmosNoSqlModelBuilder.IsVectorPropertyTypeValidCore(vectorType, out var supportedTypes)) + { + throw new NotSupportedException( + $"The provided vector type {vectorType.FullName} is not supported by the Azure CosmosDB NoSQL connector. Supported types are: {supportedTypes}"); + } + } + + private Task RunOperationAsync(string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync( + this._collectionMetadata, + operationName, + operation); + + /// + /// Gets the document ID and partition key from a key. + /// + private (string DocumentId, PartitionKey PartitionKey) GetDocumentIdAndPartitionKey(TKey key) + => key switch + { + CosmosNoSqlKey cosmosKey => (cosmosKey.DocumentId, cosmosKey.PartitionKey), + string s => (s, new PartitionKey(s)), + Guid g => (g.ToString(), new PartitionKey(g.ToString())), + _ => throw new InvalidOperationException( + $"Keys must be of type string, Guid, or CosmosNoSqlKey. Received key of type '{key.GetType().FullName}'.") + }; + + /// + /// Returns instance of with applied indexing policy. + /// More information here: . + /// + private ContainerProperties GetContainerProperties() + { + var indexingPolicy = new IndexingPolicy + { + IndexingMode = this._indexingMode, + Automatic = this._automatic + }; + + var containerProperties = new ContainerProperties( + this.Name, + partitionKeyPaths: this._partitionKeyProperties.Count > 0 + ? this._partitionKeyProperties.Select(p => $"/{p.StorageName}").ToList() + : ["/id"]) + { + IndexingPolicy = indexingPolicy + }; + + if (this._indexingMode == IndexingMode.None) + { + return containerProperties; + } + + // Process Vector properties. + var embeddings = new Collection(); + var vectorIndexPaths = new Collection(); + + foreach (var property in this._model.VectorProperties) + { + var path = $"/{property.StorageName}"; + + var embedding = new Microsoft.Azure.Cosmos.Embedding + { + DataType = GetDataType(property.EmbeddingType, property.StorageName), + Dimensions = (int)property.Dimensions, + DistanceFunction = GetDistanceFunction(property.DistanceFunction, property.StorageName), + Path = path + }; + + var vectorIndexPath = new VectorIndexPath + { + Type = GetIndexKind(property.IndexKind, property.StorageName), + Path = path + }; + + embeddings.Add(embedding); + vectorIndexPaths.Add(vectorIndexPath); + } + + indexingPolicy.VectorIndexes = vectorIndexPaths; + + var fullTextPolicy = new FullTextPolicy() { FullTextPaths = [] }; + var vectorEmbeddingPolicy = new VectorEmbeddingPolicy(embeddings); + + // Process Data properties. + foreach (var property in this._model.DataProperties) + { + if (property.IsIndexed || property.IsFullTextIndexed) + { + indexingPolicy.IncludedPaths.Add(new IncludedPath { Path = $"/{property.StorageName}/?" }); + } + if (property.IsFullTextIndexed) + { + indexingPolicy.FullTextIndexes.Add(new FullTextIndexPath { Path = $"/{property.StorageName}" }); + // TODO: Switch to using language from a setting. + fullTextPolicy.FullTextPaths.Add(new FullTextPath { Path = $"/{property.StorageName}", Language = "en-US" }); + } + } + + // Adding special mandatory indexing path. + indexingPolicy.IncludedPaths.Add(new IncludedPath { Path = "/*" }); + + // Exclude vector paths to ensure optimized performance. + foreach (var vectorIndexPath in vectorIndexPaths) + { + indexingPolicy.ExcludedPaths.Add(new ExcludedPath { Path = $"{vectorIndexPath.Path}/*" }); + } + + containerProperties.VectorEmbeddingPolicy = vectorEmbeddingPolicy; + containerProperties.FullTextPolicy = fullTextPolicy; + + return containerProperties; + } + + /// + /// More information about Azure CosmosDB NoSQL index kinds here: . + /// + private static VectorIndexType GetIndexKind(string? indexKind, string vectorPropertyName) + => indexKind switch + { + "DiskAnn" or null => VectorIndexType.DiskANN, + "Flat" => VectorIndexType.Flat, + "QuantizedFlat" => VectorIndexType.QuantizedFlat, + _ => throw new InvalidOperationException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB NoSQL VectorStore.") + }; + + /// + /// More information about Azure CosmosDB NoSQL distance functions here: . + /// + private static DistanceFunction GetDistanceFunction(string? distanceFunction, string vectorPropertyName) + => distanceFunction switch + { + SKDistanceFunction.CosineSimilarity or null => DistanceFunction.Cosine, + SKDistanceFunction.DotProductSimilarity => DistanceFunction.DotProduct, + SKDistanceFunction.EuclideanDistance => DistanceFunction.Euclidean, + + _ => throw new NotSupportedException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB NoSQL VectorStore.") + }; + + /// + /// Returns based on vector property type. + /// + private static VectorDataType GetDataType(Type vectorDataType, string vectorPropertyName) + => (Nullable.GetUnderlyingType(vectorDataType) ?? vectorDataType) switch + { + Type type when type == typeof(ReadOnlyMemory) || type == typeof(Embedding) || type == typeof(float[]) + => VectorDataType.Float32, + + Type type when type == typeof(ReadOnlyMemory) || type == typeof(Embedding) || type == typeof(byte[]) + => VectorDataType.Uint8, + + Type type when type == typeof(ReadOnlyMemory) || type == typeof(Embedding) || type == typeof(sbyte[]) + => VectorDataType.Int8, + + _ => throw new InvalidOperationException($"Data type '{vectorDataType}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB NoSQL VectorStore.") + }; + + private async IAsyncEnumerable GetItemsAsync(QueryDefinition queryDefinition, string operationName, [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var feedIterator = VectorStoreErrorHandler.RunOperation, CosmosException>( + this._collectionMetadata, + operationName, + () => this._database + .GetContainer(this.Name) + .GetItemQueryIterator(queryDefinition)); + using var errorHandlingFeedIterator = new ErrorHandlingFeedIterator(feedIterator, this._collectionMetadata, operationName); + + while (errorHandlingFeedIterator.HasMoreResults) + { + var response = await errorHandlingFeedIterator.ReadNextAsync(cancellationToken).ConfigureAwait(false); + + foreach (var record in response.Resource) + { + if (record is not null) + { + yield return record; + } + } + } + } + + private async IAsyncEnumerable> MapSearchResultsAsync( + IAsyncEnumerable jsonObjects, + string scorePropertyName, + string operationName, + bool includeVectors, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await foreach (var jsonObject in jsonObjects.ConfigureAwait(false)) + { + var score = jsonObject[scorePropertyName]?.AsValue().GetValue(); + + // Remove score from result object for mapping. + jsonObject.Remove(scorePropertyName); + + var record = this._mapper.MapFromStorageToDataModel(jsonObject, includeVectors); + + yield return new VectorSearchResult(record, score); + } + } + + /// + /// Builds a PartitionKey from the partition key property values in the record. + /// Supports single and hierarchical partition keys. + /// + private PartitionKey BuildPartitionKey(TRecord record) + { + switch (this._partitionKeyProperties) + { + case []: + return PartitionKey.None; + + // Single partition key + case [var property]: + { + return property.Type switch + { + Type t when t == typeof(string) => new PartitionKey(property.GetValue(record)), + Type t when t == typeof(bool) => new PartitionKey(property.GetValue(record)), + Type t when t == typeof(double) => new PartitionKey(property.GetValue(record)), + Type t when t == typeof(Guid) => new PartitionKey(property.GetValue(record).ToString()), + _ => throw new InvalidOperationException($"Unsupported partition key type: {property.Type.Name}") + }; + } + + // Hierarchical partition key (2 or 3 levels) + default: + var builder = new PartitionKeyBuilder(); + + foreach (var property in this._partitionKeyProperties) + { + _ = property.Type switch + { + Type t when t == typeof(string) => builder.Add(property.GetValue(record)), + Type t when t == typeof(bool) => builder.Add(property.GetValue(record)), + Type t when t == typeof(double) => builder.Add(property.GetValue(record)), + Type t when t == typeof(Guid) => builder.Add(property.GetValue(record).ToString()), + _ => throw new InvalidOperationException($"Unsupported partition key type: {property.Type.Name}") + }; + } + + return builder.Build(); + } + } + + #endregion +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionOptions.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionOptions.cs new file mode 100644 index 0000000..b26e936 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionOptions.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Options when creating a . +/// +public sealed class CosmosNoSqlCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly CosmosNoSqlCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public CosmosNoSqlCollectionOptions() + { + } + + internal CosmosNoSqlCollectionOptions(CosmosNoSqlCollectionOptions? source) : base(source) + { + this.JsonSerializerOptions = source?.JsonSerializerOptions; + this.PartitionKeyProperties = source?.PartitionKeyProperties is null ? null : [.. source.PartitionKeyProperties]; + this.IndexingMode = source?.IndexingMode ?? Default.IndexingMode; + this.Automatic = source?.Automatic ?? Default.Automatic; + } + + /// + /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure CosmosDB NoSQL record. + /// + public JsonSerializerOptions? JsonSerializerOptions { get; set; } + + /// + /// Gets or sets the property names to use as partition key components. + /// + /// + /// + /// Selecting a partition key is critical for performance and scalability. Choose properties with high cardinality + /// that evenly distribute requests. See for guidance. + /// + /// + /// When (the default), the key property (document ID) is automatically used as the partition key - a common + /// Cosmos DB strategy; in this mode, the collection key type must be or . + /// To use a different partition key (or hierarchical partition keys), specify the key properties here and use + /// as the key type. + /// + /// + public IReadOnlyList? PartitionKeyProperties { get; set; } + + /// + /// Specifies the indexing mode in the Azure Cosmos DB service. + /// More information here: . + /// + /// + /// Default is . + /// + public IndexingMode IndexingMode { get; set; } = IndexingMode.Consistent; + + /// + /// Gets or sets a value that indicates whether automatic indexing is enabled for a collection in the Azure Cosmos DB service. + /// + /// + /// Default is . + /// + public bool Automatic { get; set; } = true; +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs new file mode 100644 index 0000000..ccdceae --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs @@ -0,0 +1,268 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Azure.Cosmos; +using MEAI = Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Contains helpers to build queries for Azure CosmosDB NoSQL. +/// +internal static class CosmosNoSqlCollectionQueryBuilder +{ + /// + /// Builds to get items from Azure CosmosDB NoSQL using vector search. + /// + public static QueryDefinition BuildSearchQuery( + object vector, + ICollection? keywords, + CollectionModel model, + string vectorPropertyName, + string? distanceFunction, + string? textPropertyName, + string scorePropertyName, + Expression>? filter, + double? scoreThreshold, + int top, + int skip, + bool includeVectors) + { + Throw.IfNull(vector); + + const string VectorVariableName = "@vector"; + const string KeywordVariablePrefix = "@keyword"; + + var tableVariableName = CosmosNoSqlConstants.ContainerAlias; + + IEnumerable projectionProperties = model.Properties; + if (!includeVectors) + { + projectionProperties = projectionProperties.Where(p => p is not VectorPropertyModel); + } + var fieldsArgument = projectionProperties.Select(p => GeneratePropertyAccess(tableVariableName, p.StorageName)); + var vectorDistanceArgument = $"VectorDistance({GeneratePropertyAccess(tableVariableName, vectorPropertyName)}, {VectorVariableName})"; + var vectorDistanceArgumentWithAlias = $"{vectorDistanceArgument} AS {scorePropertyName}"; + + var selectClauseArguments = string.Join(",", [.. fieldsArgument, vectorDistanceArgumentWithAlias]); + + // Build filter object. + var (filterClause, filterParameters) = filter is not null + ? new CosmosNoSqlFilterTranslator().Translate(filter, model) + : ((string?)null, new Dictionary()); + + var queryParameters = new Dictionary + { + [VectorVariableName] = ConvertToVectorDistanceParameter(vector) + }; + + string? fullTextScoreArgument = null; + if (textPropertyName is not null && keywords is not null) + { + var fullTextScoreBuilder = new StringBuilder(); + fullTextScoreBuilder.Append($"FullTextScore({GeneratePropertyAccess(tableVariableName, textPropertyName)}"); + var i = 0; + foreach (var keyword in keywords) + { + var paramName = $"{KeywordVariablePrefix}{i}"; + fullTextScoreBuilder.Append(", ").Append(paramName); + queryParameters[paramName] = keyword; + i++; + } + fullTextScoreBuilder.Append(')'); + fullTextScoreArgument = fullTextScoreBuilder.ToString(); + } + + var rankingArgument = fullTextScoreArgument is null ? vectorDistanceArgument : $"RANK RRF({vectorDistanceArgument}, {fullTextScoreArgument})"; + + // Add score threshold filter if specified. + // For similarity functions (CosineSimilarity, DotProductSimilarity), higher scores are better, so filter with >=. + // For distance functions (EuclideanDistance), lower scores are better, so filter with <=. + const string ScoreThresholdVariableName = "@scoreThreshold"; + string? scoreThresholdClause = null; + if (scoreThreshold.HasValue) + { + var comparisonOperator = distanceFunction switch + { + Microsoft.Extensions.VectorData.DistanceFunction.CosineSimilarity => ">=", + Microsoft.Extensions.VectorData.DistanceFunction.DotProductSimilarity => ">=", + Microsoft.Extensions.VectorData.DistanceFunction.EuclideanDistance => "<=", + _ => throw new NotSupportedException($"Score threshold is not supported for distance function '{distanceFunction}'.") + }; + scoreThresholdClause = $"{vectorDistanceArgument} {comparisonOperator} {ScoreThresholdVariableName}"; + queryParameters[ScoreThresholdVariableName] = scoreThreshold.Value; + } + + // If Offset is not configured, use Top parameter instead of Limit/Offset + // since it's more optimized. Hybrid search doesn't allow top to be passed as a parameter + // so directly add it to the query here. + var topArgument = skip == 0 ? $"TOP {top} " : string.Empty; + + var builder = new StringBuilder(); + + builder.AppendLine($"SELECT {topArgument}{selectClauseArguments}"); + builder.AppendLine($"FROM {tableVariableName}"); + + if (filterClause is not null || scoreThresholdClause is not null) + { + builder.Append("WHERE "); + + if (filterClause is not null) + { + builder.Append(filterClause); + if (scoreThresholdClause is not null) + { + builder.Append(" AND "); + } + } + + if (scoreThresholdClause is not null) + { + builder.Append(scoreThresholdClause); + } + + builder.AppendLine(); + } + + builder.AppendLine($"ORDER BY {rankingArgument}"); + + if (string.IsNullOrEmpty(topArgument)) + { + // Hybrid search doesn't allow offset and limit to be passed as parameters + // so directly add it to the query here. + builder.AppendLine($"OFFSET {skip} LIMIT {top}"); + } + + var queryDefinition = new QueryDefinition(builder.ToString()); + + if (filterParameters is { Count: > 0 }) + { + queryParameters = queryParameters.Union(filterParameters).ToDictionary(k => k.Key, v => v.Value); + } + + foreach (var queryParameter in queryParameters) + { + queryDefinition.WithParameter(queryParameter.Key, queryParameter.Value); + } + + return queryDefinition; + } + + internal static QueryDefinition BuildSearchQuery( + CollectionModel model, + string whereClause, Dictionary filterParameters, + FilteredRecordRetrievalOptions filterOptions, + int top) + { + var tableVariableName = CosmosNoSqlConstants.ContainerAlias; + + IEnumerable projectionProperties = model.Properties; + if (!filterOptions.IncludeVectors) + { + projectionProperties = projectionProperties.Where(p => p is not VectorPropertyModel); + } + + var fieldsArgument = projectionProperties.Select(field => GeneratePropertyAccess(tableVariableName, field.StorageName)); + + var selectClauseArguments = string.Join(",", [.. fieldsArgument]); + + // If Offset is not configured, use Top parameter instead of Limit/Offset + // since it's more optimized. + var topArgument = filterOptions.Skip == 0 ? $"TOP {top} " : string.Empty; + + var builder = new StringBuilder(); + + builder.AppendLine($"SELECT {topArgument}{selectClauseArguments}"); + builder.AppendLine($"FROM {tableVariableName}"); + builder.Append("WHERE ").AppendLine(whereClause); + + var orderBy = filterOptions.OrderBy?.Invoke(new()).Values; + if (orderBy is { Count: > 0 }) + { + builder.Append("ORDER BY "); + + foreach (var sortInfo in orderBy) + { + builder + .Append(GeneratePropertyAccess(tableVariableName, model.GetDataOrKeyProperty(sortInfo.PropertySelector).StorageName)) + .Append(sortInfo.Ascending ? " ASC," : " DESC,"); + } + + builder.Length--; // remove the last comma + builder.AppendLine(); + } + + if (string.IsNullOrEmpty(topArgument)) + { + builder.AppendLine($"OFFSET {filterOptions.Skip} LIMIT {top}"); + } + + var queryDefinition = new QueryDefinition(builder.ToString()); + + foreach (var queryParameter in filterParameters) + { + queryDefinition.WithParameter(queryParameter.Key, queryParameter.Value); + } + + return queryDefinition; + } + + #region private + + private static string GetStoragePropertyName(string propertyName, CollectionModel model) + { + if (!model.PropertyMap.TryGetValue(propertyName, out var property)) + { + throw new InvalidOperationException($"Property name '{propertyName}' provided as part of the filter clause is not a valid property name."); + } + + return property.StorageName; + } + + /// + /// Escapes a JSON property name for use in Cosmos NoSQL SQL queries. + /// JSON property names within bracket notation need backslash and double-quote escaping. + /// + private static string EscapeJsonPropertyName(string propertyName) + => propertyName.Replace(@"\", @"\\").Replace("\"", "\\\""); + + /// + /// Generates a property access expression using bracket notation with proper escaping. + /// + private static string GeneratePropertyAccess(char alias, string propertyName) + => $"{alias}[\"{EscapeJsonPropertyName(propertyName)}\"]"; + + private static object ConvertToVectorDistanceParameter(object vector) + => vector switch + { + ReadOnlyMemory floatMemory => floatMemory.ToArray(), + MEAI.Embedding floatEmbedding => floatEmbedding.Vector.ToArray(), + ReadOnlyMemory byteMemory => ConvertToIntArray(byteMemory.Span), + byte[] byteArray => ConvertToIntArray(byteArray), + MEAI.Embedding byteEmbedding => ConvertToIntArray(byteEmbedding.Vector.Span), + ReadOnlyMemory sbyteMemory => sbyteMemory.ToArray(), + MEAI.Embedding sbyteEmbedding => sbyteEmbedding.Vector.ToArray(), + _ => vector + }; + + private static int[] ConvertToIntArray(ReadOnlySpan bytes) + { + var result = new int[bytes.Length]; + for (var i = 0; i < bytes.Length; i++) + { + result[i] = bytes[i]; + } + + return result; + } + + #endregion +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlConstants.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlConstants.cs new file mode 100644 index 0000000..9b35882 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlConstants.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +internal static class CosmosNoSqlConstants +{ + internal const string VectorStoreSystemName = "azure.cosmosdbnosql"; + + /// + /// Reserved key property name in Azure CosmosDB NoSQL. + /// + internal const string ReservedKeyPropertyName = "id"; + + /// + /// Variable name for table in Azure CosmosDB NoSQL queries. + /// Can be any string. Example: "SELECT x.Name FROM x". + /// + internal const char ContainerAlias = 'x'; +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicCollection.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicCollection.cs new file mode 100644 index 0000000..f53f9bc --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicCollection.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Microsoft.Azure.Cosmos; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Represents a collection of vector store records in a CosmosNoSql database, mapped to a dynamic Dictionary<string, object?>. +/// +/// +/// +/// This collection accepts keys of type , but only instances +/// are supported at runtime. Passing any other key type will result in an . +/// +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class CosmosNoSqlDynamicCollection : CosmosNoSqlCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB NoSQL. + /// The name of the collection. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] + public CosmosNoSqlDynamicCollection(Database database, string name, CosmosNoSqlCollectionOptions options) + : this( + new(database.Client, ownsClient: false), + _ => database, + name, + options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Connection string required to connect to Azure CosmosDB NoSQL. + /// Database name for Azure CosmosDB NoSQL. + /// The name of the collection that this will access. + /// Optional configuration options for . + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] + public CosmosNoSqlDynamicCollection( + string connectionString, + string databaseName, + string collectionName, + CosmosClientOptions? clientOptions = null, + CosmosNoSqlCollectionOptions? options = null) + : this( + new(new CosmosClient(connectionString, clientOptions), ownsClient: true), + client => client.GetDatabase(databaseName), + collectionName, + options) + { + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + Throw.IfNullOrWhitespace(collectionName); + } + + internal CosmosNoSqlDynamicCollection( + ClientWrapper clientWrapper, + Func databaseProvider, + string name, + CosmosNoSqlCollectionOptions? options) + : base( + clientWrapper, + databaseProvider, + name, + static options => new CosmosNoSqlModelBuilder() + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator, + options.JsonSerializerOptions ?? JsonSerializerOptions.Default), + options) + { + } +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicMapper.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicMapper.cs new file mode 100644 index 0000000..d90c1e7 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlDynamicMapper.cs @@ -0,0 +1,197 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using MEAI = Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within Azure CosmosDB NoSQL. +/// +internal sealed class CosmosNoSqlDynamicMapper(CollectionModel model, JsonSerializerOptions jsonSerializerOptions) + : ICosmosNoSqlMapper> +{ + public JsonObject MapFromDataToStorageModel(Dictionary dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + Throw.IfNull(dataModel); + + var jsonObject = new JsonObject + { + [CosmosNoSqlConstants.ReservedKeyPropertyName] = !dataModel.TryGetValue(model.KeyProperty.ModelName, out var keyValue) + ? throw new InvalidOperationException($"Missing value for key property '{model.KeyProperty.ModelName}") + : keyValue switch + { + string s => s, + Guid g => g.ToString(), + + null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."), + _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string.") + } + }; + + foreach (var dataProperty in model.DataProperties) + { + if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue)) + { + jsonObject[dataProperty.StorageName] = dataValue is not null ? + JsonSerializer.SerializeToNode(dataValue, dataProperty.Type, jsonSerializerOptions) : + null; + } + } + + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + + // Don't create a property if it doesn't exist in the dictionary + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + var vector = (object?)generatedEmbeddings?[i]?[recordIndex] ?? vectorValue; + + if (vector is null) + { + jsonObject[property.StorageName] = null; + continue; + } + + var jsonArray = new JsonArray(); + + switch (vector) + { + case var _ when TryGetReadOnlyMemory(vector, out var floatMemory): + foreach (var item in floatMemory.Value.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + case var _ when TryGetReadOnlyMemory(vector, out var byteMemory): + foreach (var item in byteMemory.Value.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + case var _ when TryGetReadOnlyMemory(vector, out var sbyteMemory): + foreach (var item in sbyteMemory.Value.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + default: + throw new UnreachableException(); + } + + jsonObject.Add(property.StorageName, jsonArray); + } + } + + return jsonObject; + + static bool TryGetReadOnlyMemory(object value, [NotNullWhen(true)] out ReadOnlyMemory? memory) + { + memory = value switch + { + ReadOnlyMemory m => m, + Embedding e => e.Vector, + T[] a => a, + _ => (ReadOnlyMemory?)null + }; + + return memory is not null; + } + } + + public Dictionary MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) + { + Throw.IfNull(storageModel); + + var result = new Dictionary(); + + // Loop through all known properties and map each from the storage model to the data model. + foreach (var property in model.Properties) + { + switch (property) + { + case KeyPropertyModel keyProperty: + var key = (string?)storageModel[CosmosNoSqlConstants.ReservedKeyPropertyName] + ?? throw new InvalidOperationException($"The key property '{keyProperty.StorageName}' is missing from the record retrieved from storage."); + + result[keyProperty.ModelName] = keyProperty.Type switch + { + var t when t == typeof(string) => key, + var t when t == typeof(Guid) => Guid.Parse(key), + _ => throw new UnreachableException() + }; + + continue; + + case DataPropertyModel dataProperty: + if (storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dataValue)) + { + result.Add(property.ModelName, dataValue.Deserialize(property.Type, jsonSerializerOptions)); + } + continue; + + case VectorPropertyModel vectorProperty: + if (includeVectors && storageModel.TryGetPropertyValue(vectorProperty.StorageName, out var vectorValue)) + { + if (vectorValue is not null) + { + result.Add( + vectorProperty.ModelName, + (Nullable.GetUnderlyingType(vectorProperty.Type) ?? vectorProperty.Type) switch + { + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(ToArray(vectorValue)), + Type t when t == typeof(Embedding) => new Embedding(ToArray(vectorValue)), + Type t when t == typeof(float[]) => ToArray(vectorValue), + + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(ToArray(vectorValue)), + Type t when t == typeof(Embedding) => new Embedding(ToArray(vectorValue)), + Type t when t == typeof(byte[]) => ToArray(vectorValue), + + Type t when t == typeof(ReadOnlyMemory) => new ReadOnlyMemory(ToArray(vectorValue)), + Type t when t == typeof(Embedding) => new Embedding(ToArray(vectorValue)), + Type t when t == typeof(sbyte[]) => ToArray(vectorValue), + + _ => throw new UnreachableException() + }); + } + else + { + result.Add(vectorProperty.ModelName, null); + } + } + continue; + + default: + throw new UnreachableException(); + } + } + + return result; + + static T[] ToArray(JsonNode jsonNode) + { + var jsonArray = jsonNode.AsArray(); + var array = new T[jsonArray.Count]; + + for (var i = 0; i < jsonArray.Count; i++) + { + array[i] = jsonArray[i]!.GetValue(); + } + + return array; + } + } +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlFilterTranslator.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlFilterTranslator.cs new file mode 100644 index 0000000..7bed782 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlFilterTranslator.cs @@ -0,0 +1,371 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +#pragma warning disable MEVD9001 // Experimental: filter translation base types + +internal class CosmosNoSqlFilterTranslator : FilterTranslatorBase +{ + private readonly Dictionary _parameters = []; + private readonly StringBuilder _sql = new(); + + internal (string WhereClause, Dictionary Parameters) Translate(LambdaExpression lambdaExpression, CollectionModel model) + { + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true }); + + this.Translate(preprocessedExpression); + + return (this._sql.ToString(), this._parameters); + } + + private void Translate(Expression? node) + { + switch (node) + { + case BinaryExpression binary: + this.TranslateBinary(binary); + return; + + case ConstantExpression constant: + this.TranslateConstant(constant); + return; + + case QueryParameterExpression { Name: var name, Value: var value }: + this.TranslateQueryParameter(name, value); + return; + + case MemberExpression member: + this.TranslateMember(member); + return; + + case NewArrayExpression newArray: + this.TranslateNewArray(newArray); + return; + + case MethodCallExpression methodCall: + this.TranslateMethodCall(methodCall); + return; + + case UnaryExpression unary: + this.TranslateUnary(unary); + return; + + default: + throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType); + } + } + + private void TranslateBinary(BinaryExpression binary) + { + this._sql.Append('('); + this.Translate(binary.Left); + + this._sql.Append(binary.NodeType switch + { + ExpressionType.Equal => " = ", + ExpressionType.NotEqual => " <> ", + + ExpressionType.GreaterThan => " > ", + ExpressionType.GreaterThanOrEqual => " >= ", + ExpressionType.LessThan => " < ", + ExpressionType.LessThanOrEqual => " <= ", + + ExpressionType.AndAlso => " AND ", + ExpressionType.OrElse => " OR ", + + _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType) + }); + + this.Translate(binary.Right); + this._sql.Append(')'); + } + + private void TranslateConstant(ConstantExpression constant) + => this.TranslateConstant(constant.Value); + + private void TranslateConstant(object? value) + { + switch (value) + { + case byte v: + this._sql.Append(v); + return; + case short v: + this._sql.Append(v); + return; + case int v: + this._sql.Append(v); + return; + case long v: + this._sql.Append(v); + return; + + case float v: + this._sql.Append(v); + return; + case double v: + this._sql.Append(v); + return; + + case string v: + this._sql.Append('"').Append(v.Replace(@"\", @"\\").Replace("\"", "\\\"")).Append('"'); + return; + case bool v: + this._sql.Append(v ? "true" : "false"); + return; + case Guid v: + this._sql.Append('"').Append(v.ToString()).Append('"'); + return; + + case DateTimeOffset v: + // Cosmos doesn't support DateTimeOffset with non-zero offset, so we convert it to UTC. + // See https://github.com/dotnet/efcore/issues/35310 + this._sql + .Append('"') + .Append(v.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)) + .Append("Z\""); + return; + + case DateTime v: + this._sql + .Append('"') + .Append(v.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)) + .Append('"'); + return; + +#if NET + case DateOnly v: + this._sql + .Append('"') + .Append(v.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)) + .Append('"'); + return; +#endif + + case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)): + this._sql.Append('['); + + var i = 0; + foreach (var element in v) + { + if (i++ > 0) + { + this._sql.Append(','); + } + + this.TranslateConstant(element); + } + + this._sql.Append(']'); + return; + + case null: + this._sql.Append("null"); + return; + + default: + throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name); + } + } + + private void TranslateMember(MemberExpression memberExpression) + { + if (this.TryBindProperty(memberExpression, out var property)) + { + this.GeneratePropertyAccess(property); + return; + } + + throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported"); + } + + private void TranslateNewArray(NewArrayExpression newArray) + { + this._sql.Append('['); + + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (i > 0) + { + this._sql.Append(", "); + } + + this.Translate(newArray.Expressions[i]); + } + + this._sql.Append(']'); + } + + private void TranslateMethodCall(MethodCallExpression methodCall) + { + // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") + if (this.TryBindProperty(methodCall, out var property)) + { + this.GeneratePropertyAccess(property); + return; + } + + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): + this.TranslateContains(source, item); + return; + + // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any + when any.Method.DeclaringType == typeof(Enumerable): + this.TranslateAny(anySource, lambda); + return; + + default: + throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); + } + } + + private void TranslateContains(Expression source, Expression item) + { + this._sql.Append("ARRAY_CONTAINS("); + this.Translate(source); + this._sql.Append(", "); + this.Translate(item); + this._sql.Append(')'); + } + + /// + /// Translates an Any() call with a Contains predicate, e.g. r.Strings.Any(s => array.Contains(s)). + /// This checks whether any element in the array field is contained in the given values. + /// + private void TranslateAny(Expression source, LambdaExpression lambda) + { + // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) + // Translates to: EXISTS(SELECT VALUE t FROM t IN c["Field"] WHERE ARRAY_CONTAINS(@values, t)) + if (!this.TryBindProperty(source, out var property) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Verify that the item is the lambda parameter + if (itemExpression != lambda.Parameters[0]) + { + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + + // Now extract the values from valuesExpression + // Generate: EXISTS(SELECT VALUE t FROM t IN c["Field"] WHERE ARRAY_CONTAINS(@values, t)) + switch (valuesExpression) + { + // Inline array: r.Strings.Any(s => new[] { "a", "b" }.Contains(s)) + case NewArrayExpression newArray: + { + var values = new object?[newArray.Expressions.Count]; + for (var i = 0; i < newArray.Expressions.Count; i++) + { + values[i] = newArray.Expressions[i] switch + { + ConstantExpression { Value: var v } => v, + QueryParameterExpression { Value: var v } => v, + _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") + }; + } + + this.GenerateAnyContains(property, values); + return; + } + + // Captured/parameterized array or list: r.Strings.Any(s => capturedArray.Contains(s)) + case QueryParameterExpression queryParameter: + this.GenerateAnyContains(property, queryParameter); + return; + + // Constant array: shouldn't normally happen, but handle it + case ConstantExpression { Value: var value }: + this.GenerateAnyContains(property, value); + return; + + default: + throw new NotSupportedException("Unsupported method call: Enumerable.Any"); + } + } + + private void GenerateAnyContains(PropertyModel property, object? values) + { + this._sql.Append("EXISTS(SELECT VALUE t FROM t IN "); + this.GeneratePropertyAccess(property); + this._sql.Append(" WHERE ARRAY_CONTAINS("); + this.TranslateConstant(values); + this._sql.Append(", t))"); + } + + private void GenerateAnyContains(PropertyModel property, QueryParameterExpression queryParameter) + { + this._sql.Append("EXISTS(SELECT VALUE t FROM t IN "); + this.GeneratePropertyAccess(property); + this._sql.Append(" WHERE ARRAY_CONTAINS("); + this.TranslateQueryParameter(queryParameter.Name, queryParameter.Value); + this._sql.Append(", t))"); + } + + private void TranslateUnary(UnaryExpression unary) + { + switch (unary.NodeType) + { + // Special handling for !(a == b) and !(a != b) + case ExpressionType.Not: + if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) + { + this.TranslateBinary( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + return; + } + + this._sql.Append("(NOT "); + this.Translate(unary.Operand); + this._sql.Append(')'); + return; + + // Handle converting non-nullable to nullable; such nodes are found in e.g. r => r.Int == nullableInt + case ExpressionType.Convert when Nullable.GetUnderlyingType(unary.Type) == unary.Operand.Type: + this.Translate(unary.Operand); + return; + + // Handle convert over member access, for dynamic dictionary access (r => (int)r["SomeInt"] == 8) + case ExpressionType.Convert when this.TryBindProperty(unary.Operand, out var property) && unary.Type == property.Type: + this.GeneratePropertyAccess(property); + return; + + default: + throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); + } + } + + protected void TranslateQueryParameter(string name, object? value) + { + name = '@' + name; + this._parameters.Add(name, value); + this._sql.Append(name); + } + + protected virtual void GeneratePropertyAccess(PropertyModel property) + => this._sql + .Append(CosmosNoSqlConstants.ContainerAlias) + .Append("[\"") + .Append(property.StorageName.Replace(@"\", @"\\").Replace("\"", "\\\"")) + .Append("\"]"); +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlKey.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlKey.cs new file mode 100644 index 0000000..90709bc --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlKey.cs @@ -0,0 +1,184 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Azure.Cosmos; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Represents a key for Azure CosmosDB NoSQL, containing both the record key (document ID) and the partition key. +/// +/// +/// +/// Azure Cosmos DB requires both a record key (the unique document identifier within a partition) and a partition key +/// to identify a document. This struct encapsulates both values together with the Cosmos SDK's type. +/// +/// +/// For simple partition keys, use the convenience constructors that accept , , or +/// partition key values. For hierarchical partition keys (up to 3 levels), construct a using +/// and pass it to the primary constructor. +/// +/// +// This is conceptually a record, but we're targeting .NET Standard 2.0 too. +public readonly struct CosmosNoSqlKey : IEquatable +{ + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(string documentId, string partitionKey) + { + this.DocumentId = documentId; + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(Guid documentId, string partitionKey) + { + this.DocumentId = documentId.ToString(); + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(string documentId, Guid partitionKey) + { + this.DocumentId = documentId; + this.PartitionKey = new PartitionKey(partitionKey.ToString()); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(Guid documentId, Guid partitionKey) + { + this.DocumentId = documentId.ToString(); + this.PartitionKey = new PartitionKey(partitionKey.ToString()); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(string documentId, bool partitionKey) + { + this.DocumentId = documentId; + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(Guid documentId, bool partitionKey) + { + this.DocumentId = documentId.ToString(); + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(string documentId, double partitionKey) + { + this.DocumentId = documentId; + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + public CosmosNoSqlKey(Guid documentId, double partitionKey) + { + this.DocumentId = documentId.ToString(); + this.PartitionKey = new PartitionKey(partitionKey); + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + /// + /// Use this constructor for hierarchical partition keys or special partition key values like or . + /// For hierarchical partition keys, construct the using . + /// + public CosmosNoSqlKey(string documentId, PartitionKey partitionKey) + { + this.DocumentId = documentId; + this.PartitionKey = partitionKey; + } + + /// + /// Initializes a new instance of the struct with a document ID and partition key. + /// + /// The document ID. + /// The Cosmos DB partition key. + /// + /// Use this constructor for hierarchical partition keys or special partition key values like or . + /// For hierarchical partition keys, construct the using . + /// + public CosmosNoSqlKey(Guid documentId, PartitionKey partitionKey) + { + this.DocumentId = documentId.ToString(); + this.PartitionKey = partitionKey; + } + + /// + /// Gets the document ID. + /// + /// + /// The document ID uniquely identifies a document within a partition. It is stored in the id property of the Cosmos DB document. + /// + public string DocumentId { get; } + + /// + /// Gets the partition key. + /// + /// + /// The partition key determines which logical partition the document belongs to. + /// See for guidance on choosing partition keys. + /// + public PartitionKey PartitionKey { get; } + + /// + public bool Equals(CosmosNoSqlKey other) + => Equals(this.DocumentId, other.DocumentId) && this.PartitionKey.Equals(other.PartitionKey); + + /// + public override bool Equals(object? obj) + => obj is CosmosNoSqlKey other && this.Equals(other); + + /// + public override int GetHashCode() + => HashCode.Combine(this.DocumentId, this.PartitionKey); + + /// + /// Determines whether two instances are equal. + /// + public static bool operator ==(CosmosNoSqlKey left, CosmosNoSqlKey right) + => left.Equals(right); + + /// + /// Determines whether two instances are not equal. + /// + public static bool operator !=(CosmosNoSqlKey left, CosmosNoSqlKey right) + => !left.Equals(right); +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlMapper.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlMapper.cs new file mode 100644 index 0000000..7b97536 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlMapper.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; +using MEAI = Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Class for mapping between a json node stored in Azure CosmosDB NoSQL and the consumer data model. +/// +/// The consumer data model to map to or from. +internal sealed class CosmosNoSqlMapper : ICosmosNoSqlMapper + where TRecord : class +{ + private readonly CollectionModel _model; + private readonly KeyPropertyModel _keyProperty; + private readonly JsonSerializerOptions _jsonSerializerOptions; + + public CosmosNoSqlMapper(CollectionModel model, JsonSerializerOptions? jsonSerializerOptions) + { + this._model = model; + this._keyProperty = model.KeyProperty; + + // Add byte array converters to serialize byte[] and ReadOnlyMemory as JSON arrays of numbers + // instead of base64-encoded strings, which is required for Cosmos DB vector operations. + var newOptions = new JsonSerializerOptions(jsonSerializerOptions ?? JsonSerializerOptions.Default); + newOptions.Converters.Add(new ByteArrayJsonConverter()); + newOptions.Converters.Add(new ReadOnlyMemoryByteJsonConverter()); + this._jsonSerializerOptions = newOptions; + } + + public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings) + { + var jsonObject = JsonSerializer.SerializeToNode(dataModel, this._jsonSerializerOptions)!.AsObject(); + + // The key property in Azure CosmosDB NoSQL is always named 'id'. + // But the external JSON serializer used just above isn't aware of that, and will produce a JSON object with another name, taking into + // account e.g. naming policies. SerializedKeyName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set + RenameJsonProperty(jsonObject, this._keyProperty.SerializedKeyName!, CosmosNoSqlConstants.ReservedKeyPropertyName); + + // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above. + // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object). + for (var i = 0; i < this._model.VectorProperties.Count; i++) + { + var property = this._model.VectorProperties[i]; + + object? embedding = generatedEmbeddings?[i]?[recordIndex]; + + if (embedding is null) + { + switch (Nullable.GetUnderlyingType(property.Type) ?? property.Type) + { + // For raw array/memory types, JsonSerializer already serialized them correctly above + // (including byte[] thanks to our custom converter). Nothing more to do. + case var t when t == typeof(ReadOnlyMemory): + case var t2 when t2 == typeof(float[]): + case var t3 when t3 == typeof(ReadOnlyMemory): + case var t4 when t4 == typeof(byte[]): + case var t5 when t5 == typeof(ReadOnlyMemory): + case var t6 when t6 == typeof(sbyte[]): + continue; + + // For Embedding types, we need to extract the vector and serialize it as a simple array + case var t when t == typeof(Embedding): + case var t1 when t1 == typeof(Embedding): + case var t2 when t2 == typeof(Embedding): + embedding = property.GetValueAsObject(dataModel); + break; + + default: + throw new UnreachableException(); + } + } + + var jsonArray = new JsonArray(); + + switch (embedding) + { + case Embedding e: + foreach (var item in e.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + case Embedding e: + foreach (var item in e.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + case Embedding e: + foreach (var item in e.Vector.Span) + { + jsonArray.Add(JsonValue.Create(item)); + } + break; + + default: + throw new UnreachableException(); + } + + jsonObject[property.StorageName] = jsonArray; + } + + return jsonObject; + } + + public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) + { + // See above comment. + RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.SerializedKeyName!); + + foreach (var vectorProperty in this._model.VectorProperties) + { + if (!includeVectors) + { + // Remove vector properties so they deserialize as default (e.g. empty ReadOnlyMemory). + storageModel.Remove(vectorProperty.StorageName); + continue; + } + + var arrayNode = storageModel[vectorProperty.StorageName]; + if (arrayNode is null) + { + continue; + } + + // Embedding is stored as a simple JSON array, so convert it to the expected object shape for deserialization + if (vectorProperty.Type == typeof(Embedding) + || vectorProperty.Type == typeof(Embedding) + || vectorProperty.Type == typeof(Embedding)) + { + storageModel[vectorProperty.StorageName] = new JsonObject + { + [nameof(Embedding<>.Vector)] = arrayNode.DeepClone() + }; + } + + // For byte[], ReadOnlyMemory, sbyte[], ReadOnlyMemory, float[], ReadOnlyMemory, + // the custom converters (for byte) and default converters (for others) handle deserialization correctly. + } + + return storageModel.Deserialize(this._jsonSerializerOptions)!; + } + + #region private + + private static void RenameJsonProperty(JsonObject jsonObject, string oldKey, string newKey) + { + if (jsonObject is not null && jsonObject.ContainsKey(oldKey)) + { + JsonNode? value = jsonObject[oldKey]; + + jsonObject.Remove(oldKey); + + jsonObject[newKey] = value; + } + } + + #endregion +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlModelBuilder.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlModelBuilder.cs new file mode 100644 index 0000000..9184a92 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlModelBuilder.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +internal class CosmosNoSqlModelBuilder() : CollectionJsonModelBuilder(s_modelBuildingOptions) +{ + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, byte[], ReadOnlyMemory, Embedding, sbyte[]"; + + private static readonly CollectionModelBuildingOptions s_modelBuildingOptions = new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleVectors = true, + UsesExternalSerializer = true, + ReservedKeyStorageName = CosmosNoSqlConstants.ReservedKeyPropertyName + }; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + // CosmosNoSqlKey is a composite key type (document ID + partition key) that doesn't correspond to any single key property type; + // skip the base TKey-to-key-property validation when it's used. + if (this.KeyType != typeof(CosmosNoSqlKey)) + { + base.ValidateKeyProperty(keyProperty); + } + + // Note that the key property in Cosmos NoSQL refers to the document ID, not to the CosmosNoSqlKey structure which includes both + // the document ID and the partition key (and which is the generic TKey type parameter of the collection). + var type = keyProperty.Type; + + if (type != typeof(string) && type != typeof(Guid)) + { + throw new NotSupportedException( + $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string or Guid."); + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return IsValid(type) + || (type.IsArray && IsValid(type.GetElementType()!)) + || (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) && IsValid(type.GenericTypeArguments[0])); + + static bool IsValid(Type type) + => type == typeof(bool) || + type == typeof(string) || + type == typeof(int) || + type == typeof(long) || + type == typeof(float) || + type == typeof(double) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif + type == typeof(DateTimeOffset); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + /// + protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = + [ + EmbeddingGenerationDispatcher.Create>(), + EmbeddingGenerationDispatcher.Create>(), + EmbeddingGenerationDispatcher.Create>() + ]; + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + + if (Nullable.GetUnderlyingType(type) is Type underlyingType) + { + type = underlyingType; + } + + return type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(float[]) + || type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(byte[]) + || type == typeof(ReadOnlyMemory) + || type == typeof(Embedding) + || type == typeof(sbyte[]); + } +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlServiceCollectionExtensions.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlServiceCollectionExtensions.cs new file mode 100644 index 0000000..0c6b045 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlServiceCollectionExtensions.cs @@ -0,0 +1,346 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.Extensions.DependencyInjection; +#pragma warning restore IDE0130 // Namespace does not match folder structure + +/// +/// Extension methods to register Azure CosmosDB NoSQL instances on an . +/// +public static class CosmosNoSqlServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "The Cosmos NoSQL provider is currently incompatible with NativeAOT."; + private const string UnreferencedCodeMessage = "The Cosmos NoSQL provider is currently incompatible with trimming."; + + /// + /// Registers a as + /// with retrieved from the dependency injection container. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosNoSqlVectorStore( + this IServiceCollection services, + CosmosNoSqlVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedCosmosNoSqlVectorStore(services, serviceKey: null, options, lifetime); + + /// + /// Registers a keyed as + /// with retrieved from the dependency injection container. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosNoSqlVectorStore( + this IServiceCollection services, + object? serviceKey, + CosmosNoSqlVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(CosmosNoSqlVectorStore), serviceKey, (sp, _) => + { + var database = sp.GetRequiredService(); + options = GetStoreOptions(sp, _ => options); + + return new CosmosNoSqlVectorStore(database, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosNoSqlVectorStore( + this IServiceCollection services, + string connectionString, + string databaseName, + CosmosNoSqlVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedCosmosNoSqlVectorStore(services, serviceKey: null, connectionString, databaseName, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// Connection string required to connect to Azure CosmosDB NoSQL. + /// Database name for Azure CosmosDB NoSQL. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosNoSqlVectorStore( + this IServiceCollection services, + object? serviceKey, + string connectionString, + string databaseName, + CosmosNoSqlVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + + services.Add(new ServiceDescriptor(typeof(CosmosNoSqlVectorStore), serviceKey, (sp, _) => + { + options = GetStoreOptions(sp, _ => options); + var clientOptions = CreateClientOptions(options?.JsonSerializerOptions); + + return new CosmosNoSqlVectorStore(connectionString, databaseName, clientOptions, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with retrieved from the dependency injection container. + /// + /// + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddCosmosNoSqlCollection( + this IServiceCollection services, + string name, + CosmosNoSqlCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedCosmosNoSqlCollection(services, serviceKey: null, name, options, lifetime); + + /// + /// Registers a keyed as + /// with retrieved from the dependency injection container. + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(DynamicCodeMessage)] + [RequiresDynamicCode(UnreferencedCodeMessage)] + public static IServiceCollection AddKeyedCosmosNoSqlCollection( + this IServiceCollection services, + object? serviceKey, + string name, + CosmosNoSqlCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(CosmosNoSqlCollection), serviceKey, (sp, _) => + { + var database = sp.GetRequiredService(); + options = GetCollectionOptions(sp, _ => options); + + return new CosmosNoSqlCollection(database, name, options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddCosmosNoSqlCollection( + this IServiceCollection services, + string name, + string connectionString, + string databaseName, + CosmosNoSqlCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedCosmosNoSqlCollection(services, serviceKey: null, name, connectionString, databaseName, options, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// Connection string required to connect to Azure CosmosDB NoSQL. + /// Database name for Azure CosmosDB NoSQL. + /// Optional options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedCosmosNoSqlCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string connectionString, + string databaseName, + CosmosNoSqlCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + + return AddKeyedCosmosNoSqlCollection(services, serviceKey, name, _ => connectionString, _ => databaseName, _ => options!, lifetime); + } + + /// + /// Registers a as + /// using the provided and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddCosmosNoSqlCollection( + this IServiceCollection services, + string name, + Func connectionStringProvider, + Func databaseNameProvider, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedCosmosNoSqlCollection(services, serviceKey: null, name, connectionStringProvider, databaseNameProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// using the provided and . + /// + /// The type of the key. + /// The type of the record. + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The connection string provider. + /// The database name provider. + /// Options provider to further configure the collection. + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedCosmosNoSqlCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func connectionStringProvider, + Func databaseNameProvider, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + Throw.IfNull(connectionStringProvider); + Throw.IfNull(databaseNameProvider); + + services.Add(new ServiceDescriptor(typeof(CosmosNoSqlCollection), serviceKey, (sp, _) => + { + var options = GetCollectionOptions(sp, optionsProvider); + var clientOptions = CreateClientOptions(options?.JsonSerializerOptions); + + return new CosmosNoSqlCollection( + connectionString: connectionStringProvider(sp), + databaseName: databaseNameProvider(sp), + name: name, + clientOptions, + options); + }, lifetime)); + + AddAbstractions(services, serviceKey, lifetime); + + return services; + } + + private static void AddAbstractions(IServiceCollection services, object? serviceKey, ServiceLifetime lifetime) + where TKey : notnull + where TRecord : class + { + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + } + + private static CosmosNoSqlVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static CosmosNoSqlCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static CosmosClientOptions CreateClientOptions(JsonSerializerOptions? jsonSerializerOptions) => new() + { + ApplicationName = "CommunityToolkit.VectorData.CosmosNoSql", + UseSystemTextJsonSerializerWithOptions = jsonSerializerOptions ?? JsonSerializerOptions.Default, + }; +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStore.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStore.cs new file mode 100644 index 0000000..020fb57 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStore.cs @@ -0,0 +1,200 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Class for accessing the list of collections in a Azure CosmosDB NoSQL vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class CosmosNoSqlVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// that can be used to manage the collections in Azure CosmosDB NoSQL. + private readonly Database _database; + + private readonly ClientWrapper _clientWrapper; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string))] }; + + private readonly JsonSerializerOptions? _jsonSerializerOptions; + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB NoSQL. + /// Optional configuration options for this class. + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] + public CosmosNoSqlVectorStore(Database database, CosmosNoSqlVectorStoreOptions? options = null) + : this(new(database.Client, ownsClient: false), _ => database, options) + { + Throw.IfNull(database); + } + + /// + /// Initializes a new instance of the class. + /// + /// Connection string required to connect to Azure CosmosDB NoSQL. + /// Database name for Azure CosmosDB NoSQL. + /// Optional configuration options for . + /// Optional configuration options for . + public CosmosNoSqlVectorStore(string connectionString, string databaseName, + CosmosClientOptions? clientOptions = null, CosmosNoSqlVectorStoreOptions? storeOptions = null) + : this(new ClientWrapper(new CosmosClient(connectionString, clientOptions), ownsClient: true), client => client.GetDatabase(databaseName), storeOptions) + { + Throw.IfNullOrWhitespace(connectionString); + Throw.IfNullOrWhitespace(databaseName); + } + + private CosmosNoSqlVectorStore(ClientWrapper clientWrapper, + Func databaseProvider, CosmosNoSqlVectorStoreOptions? options) + { + try + { + this._database = databaseProvider(clientWrapper.Client); + this._embeddingGenerator = options?.EmbeddingGenerator; + this._jsonSerializerOptions = options?.JsonSerializerOptions; + + this._metadata = new() + { + VectorStoreSystemName = CosmosNoSqlConstants.VectorStoreSystemName, + VectorStoreName = this._database.Id + }; + } + catch (Exception) + { + // Something went wrong, we dispose the client and don't store a reference. + clientWrapper.Dispose(); + + throw; + } + + this._clientWrapper = clientWrapper; + } + + /// + protected override void Dispose(bool disposing) + { + this._clientWrapper.Dispose(); + base.Dispose(disposing); + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] +#if NET + public override CosmosNoSqlCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new CosmosNoSqlCollection( + this._clientWrapper.Share(), + _ => this._database, + name, + new() + { + Definition = definition, + JsonSerializerOptions = this._jsonSerializerOptions, + EmbeddingGenerator = this._embeddingGenerator + }); + + /// + [RequiresUnreferencedCode("The Cosmos NoSQL provider is currently incompatible with trimming.")] + [RequiresDynamicCode("The Cosmos NoSQL provider is currently incompatible with NativeAOT.")] +#if NET + public override CosmosNoSqlDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new CosmosNoSqlDynamicCollection( + this._clientWrapper.Share(), + _ => this._database, + name, + new() + { + Definition = definition, + JsonSerializerOptions = this._jsonSerializerOptions, + EmbeddingGenerator = this._embeddingGenerator + } + ); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string Query = "SELECT c.id FROM c"; + + const string OperationName = "ListCollectionNamesAsync"; + using var feedIterator = VectorStoreErrorHandler.RunOperation, CosmosException>( + this._metadata, + OperationName, + () => this._database.GetContainerQueryIterator(Query)); + + while (feedIterator.HasMoreResults) + { + var next = await feedIterator.ReadNextAsync(cancellationToken).ConfigureAwait(false); + + foreach (var result in next.Resource) + { + yield return result.Id; + } + } + } + + private sealed class ContainerIdResult + { + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + } + + /// + public override Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + var collection = this.GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.CollectionExistsAsync(cancellationToken); + } + + /// + public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + var collection = this.GetDynamicCollection(name, s_generalPurposeDefinition); + return collection.EnsureCollectionDeletedAsync(cancellationToken); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStoreOptions.cs b/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStoreOptions.cs new file mode 100644 index 0000000..76c8765 --- /dev/null +++ b/MEVD/src/CosmosNoSql/CosmosNoSqlVectorStoreOptions.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +/// +/// Options when creating a . +/// +public sealed class CosmosNoSqlVectorStoreOptions +{ + /// + /// Initializes a new instance of the class. + /// + public CosmosNoSqlVectorStoreOptions() + { + } + + internal CosmosNoSqlVectorStoreOptions(CosmosNoSqlVectorStoreOptions? source) + { + this.JsonSerializerOptions = source?.JsonSerializerOptions; + this.EmbeddingGenerator = source?.EmbeddingGenerator; + } + + /// + /// Gets or sets the JSON serializer options to use when converting between the data model and the Azure CosmosDB NoSQL record. + /// + public JsonSerializerOptions? JsonSerializerOptions { get; set; } + + /// + /// Gets or sets the default embedding generator to use when generating vectors embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/CosmosNoSql/ErrorHandlingFeedIterator.cs b/MEVD/src/CosmosNoSql/ErrorHandlingFeedIterator.cs new file mode 100644 index 0000000..e79d760 --- /dev/null +++ b/MEVD/src/CosmosNoSql/ErrorHandlingFeedIterator.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +internal class ErrorHandlingFeedIterator : FeedIterator +{ + private readonly FeedIterator _internalFeedIterator; + private readonly string _operationName; + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + public ErrorHandlingFeedIterator( + FeedIterator internalFeedIterator, + VectorStoreCollectionMetadata collectionMetadata, + string operationName) + { + this._internalFeedIterator = internalFeedIterator; + this._operationName = operationName; + this._collectionMetadata = collectionMetadata; + } + + public ErrorHandlingFeedIterator( + FeedIterator internalFeedIterator, + VectorStoreMetadata metadata, + string operationName) + { + this._internalFeedIterator = internalFeedIterator; + this._operationName = operationName; + this._collectionMetadata = new VectorStoreCollectionMetadata() + { + CollectionName = null, + VectorStoreName = metadata.VectorStoreName, + VectorStoreSystemName = metadata.VectorStoreSystemName, + }; + } + + public override bool HasMoreResults => this._internalFeedIterator.HasMoreResults; + + public override Task> ReadNextAsync(CancellationToken cancellationToken = default) + { + return VectorStoreErrorHandler.RunOperationAsync, CosmosException>( + this._collectionMetadata, + this._operationName, + () => this._internalFeedIterator.ReadNextAsync(cancellationToken)); + } +} diff --git a/MEVD/src/CosmosNoSql/ICosmosNoSQLMapper.cs b/MEVD/src/CosmosNoSql/ICosmosNoSQLMapper.cs new file mode 100644 index 0000000..957fcaa --- /dev/null +++ b/MEVD/src/CosmosNoSql/ICosmosNoSQLMapper.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Nodes; +using MEAI = Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.CosmosNoSql; + +internal interface ICosmosNoSqlMapper +{ + /// + /// Maps from the consumer record data model to the storage model. + /// + JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, IReadOnlyList?[]? generatedEmbeddings); + + /// + /// Maps from the storage model to the consumer record data model. + /// + TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors); +} diff --git a/MEVD/src/CosmosNoSql/README.md b/MEVD/src/CosmosNoSql/README.md new file mode 100644 index 0000000..c850314 --- /dev/null +++ b/MEVD/src/CosmosNoSql/README.md @@ -0,0 +1,72 @@ +# CommunityToolkit.VectorData.CosmosNoSql + +`CommunityToolkit.VectorData.CosmosNoSql` is the Azure Cosmos DB for NoSQL provider for `Microsoft.Extensions.VectorData`. + +## Install + +```xml + +``` + +## Get started + +```csharp +using Microsoft.Azure.Cosmos; +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Extensions.VectorData; + +CosmosClient client = new(""); +Database database = client.GetDatabase("vector-database"); + +VectorStore vectorStore = new CosmosNoSqlVectorStore(database); +VectorStoreCollection collection = + vectorStore.GetCollection("hotels"); +``` + +You can also register the provider with dependency injection: + +```csharp +using Microsoft.Extensions.DependencyInjection; + +services.AddCosmosNoSqlVectorStore("", "vector-database"); +services.AddCosmosNoSqlCollection( + name: "hotels", + connectionString: "", + databaseName: "vector-database"); +``` + +## Define a record model + +```csharp +using Microsoft.Extensions.VectorData; + +public sealed class Hotel +{ + [VectorStoreKey] + public string HotelId { get; set; } = string.Empty; + + [VectorStoreData(IsIndexed = true)] + public string HotelName { get; set; } = string.Empty; + + [VectorStoreVector(Dimensions: 1536, DistanceFunction = DistanceFunction.CosineSimilarity)] + public ReadOnlyMemory DescriptionEmbedding { get; set; } +} +``` + +## Create and query a collection + +```csharp +await collection.EnsureCollectionExistsAsync(); + +await collection.UpsertAsync(new Hotel +{ + HotelId = "hotel-1", + HotelName = "Contoso Suites", + DescriptionEmbedding = embedding +}); + +await foreach (VectorSearchResult result in collection.SearchAsync(embedding, top: 3)) +{ + Console.WriteLine($"{result.Record.HotelName}: {result.Score}"); +} +``` diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj new file mode 100644 index 0000000..36d37da --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj @@ -0,0 +1,30 @@ + + + + net10.0 + enable + enable + true + false + CosmosNoSql.ConformanceTests + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlBasicModelTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlBasicModelTests.cs new file mode 100644 index 0000000..cf8ffb2 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlBasicModelTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlBasicModelTests(CosmosNoSqlBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override Task SearchAsync_with_Skip() + { + // The vNext emulator's DiskANN index does not support OFFSET in vector search. + if (!((CosmosNoSqlTestStore)fixture.TestStore).UsesLocalEmulator) + { + return base.SearchAsync_with_Skip(); + } + + return Task.CompletedTask; + } + + public new sealed class Fixture : BasicModelTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlBasicModelTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionManagementTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionManagementTests.cs new file mode 100644 index 0000000..5c3664e --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlCollectionManagementTests.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlCollectionManagementTests(CosmosNoSqlFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDataTypeTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDataTypeTests.cs new file mode 100644 index 0000000..26357bd --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDataTypeTests.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +// The type is internal to disable the tests due to emulator limitations +internal sealed class CosmosNoSqlDataTypeTests(CosmosNoSqlDataTypeTests.Fixture fixture) + : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + public override Task DateTimeOffset() + => Task.CompletedTask; + + public new sealed class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlDataTypeTests))); + + public override TestStore TestStore => _store.Value; + + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(decimal), + typeof(Guid), + typeof(TimeOnly) + ]; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDependencyInjectionTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDependencyInjectionTests.cs new file mode 100644 index 0000000..d269e77 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDependencyInjectionTests.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using VectorData.ConformanceTests; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlDependencyInjectionTests + : DependencyInjectionTests.Record>, string, DependencyInjectionTests.Record> +{ + private const string TestConnectionString = "AccountEndpoint=https://localhost:8081;AccountKey=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="; + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddCosmosNoSqlVectorStore(TestConnectionString, nameof(CosmosNoSqlDependencyInjectionTests), lifetime: lifetime) + : services.AddKeyedCosmosNoSqlVectorStore(serviceKey, TestConnectionString, nameof(CosmosNoSqlDependencyInjectionTests), lifetime: lifetime); + } + } + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddCosmosNoSqlCollection(name, TestConnectionString, nameof(CosmosNoSqlDependencyInjectionTests), lifetime: lifetime) + : services.AddKeyedCosmosNoSqlCollection(serviceKey, name, TestConnectionString, nameof(CosmosNoSqlDependencyInjectionTests), lifetime: lifetime); + } + } + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey) + { + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDistanceFunctionTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDistanceFunctionTests.cs new file mode 100644 index 0000000..3bd747a --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDistanceFunctionTests.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlDistanceFunctionTests(CosmosNoSqlDistanceFunctionTests.Fixture fixture) + : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task CosineDistance() => Assert.ThrowsAsync(base.CosineDistance); + public override Task EuclideanSquaredDistance() => Assert.ThrowsAsync(base.EuclideanSquaredDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + + public override Task EuclideanDistance() + { + if (!((CosmosNoSqlTestStore)fixture.TestStore).UsesLocalEmulator) + { + // Fails on emulator, most likely due to emulator bug + return base.EuclideanDistance(); + } + + return Task.CompletedTask; + } + + public new sealed class Fixture : DistanceFunctionTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlDistanceFunctionTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDynamicModelTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDynamicModelTests.cs new file mode 100644 index 0000000..fe51911 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlDynamicModelTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlDynamicModelTests(CosmosNoSqlDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override Task SearchAsync_with_Skip() + { + // The vNext emulator's DiskANN index does not support OFFSET in vector search. + if (!((CosmosNoSqlTestStore)fixture.TestStore).UsesLocalEmulator) + { + return base.SearchAsync_with_Skip(); + } + + return Task.CompletedTask; + } + + public new sealed class Fixture : DynamicModelTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlDynamicModelTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingGenerationTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingGenerationTests.cs new file mode 100644 index 0000000..bf210af --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingGenerationTests.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlEmbeddingGenerationTests( + CosmosNoSqlEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, + CosmosNoSqlEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), + IClassFixture, + IClassFixture +{ + public new sealed class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override string DefaultIndexKind => "DiskAnn"; + + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlEmbeddingGenerationTests))); + + public override TestStore TestStore => _store.Value; + + public override VectorStoreCollectionDefinition CreateRecordDefinition() + => CosmosNoSqlConformanceTestHelpers.UseLowerCaseVectorStorageName(base.CreateRecordDefinition(), "embedding"); + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null) + => new CosmosNoSqlVectorStore( + ((CosmosNoSqlTestStore)TestStore).Database, + new() { EmbeddingGenerator = embeddingGenerator, JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(((CosmosNoSqlTestStore)TestStore).Database) + .AddCosmosNoSqlVectorStore(new() + { + JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions, + }), + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(((CosmosNoSqlTestStore)TestStore).Database) + .AddCosmosNoSqlCollection(CollectionName, new() + { + JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions, + }), + ]; + } + + public new sealed class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override string DefaultIndexKind => "DiskAnn"; + + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlEmbeddingGenerationTests))); + + public override TestStore TestStore => _store.Value; + + public override VectorStoreCollectionDefinition CreateRecordDefinition() + => CosmosNoSqlConformanceTestHelpers.UseLowerCaseVectorStorageName(base.CreateRecordDefinition(), "embedding"); + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null) + => new CosmosNoSqlVectorStore( + ((CosmosNoSqlTestStore)TestStore).Database, + new() { EmbeddingGenerator = embeddingGenerator, JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services.AddCosmosNoSqlVectorStore( + ((CosmosNoSqlTestStore)TestStore).ConnectionString, + nameof(CosmosNoSqlEmbeddingGenerationTests), + new() { JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions }) + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services.AddCosmosNoSqlCollection( + CollectionName, + ((CosmosNoSqlTestStore)TestStore).ConnectionString, + nameof(CosmosNoSqlEmbeddingGenerationTests), + new() { JsonSerializerOptions = CosmosNoSqlTestStore.SerializerOptions }) + ]; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingTypeTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingTypeTests.cs new file mode 100644 index 0000000..d551e7f --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingTypeTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlEmbeddingTypeTests(CosmosNoSqlEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new sealed class Fixture : EmbeddingTypeTests.Fixture + { + public override string DefaultIndexKind => "DiskAnn"; + + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlEmbeddingTypeTests))); + + public override TestStore TestStore => _store.Value; + + public override VectorStoreCollectionDefinition CreateRecordDefinition( + IEmbeddingGenerator? embeddingGenerator, + string? distanceFunction, + int dimensions) + => CosmosNoSqlConformanceTestHelpers.UseLowerCaseVectorStorageName( + base.CreateRecordDefinition(embeddingGenerator, distanceFunction, dimensions), + "vector"); + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlFilterTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlFilterTests.cs new file mode 100644 index 0000000..5e95100 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlFilterTests.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +// The type is internal to disable the tests due to emulator limitations +internal sealed class CosmosNoSqlFilterTests(CosmosNoSqlFilterTests.Fixture fixture) + : FilterTests(fixture), IClassFixture +{ + public new sealed class Fixture : FilterTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlFilterTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlHybridSearchTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlHybridSearchTests.cs new file mode 100644 index 0000000..94151ee --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlHybridSearchTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +// The type is internal to disable the tests due to emulator limitations +internal sealed class CosmosNoSqlHybridSearchTests( + CosmosNoSqlHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, + CosmosNoSqlHybridSearchTests.MultiTextFixture multiTextFixture) + : HybridSearchTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new sealed class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(VectorAndStringFixture))); + + public override TestStore TestStore => _store.Value; + } + + public new sealed class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(MultiTextFixture))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs new file mode 100644 index 0000000..b5ed349 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlIndexKindTests.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlIndexKindTests(CosmosNoSqlIndexKindTests.Fixture fixture) + : IndexKindTests(fixture), IClassFixture +{ + [Fact] + public Task DiskAnn() => Test("DiskAnn"); + + public override Task Flat() + { + if (!((CosmosNoSqlTestStore)fixture.TestStore).UsesLocalEmulator) + { + return base.Flat(); + } + + return Task.CompletedTask; + } + + public new sealed class Fixture : IndexKindTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlIndexKindTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlKeyTypeTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlKeyTypeTests.cs new file mode 100644 index 0000000..57a46ed --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlKeyTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlKeyTypeTests(CosmosNoSqlKeyTypeTests.Fixture fixture) + : KeyTypeTests(fixture), IClassFixture +{ + public new sealed class Fixture : KeyTypeTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlKeyTypeTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlMultiVectorModelTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlMultiVectorModelTests.cs new file mode 100644 index 0000000..659f76e --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlMultiVectorModelTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlMultiVectorModelTests(CosmosNoSqlMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new sealed class Fixture : MultiVectorModelTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlMultiVectorModelTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoDataModelTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoDataModelTests.cs new file mode 100644 index 0000000..944fc09 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoDataModelTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlNoDataModelTests(CosmosNoSqlNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new sealed class Fixture : NoDataModelTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlNoDataModelTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoVectorModelTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoVectorModelTests.cs new file mode 100644 index 0000000..7268f04 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlNoVectorModelTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CosmosNoSql.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlNoVectorModelTests(CosmosNoSqlNoVectorModelTests.Fixture fixture) + : NoVectorModelTests(fixture), IClassFixture +{ + public new sealed class Fixture : NoVectorModelTests.Fixture + { + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlNoVectorModelTests))); + + public override TestStore TestStore => _store.Value; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlTestSuiteImplementationTests.cs b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlTestSuiteImplementationTests.cs new file mode 100644 index 0000000..a84ae45 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSqlTestSuiteImplementationTests.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.TypeTests; + +namespace CosmosNoSql.ConformanceTests; + +public sealed class CosmosNoSqlTestSuiteImplementationTests : TestSuiteImplementationTests +{ + protected override ICollection IgnoredTestBases { get; } = + [ + // Not supported by current emulator version + typeof(FilterTests<>), + typeof(DataTypeTests<>), + typeof(DataTypeTests<,>), + typeof(HybridSearchTests<>), + ]; +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlConformanceTestHelpers.cs b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlConformanceTestHelpers.cs new file mode 100644 index 0000000..91d2106 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlConformanceTestHelpers.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CosmosNoSql.ConformanceTests.Support; + +internal static class CosmosNoSqlConformanceTestHelpers +{ + public static VectorStoreCollectionDefinition UseLowerCaseVectorStorageName( + VectorStoreCollectionDefinition definition, + string storageName) + { + foreach (var vectorProperty in definition.Properties.OfType()) + { + vectorProperty.StorageName = storageName; + } + + return definition; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlFixture.cs b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlFixture.cs new file mode 100644 index 0000000..f57f66c --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlFixture.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace CosmosNoSql.ConformanceTests.Support; + +public sealed class CosmosNoSqlFixture : VectorStoreFixture +{ + private readonly Lazy _store = new(() => new CosmosNoSqlTestStore(nameof(CosmosNoSqlFixture))); + + public override TestStore TestStore => _store.Value; +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs new file mode 100644 index 0000000..faa6157 --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestEnvironment.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; + +namespace CosmosNoSql.ConformanceTests.Support; + +#pragma warning disable CA1810 // Initialize all static fields when those fields are declared + +internal static class CosmosNoSqlTestEnvironment +{ + public static readonly string? ConnectionString; + + public static bool IsConnectionStringDefined => ConnectionString is not null; + + static CosmosNoSqlTestEnvironment() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: true) + .AddJsonFile(path: "testsettings.development.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + var cosmosSection = configuration.GetSection("CosmosDb"); + ConnectionString = cosmosSection["ConnectionString"]; + } +} diff --git a/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestStore.cs b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestStore.cs new file mode 100644 index 0000000..767dd3b --- /dev/null +++ b/MEVD/test/CosmosNoSql.ConformanceTests/Support/CosmosNoSqlTestStore.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.VectorData; +using Testcontainers.CosmosDb; +using VectorData.ConformanceTests.Support; + +namespace CosmosNoSql.ConformanceTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable + +/// The emulator does not support running multiple queries simultaneously, so we create a unique database for each test store instance. +internal sealed class CosmosNoSqlTestStore(string uniqueDatabaseName) : TestStore +{ + private CosmosDbContainer? _container; + private CosmosClient? _client; + private Database? _database; + private string? _connectionString; + + + public bool UsesLocalEmulator => _container is not null; + + public override string DefaultIndexKind => "DiskAnn"; + + public string ConnectionString => _connectionString ?? throw new InvalidOperationException("Cosmos DB test store has not been started."); + + public static JsonSerializerOptions SerializerOptions { get; } = new(JsonSerializerDefaults.Web) + { + PropertyNamingPolicy = new CosmosNoSqlTestJsonNamingPolicy() + }; + + public Database Database => _database ?? throw new InvalidOperationException("Cosmos DB test store has not been started."); + + public override VectorStoreCollection CreateCollection( + string name, + VectorStoreCollectionDefinition definition) + => new CosmosNoSqlCollection( + Database, + name, + new() + { + Definition = definition, + JsonSerializerOptions = SerializerOptions, + }); + + public override VectorStoreCollection> CreateDynamicCollection( + string name, + VectorStoreCollectionDefinition definition) + => new CosmosNoSqlDynamicCollection( + Database, + name, + new() + { + Definition = definition, + JsonSerializerOptions = SerializerOptions, + }); + + protected override async Task StartAsync() + { + if (CosmosNoSqlTestEnvironment.IsConnectionStringDefined) + { + _connectionString = CosmosNoSqlTestEnvironment.ConnectionString!; + } + else + { + _container ??= new CosmosDbBuilder("mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-latest") + .WithEnvironment("QUERY_BUFFER_SIZE_KB", "65536") + .Build(); + + await _container.StartAsync(); + _connectionString = _container.GetConnectionString(); + } + + CosmosClientOptions clientOptions = new() + { + ConnectionMode = ConnectionMode.Gateway, + LimitToEndpoint = true, + RequestTimeout = TimeSpan.FromSeconds(10), + UseSystemTextJsonSerializerWithOptions = JsonSerializerOptions.Default, + HttpClientFactory = _container is not null + ? () => _container.HttpClient + : null, + }; + + _client = new CosmosClient(_connectionString, clientOptions); + + _database = await _client.CreateDatabaseIfNotExistsAsync(uniqueDatabaseName).ConfigureAwait(false); + DefaultVectorStore = new CosmosNoSqlVectorStore(_database, new() { JsonSerializerOptions = SerializerOptions }); + } + + protected override async Task StopAsync() + { + if (_container is not null) + { + await _container.StopAsync(); + await _container.DisposeAsync(); + } + } + + private sealed class CosmosNoSqlTestJsonNamingPolicy : JsonNamingPolicy + { + public override string ConvertName(string name) + => name is "Vector" or "Embedding" + ? "vector" + : JsonNamingPolicy.CamelCase.ConvertName(name); + } +} diff --git a/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj b/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj new file mode 100644 index 0000000..99a03e1 --- /dev/null +++ b/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj @@ -0,0 +1,30 @@ + + + + CommunityToolkit.VectorData.CosmosNoSql.UnitTests + CommunityToolkit.VectorData.CosmosNoSql.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001 + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSqlOptionsTests.cs b/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSqlOptionsTests.cs new file mode 100644 index 0000000..3d5febf --- /dev/null +++ b/MEVD/test/CosmosNoSql.UnitTests/CosmosNoSqlOptionsTests.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json; +using CommunityToolkit.VectorData.CosmosNoSql; +using Microsoft.Azure.Cosmos; +using Xunit; + +namespace CommunityToolkit.VectorData.CosmosNoSql.UnitTests; + +public sealed class CosmosNoSqlOptionsTests +{ + [Fact] + public void CollectionOptionsPropertiesCanBeConfigured() + { + JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + CosmosNoSqlCollectionOptions source = new() + { + JsonSerializerOptions = serializerOptions, + PartitionKeyProperties = new[] { "tenantId", "category" }, + Automatic = false, + IndexingMode = IndexingMode.None, + }; + + Assert.Same(serializerOptions, source.JsonSerializerOptions); + Assert.Equal( + new List { "tenantId", "category" }, + new List(source.PartitionKeyProperties!)); + Assert.False(source.Automatic); + Assert.Equal(IndexingMode.None, source.IndexingMode); + } + + [Fact] + public void VectorStoreOptionsPropertiesCanBeConfigured() + { + JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); + CosmosNoSqlVectorStoreOptions source = new() + { + JsonSerializerOptions = serializerOptions, + }; + + Assert.Same(serializerOptions, source.JsonSerializerOptions); + } +}