diff --git a/.gitignore b/.gitignore index 2248a7d..205be05 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ test/DataStax.AstraDB.DataAPI.IntegrationTests/appsettings.sample.json test/DataStax.AstraDB.DataApi.UnitTests/bin test/DataStax.AstraDB.DataApi.UnitTests/obj appsettings.json +latest_run.log diff --git a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs index 0d9cf4a..e31481c 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs @@ -21,7 +21,6 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -413,6 +412,6 @@ internal Task GetDatabaseInfoAsync(Guid dbGuid, CommandOptions opt private Command CreateCommand() { - return new Command(_client, OptionsTree, new AdminCommandUrlBuilder(OptionsTree)); + return new Command(_client, OptionsTree, new AdminCommandUrlBuilder()); } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs b/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs index 3b0d58b..e404726 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs @@ -1,3 +1,19 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + using DataStax.AstraDB.DataApi.Core; using System.Text.Json.Serialization; diff --git a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs index ed133af..6000ad1 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs @@ -16,13 +16,25 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Core.Query; using DataStax.AstraDB.DataApi.Core.Results; +using DataStax.AstraDB.DataApi.SerDes; using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Collections; -public class Collection where T : class +public class Collection : Collection where T : class +{ + internal Collection(string collectionName, Database database, CommandOptions commandOptions) + : base(collectionName, database, commandOptions) { } +} + +public class Collection where T : class { private readonly string _collectionName; private readonly Database _database; @@ -39,34 +51,132 @@ internal Collection(string collectionName, Database database, CommandOptions com _commandOptions = commandOptions; } - public CollectionInsertOneResult InsertOne(T document) + public CollectionInsertOneResult InsertOne(T document) { return InsertOne(document, null); } - public CollectionInsertOneResult InsertOne(T document, CommandOptions commandOptions) + public CollectionInsertOneResult InsertOne(T document, CommandOptions commandOptions) { return InsertOneAsync(document, commandOptions, runSynchronously: true).ResultSync(); } - public Task InsertOneAsync(T document) + public Task> InsertOneAsync(T document) { return InsertOneAsync(document, new CommandOptions()); } - public Task InsertOneAsync(T document, CommandOptions commandOptions) + public Task> InsertOneAsync(T document, CommandOptions commandOptions) { return InsertOneAsync(document, commandOptions, runSynchronously: false); } - private async Task InsertOneAsync(T document, CommandOptions commandOptions, bool runSynchronously) + private async Task> InsertOneAsync(T document, CommandOptions commandOptions, bool runSynchronously) { Guard.NotNull(document, nameof(document)); + var payload = new { document }; + var command = CreateCommand("insertOne").WithPayload(payload).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); + return new CollectionInsertOneResult { InsertedId = response.Result.InsertedIds[0] }; + } + + public CollectionInsertManyResult InsertMany(List documents) + { + return InsertMany(documents, null, null); + } - var command = CreateCommand("insertOne").WithDocument(document).AddCommandOptions(commandOptions); - var response = await command.RunAsync(runSynchronously).ConfigureAwait(false); + public CollectionInsertManyResult InsertMany(List documents, InsertManyOptions insertOptions) + { + return InsertMany(documents, insertOptions, null); + } - return new CollectionInsertOneResult { InsertedId = response.Result.InsertedIds[0] }; + public CollectionInsertManyResult InsertMany(List documents, CommandOptions commandOptions) + { + return InsertMany(documents, null, commandOptions); + } + + public CollectionInsertManyResult InsertMany(List documents, InsertManyOptions insertOptions, CommandOptions commandOptions) + { + return InsertManyAsync(documents, insertOptions, commandOptions, runSynchronously: true).ResultSync(); + } + + public Task> InsertManyAsync(List documents) + { + return InsertManyAsync(documents, new CommandOptions()); + } + + public Task> InsertManyAsync(List documents, InsertManyOptions insertOptions) + { + return InsertManyAsync(documents, insertOptions, null, runSynchronously: false); + } + + public Task> InsertManyAsync(List documents, CommandOptions commandOptions) + { + return InsertManyAsync(documents, null, commandOptions, runSynchronously: false); + } + + private async Task> InsertManyAsync(List documents, InsertManyOptions insertOptions, CommandOptions commandOptions, bool runSynchronously) + { + Guard.NotNullOrEmpty(documents, nameof(documents)); + + if (insertOptions == null) insertOptions = new InsertManyOptions(); + if (insertOptions.Concurrency > 1 && insertOptions.InsertInOrder) + { + throw new ArgumentException("Cannot run ordered insert_many concurrently."); + } + if (insertOptions.ChunkSize > InsertManyOptions.MaxChunkSize) + { + throw new ArgumentException("Chunk size cannot be greater than the max chunk size of " + InsertManyOptions.MaxChunkSize + "."); + } + + var start = DateTime.Now; + + var result = new CollectionInsertManyResult(); + var tasks = new List(); + var semaphore = new SemaphoreSlim(insertOptions.Concurrency); + + var chunks = documents.Chunk(insertOptions.ChunkSize); + + foreach (var chunk in chunks) + { + tasks.Add(Task.Run(async () => + { + await semaphore.WaitAsync(); + try + { + var runResult = await RunInsertManyAsync(chunk, insertOptions.InsertInOrder, commandOptions, runSynchronously).ConfigureAwait(false); + lock (result.InsertedIds) + { + result.InsertedIds.AddRange(runResult.InsertedIds); + } + } + finally + { + semaphore.Release(); + } + })); + } + + await Task.WhenAll(tasks); + return result; + } + + private async Task> RunInsertManyAsync(List documents, bool insertOrdered, CommandOptions commandOptions, bool runSynchronously) + { + var payload = new + { + documents, + options = new + { + ordered = insertOrdered + } + }; + commandOptions ??= new CommandOptions(); + var outputConverter = typeof(TId) == typeof(object) ? new IdListConverter() : null; + commandOptions.SetConvertersIfNull(new DocumentConverter(), outputConverter); + var command = CreateCommand("insertMany").WithPayload(payload).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); + return new CollectionInsertManyResult { InsertedIds = response.Result.InsertedIds.ToList() }; } public void Drop() @@ -79,8 +189,456 @@ public async Task DropAsync() await _database.DropCollectionAsync(_collectionName).ConfigureAwait(false); } + public T FindOne() + { + return FindOne(null, new FindOptions(), null); + } + + public T FindOne(CommandOptions commandOptions) + { + return FindOne(null, new FindOptions(), commandOptions); + } + + public T FindOne(Filter filter) + { + return FindOne(filter, new FindOptions(), null); + } + + public T FindOne(FindOptions findOptions) + { + return FindOne(null, findOptions, null); + } + + public T FindOne(FindOptions findOptions, CommandOptions commandOptions) + { + return FindOne(null, findOptions, commandOptions); + } + + public T FindOne(Filter filter, CommandOptions commandOptions) + { + return FindOne(filter, new FindOptions(), commandOptions); + } + + public T FindOne(Filter filter, FindOptions findOptions) + { + return FindOne(filter, findOptions, null); + } + + public T FindOne(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); + } + + public TResult FindOne() + { + return FindOne(null, new FindOptions(), null); + } + + public TResult FindOne(CommandOptions commandOptions) + { + return FindOne(null, new FindOptions(), commandOptions); + } + + public TResult FindOne(Filter filter) + { + return FindOne(filter, new FindOptions(), null); + } + + public TResult FindOne(FindOptions findOptions) + { + return FindOne(null, findOptions, null); + } + + public TResult FindOne(FindOptions findOptions, CommandOptions commandOptions) + { + return FindOne(null, findOptions, commandOptions); + } + + public TResult FindOne(Filter filter, CommandOptions commandOptions) + { + return FindOne(filter, new FindOptions(), commandOptions); + } + + public TResult FindOne(Filter filter, FindOptions findOptions) + { + return FindOne(filter, findOptions, null); + } + + public TResult FindOne(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); + } + + public Task FindOneAsync() + { + return FindOneAsync(null, new FindOptions(), null); + } + + public Task FindOneAsync(CommandOptions commandOptions) + { + return FindOneAsync(null, new FindOptions(), commandOptions); + } + + public Task FindOneAsync(FindOptions findOptions) + { + return FindOneAsync(null, findOptions, null); + } + + public Task FindOneAsync(FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(null, findOptions, commandOptions); + } + + public Task FindOneAsync(Filter filter) + { + return FindOneAsync(filter, new FindOptions(), null); + } + + public Task FindOneAsync(Filter filter, CommandOptions commandOptions) + { + return FindOneAsync(filter, new FindOptions(), commandOptions); + } + + public Task FindOneAsync(Filter filter, FindOptions findOptions) + { + return FindOneAsync(filter, findOptions, null); + } + + public Task FindOneAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(filter, findOptions, commandOptions, false); + } + + public Task FindOneAsync() + { + return FindOneAsync(null, new FindOptions(), null); + } + + public Task FindOneAsync(CommandOptions commandOptions) + { + return FindOneAsync(null, new FindOptions(), commandOptions); + } + + public Task FindOneAsync(FindOptions findOptions) + { + return FindOneAsync(null, findOptions, null); + } + + public Task FindOneAsync(FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(null, findOptions, commandOptions); + } + + public Task FindOneAsync(Filter filter) + { + return FindOneAsync(filter, new FindOptions(), null); + } + + public Task FindOneAsync(Filter filter, CommandOptions commandOptions) + { + return FindOneAsync(filter, new FindOptions(), commandOptions); + } + + public Task FindOneAsync(Filter filter, FindOptions findOptions) + { + return FindOneAsync(filter, findOptions, null); + } + + public Task FindOneAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindOneAsync(filter, findOptions, commandOptions, false); + } + + private async Task FindOneAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions, bool runSynchronously) + { + findOptions.Filter = filter; + var command = CreateCommand("findOne").WithPayload(findOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); + return response.Data.Document; + } + + public FluentFind Find() + { + return new FluentFind(this, null); + } + + public FluentFind Find(Filter filter) + { + return new FluentFind(this, filter); + } + + public FluentFind Find() where TResult : class + { + return new FluentFind(this, null); + } + + public FluentFind Find(Filter filter) where TResult : class + { + return new FluentFind(this, filter); + } + + public Cursor FindMany() + { + return FindMany(null, new FindOptions(), null); + } + + public Cursor FindMany(CommandOptions commandOptions) + { + return FindMany(null, new FindOptions(), commandOptions); + } + + public Cursor FindMany(Filter filter) + { + return FindMany(filter, new FindOptions(), null); + } + + public Cursor FindMany(FindOptions findOptions) + { + return FindMany(null, findOptions, null); + } + + public Cursor FindMany(FindOptions findOptions, CommandOptions commandOptions) + { + return FindMany(null, findOptions, commandOptions); + } + + public Cursor FindMany(Filter filter, CommandOptions commandOptions) + { + return FindMany(filter, new FindOptions(), commandOptions); + } + + public Cursor FindMany(Filter filter, FindOptions findOptions) + { + return FindMany(filter, findOptions, null); + } + + public Cursor FindMany(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsyncCursor(filter, findOptions, commandOptions, true).ResultSync(); + } + + public Cursor FindMany() + { + return FindManyAsyncCursor(null, new FindOptions(), null, true).ResultSync(); + } + + public Cursor FindMany(CommandOptions commandOptions) + { + return FindMany(null, new FindOptions(), commandOptions); + } + + public Cursor FindMany(Filter filter) + { + return FindMany(filter, new FindOptions(), null); + } + + public Cursor FindMany(FindOptions findOptions) + { + return FindMany(null, findOptions, null); + } + + public Cursor FindMany(FindOptions findOptions, CommandOptions commandOptions) + { + return FindMany(null, findOptions, commandOptions); + } + + public Cursor FindMany(Filter filter, CommandOptions commandOptions) + { + return FindMany(filter, new FindOptions(), commandOptions); + } + + public Cursor FindMany(Filter filter, FindOptions findOptions) + { + return FindMany(filter, findOptions, null); + } + + public Cursor FindMany(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsyncCursor(filter, findOptions, commandOptions, true).ResultSync(); + } + + public Task> FindManyAsync() + { + return FindManyAsync(null, new FindOptions(), null); + } + + public Task> FindManyAsync(CommandOptions commandOptions) + { + return FindManyAsync(null, new FindOptions(), commandOptions); + } + + public Task> FindManyAsync(FindOptions findOptions) + { + return FindManyAsync(null, findOptions, null); + } + + public Task> FindManyAsync(FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsync(null, findOptions, commandOptions); + } + + public Task> FindManyAsync(Filter filter) + { + return FindManyAsync(filter, new FindOptions(), null); + } + + public Task> FindManyAsync(Filter filter, CommandOptions commandOptions) + { + return FindManyAsync(filter, new FindOptions(), commandOptions); + } + + public Task> FindManyAsync(Filter filter, FindOptions findOptions) + { + return FindManyAsync(filter, findOptions, null); + } + + public Task> FindManyAsync() + { + return FindManyAsync(null, new FindOptions(), null); + } + + public Task> FindManyAsync(CommandOptions commandOptions) + { + return FindManyAsync(null, new FindOptions(), commandOptions); + } + + public Task> FindManyAsync(FindOptions findOptions) + { + return FindManyAsync(null, findOptions, null); + } + + public Task> FindManyAsync(FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsync(null, findOptions, commandOptions); + } + + public Task> FindManyAsync(Filter filter) + { + return FindManyAsync(filter, new FindOptions(), null); + } + + public Task> FindManyAsync(Filter filter, CommandOptions commandOptions) + { + return FindManyAsync(filter, new FindOptions(), commandOptions); + } + + public Task> FindManyAsync(Filter filter, FindOptions findOptions) + { + return FindManyAsync(filter, findOptions, null); + } + + public Task> FindManyAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsyncCursor(filter, findOptions, commandOptions, false); + } + + public Task> FindManyAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions) + { + return FindManyAsyncCursor(filter, findOptions, commandOptions, false); + } + + internal async Task> FindManyAsyncCursor(Filter filter, FindOptions findOptions, CommandOptions commandOptions, bool runSynchronously) + { + findOptions.Filter = filter; + var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); + return new Cursor(response, (string pageState, bool runSynchronously) => + { + findOptions ??= new FindOptions(); + findOptions.PageState = pageState; + return FindManyAsync(filter, findOptions, commandOptions, runSynchronously); + }); + } + + internal async Task, FindStatusResult>> FindManyAsync(Filter filter, FindOptions findOptions, CommandOptions commandOptions, bool runSynchronously) + { + findOptions.Filter = filter; + var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); + return response; + } + + public DocumentsCountResult CountDocuments() + { + return CountDocumentsAsync(null, null, true).ResultSync(); + } + + public DocumentsCountResult CountDocuments(CommandOptions commandOptions) + { + return CountDocumentsAsync(null, commandOptions, true).ResultSync(); + } + + public Task CountDocumentsAsync() + { + return CountDocumentsAsync(null, null, false); + } + + public Task CountDocumentsAsync(CommandOptions commandOptions) + { + return CountDocumentsAsync(null, commandOptions, false); + } + + public DocumentsCountResult CountDocuments(Filter filter) + { + return CountDocumentsAsync(filter, null, true).ResultSync(); + } + + public DocumentsCountResult CountDocuments(Filter filter, CommandOptions commandOptions) + { + return CountDocumentsAsync(filter, commandOptions, true).ResultSync(); + } + + public Task CountDocumentsAsync(Filter filter) + { + return CountDocumentsAsync(filter, null, false); + } + + public Task CountDocumentsAsync(Filter filter, CommandOptions commandOptions) + { + return CountDocumentsAsync(filter, commandOptions, false); + } + + internal async Task CountDocumentsAsync(Filter filter, CommandOptions commandOptions, bool runSynchronously) + { + var findOptions = new FindOptions() + { + Filter = filter, + }; + var command = CreateCommand("countDocuments").WithPayload(findOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + return response.Result; + } + + public EstimatedDocumentsCountResult EstimateDocumentCount() + { + return EstimateDocumentCountAsync(null, true).ResultSync(); + } + + public EstimatedDocumentsCountResult EstimateDocumentCount(CommandOptions commandOptions) + { + return EstimateDocumentCountAsync(commandOptions, true).ResultSync(); + } + + public Task EstimateDocumentCountAsync() + { + return EstimateDocumentCountAsync(null, false); + } + + public Task EstimateDocumentCountAsync(CommandOptions commandOptions) + { + return EstimateDocumentCountAsync(commandOptions, false); + } + + internal async Task EstimateDocumentCountAsync(CommandOptions commandOptions, bool runSynchronously) + { + var command = CreateCommand("estimatedDocumentCount").WithPayload(new { }).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + return response.Result; + } + internal Command CreateCommand(string name) { - return new Command(name, _database.Client, _database.OptionsTree, new DatabaseCommandUrlBuilder(_database, _database.OptionsTree, _collectionName)); + var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }).ToArray(); + return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _collectionName)); } -} \ No newline at end of file +} diff --git a/src/DataStax.AstraDB.DataApi/Core/SerDes/IDataSerializer.cs b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs similarity index 69% rename from src/DataStax.AstraDB.DataApi/Core/SerDes/IDataSerializer.cs rename to src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs index ed61db7..591893c 100644 --- a/src/DataStax.AstraDB.DataApi/Core/SerDes/IDataSerializer.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs @@ -16,15 +16,9 @@ using System.Collections.Generic; -namespace DataStax.AstraDB.DataApi.Core.SerDes; +namespace DataStax.AstraDB.DataApi.Collections; -public interface IDataSerializer +public class CollectionInsertManyResult { - string Serialize(T obj); - - T Deserialize(string json); - - Dictionary SerializeToMap(T obj); - - T DeserializeFromMap(Dictionary map); -} + public List InsertedIds { get; internal set; } = new List(); +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertOneResult.cs b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertOneResult.cs index d6ac10c..befd861 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertOneResult.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertOneResult.cs @@ -16,7 +16,7 @@ namespace DataStax.AstraDB.DataApi.Collections; -public class CollectionInsertOneResult +public class CollectionInsertOneResult { - public object InsertedId { get; internal set; } + public T InsertedId { get; internal set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Collections/Document.cs b/src/DataStax.AstraDB.DataApi/Collections/Document.cs index 280f813..8535688 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/Document.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/Document.cs @@ -17,6 +17,7 @@ namespace DataStax.AstraDB.DataApi.Collections; +//TODO: flesh this out (dictionary?) public class Document { diff --git a/src/DataStax.AstraDB.DataApi/Core/ApiResponse.cs b/src/DataStax.AstraDB.DataApi/Core/ApiResponse.cs index f46a71d..4e1cbd2 100644 --- a/src/DataStax.AstraDB.DataApi/Core/ApiResponse.cs +++ b/src/DataStax.AstraDB.DataApi/Core/ApiResponse.cs @@ -19,22 +19,25 @@ namespace DataStax.AstraDB.DataApi.Core; -internal class ApiResponse +internal class ApiResponseWithStatus { [JsonPropertyName("status")] - public TResponse Result { get; set; } + public T Result { get; set; } [JsonPropertyName("errors")] public List Errors { get; set; } - - // TODO: remove? - // [JsonPropertyName("data")] - // public ApiData Data { get; set; } } -internal class ApiResponseDictionary : Dictionary +internal class ApiResponseWithData { + [JsonPropertyName("errors")] + public List Errors { get; set; } + [JsonPropertyName("data")] + public T Data { get; set; } + + [JsonPropertyName("status")] + public TStatus Status { get; set; } } diff --git a/src/DataStax.AstraDB.DataApi/Core/ApiVector.cs b/src/DataStax.AstraDB.DataApi/Core/ApiResponseDictionary.cs similarity index 71% rename from src/DataStax.AstraDB.DataApi/Core/ApiVector.cs rename to src/DataStax.AstraDB.DataApi/Core/ApiResponseDictionary.cs index 088a240..bfa8d1f 100644 --- a/src/DataStax.AstraDB.DataApi/Core/ApiVector.cs +++ b/src/DataStax.AstraDB.DataApi/Core/ApiResponseDictionary.cs @@ -14,23 +14,13 @@ * limitations under the License. */ -using System; -using System.Linq; +using System.Collections.Generic; namespace DataStax.AstraDB.DataApi.Core; -//TODO: placeholder -public class ApiVector +internal class ApiResponseDictionary : Dictionary { - private readonly float[] _vector; - public ApiVector(float[] vector) - { - _vector = vector; - } - - public int dimension() - { - return _vector.Count(); - } } + + diff --git a/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs b/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs index 6c32748..e7c1576 100644 --- a/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs @@ -16,6 +16,7 @@ using DataStax.AstraDB.DataApi.Utils; using System.Linq; +using System.Text.Json.Serialization; using System.Threading; namespace DataStax.AstraDB.DataApi.Core; @@ -26,6 +27,8 @@ public class CommandOptions internal DBEnvironment? Environment { get; set; } internal RunMode? RunMode { get; set; } internal string Keyspace { get; set; } + internal JsonConverter InputConverter { get; set; } + internal JsonConverter OutputConverter { get; set; } public string Token { get; internal set; } public DataApiDestination? Destination { get; set; } @@ -34,6 +37,12 @@ public class CommandOptions public ApiVersion? ApiVersion { get; set; } public CancellationToken? CancellationToken { get; set; } + public void SetConvertersIfNull(JsonConverter inputConverter, JsonConverter outputConverter) + { + InputConverter ??= inputConverter; + OutputConverter ??= outputConverter; + } + public static CommandOptions Merge(params CommandOptions[] arr) { var list = arr.ToList(); @@ -48,7 +57,9 @@ public static CommandOptions Merge(params CommandOptions[] arr) TimeoutOptions = list.Select(o => o.TimeoutOptions).Merge(), ApiVersion = list.Select(o => o.ApiVersion).Merge(), CancellationToken = list.Select(o => o.CancellationToken).Merge(), - Keyspace = list.Select(o => o.Keyspace).Merge() + Keyspace = list.Select(o => o.Keyspace).Merge(), + InputConverter = list.Select(o => o.InputConverter).Merge(), + OutputConverter = list.Select(o => o.OutputConverter).Merge(), }; return options; } diff --git a/src/DataStax.AstraDB.DataApi/Core/CommandUrlBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/CommandUrlBuilder.cs index 70f0a5f..865b56d 100644 --- a/src/DataStax.AstraDB.DataApi/Core/CommandUrlBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/CommandUrlBuilder.cs @@ -14,35 +14,27 @@ * limitations under the License. */ -using DataStax.AstraDB.DataApi.Utils; - namespace DataStax.AstraDB.DataApi.Core; abstract class CommandUrlBuilder { - internal abstract string BuildUrl(); + internal abstract string BuildUrl(CommandOptions options); } internal class DatabaseCommandUrlBuilder : CommandUrlBuilder { private readonly Database _database; - private readonly CommandOptions[] _optionsTree; private readonly string _urlPostfix; - //TODO: refactor once we get more usages - internal DatabaseCommandUrlBuilder(Database database, CommandOptions[] optionsTree, string urlPostfix) + internal DatabaseCommandUrlBuilder(Database database, string urlPostfix) { _database = database; - _optionsTree = optionsTree; _urlPostfix = urlPostfix; } - internal override string BuildUrl() + internal override string BuildUrl(CommandOptions options) { - var options = CommandOptions.Merge(_optionsTree); - //TODO: Is this how we want to get the keyspace? (I think not...) - //TODO: factor in environment var url = $"{_database.ApiEndpoint}/api/json/{options.ApiVersion.Value.ToUrlString()}" + $"/{options.Keyspace}/{_urlPostfix}"; return url; @@ -51,25 +43,21 @@ internal override string BuildUrl() internal class AdminCommandUrlBuilder : CommandUrlBuilder { - private readonly CommandOptions[] _optionsTree; private readonly string _urlPostfix; //TODO: refactor once we get more usages - internal AdminCommandUrlBuilder(CommandOptions[] optionsTree, string urlPostfix) + internal AdminCommandUrlBuilder(string urlPostfix) { - _optionsTree = optionsTree; _urlPostfix = urlPostfix; } - internal AdminCommandUrlBuilder(CommandOptions[] optionsTree) : this(optionsTree, null) + internal AdminCommandUrlBuilder() : this(null) { } - internal override string BuildUrl() + internal override string BuildUrl(CommandOptions options) { - var options = CommandOptions.Merge(_optionsTree); - string url = null; switch (options.Environment) { diff --git a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs index 530e112..c756082 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs @@ -14,6 +14,7 @@ * limitations under the License. */ +using DataStax.AstraDB.DataApi.SerDes; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; @@ -51,7 +52,6 @@ internal Command(DataApiClient client, CommandOptions[] options, CommandUrlBuild internal Command(string name, DataApiClient client, CommandOptions[] options, CommandUrlBuilder urlBuilder) { - //TODO include database-specific options (and maybe collection-specific as well) _commandOptionsTree = options.ToList(); _client = client; _name = name; @@ -68,12 +68,6 @@ internal Command AddCommandOptions(CommandOptions options) return this; } - internal Command WithDocument(object document) - { - Payload = new { document }; - return this; - } - internal Command WithPayload(object document) { Payload = document; @@ -99,14 +93,28 @@ internal object BuildContent() return dictionary; } - internal async Task> RunAsync(bool runSynchronously) + internal async Task> RunAsyncReturnDictionary(bool runSynchronously) { - return await RunAsync(runSynchronously).ConfigureAwait(false); + return await RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); } - internal async Task> RunAsync(bool runSynchronously) + internal async Task> RunAsyncReturnStatus(bool runSynchronously) { - var response = await RunCommandAsync>(HttpMethod.Post, runSynchronously).ConfigureAwait(false); + var response = await RunCommandAsync>(HttpMethod.Post, runSynchronously).ConfigureAwait(false); + if (response.Errors != null && response.Errors.Count > 0) + { + throw new CommandException(response.Errors); + } + return response; + } + + internal async Task> RunAsyncReturnData(bool runSynchronously) + { + _commandOptionsTree.Add(new CommandOptions() + { + OutputConverter = new DocumentConverter() + }); + var response = await RunCommandAsync>(HttpMethod.Post, runSynchronously).ConfigureAwait(false); if (response.Errors != null && response.Errors.Count > 0) { throw new CommandException(response.Errors); @@ -127,9 +135,19 @@ internal async Task RunAsyncRaw(HttpMethod httpMethod, bool runSynchronous private async Task RunCommandAsync(HttpMethod method, bool runSynchronously) { var commandOptions = CommandOptions.Merge(_commandOptionsTree.ToArray()); - var content = new StringContent(JsonSerializer.Serialize(BuildContent()), Encoding.UTF8, "application/json"); + var serializeOptions = commandOptions.InputConverter == null ? + new JsonSerializerOptions() + { + Converters = { new ObjectIdConverter() } + } : + new JsonSerializerOptions() + { + Converters = { commandOptions.InputConverter, new ObjectIdConverter() } + }; + + var content = new StringContent(JsonSerializer.Serialize(BuildContent(), serializeOptions), Encoding.UTF8, "application/json"); - var url = _urlBuilder.BuildUrl(); + var url = _urlBuilder.BuildUrl(commandOptions); if (_urlPaths.Any()) { url += "/" + string.Join("/", _urlPaths); @@ -190,10 +208,10 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl if (runSynchronously) { #if NET5_0_OR_GREATER - response = httpClient.Send(request, linkedCts.Token); - var contentTask = Task.Run(() => response.Content.ReadAsStringAsync()); - contentTask.Wait(); - responseContent = contentTask.Result; + response = httpClient.Send(request, linkedCts.Token); + var contentTask = Task.Run(() => response.Content.ReadAsStringAsync()); + contentTask.Wait(); + responseContent = contentTask.Result; #else var requestTask = Task.Run(() => httpClient.SendAsync(request, linkedCts.Token)); requestTask.Wait(); @@ -231,7 +249,16 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl return default; } - return JsonSerializer.Deserialize(responseContent); + var deserializeOptions = commandOptions.OutputConverter == null ? + new JsonSerializerOptions() + { + Converters = { new ObjectIdConverter() } + } : + new JsonSerializerOptions() + { + Converters = { commandOptions.OutputConverter, new ObjectIdConverter() } + }; + return JsonSerializer.Deserialize(responseContent, deserializeOptions); } } diff --git a/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs b/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs new file mode 100644 index 0000000..892bbd0 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs @@ -0,0 +1,33 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Commands; + +public static class DataApiKeywords +{ + public const string Id = "_id"; + public const string All = "$all"; + public const string Date = "$date"; + public const string Uuid = "$uuid"; + public const string ObjectId = "$objectId"; + public const string Size = "$size"; + public const string Exists = "$exists"; + public const string Slice = "$slice"; + public const string Similarity = "$similarity"; + public const string Vector = "$vector"; + public const string SortVector = "sortVector"; + public const string Vectorize = "$vectorize"; +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs new file mode 100644 index 0000000..1f11749 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs @@ -0,0 +1,212 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using DataStax.AstraDB.DataApi.Core.Results; + +namespace DataStax.AstraDB.DataApi.Core; + +public class Cursor +{ + private DocumentsResult _currentBatch; + private Func, FindStatusResult>>> FetchNextBatch { get; } + + public IEnumerable Current + { + get + { + //TODO: handle states, for example: throw exception if MoveNext() returned false (i.e. no more documents) + return _currentBatch.Documents; + } + } + + public float[] SortVector { get; internal set; } = Array.Empty(); + + internal Cursor(ApiResponseWithData, FindStatusResult> currentResult, Func, FindStatusResult>>> fetchNextBatch) + { + _currentBatch = currentResult.Data; + if (_currentBatch != null && currentResult.Status != null) + { + SortVector = currentResult.Status.SortVector; + } + FetchNextBatch = fetchNextBatch; + } + + public bool MoveNext() + { + if (string.IsNullOrEmpty(_currentBatch.NextPageState)) + { + return false; + } + var nextResult = FetchNextBatch(_currentBatch.NextPageState, true).ResultSync(); + _currentBatch = nextResult.Data; + if (nextResult.Status != null && nextResult.Status.SortVector != null) + { + SortVector = SortVector.Concat(nextResult.Status.SortVector).ToArray(); + } + return true; + } + + public async Task MoveNextAsync() + { + if (string.IsNullOrEmpty(_currentBatch.NextPageState)) + { + return false; + } + var nextResult = await FetchNextBatch(_currentBatch.NextPageState, false).ConfigureAwait(false); + _currentBatch = nextResult.Data; + if (nextResult.Status != null && nextResult.Status.SortVector != null) + { + SortVector = SortVector.Concat(nextResult.Status.SortVector).ToArray(); + } + return true; + } +} + +public static class CursorExtensions +{ + public static async IAsyncEnumerable ToAsyncEnumerable(this Cursor cursor) where T : class + { + bool hasNext; + do + { + var currentItems = cursor.Current; + if (currentItems == null) + { + yield break; + } + + hasNext = await cursor.MoveNextAsync().ConfigureAwait(false); + + foreach (var item in currentItems) + { + yield return item; + } + } while (hasNext); + } + + public static IEnumerable ToEnumerable(this Cursor cursor) where T : class + { + return new CursorEnumerable(cursor); + } + + public static List ToList(this Cursor cursor) where T : class + { + return new CursorEnumerable(cursor).ToList(); + } +} + +public class CursorEnumerable : IEnumerable where T : class +{ + private Cursor _cursor; + public CursorEnumerable(Cursor cursor) + { + _cursor = cursor; + } + + public IEnumerator GetEnumerator() + { + return new CursorEnumerator(_cursor); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } +} + +public class CursorEnumerator : IEnumerator +{ + private Cursor _cursor; + private List _currentPageItems; + private int _currentIndex = -1; + private bool _isFinished = false; + + public CursorEnumerator(Cursor cursor) + { + _cursor = cursor; + _currentPageItems = cursor.Current.ToList(); + } + + public T Current + { + get + { + if (_currentPageItems == null || _currentIndex < 0 || _currentIndex >= _currentPageItems.Count) + { + throw new InvalidOperationException("Current is not valid before MoveNext or after enumeration is finished."); + } + return _currentPageItems[_currentIndex]; + } + } + + object IEnumerator.Current => Current; + + public void Dispose() + { + _cursor = null; + } + + public bool MoveNext() + { + if (_isFinished) + { + return false; + } + + _currentIndex++; + + if (_currentPageItems == null || _currentIndex >= _currentPageItems.Count) + { + return FetchNextPage(); + } + + return true; + } + + private bool FetchNextPage() + { + var hasNext = _cursor.MoveNext(); + if (!hasNext) + { + _isFinished = true; + return false; + } + + _currentPageItems = _cursor.Current.ToList(); + _currentIndex = 0; + + if (_currentPageItems == null || _currentPageItems.Count == 0) + { + _isFinished = true; + return false; + } + + return true; + } + + public void Reset() + { + _currentIndex = -1; + _currentPageItems = new List(); + _isFinished = false; + } + +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Database.cs b/src/DataStax.AstraDB.DataApi/Core/Database.cs index 7736e7f..abb8dcd 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Database.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Database.cs @@ -18,7 +18,8 @@ using DataStax.AstraDB.DataApi.Core.Commands; using DataStax.AstraDB.DataApi.Core.Results; using DataStax.AstraDB.DataApi.Utils; -using System.Linq; +using System.Collections; +using System.Collections.Generic; using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Core; @@ -73,44 +74,47 @@ public async Task DoesCollectionExistAsync(string collectionName, CommandO return collectionNames.CollectionNames.Count > 0 && collectionNames.CollectionNames.Contains(collectionName); } - public ListCollectionNamesResult ListCollectionNames() + public List ListCollectionNames() { return ListCollectionNames(null); } - public ListCollectionNamesResult ListCollectionNames(CommandOptions commandOptions) + public List ListCollectionNames(CommandOptions commandOptions) { return ListCollectionNamesAsync(commandOptions).ResultSync(); } - public Task ListCollectionNamesAsync() + public Task> ListCollectionNamesAsync() { return ListCollectionNamesAsync(null); } - public Task ListCollectionNamesAsync(CommandOptions commandOptions) + public async Task> ListCollectionNamesAsync(CommandOptions commandOptions) { - return ListCollectionsAsync(includeDetails: false, commandOptions, runSynchronously: false); + var result = await ListCollectionsAsync(includeDetails: false, commandOptions, runSynchronously: false).ConfigureAwait(false); + return result.CollectionNames; } - public ListCollectionsResult ListCollections() + public IEnumerable ListCollections() { return ListCollections(null); } - public ListCollectionsResult ListCollections(CommandOptions commandOptions) + public IEnumerable ListCollections(CommandOptions commandOptions) { - return ListCollectionsAsync(includeDetails: true, commandOptions, runSynchronously: true).ResultSync(); + var result = ListCollectionsAsync(includeDetails: true, commandOptions, runSynchronously: true).ResultSync(); + return result.Collections; } - public Task ListCollectionsAsync() + public Task> ListCollectionsAsync() { return ListCollectionsAsync(null); } - public Task ListCollectionsAsync(CommandOptions commandOptions) + public async Task> ListCollectionsAsync(CommandOptions commandOptions) { - return ListCollectionsAsync(true, commandOptions, runSynchronously: false); + var result = await ListCollectionsAsync(true, commandOptions, runSynchronously: false).ConfigureAwait(false); + return result.Collections; } private async Task ListCollectionsAsync(bool includeDetails, CommandOptions commandOptions, bool runSynchronously) @@ -120,7 +124,7 @@ private async Task ListCollectionsAsync(bool includeDetails, CommandOption options = new { explain = includeDetails } }; var command = CreateCommand("findCollections").WithPayload(payload).AddCommandOptions(commandOptions); - var response = await command.RunAsync(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); return response.Result; } @@ -194,6 +198,36 @@ public Task> CreateCollectionAsync(string collectionName, Colle return CreateCollectionAsync(collectionName, definition, options, false); } + public Collection CreateCollection(string collectionName) where T : class + { + return CreateCollection(collectionName, null, null); + } + + public Collection CreateCollection(string collectionName, CollectionDefinition definition) where T : class + { + return CreateCollection(collectionName, definition, null); + } + + public Collection CreateCollection(string collectionName, CollectionDefinition definition, CommandOptions options) where T : class + { + return CreateCollectionAsync(collectionName, definition, options, false).ResultSync(); + } + + public Task> CreateCollectionAsync(string collectionName) where T : class + { + return CreateCollectionAsync(collectionName, null, null); + } + + public Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition) where T : class + { + return CreateCollectionAsync(collectionName, definition, null); + } + + public Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition, CommandOptions options) where T : class + { + return CreateCollectionAsync(collectionName, definition, options, false); + } + private async Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition, CommandOptions options, bool runSynchronously) where T : class { object payload = definition == null ? new @@ -205,18 +239,28 @@ private async Task> CreateCollectionAsync(string collectionName options = definition }; var command = CreateCommand("createCollection").WithPayload(payload).AddCommandOptions(options); - await command.RunAsync(runSynchronously).ConfigureAwait(false); + await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); return GetCollection(collectionName); } - public Collection GetCollection(string collectionName) + private async Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition, CommandOptions options, bool runSynchronously) where T : class { - return GetCollection(collectionName, new CommandOptions()); + object payload = definition == null ? new + { + name = collectionName + } : new + { + name = collectionName, + options = definition + }; + var command = CreateCommand("createCollection").WithPayload(payload).AddCommandOptions(options); + await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); + return GetCollection(collectionName); } - public Collection GetCollection(string collectionName) where T : class + public Collection GetCollection(string collectionName) { - return GetCollection(collectionName, new CommandOptions()); + return GetCollection(collectionName, new CommandOptions()); } public Collection GetCollection(string collectionName, CommandOptions options) @@ -224,12 +268,28 @@ public Collection GetCollection(string collectionName, CommandOptions return GetCollection(collectionName, options); } + public Collection GetCollection(string collectionName) where T : class + { + return GetCollection(collectionName, new CommandOptions()); + } + public Collection GetCollection(string collectionName, CommandOptions options) where T : class { Guard.NotNullOrEmpty(collectionName, nameof(collectionName)); return new Collection(collectionName, this, options); } + public Collection GetCollection(string collectionName) where T : class + { + return GetCollection(collectionName, new CommandOptions()); + } + + public Collection GetCollection(string collectionName, CommandOptions options) where T : class + { + Guard.NotNullOrEmpty(collectionName, nameof(collectionName)); + return new Collection(collectionName, this, options); + } + public void DropCollection(string collectionName) { DropCollection(collectionName, null); @@ -257,12 +317,12 @@ private async Task DropCollectionAsync(string collectionName, CommandOptions opt { name = collectionName }).AddCommandOptions(options); - await command.RunAsync(runSynchronously).ConfigureAwait(false); + await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); } internal Command CreateCommand(string name) { - return new Command(name, _client, OptionsTree, new DatabaseCommandUrlBuilder(this, OptionsTree, _urlPostfix)); + return new Command(name, _client, OptionsTree, new DatabaseCommandUrlBuilder(this, _urlPostfix)); } } diff --git a/src/DataStax.AstraDB.DataApi/Core/DefaultIdOptions.cs b/src/DataStax.AstraDB.DataApi/Core/DefaultIdOptions.cs index b44bce4..5398044 100644 --- a/src/DataStax.AstraDB.DataApi/Core/DefaultIdOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/DefaultIdOptions.cs @@ -14,7 +14,6 @@ * limitations under the License. */ -using DataStax.AstraDB.DataApi.Core; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core; diff --git a/src/DataStax.AstraDB.DataApi/Core/Extensions.cs b/src/DataStax.AstraDB.DataApi/Core/Extensions.cs index d0d5889..0e3ce97 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Extensions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Extensions.cs @@ -15,6 +15,8 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Core; @@ -39,4 +41,12 @@ public static void ResultSync(this Task task) { task.GetAwaiter().GetResult(); } + + public static IEnumerable> Chunk(this List list, int chunkSize) + { + for (int i = 0; i < list.Count; i += chunkSize) + { + yield return list.GetRange(i, Math.Min(chunkSize, list.Count - i)); + } + } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/InsertManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/InsertManyOptions.cs new file mode 100644 index 0000000..ffefcf9 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/InsertManyOptions.cs @@ -0,0 +1,36 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core; + +public class InsertManyOptions +{ + public const int MaxChunkSize = 50; + public const int MaxConcurrency = int.MaxValue; + + private bool _insertInOrder = false; + public bool InsertInOrder + { + get => _insertInOrder; + set + { + if (value) Concurrency = 1; + _insertInOrder = value; + } + } + public int Concurrency { get; set; } = MaxConcurrency; + public int ChunkSize { get; set; } = MaxChunkSize; +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Builders.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Builders.cs new file mode 100644 index 0000000..27eaf22 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Builders.cs @@ -0,0 +1,24 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class Builders +{ + public static FilterBuilder Filter => new(); + public static ProjectionBuilder Projection => new(); + public static SortBuilder Sort => new(); +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs new file mode 100644 index 0000000..590db72 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs @@ -0,0 +1,51 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class Filter +{ + public virtual object Name { get; } + public virtual object Value { get; } + + internal Filter(string filterName, object value) + { + Name = filterName; + Value = value; + } + + internal Filter(string filterOperator, string fieldName, object value) + { + Name = filterOperator; + Value = new Filter(fieldName, value); + } + + public static Filter operator &(Filter left, Filter right) + { + return new LogicalFilter(LogicalOperator.And, new[] { left, right }); + } + + public static Filter operator |(Filter left, Filter right) + { + return new LogicalFilter(LogicalOperator.Or, new[] { left, right }); + } + + public static Filter operator !(Filter notFilter) + { + return new LogicalFilter(LogicalOperator.Not, notFilter); + } + +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs new file mode 100644 index 0000000..784f67c --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs @@ -0,0 +1,160 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections.Generic; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class FilterBuilder +{ + public Filter And(params Filter[] filters) + { + return new LogicalFilter(LogicalOperator.And, filters); + } + + public Filter Or(params Filter[] filters) + { + return new LogicalFilter(LogicalOperator.Or, filters); + } + + public Filter Not(Filter filter) + { + return new LogicalFilter(LogicalOperator.Not, filter); + } + + public Filter Gt(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.GreaterThan, value); + } + + public Filter Gt(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.GreaterThan, value); + } + + public Filter Gte(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.GreaterThanOrEqualTo, value); + } + + public Filter Gte(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.GreaterThanOrEqualTo, value); + } + + public Filter Lt(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.LessThan, value); + } + + public Filter Lt(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.LessThan, value); + } + + public Filter Lte(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.LessThanOrEqualTo, value); + } + + public Filter Lte(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.LessThanOrEqualTo, value); + } + + public Filter Eq(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.EqualsTo, value); + } + + public Filter Eq(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.EqualsTo, value); + } + + public Filter Ne(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.NotEqualsTo, value); + } + + public Filter Ne(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.NotEqualsTo, value); + } + + public Filter In(string fieldName, T2[] values) + { + return new Filter(fieldName, FilterOperator.In, values); + } + + public Filter In(Expression> expression, TField[] array) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.In, array); + } + + public Filter In(Expression> expression, TField[] array) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.In, array); + } + + public Filter Nin(string fieldName, T2[] values) + { + return new Filter(fieldName, FilterOperator.NotIn, values); + } + + public Filter Nin(Expression> expression, TField[] array) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.NotIn, array); + } + + public Filter Nin(Expression> expression, TField[] array) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.NotIn, array); + } + + public Filter Exists(string fieldName) + { + return new Filter(fieldName, FilterOperator.Exists, true); + } + + public Filter Exists(Expression>> expression) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.Exists, true); + } + + public Filter All(string fieldName, TField[] array) + { + return new Filter(fieldName, FilterOperator.All, array); + } + + public Filter All(Expression> expression, TField[] array) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.All, array); + } + + public Filter Size(string fieldName, int size) + { + return new Filter(fieldName, FilterOperator.Size, size); + } + + public Filter Size(Expression> expression, int size) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.Size, size); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs new file mode 100644 index 0000000..1db9fe2 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs @@ -0,0 +1,35 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class FilterOperator +{ + public const string GreaterThan = "$gt"; + public const string GreaterThanOrEqualTo = "$gte"; + public const string LessThan = "$lt"; + public const string LessThanOrEqualTo = "$lte"; + public const string EqualsTo = "$eq"; + public const string NotEqualsTo = "$ne"; + public const string In = "$in"; + public const string NotIn = "$nin"; + public const string Exists = "$exists"; + public const string All = "$all"; + public const string Size = "$size"; + public const string Contains = "$contains"; + public const string ContainsKey = "$containsKey"; + public const string ContainsEntry = "$containsEntry"; +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs new file mode 100644 index 0000000..fce23fe --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs @@ -0,0 +1,134 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using MongoDB.Bson; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class FindOptions +{ + [JsonIgnore] + public SortBuilder Sort { get; set; } + + [JsonIgnore] + public IProjectionBuilder Projection { get; set; } + + [JsonIgnore] + public int? Skip { get; set; } + + [JsonIgnore] + public int? Limit { get; set; } + + [JsonIgnore] + public bool? IncludeSimilarity { get; set; } + + [JsonIgnore] + public bool? IncludeSortVector { get; set; } + + [JsonIgnore] + internal Filter Filter { get; set; } + + [JsonIgnore] + internal string PageState { get; set; } + + [JsonInclude] + [JsonPropertyName("filter")] + internal Dictionary FilterMap => Filter == null ? null : SerializeFilter(Filter); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + internal Dictionary ProjectionMap => Projection == null ? null : Projection.Projections.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + internal FindApiOptions Options + { + get + { + if (IncludeSimilarity == null && + IncludeSortVector == null && + PageState == null && + Skip == null && + Limit == null && + string.IsNullOrEmpty(PageState)) + { + return null; + } + return new FindApiOptions + { + IncludeSimilarity = IncludeSimilarity, + IncludeSortVector = IncludeSortVector, + PageState = PageState, + Skip = Skip, + Limit = Limit + }; + } + } + + private Dictionary SerializeFilter(Filter filter) + { + var result = new Dictionary(); + if (filter.Value is Filter[] filtersArray) + { + var serializedArray = new List(); + foreach (var nestedFilter in filtersArray) + { + serializedArray.Add(SerializeFilter(nestedFilter)); + } + result[filter.Name.ToString()] = serializedArray; + } + else + { + //TODO: abstract out ObjectId handling + result[filter.Name.ToString()] = filter.Value is Filter nestedFilter ? SerializeFilter(nestedFilter) : + filter.Value is ObjectId ? filter.Value.ToString() : filter.Value; + } + return result; + } +} + +internal class FindApiOptions +{ + [JsonPropertyName("skip")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Skip { get; set; } + + [JsonPropertyName("limit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Limit { get; set; } + + [JsonPropertyName("includeSimilarity")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? IncludeSimilarity { get; set; } + + [JsonPropertyName("includeSortVector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? IncludeSortVector { get; set; } + + [JsonPropertyName("pageState")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string PageState { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FluentFind.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FluentFind.cs new file mode 100644 index 0000000..b6b9959 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FluentFind.cs @@ -0,0 +1,124 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core.Results; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class FluentFind where T : class where TResult : class +{ + private readonly Filter _filter; + private readonly Collection _collection; + private FindOptions _findOptions; + + internal Filter Filter => _filter; + internal Collection Collection => _collection; + + private FindOptions FindOptions + { + get { return _findOptions ??= new FindOptions(); } + } + + internal FluentFind(Collection collection, Filter filter) + { + _filter = filter; + _collection = collection; + } + + public FluentFind Project(IProjectionBuilder projection) + { + FindOptions.Projection = projection; + return this; + } + + public FluentFind Sort(SortBuilder sortBuilder) + { + FindOptions.Sort = sortBuilder; + return this; + } + + public FluentFind Limit(int limit) + { + FindOptions.Limit = limit; + return this; + } + + public FluentFind Skip(int skip) + { + FindOptions.Skip = skip; + return this; + } + + public FluentFind IncludeSimilarity(bool includeSimilarity) + { + FindOptions.IncludeSimilarity = includeSimilarity; + return this; + } + + public FluentFind IncludeSortVector(bool includeSortVector) + { + FindOptions.IncludeSortVector = includeSortVector; + return this; + } + + internal Task, FindStatusResult>> RunAsync(string pageState = null, bool runSynchronously = false) + { + FindOptions.PageState = pageState; + return _collection.FindManyAsync(_filter, FindOptions, null, runSynchronously); + } + +} + +public static class FluentFindExtensions +{ + public static async Task> ToCursorAsync(this FluentFind fluentFind) where T : class where TResult : class + { + var initialResults = await fluentFind.RunAsync().ConfigureAwait(false); + var cursor = new Cursor(initialResults, (string pageState, bool runSynchronously) => fluentFind.RunAsync(pageState, runSynchronously)); + return cursor; + } + + public static async IAsyncEnumerable ToAsyncEnumerable(this FluentFind fluentFind) where T : class where TResult : class + { + var cursor = await fluentFind.ToCursorAsync().ConfigureAwait(false); + await foreach (var item in cursor.ToAsyncEnumerable()) + { + yield return item; + } + } + + public static async Task> ToEnumerableAsync(this FluentFind fluentFind) where T : class where TResult : class + { + var cursor = await fluentFind.ToCursorAsync().ConfigureAwait(false); + return cursor.ToEnumerable(); + } + + public static async Task> ToListAsync(this FluentFind fluentFind) where T : class where TResult : class + { + var cursor = await fluentFind.ToCursorAsync().ConfigureAwait(false); + return cursor.ToList(); + } + + public static List ToList(this FluentFind fluentFind) where T : class where TResult : class + { + var initialResults = fluentFind.RunAsync(null, true).ResultSync(); + var cursor = new Cursor(initialResults, (string pageState, bool runSynchronously) => fluentFind.RunAsync(pageState, runSynchronously)); + return cursor.ToList(); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs new file mode 100644 index 0000000..00ed80b --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs @@ -0,0 +1,32 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Linq; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +internal class LogicalFilter : Filter +{ + internal LogicalFilter(LogicalOperator logicalOperator, Filter[] filters) : + base(logicalOperator.ToApiString(), filters) + { + } + + internal LogicalFilter(LogicalOperator logicalOperator, Filter filter) : + base(logicalOperator.ToApiString(), filter) + { + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs new file mode 100644 index 0000000..3827472 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs @@ -0,0 +1,40 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public enum LogicalOperator : int +{ + And, + Or, + Not +} + +public static class LogicalOperatorExtensions +{ + public static string ToApiString(this LogicalOperator value) + { + return value switch + { + LogicalOperator.And => "$and", + LogicalOperator.Or => "$or", + LogicalOperator.Not => "$not", + _ => throw new ArgumentException("Invalid Logical Operator"), + }; + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs new file mode 100644 index 0000000..55faa0d --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs @@ -0,0 +1,45 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + + +public class Projection +{ + public string FieldName { get; set; } + public bool Include { get; set; } + public int? SliceStart { get; set; } + public int? SliceEnd { get; set; } + + internal object Value + { + get + { + if (SliceStart.HasValue) + { + if (SliceEnd.HasValue) + { + return new int[] { SliceStart.Value, SliceEnd.Value }; + } + else + { + return SliceStart.Value; + } + } + return Include; + } + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs new file mode 100644 index 0000000..46314c3 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs @@ -0,0 +1,116 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Query; +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections.Generic; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class ProjectionBuilder +{ + public ProjectionBuilder() { } + + public InclusiveProjectionBuilder Include(string field) + { + var builder = new InclusiveProjectionBuilder(); + builder.Include(field); + return builder; + } + + public ExclusiveProjectionBuilder Exclude(string field) + { + var builder = new ExclusiveProjectionBuilder(); + builder.Exclude(field); + return builder; + } + + public InclusiveProjectionBuilder Include(Expression> fieldExpression) + { + var builder = new InclusiveProjectionBuilder(); + builder.Include(fieldExpression); + return builder; + } + + public ExclusiveProjectionBuilder Exclude(Expression> fieldExpression) + { + var builder = new ExclusiveProjectionBuilder(); + builder.Exclude(fieldExpression); + return builder; + } + +} + +public interface IProjectionBuilder +{ + internal List Projections { get; } +} + +public abstract class ProjectionBuilderBase : IProjectionBuilder where TBuilder : ProjectionBuilderBase +{ + protected readonly List _projections = new List(); + + List IProjectionBuilder.Projections => _projections; + + public TBuilder Include(SpecialField specialField) + { + _projections.Add(new Projection() { FieldName = specialField.ToString(), Include = true }); + return (TBuilder)this; + } + + public TBuilder Exclude(SpecialField specialField) + { + _projections.Add(new Projection() { FieldName = specialField.ToString(), Include = false }); + return (TBuilder)this; + } + +} + +public class InclusiveProjectionBuilder : ProjectionBuilderBase> +{ + internal InclusiveProjectionBuilder() { } + + public InclusiveProjectionBuilder Include(string fieldName) + { + _projections.Add(new Projection() { FieldName = fieldName, Include = true }); + return this; + } + + public InclusiveProjectionBuilder Include(Expression> fieldExpression) + { + _projections.Add(new Projection() { FieldName = fieldExpression.GetMemberNameTree(), Include = true }); + return this; + } +} + +public class ExclusiveProjectionBuilder : ProjectionBuilderBase> +{ + internal ExclusiveProjectionBuilder() { } + + public ExclusiveProjectionBuilder Exclude(string fieldName) + { + _projections.Add(new Projection() { FieldName = fieldName, Include = false }); + return this; + } + + public ExclusiveProjectionBuilder Exclude(Expression> fieldExpression) + { + _projections.Add(new Projection() { FieldName = fieldExpression.GetMemberNameTree(), Include = false }); + return this; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs new file mode 100644 index 0000000..610adc8 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs @@ -0,0 +1,60 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class Sort +{ + internal const int SortAscending = 1; + internal const int SortDescending = -1; + + internal string Name { get; set; } + internal object Value { get; set; } + + internal Sort(string sortKey, object value) + { + Name = sortKey; + Value = value; + } + + public static Sort Ascending(string field) => new(field, SortAscending); + + public static Sort Descending(string field) => new(field, SortDescending); + + public static Sort Vector(float[] vector) => new(DataApiKeywords.Vector, vector); + + public static Sort Vectorize(string valueToVectorize) => new(DataApiKeywords.Vectorize, valueToVectorize); +} + +public class Sort : Sort +{ + internal Sort(string sortKey, object value) : base(sortKey, value) { } + + public static Sort Ascending(Expression> expression) + { + return new Sort(expression.GetMemberNameTree(), SortAscending); + } + + public static Sort Descending(Expression> expression) + { + return new Sort(expression.GetMemberNameTree(), SortDescending); + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs new file mode 100644 index 0000000..9514993 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs @@ -0,0 +1,65 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class SortBuilder +{ + private List _sorts = new List(); + + internal List Sorts => _sorts; + + public SortBuilder Ascending(string fieldName) + { + _sorts.Add(Sort.Ascending(fieldName)); + return this; + } + + public SortBuilder Ascending(Expression> expression) + { + _sorts.Add(Sort.Ascending(expression)); + return this; + } + + public SortBuilder Descending(string fieldName) + { + _sorts.Add(Sort.Descending(fieldName)); + return this; + } + + public SortBuilder Descending(Expression> expression) + { + _sorts.Add(Sort.Descending(expression)); + return this; + } + + public SortBuilder Vector(float[] vector) + { + _sorts.Add(Sort.Vector(vector)); + return this; + } + + public SortBuilder Vectorize(string valueToVectorize) + { + _sorts.Add(Sort.Vectorize(valueToVectorize)); + return this; + } + +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/SpecialField.cs b/src/DataStax.AstraDB.DataApi/Core/Query/SpecialField.cs new file mode 100644 index 0000000..3da1dd5 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/SpecialField.cs @@ -0,0 +1,42 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Commands; +using System; + +namespace DataStax.AstraDB.DataApi.Query; + +public enum SpecialField +{ + Vectorize, + Vector, + Id +} + +public static class SpecialFieldExtensions +{ + public static string ToString(this SpecialField specialField) + { + return specialField switch + { + SpecialField.Vectorize => DataApiKeywords.Vectorize, + SpecialField.Vector => DataApiKeywords.Vector, + SpecialField.Id => DataApiKeywords.Id, + _ => throw new ArgumentOutOfRangeException(), + }; + + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/CollectionResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/CollectionInfo.cs similarity index 96% rename from src/DataStax.AstraDB.DataApi/Core/Results/CollectionResult.cs rename to src/DataStax.AstraDB.DataApi/Core/Results/CollectionInfo.cs index 7d69df6..902db66 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/CollectionResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/CollectionInfo.cs @@ -18,10 +18,11 @@ namespace DataStax.AstraDB.DataApi.Core.Results; -public class CollectionResult +public class CollectionInfo { [JsonPropertyName("name")] public string Name { get; set; } + [JsonPropertyName("options")] public CollectionDefinition Options { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentResult.cs new file mode 100644 index 0000000..2a2400b --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentResult.cs @@ -0,0 +1,25 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +public class DocumentResult +{ + [JsonPropertyName("document")] + public T Document { get; set; } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs new file mode 100644 index 0000000..61707e3 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs @@ -0,0 +1,28 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +public class DocumentsCountResult +{ + [JsonPropertyName("count")] + public int Count { get; set; } + + [JsonPropertyName("moreData")] + public bool MoreDocumentsExist { get; set; } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/ApiData.cs b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs similarity index 75% rename from src/DataStax.AstraDB.DataApi/Core/ApiData.cs rename to src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs index 328ce99..ea1b6a6 100644 --- a/src/DataStax.AstraDB.DataApi/Core/ApiData.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs @@ -14,19 +14,16 @@ * limitations under the License. */ -using DataStax.AstraDB.DataApi.Collections; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace DataStax.AstraDB.DataApi.Core; +namespace DataStax.AstraDB.DataApi.Core.Results; -//TODO: placeholder -internal class ApiData +public class DocumentsResult { [JsonPropertyName("documents")] - public List Documents { get; set; } - [JsonPropertyName("document")] - public Document Document { get; set; } + public List Documents { get; set; } + [JsonPropertyName("nextPageState")] public string NextPageState { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs b/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs index 2d6ed67..70144a5 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs @@ -30,12 +30,6 @@ public class EmbeddingProvider public List Parameters { get; set; } public List Models { get; set; } - // public AuthenticationMethod GetSharedSecretAuthentication() => - // SupportedAuthentication.GetValueOrDefault(AuthenticationMethodSharedSecret); - - // public AuthenticationMethod GetHeaderAuthentication() => - // SupportedAuthentication.GetValueOrDefault(AuthenticationMethodHeader); - public class Model { public string Name { get; set; } diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs new file mode 100644 index 0000000..c2954c5 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs @@ -0,0 +1,25 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +public class EstimatedDocumentsCountResult +{ + [JsonPropertyName("count")] + public int Count { get; set; } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs index 0f1059e..4dd86de 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs @@ -20,9 +20,7 @@ namespace DataStax.AstraDB.DataApi.Core.Results; public class FindEmbeddingProvidersResult { - Dictionary EmbeddingProviders { get; set; } public FindEmbeddingProvidersResult() { } - } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs new file mode 100644 index 0000000..fbf28ac --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs @@ -0,0 +1,25 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +public class FindStatusResult +{ + [JsonPropertyName("sortVector")] + public float[] SortVector { get; set; } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs b/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs index f52c8ee..3559427 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs @@ -14,12 +14,13 @@ * limitations under the License. */ +using System.Collections.Generic; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Results; -internal class InsertDocumentsCommandResponse +internal class InsertDocumentsCommandResponse { [JsonPropertyName("insertedIds")] - public object[] InsertedIds { get; set; } + public List InsertedIds { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs index 08ac16d..04e511a 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs @@ -21,5 +21,5 @@ namespace DataStax.AstraDB.DataApi.Core.Results; public class ListCollectionsResult { [JsonPropertyName("collections")] - public CollectionResult[] Collections { get; set; } + public CollectionInfo[] Collections { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/SerDes/DataSerializer.cs b/src/DataStax.AstraDB.DataApi/Core/SerDes/DataSerializer.cs deleted file mode 100644 index 0fd92da..0000000 --- a/src/DataStax.AstraDB.DataApi/Core/SerDes/DataSerializer.cs +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Collections.Generic; -using System.Text.Json; - -namespace DataStax.AstraDB.DataApi.Core.SerDes; - -public class DatabaseSerializer : IDataSerializer -{ - private readonly JsonSerializerOptions _options; - - public DatabaseSerializer() - { - _options = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true - }; - } - - public string Serialize(T obj) - { - return JsonSerializer.Serialize(obj, _options); - } - - public T Deserialize(string json) - { - return JsonSerializer.Deserialize(json, _options); - } - - public Dictionary SerializeToMap(T obj) - { - string json = Serialize(obj); - return Deserialize>(json); - } - - public T DeserializeFromMap(Dictionary map) - { - string json = Serialize(map); - return Deserialize(json); - } -} diff --git a/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs b/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs index be03cad..5831c2a 100644 --- a/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs @@ -21,7 +21,7 @@ namespace DataStax.AstraDB.DataApi.Core; public class VectorOptions { [JsonPropertyName("dimension")] - public int Dimension { get; set; } + public int? Dimension { get; set; } [JsonPropertyName("metric")] public SimilarityMetric Metric { get; set; } diff --git a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs index d011e49..4803244 100644 --- a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs +++ b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs @@ -20,7 +20,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Http; using System.Net.Http; using System; using System.Threading.Tasks; diff --git a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj index 33555d0..be1fdb6 100644 --- a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj +++ b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj @@ -10,6 +10,7 @@ + diff --git a/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs new file mode 100644 index 0000000..1a871ba --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs @@ -0,0 +1,164 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; + +using DataStax.AstraDB.DataApi.Core.Commands; +using MongoDB.Bson; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + + +public class DocumentConverter : JsonConverter +{ + private static readonly Dictionary FieldMappings; + private static readonly Dictionary ReverseMappings; + private static readonly List PropertyNamesToIgnore = new(); + + static DocumentConverter() + { + FieldMappings = new Dictionary(); + ReverseMappings = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var prop in typeof(T).GetProperties()) + { + var attr = prop.GetCustomAttribute(); + if (attr != null) + { + string jsonName = attr.Field switch + { + DocumentMappingField.Vectorize => DataApiKeywords.Vectorize, + DocumentMappingField.Vector => DataApiKeywords.Vector, + DocumentMappingField.Id => DataApiKeywords.Id, + DocumentMappingField.Similarity => DataApiKeywords.Similarity, + _ => prop.Name + }; + FieldMappings[prop] = jsonName; + ReverseMappings[jsonName] = prop; + PropertyNamesToIgnore.Add(prop.Name); + } + else + { + if (prop.Name == DataApiKeywords.Id) + { + FieldMappings[prop] = DataApiKeywords.Id; + ReverseMappings[DataApiKeywords.Id] = prop; + PropertyNamesToIgnore.Add(prop.Name); + } + } + } + } + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + throw new JsonException("Expected StartObject"); + + T instance = Activator.CreateInstance(); + PropertyInfo[] properties = typeof(T).GetProperties(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + return instance; + + if (reader.TokenType != JsonTokenType.PropertyName) + continue; + + string propertyName = reader.GetString(); + reader.Read(); // Move to value + + PropertyInfo targetProp = ReverseMappings.TryGetValue(propertyName, out var mappedProp) + ? mappedProp + : properties.FirstOrDefault(p => + p.Name.Equals(propertyName, StringComparison.OrdinalIgnoreCase) && + !FieldMappings.ContainsKey(p)); + + if (targetProp != null && targetProp.CanWrite) + { + object value = JsonSerializer.Deserialize(ref reader, targetProp.PropertyType, options); + targetProp.SetValue(instance, value); + } + else + { + reader.Skip(); + } + } + + throw new JsonException("Unexpected end of JSON"); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + foreach (var prop in typeof(T).GetProperties()) + { + if (FieldMappings.TryGetValue(prop, out string mappedName)) + { + string propertyName = mappedName; + object propValue = prop.GetValue(value); + + writer.WritePropertyName(propertyName); + + // if (propertyName == DataApiKeywords.Id && propValue?.GetType() == typeof(ObjectId)) + // { + // JsonSerializer.Serialize(writer, propValue.ToString(), typeof(string), options); + // } + // else + // { + JsonSerializer.Serialize(writer, propValue, prop.PropertyType, options); + //} + } + } + + // Delegate to default serialization for non-special properties + var defaultOptions = new JsonSerializerOptions + { + AllowTrailingCommas = options.AllowTrailingCommas, + DefaultBufferSize = options.DefaultBufferSize, + DictionaryKeyPolicy = options.DictionaryKeyPolicy, + Encoder = options.Encoder, + IgnoreReadOnlyFields = options.IgnoreReadOnlyFields, + IgnoreReadOnlyProperties = options.IgnoreReadOnlyProperties, + IncludeFields = options.IncludeFields, + MaxDepth = options.MaxDepth, + NumberHandling = options.NumberHandling, + PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive, + PropertyNamingPolicy = options.PropertyNamingPolicy, + ReadCommentHandling = options.ReadCommentHandling, + ReferenceHandler = options.ReferenceHandler, + UnknownTypeHandling = options.UnknownTypeHandling, + WriteIndented = options.WriteIndented + }; + string defaultJson = JsonSerializer.Serialize(value, typeof(T), defaultOptions); + using var doc = JsonDocument.Parse(defaultJson); + foreach (var prop in doc.RootElement.EnumerateObject()) + { + if (!PropertyNamesToIgnore.Contains(prop.Name)) + { + writer.WritePropertyName(prop.Name); + prop.Value.WriteTo(writer); + } + } + + writer.WriteEndObject(); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingAttribute.cs b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingAttribute.cs new file mode 100644 index 0000000..5b70709 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingAttribute.cs @@ -0,0 +1,30 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; + +using System; + +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false)] +public class DocumentMappingAttribute : Attribute +{ + public DocumentMappingField Field { get; } + + public DocumentMappingAttribute(DocumentMappingField field) + { + Field = field; + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs new file mode 100644 index 0000000..56daf08 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs @@ -0,0 +1,30 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; +public enum DocumentMappingField +{ + /// Serializes as "$vectorize" for a string to vectorize + Vectorize, + /// Serializes as "$vector" for vector data. + Vector, + /// Serializes as "_id" for unique identifiers + Id, + /// On read operations only, serializes the similarity result for vector comparisons + Similarity, + /// On read operations only, serializes the vector used for sorting + SortVector +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/IdAttribute.cs b/src/DataStax.AstraDB.DataApi/SerDes/IdAttribute.cs new file mode 100644 index 0000000..9a0de9a --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/IdAttribute.cs @@ -0,0 +1,24 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; + +using System; + +[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] +public class IdAttribute : Attribute +{ +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/IdListConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/IdListConverter.cs new file mode 100644 index 0000000..fee7f40 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/IdListConverter.cs @@ -0,0 +1,85 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; + +using MongoDB.Bson; +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + + +public class IdListConverter : JsonConverter> +{ + public override List Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException("Expected StartArray for ID list"); + } + + var ids = new List(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + { + return ids; + } + + ids.Add(ReadSingleValue(ref reader, typeToConvert, options)); + } + + throw new JsonException("Unexpected end of JSON while reading ID array"); + } + + private object ReadSingleValue(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + object value = reader.TokenType switch + { + JsonTokenType.Number => reader.TryGetInt32(out int intValue) ? intValue : reader.GetDouble(), + JsonTokenType.String => ParseStringValue(reader.GetString()), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Null => null, + _ => JsonSerializer.Deserialize(ref reader, typeToConvert, options), + }; + return value; + } + + private object ParseStringValue(string value) + { + if (string.IsNullOrEmpty(value)) + return value; + + if (Guid.TryParse(value, out Guid guidValue)) + return guidValue; + + if (ObjectId.TryParse(value, out ObjectId objectIdValue)) + return objectIdValue; + + if (DateTime.TryParse(value, null, System.Globalization.DateTimeStyles.RoundtripKind, out DateTime dateTimeValue)) + return dateTimeValue; + + return value; + } + + public override void Write(Utf8JsonWriter writer, List value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, value?.GetType() ?? typeof(object), options); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/ObjectIdConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/ObjectIdConverter.cs new file mode 100644 index 0000000..8214edc --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/ObjectIdConverter.cs @@ -0,0 +1,52 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.SerDes; + +using MongoDB.Bson; +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + + +public class ObjectIdConverter : JsonConverter +{ + public override ObjectId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + string objectIdString = reader.GetString(); + if (ObjectId.TryParse(objectIdString, out ObjectId objectId)) + { + return objectId; + } + else + { + throw new JsonException($"Invalid ObjectId string: {objectIdString}"); + } + } + else + { + throw new JsonException($"Expected string for ObjectId, but found {reader.TokenType}"); + } + } + + public override void Write(Utf8JsonWriter writer, ObjectId value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs b/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs index 96eeb08..985059d 100644 --- a/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs +++ b/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs @@ -14,7 +14,13 @@ * limitations under the License. */ +using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.SerDes; +using System; using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; +using System.Text; namespace DataStax.AstraDB.DataApi.Utils; @@ -34,4 +40,38 @@ internal static T Merge(this IEnumerable list) } return result; } + + public static string GetMemberNameTree(this Expression> expression) + { + if (expression.Body is MemberExpression memberExpression) + { + StringBuilder sb = new StringBuilder(); + BuildPropertyName(memberExpression, sb); + return sb.ToString(); + } + + throw new ArgumentException("Invalid property expression."); + } + + private static void BuildPropertyName(MemberExpression memberExpression, StringBuilder sb) + { + if (memberExpression.Expression is MemberExpression parentExpression) + { + BuildPropertyName(parentExpression, sb); + sb.Append('.'); + } + + var name = memberExpression.Member.Name; + if (memberExpression.Member is PropertyInfo propertyInfo) + { + var attribute = propertyInfo.GetCustomAttribute(); + if (attribute != null && attribute.Field == DocumentMappingField.Id) + { + name = DataApiKeywords.Id; + } + } + sb.Append(name); + + } + } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Utils/Guard.cs b/src/DataStax.AstraDB.DataApi/Utils/Guard.cs index 24ff4e6..043a441 100644 --- a/src/DataStax.AstraDB.DataApi/Utils/Guard.cs +++ b/src/DataStax.AstraDB.DataApi/Utils/Guard.cs @@ -15,6 +15,7 @@ */ using System; +using System.Collections; namespace DataStax.AstraDB.DataApi.Utils; @@ -28,6 +29,14 @@ public static void NotNullOrEmpty(string value, string paramName, string message } } + public static void NotNullOrEmpty(IList value, string paramName, string message = null) + { + if (value == null || value.Count == 0) + { + throw new ArgumentNullException(message.OrIfEmpty("Value cannot be null or empty."), paramName); + } + } + public static void Equals(T value, T valueTwo, string paramName, string message = null) { if (!value.Equals(valueTwo)) diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs index 64bc176..166fca2 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs @@ -4,6 +4,8 @@ using Microsoft.Extensions.Logging; using System.Text.RegularExpressions; +namespace DataStax.AstraDB.DataApi.IntegrationTests; + public class AdminFixture : IDisposable { public AdminFixture() diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/ClientFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/ClientFixture.cs deleted file mode 100644 index 89ffefe..0000000 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/ClientFixture.cs +++ /dev/null @@ -1,55 +0,0 @@ -using DataStax.AstraDB.DataApi; -using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Xunit; - -[CollectionDefinition("DatabaseAndCollections")] -public class DatabaseAndCollectionsCollection : ICollectionFixture -{ - -} - -public class ClientFixture : IDisposable, IAsyncLifetime -{ - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - - public ClientFixture() - { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddConsole()); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(databaseUrl); - - } - - public async Task InitializeAsync() - { - await Database.CreateCollectionAsync(Constants.DefaultCollection); - } - - public async Task DisposeAsync() - { - await Database.DropCollectionAsync(Constants.DefaultCollection); - } - - public void Dispose() - { - - } -} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs new file mode 100644 index 0000000..0c1dad3 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs @@ -0,0 +1,179 @@ +using DataStax.AstraDB.DataApi; +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Xunit; +using Xunit.Abstractions; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +[CollectionDefinition("DatabaseAndCollections")] +public class DatabaseAndCollectionsCollection : ICollectionFixture +{ + +} + +public class CollectionsFixture : IDisposable, IAsyncLifetime +{ + public DataApiClient Client { get; private set; } + public Database Database { get; private set; } + public string OpenAiApiKey { get; set; } + + public CollectionsFixture() + { + IConfiguration configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables(prefix: "ASTRA_DB_") + .Build(); + + var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; + var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; + OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; + + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../latest_run.log")); + ILogger logger = factory.CreateLogger("IntegrationTests"); + + var clientOptions = new CommandOptions + { + RunMode = RunMode.Debug + }; + Client = new DataApiClient(token, clientOptions, logger); + Database = Client.GetDatabase(databaseUrl); + } + + public async Task InitializeAsync() + { + await CreateSearchCollection(); + var collection = Database.GetCollection(_queryCollectionName); + SearchCollection = collection; + } + + public async Task DisposeAsync() + { + await Database.DropCollectionAsync(_queryCollectionName); + } + + public Collection SearchCollection { get; private set; } + + + private const string _queryCollectionName = "simpleObjectsQueryTests"; + private async Task CreateSearchCollection() + { + List items = new List() { + new() + { + _id = 0, + Name = "Cat", + Properties = new Properties() { + PropertyOne = "groupone", + PropertyTwo = "cat", + IntProperty = 1, + BoolProperty = true, + StringArrayProperty = new[] { "cat1", "cat2", "cat3" } + } + }, + new() + { + _id = 1, + Name = "Dog", + Properties = new Properties() { + PropertyOne = "groupone", + PropertyTwo = "dog", + IntProperty = 2, + BoolProperty = true, + StringArrayProperty = new[] { "dog1", "dog2", "dog3" } + } + }, + new() + { + _id = 2, + Name = "Horse", + Properties = new Properties() { + PropertyOne = "grouptwo", + PropertyTwo = "horse", + IntProperty = 3, + BoolProperty = true, + StringArrayProperty = new[] { "horse1", "horse2", "horse3" } + } + }, + new() + { + _id = 3, + Name = "Cow", + Properties = new Properties() { + PropertyOne = "grouptwo", + PropertyTwo = "cow", + IntProperty = 4, + BoolProperty = true, + StringArrayProperty = new[] { "cow1", "cow2", "cow3" } + } + }, + new() + { + _id = 4, + Name = "Alligator", + Properties = new Properties() { + PropertyOne = "grouptwo", + PropertyTwo = "alligator", + IntProperty = 5, + BoolProperty = true, + StringArrayProperty = new[] { "alligator1", "alligator2", "alligator3" } + } + }, + }; + + for (var i = 5; i <= 30; i++) + { + items.Add(new() + { + _id = i, + Name = $"Animal{i}", + Properties = new Properties() + { + PropertyOne = "groupthree", + PropertyTwo = $"animal{i}", + IntProperty = i + 1, + BoolProperty = true, + StringArrayProperty = new[] { $"animal{i}1", $"animal{i}2" } + } + }); + } + items.Add(new() + { + _id = 31, + Name = "Cow Group 4", + Properties = new Properties() + { + PropertyOne = "groupfour", + PropertyTwo = "cow", + IntProperty = 32, + BoolProperty = true, + StringArrayProperty = new[] { "cow1", "cow2" } + } + }); + items.Add(new() + { + _id = 32, + Name = "Alligator Group 4", + Properties = new Properties() + { + PropertyOne = "groupfour", + PropertyTwo = "alligator", + IntProperty = 33, + BoolProperty = true, + StringArrayProperty = new[] { "alligator1", "alligator2" } + } + }); + var collection = await Database.CreateCollectionAsync(_queryCollectionName); + await collection.InsertManyAsync(items); + + SearchCollection = collection; + } + + public void Dispose() + { + //nothing needed + } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Constants.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Constants.cs index b21e581..87b5f34 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Constants.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Constants.cs @@ -1,4 +1,6 @@ +namespace DataStax.AstraDB.DataApi.IntegrationTests; + public static class Constants { public static string DefaultCollection = "testCollection"; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj b/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj index 4714e52..ea94ae1 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj @@ -15,6 +15,8 @@ + + diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/FileLogger.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/FileLogger.cs new file mode 100644 index 0000000..4acefb8 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/FileLogger.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Logging; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +public class FileLogger : ILogger +{ + private readonly string _filePath; + private readonly LogLevel _minLogLevel; + private readonly object _lock = new object(); + + public FileLogger(string filePath, LogLevel minLogLevel = LogLevel.Trace) + { + _filePath = filePath; + _minLogLevel = minLogLevel; + + string directory = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + } + + public IDisposable BeginScope(TState state) => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= _minLogLevel; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception exception, + Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + string message = formatter(state, exception); + string timestamp = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"); + string logEntry = $"{timestamp} [{logLevel}] {message}"; + + if (exception != null) + { + logEntry += $"\nException: {exception}"; + } + + lock (_lock) + { + try + { + File.AppendAllText(_filePath, logEntry + Environment.NewLine); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to write to log file: {ex.Message}"); + } + } + } +} + +public class FileLoggerProvider : ILoggerProvider +{ + private readonly string _filePath; + private readonly LogLevel _minLogLevel; + + public FileLoggerProvider(string filePath, LogLevel minLogLevel = LogLevel.Information) + { + _filePath = filePath; + _minLogLevel = minLogLevel; + } + + public ILogger CreateLogger(string categoryName) + { + return new FileLogger(_filePath, _minLogLevel); + } + + public void Dispose() + { + // Nothing to dispose + } +} + +public static class FileLoggerExtensions +{ + public static ILoggingBuilder AddFileLogger( + this ILoggingBuilder factory, + string filePath, + LogLevel minLogLevel = LogLevel.Information) + { + factory.AddProvider(new FileLoggerProvider(filePath, minLogLevel)); + return factory; + } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs new file mode 100644 index 0000000..f12f335 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs @@ -0,0 +1,99 @@ + + +using DataStax.AstraDB.DataApi.SerDes; +using MongoDB.Bson; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +public class SimpleObjectWithVector +{ + [DocumentMapping(DocumentMappingField.Id)] + public int Id { get; set; } + public string Name { get; set; } + [DocumentMapping(DocumentMappingField.Vector)] + public float[] VectorEmbeddings { get; set; } +} + +public class SimpleObjectWithVectorize +{ + [DocumentMapping(DocumentMappingField.Id)] + public int Id { get; set; } + public string Name { get; set; } + [DocumentMapping(DocumentMappingField.Vectorize)] + public string StringToVectorize => Name; +} + +public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize +{ + [DocumentMapping(DocumentMappingField.Similarity)] + public double? Similarity { get; set; } + [DocumentMapping(DocumentMappingField.SortVector)] + public float[] Vector { get; set; } +} + +public class SimpleObjectWithObjectId +{ + public ObjectId _id { get; set; } + public string Name { get; set; } +} + +public class SimpleObject +{ + public int _id { get; set; } + public string Name { get; set; } + public Properties Properties { get; set; } +} + +public class Properties +{ + public string PropertyOne { get; set; } + public string PropertyTwo { get; set; } + public int IntProperty { get; set; } + public string[] StringArrayProperty { get; set; } + public bool BoolProperty { get; set; } +} + +public class SimpleObjectSkipNulls +{ + public int _id { get; set; } + public string Name { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string PropertyOne { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string PropertyTwo { get; set; } +} + +public class DifferentIdsObject +{ + [DocumentMapping(DocumentMappingField.Id)] + public object TheId { get; set; } + public string Name { get; set; } +} + +public class Restaurant +{ + public Restaurant() { } + public Guid Id { get; set; } + public string Name { get; set; } + public string RestaurantId { get; set; } + public string Cuisine { get; set; } + public Address Address { get; set; } + public string Borough { get; set; } + public List Grades { get; set; } +} + +public class Address +{ + public string Building { get; set; } + public double[] Coordinates { get; set; } + public string Street { get; set; } + public string ZipCode { get; set; } +} + +public class GradeEntry +{ + public DateTime Date { get; set; } + public string Grade { get; set; } + public float? Score { get; set; } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs index d78d7ed..c79de08 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs @@ -1,10 +1,8 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Admin; -using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using Xunit; -namespace DataStax.AstraDB.DataApi.IntegrationTests.Tests; +namespace DataStax.AstraDB.DataApi.IntegrationTests; [CollectionDefinition("Admin Collection")] public class AdminCollection : ICollectionFixture diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs index 8f349ed..ef2ccee 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs @@ -1,16 +1,17 @@ -using DataStax.AstraDB.DataApi; -using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Query; +using MongoDB.Bson; +using UUIDNext; using Xunit; -namespace DataStax.AstraDB.DataApi.IntegrationTests.Tests; +namespace DataStax.AstraDB.DataApi.IntegrationTests; [Collection("DatabaseAndCollections")] public class CollectionTests { - ClientFixture fixture; + CollectionsFixture fixture; - public CollectionTests(ClientFixture fixture) + public CollectionTests(CollectionsFixture fixture) { this.fixture = fixture; } @@ -18,6 +19,7 @@ public CollectionTests(ClientFixture fixture) [Fact] public async Task InsertDocumentAsync() { + var collectionName = "restaurants"; try { Console.WriteLine("Inserting a document..."); @@ -33,44 +35,191 @@ public async Task InsertDocumentAsync() }, Borough = "Manhattan", }; - var collectionName = "restaurants"; var collection = await fixture.Database.CreateCollectionAsync(collectionName); var result = await collection.InsertOneAsync(newRestaurant); - await fixture.Database.DropCollectionAsync(collectionName); var newId = result.InsertedId; Assert.NotNull(newId); } - catch (Exception e) + finally { - Assert.Fail(e.Message); + await fixture.Database.DropCollectionAsync(collectionName); } } -} + [Fact] + public async Task InsertDocument_UsingDefaultId() + { + var collectionName = "defaultId"; + try + { + Console.WriteLine("Inserting a document..."); + var options = new CollectionDefinition + { + DefaultId = new DefaultIdOptions + { + Type = DefaultIdType.ObjectId + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + var newItem = new SimpleObjectWithObjectId + { + Name = "Test Object", + }; + var result = await collection.InsertOneAsync(newItem); + var newId = result.InsertedId; + var parsed = ObjectId.TryParse(newId.ToString(), out var newIdAsObjectId); + Assert.True(parsed); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } -public class Restaurant -{ - public Restaurant() { } - public Guid Id { get; set; } - public string Name { get; set; } - public string RestaurantId { get; set; } - public string Cuisine { get; set; } - public Address Address { get; set; } - public string Borough { get; set; } - public List Grades { get; set; } -} + [Fact] + public async Task InsertDocumentsNotOrderedAsync() + { + var collectionName = "simpleObjects"; + try + { + List items = new List(); + for (var i = 0; i < 10; i++) + { + items.Add(new SimpleObject() + { + _id = i, + Name = $"Test Object {i}" + }); + } + ; + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + var result = await collection.InsertManyAsync(items); + await fixture.Database.DropCollectionAsync(collectionName); + Assert.Equal(items.Count, result.InsertedIds.Count); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + + [Fact] + public async Task InsertDocumentsIdsInOrder() + { + var collectionName = "differentIdsObjects"; + + try + { + var date = DateTime.Now; + var uuid4 = Uuid.NewRandom(); + Guid urlNamespaceId = Guid.Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8"); + var uuid5 = Uuid.NewNameBased(urlNamespaceId, "https://github.com/uuid6/uuid6-ietf-draft"); + var uuid7 = Uuid.NewDatabaseFriendly(UUIDNext.Database.PostgreSql); + var uuid8 = Uuid.NewDatabaseFriendly(UUIDNext.Database.SqlServer); + var objectId = new ObjectId(); + + List items = new List + { + new DifferentIdsObject() + { + TheId = 1, + Name = $"Test Object Int" + }, + new DifferentIdsObject() + { + TheId = objectId, + Name = $"Test Object ObjectId" + }, + new DifferentIdsObject() + { + TheId = uuid4, + Name = $"Test Object UUID4" + }, + new DifferentIdsObject() + { + TheId = uuid5, + Name = $"Test Object UUID5" + }, + new DifferentIdsObject() + { + TheId = uuid7, + Name = $"Test Object UUID7" + }, + new DifferentIdsObject() + { + TheId = uuid8, + Name = $"Test Object UUID8" + }, + new DifferentIdsObject() + { + TheId = "This is an id string", + Name = $"Test Object String" + } + }; + + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + var result = await collection.InsertManyAsync(items, new InsertManyOptions() { InsertInOrder = true }); + + Assert.Equal(items.Count, result.InsertedIds.Count); + for (var i = 0; i < result.InsertedIds.Count; i++) + { + Assert.Equal((dynamic)items[i].TheId, (dynamic)result.InsertedIds[i]); + } + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + + [Fact] + public async Task InsertDocumentsOrderedAsync() + { + var collectionName = "simpleObjects"; + try + { + List items = new List(); + for (var i = 0; i < 10; i++) + { + items.Add(new SimpleObject() + { + _id = i, + Name = $"Test Object {i}" + }); + } + ; + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + var result = await collection.InsertManyAsync(items, new InsertManyOptions() { InsertInOrder = true }); + await fixture.Database.DropCollectionAsync(collectionName); + Assert.Equal(items.Count, result.InsertedIds.Count); + for (var i = 0; i < 10; i++) + { + var id = result.InsertedIds[i]; + Assert.Equal(i, id); + } + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + + [Fact] + public async Task CountDocuments_NoFilter_ReturnsCorrectCount() + { + var collection = fixture.SearchCollection; + var count = await collection.CountDocumentsAsync(); + Assert.Equal(33, count.Count); + } + + [Fact] + public async Task CountDocuments_Filter_ReturnsCorrectCount() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq("Properties.PropertyOne", "grouptwo"); + var count = await collection.CountDocumentsAsync(filter); + Assert.Equal(3, count.Count); + } -public class Address -{ - public string Building { get; set; } - public double[] Coordinates { get; set; } - public string Street { get; set; } - public string ZipCode { get; set; } } -public class GradeEntry -{ - public DateTime Date { get; set; } - public string Grade { get; set; } - public float? Score { get; set; } -} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs index 3906d2e..bd1f0ec 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs @@ -3,14 +3,14 @@ using DataStax.AstraDB.DataApi.Core; using Xunit; -namespace DataStax.AstraDB.DataApi.IntegrationTests.Tests; +namespace DataStax.AstraDB.DataApi.IntegrationTests; [Collection("DatabaseAndCollections")] public class DatabaseTests { - ClientFixture fixture; + CollectionsFixture fixture; - public DatabaseTests(ClientFixture fixture) + public DatabaseTests(CollectionsFixture fixture) { this.fixture = fixture; } @@ -59,10 +59,9 @@ public async Task CreateCollection_CancellationToken() } catch (OperationCanceledException) { - // Optionally re-throw if you want the test to fail if it's caught later. - throw; // Important for assertion! + throw; } - }, cts.Token); // Pass the token to Task.Run as well + }, cts.Token); cts.Cancel(); Assert.Null(collection); var exists = await fixture.Database.DoesCollectionExistAsync(collectionName); @@ -159,8 +158,10 @@ public async Task CreateCollection_WithVectorEuclidean() [Fact] public async Task DoesCollectionExistAsync_ExistingCollection_ReturnsTrue() { + await fixture.Database.CreateCollectionAsync(Constants.DefaultCollection); var exists = await fixture.Database.DoesCollectionExistAsync(Constants.DefaultCollection); Assert.True(exists); + await fixture.Database.DropCollectionAsync(Constants.DefaultCollection); } [Fact] @@ -180,74 +181,128 @@ public async Task DoesCollectionExistAsync_InvalidCollectionName_ReturnsFalse(st Assert.False(exists); } + [Fact] + public async Task Create_Drop_FlowWorks() + { + const string collectionName = "createAndDropCollection"; + await fixture.Database.CreateCollectionAsync(collectionName); + var exists = await fixture.Database.DoesCollectionExistAsync(collectionName); + Assert.True(exists); + await fixture.Database.DropCollectionAsync(Constants.DefaultCollection); + exists = await fixture.Database.DoesCollectionExistAsync(Constants.DefaultCollection); + Assert.False(exists); + } + [Fact] public async Task ListCollectionNamesAsync_ShouldReturnCollectionNames() { + await fixture.Database.CreateCollectionAsync(Constants.DefaultCollection); var result = await fixture.Database.ListCollectionNamesAsync(); Assert.NotNull(result); - Assert.Contains(Constants.DefaultCollection, result.CollectionNames); + Assert.Contains(Constants.DefaultCollection, result); + await fixture.Database.DropCollectionAsync(Constants.DefaultCollection); } [Fact] public async Task ListCollectionNamesAsync_WithCommandOptions_ShouldReturnCollectionNames() { + await fixture.Database.CreateCollectionAsync(Constants.DefaultCollection); var commandOptions = new CommandOptions { /* Initialize with necessary options */ }; var result = await fixture.Database.ListCollectionNamesAsync(commandOptions); Assert.NotNull(result); - Assert.Contains(Constants.DefaultCollection, result.CollectionNames); - } - - // [Fact] - // public async Task CreateCollection_WithVectorizeHeader() - // { - // var collectionName = "collectionVectorizeHeader"; - // var options = new CollectionDefinition - // { - // Vector = new VectorOptions - // { - // Dimension = 1536, - // Metric = SimilarityMetric.DotProduct, - // Service = new VectorServiceOptions - // { - // Provider = "openai", - // ModelName = "text-embedding-ada-002", - // Authentication = new Dictionary - // { - // { "openai", "OPENAI_API_KEY" } - // } - // } - // } - // }; - // var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); - // Assert.NotNull(collection); - // Assert.Equal(collectionName, collection.CollectionName); - // await fixture.Database.DropCollectionAsync(collectionName); - // } - - // [Fact] - // public async Task CreateCollection_WithVectorizeSharedKey() - // { - // var collectionName = "collectionVectorizeSharedKey"; - // var options = new CollectionDefinition - // { - // Vector = new VectorOptions - // { - // Dimension = 1536, - // Metric = SimilarityMetric.DotProduct, - // Service = new VectorServiceOptions - // { - // Provider = "openai", - // ModelName = "text-embedding-ada-002", - // Authentication = new Dictionary - // { - // { "openai", "OPENAI_API_KEY" } - // } - // } - // } - // }; - // var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); - // Assert.NotNull(collection); - // Assert.Equal(collectionName, collection.CollectionName); - // await fixture.Database.DropCollectionAsync(collectionName); - // } + Assert.Contains(Constants.DefaultCollection, result); + await fixture.Database.DropCollectionAsync(Constants.DefaultCollection); + } + + [Fact] + public async Task CreateCollection_WithVectorizeHeader() + { + var collectionName = "collectionVectorizeHeader"; + var options = new CollectionDefinition + { + Vector = new VectorOptions + { + Dimension = 1536, + Metric = SimilarityMetric.DotProduct, + Service = new VectorServiceOptions + { + Provider = "openai", + ModelName = "text-embedding-ada-002", + Authentication = new Dictionary + { + { "providerKey", fixture.OpenAiApiKey } + } + } + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + Assert.NotNull(collection); + Assert.Equal(collectionName, collection.CollectionName); + await fixture.Database.DropCollectionAsync(collectionName); + } + + [Fact] + public async Task CreateCollection_WithVectorizeSharedKey() + { + var collectionName = "collectionVectorizeSharedKey"; + var options = new CollectionDefinition + { + Vector = new VectorOptions + { + Dimension = 1536, + Metric = SimilarityMetric.DotProduct, + Service = new VectorServiceOptions + { + Provider = "openai", + ModelName = "text-embedding-ada-002", + Authentication = new Dictionary + { + { "providerKey", fixture.OpenAiApiKey } + } + } + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + Assert.NotNull(collection); + Assert.Equal(collectionName, collection.CollectionName); + await fixture.Database.DropCollectionAsync(collectionName); + } + + [Fact] + public async Task GetCollectionMetadata() + { + var collectionName = "collectionMetadataTest"; + try + { + var options = new CollectionDefinition + { + Indexing = new IndexingOptions + { + Allow = new List { "metadata" } + }, + Vector = new VectorOptions + { + Dimension = 14, + Metric = SimilarityMetric.DotProduct, + }, + DefaultId = new DefaultIdOptions + { + Type = DefaultIdType.ObjectId + } + }; + await fixture.Database.CreateCollectionAsync(collectionName, options); + var collections = await fixture.Database.ListCollectionsAsync(); + var collectionMetadata = collections.FirstOrDefault(c => c.Name == collectionName); + Assert.NotNull(collectionMetadata); + Assert.Equal(14, collectionMetadata.Options.Vector.Dimension); + Assert.Equal(SimilarityMetric.DotProduct, collectionMetadata.Options.Vector.Metric); + Assert.Contains("metadata", collectionMetadata.Options.Indexing.Allow); + Assert.Equal(DefaultIdType.ObjectId, collectionMetadata.Options.DefaultId.Type); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs new file mode 100644 index 0000000..295fd26 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs @@ -0,0 +1,576 @@ +using System.Linq; +using System.Text.Json; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Query; +using MongoDB.Bson; +using UUIDNext; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +[Collection("DatabaseAndCollections")] +public class SearchTests +{ + CollectionsFixture fixture; + + public SearchTests(CollectionsFixture fixture) + { + this.fixture = fixture; + } + + [Fact] + public async Task ById() + { + var collectionName = "idsSearchCollection"; + + try + { + var date = DateTime.Now; + var uuid4 = Uuid.NewRandom(); + Guid urlNamespaceId = Guid.Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8"); + var uuid5 = Uuid.NewNameBased(urlNamespaceId, "https://github.com/uuid6/uuid6-ietf-draft"); + var uuid7 = Uuid.NewDatabaseFriendly(UUIDNext.Database.PostgreSql); + var uuid8 = Uuid.NewDatabaseFriendly(UUIDNext.Database.SqlServer); + var objectId = new ObjectId(); + + List items = new List + { + new DifferentIdsObject() + { + TheId = 1, + Name = $"Test Object Int" + }, + new DifferentIdsObject() + { + TheId = objectId, + Name = $"Test Object ObjectId" + }, + new DifferentIdsObject() + { + TheId = uuid4, + Name = $"Test Object UUID4" + }, + new DifferentIdsObject() + { + TheId = uuid5, + Name = $"Test Object UUID5" + }, + new DifferentIdsObject() + { + TheId = uuid7, + Name = $"Test Object UUID7" + }, + new DifferentIdsObject() + { + TheId = uuid8, + Name = $"Test Object UUID8" + }, + new DifferentIdsObject() + { + TheId = "This is an id string", + Name = $"Test Object String" + }, + new DifferentIdsObject() + { + TheId = date, + Name = $"Test Object DateTime" + } + }; + + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + await collection.InsertManyAsync(items, new InsertManyOptions() { InsertInOrder = true }); + + //Search using Expression + var filter = Builders.Filter.Eq(d => d.TheId, 1); + var searchResult = await collection.FindOneAsync(filter); + Assert.Equal(1.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object Int", searchResult.Name); + + //Search using String + filter = Builders.Filter.Eq("_id", 1); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(1.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object Int", searchResult.Name); + + //objectId + filter = Builders.Filter.Eq(d => d.TheId, objectId); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(objectId.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object ObjectId", searchResult.Name); + + //uuid4 + filter = Builders.Filter.Eq(d => d.TheId, uuid4); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(uuid4.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object UUID4", searchResult.Name); + + //uuid5 + filter = Builders.Filter.Eq(d => d.TheId, uuid5); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(uuid5.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object UUID5", searchResult.Name); + + //uuid7 + filter = Builders.Filter.Eq(d => d.TheId, uuid7); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(uuid7.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object UUID7", searchResult.Name); + + //uuid8 + filter = Builders.Filter.Eq(d => d.TheId, uuid8); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(uuid8.ToString(), searchResult.TheId.ToString()); + Assert.Equal("Test Object UUID8", searchResult.Name); + + //string + filter = Builders.Filter.Eq(d => d.TheId, "This is an id string"); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal("This is an id string", searchResult.TheId.ToString()); + Assert.Equal("Test Object String", searchResult.Name); + + //date + filter = Builders.Filter.Eq(d => d.TheId, date); + searchResult = await collection.FindOneAsync(filter); + Assert.Equal(JsonSerializer.Serialize(date).Replace("\"", ""), searchResult.TheId.ToString()); + Assert.Equal("Test Object DateTime", searchResult.Name); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + + [Fact] + public async Task SimpleStringFilter() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq("Properties.PropertyOne", "grouptwo"); + var results = await collection.Find(filter).ToEnumerableAsync(); + var expectedArray = new[] { "horse", "cow", "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task SimpleExpressionFilter() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var results = await collection.Find(filter).ToEnumerableAsync(); + var expectedArray = new[] { "horse", "cow", "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task Limit_RunsAsync_ReturnsLimitedResult() + { + var collection = fixture.SearchCollection; + var results = await collection.Find().Limit(1).ToEnumerableAsync(); + Assert.Single(results); + } + + [Fact] + public void InclusiveProjection_RunsSync_ReturnsIncludedProperties() + { + var collection = fixture.SearchCollection; + var inclusiveProjection = Builders.Projection + .Include("Properties.PropertyTwo"); + var results = collection.Find().Limit(1).Project(inclusiveProjection).ToList(); + var result = results.First(); + Assert.True(string.IsNullOrEmpty(result.Name)); + Assert.True(string.IsNullOrEmpty(result.Properties.PropertyOne)); + Assert.False(string.IsNullOrEmpty(result.Properties.PropertyTwo)); + } + + [Fact] + public void ExclusiveProjection_RunsSync_ExcludesProperties() + { + var collection = fixture.SearchCollection; + var exclusiveProjection = Builders.Projection + .Exclude("Properties.PropertyTwo"); + var results = collection.Find().Limit(1).Project(exclusiveProjection).ToList(); + var result = results.First(); + Assert.False(string.IsNullOrEmpty(result.Name)); + Assert.False(string.IsNullOrEmpty(result.Properties.PropertyOne)); + Assert.True(string.IsNullOrEmpty(result.Properties.PropertyTwo)); + } + + [Fact] + public async Task Sort_RunsAsync_ReturnsSortedResult() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var sort = Builders.Sort.Ascending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).ToEnumerableAsync(); + var expectedArray = new[] { "alligator", "cow", "horse" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task SortDescending_RunsAsync_ReturnsResultsDescending() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var sort = Builders.Sort.Descending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).ToEnumerableAsync(); + var expectedArray = new[] { "horse", "cow", "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task Skip_RunsAsync_ReturnsResultsAfterSkip() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var sort = Builders.Sort.Descending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).Skip(1).ToEnumerableAsync(); + var expectedArray = new[] { "cow", "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task LimitAndSkip_RunsAsync_ReturnsExpectedResults() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var sort = Builders.Sort.Descending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).Skip(2).Limit(1).ToEnumerableAsync(); + var expectedArray = new[] { "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task NotFluent_RunsAsync_ReturnsExpectedResults() + { + var collection = fixture.SearchCollection; + var filter = Builders.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); + var sort = Builders.Sort.Descending(o => o.Properties.PropertyTwo); + var inclusiveProjection = Builders.Projection + .Include("Properties.PropertyTwo"); + var findOptions = new FindOptions() + { + Sort = sort, + Limit = 1, + Skip = 2, + Projection = inclusiveProjection + }; + var cursor = await collection.FindManyAsync(filter, findOptions); + var results = cursor.ToEnumerable(); + var expectedArray = new[] { "alligator" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + var result = results.First(); + Assert.True(string.IsNullOrEmpty(result.Name)); + Assert.True(string.IsNullOrEmpty(result.Properties.PropertyOne)); + Assert.False(string.IsNullOrEmpty(result.Properties.PropertyTwo)); + } + + [Fact] + public async Task LogicalAnd_MongoStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Eq(so => so.Properties.PropertyOne, "grouptwo") & builder.Eq(so => so.Properties.PropertyTwo, "cow"); + var results = await collection.Find(filter).ToListAsync(); + var expectedArray = new[] { "cow" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task LogicalAnd_AstraStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.And(builder.Eq(so => so.Properties.PropertyOne, "grouptwo"), builder.Eq(so => so.Properties.PropertyTwo, "cow")); + var results = await collection.Find(filter).ToListAsync(); + var expectedArray = new[] { "cow" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task LogicalOr_MongoStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Eq(so => so.Properties.PropertyTwo, "alligator") | builder.Eq(so => so.Properties.PropertyTwo, "cow"); + var sort = Builders.Sort.Ascending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + var expectedArray = new[] { "alligator", "alligator", "cow", "cow" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task LogicalOr_AstraStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Or(builder.Eq(so => so.Properties.PropertyOne, "groupone"), builder.Eq(so => so.Properties.PropertyOne, "grouptwo")); + var sort = Builders.Sort.Ascending(o => o.Properties.PropertyTwo); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + var expectedArray = new[] { "alligator", "cat", "cow", "dog", "horse" }; + var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); + Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); + } + + [Fact] + public async Task LogicalNot_MongoStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = !(builder.Eq(so => so.Properties.PropertyTwo, "alligator") | builder.Eq(so => so.Properties.PropertyTwo, "cow")); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(29, results.Count); + } + + [Fact] + public async Task LogicalNot_AstraStyle() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Not(builder.Eq(so => so.Properties.PropertyTwo, "alligator") | builder.Eq(so => so.Properties.PropertyTwo, "cow")); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(29, results.Count); + } + + [Fact] + public async Task GreaterThan() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Gt(so => so.Properties.IntProperty, 20); + var sort = Builders.Sort.Ascending(o => o.Properties.IntProperty); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + Assert.Equal(13, results.Count); + Assert.Equal(21, results.First().Properties.IntProperty); + } + + [Fact] + public async Task GreaterThanOrEqual() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Gte(so => so.Properties.IntProperty, 20); + var sort = Builders.Sort.Ascending(o => o.Properties.IntProperty); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + Assert.Equal(14, results.Count); + Assert.Equal(20, results.First().Properties.IntProperty); + } + + [Fact] + public async Task LessThan() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Lt(so => so.Properties.IntProperty, 20); + var sort = Builders.Sort.Descending(o => o.Properties.IntProperty); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + Assert.Equal(19, results.Count); + Assert.Equal(19, results.First().Properties.IntProperty); + } + + [Fact] + public async Task LessThanOrEqual() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Lte(so => so.Properties.IntProperty, 20); + var sort = Builders.Sort.Descending(o => o.Properties.IntProperty); + var results = await collection.Find(filter).Sort(sort).ToListAsync(); + Assert.Equal(20, results.Count); + Assert.Equal(20, results.First().Properties.IntProperty); + } + + [Fact] + public async Task NotEqualTo() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Ne(so => so.Properties.PropertyOne, "groupthree"); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(7, results.Count); + } + + [Fact] + public async Task InArray() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.In(so => so.Properties.PropertyOne, new[] { "groupone", "grouptwo" }); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(5, results.Count); + } + + [Fact] + public async Task NotInArray() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Nin(so => so.Properties.PropertyOne, new[] { "groupone", "grouptwo" }); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(28, results.Count); + } + + [Fact] + public async Task InArray_WithArrays() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.In(so => so.Properties.StringArrayProperty, new[] { "cat1", "dog1" }); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(2, results.Count); + } + + [Fact] + public async Task PropertyExists() + { + var collectionName = "differentProperties"; + + List items = new List(); + for (var i = 0; i < 5; i++) + { + items.Add(new SimpleObjectSkipNulls() + { + _id = i, + Name = $"Test Object {i}", + PropertyOne = i % 2 == 0 ? "groupone" : "grouptwo", + PropertyTwo = i % 2 == 0 ? "hasvalue" : null + }); + } + ; + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + var result = await collection.InsertManyAsync(items); + var builder = Builders.Filter; + var filter = builder.Exists(so => so.PropertyTwo); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(3, results.Count); + await fixture.Database.DropCollectionAsync(collectionName); + } + + [Fact] + public async Task ArrayContainsAll() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.All(so => so.Properties.StringArrayProperty, new[] { "alligator1", "alligator2", "alligator3" }); + var results = await collection.Find(filter).ToListAsync(); + Assert.Single(results); + } + + [Fact] + public async Task ArrayHasSize() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Size(so => so.Properties.StringArrayProperty, 3); + var results = await collection.Find(filter).ToListAsync(); + Assert.Equal(5, results.Count); + } + + [Fact] + public async Task QueryDocumentsWithVectorsAsync() + { + var collectionName = "simpleObjectsWithVectors"; + try + { + List items = new List() { + new() + { + Id = 0, + Name = "This is about a cat.", + VectorEmbeddings = (new double[] { 0.089152224,0.041651253,0.006238773,0.043506123,-0.08882473,-0.028041942,0.057035644,0.021510456,-0.0335324,-0.015038743,-0.010946267,-0.033052385,-0.004451128,0.024450185,-0.10682846,-0.008512233,-0.06361482,0.011363489,-0.018674858,0.05755979,0.010276637,0.07108201,0.02609113,0.0051583643,-0.099871725,0.064882025,-0.04566266,-0.0447845,0.015848588,0.05461174,-0.09496783,0.00975987,-0.0068527046,0.058530692,-0.077218644,-0.07914207,0.015511969,0.069497555,0.065023474,0.08405613,-0.009922312,-0.055714585,0.014438505,-0.013197334,-0.057995494,0.034810554,-0.0021572637,-0.07460552,0.05961566,-0.06183412,-0.079871744,-0.011298,-0.049480535,-0.024539579,-0.021874059,-0.0039382433,0.051196072,-0.048952673,-0.026021041,-0.016276214,0.02319573,0.041072488,0.008482071,0.09586513,0.038031645,-0.024842998,0.003575021,-0.011547007,0.006391744,-0.049402386,0.0467421,0.00959334,-0.03264049,0.026778983,-0.009813442,-0.06520412,0.07849598,-0.0003208971,0.0959958,0.0781274,-0.042293962,-0.014687874,0.03163216,0.07175497,0.008732745,0.06964863,0.031615674,-0.05005147,-0.047878552,-0.0005746399,-0.006433992,-0.020098396,-0.0030210759,0.0011368011,-0.0495901,-0.011242182,-0.003216857,-0.06091114,-0.025426276,0.076356426,0.023855325,-0.0024857672,0.036267877,0.07782738,-0.035250574,-0.0038455587,0.02195504,0.03382706,0.021517783,0.0117069045,0.028399047,-0.10053743,-0.05483047,0.076560654,0.08378389,0.005169428,0.022630876,-0.051714625,0.06101812,-0.012016718,0.08596926,-0.045620713,-0.0357822,0.026691968,-0.00868368,-0.06845361,-0.0064422367,-1.0561074e-32,0.018288853,-0.03167005,0.01725241,-0.016210299,0.103539616,0.023504676,-0.013519004,0.003982343,-0.029885074,0.033861946,-0.09327008,-0.05178391,-0.0436475,-0.059817906,-0.009489782,-0.08695476,-0.0659479,-0.005502922,-0.0057627205,0.03876865,0.011172102,0.08398976,0.060095742,-0.032664955,-0.0024455208,-0.06746624,-0.080734365,-0.079751015,-0.023598457,0.0002563353,0.053381264,0.026176691,0.07066675,-0.020418348,-0.11429579,-0.047344334,-0.013287097,-0.01864271,-0.0063428497,0.037078038,0.043008555,-0.004289443,0.03631616,0.033966597,0.022350233,0.031604636,-0.024446608,0.019221745,-0.046859663,0.060180783,0.12533426,-0.00479271,0.035111945,-0.040028594,-0.031212045,0.010197592,0.030907258,-0.030999627,-0.007732326,0.13904367,0.04940509,0.010669046,0.016782433,-0.025963195,0.023240745,-0.063346595,0.019595988,-0.021747895,0.089717746,0.07910935,-0.0657273,0.019861227,0.03558573,-0.086711794,-0.016101884,-0.019212069,0.02883235,-0.011353625,-0.08117944,0.08045923,-0.03617811,-0.015651839,0.11304874,-0.011179125,-0.03710327,0.041835453,0.02170425,-0.035943862,-0.059953246,0.07860478,-0.083585985,0.08917205,-0.027793054,-0.1512843,0.06380901,6.228111e-33,-0.026383527,-0.0017929141,0.023303084,0.02218534,-0.06251773,0.048760373,0.010181715,0.07173921,-0.07172141,0.07956989,-0.04191949,0.07540622,0.12423252,-0.0070834314,0.09814076,0.039843906,-0.018787384,0.007695833,0.0671187,-0.03075603,-0.04254027,0.05959335,-0.06380347,0.038309086,0.023116969,0.014789075,0.031658806,-0.055602986,-0.0071550673,-0.19415864,0.01743854,-0.13140139,-0.0144067705,-0.04930145,-0.050060764,0.057865113,-0.053838655,-0.09110394,-0.022664431,-0.014732251,0.0136811845,-0.008339691,0.043517523,-0.0043350235,-0.0198879,-0.037622113,-0.0013067131,-0.009471046,-0.0052737775,0.032179937,-0.03113369,-0.09677153,0.035852194,0.01569161,0.028246677,0.009729108,-0.03177169,-0.009778319,0.017784022,0.032719884,-0.009280669,0.030814694,-0.033533014,0.052525237,-0.06690553,-0.02933622,-0.02315666,-0.0909028,0.009725973,-0.056958094,0.12035951,0.03881476,-0.072808184,-0.07154062,-0.014807507,-0.0012886004,0.053991668,-0.022750335,-0.031523474,-0.05695466,-0.019918608,-0.041242998,0.008761982,0.011521017,0.021141063,-0.032588396,0.027337488,0.092536,-0.019304343,0.05793468,-0.0062565147,0.0122958785,0.034887247,-0.120099336,0.013632101,-1.8641153e-8,-0.03772457,-0.048523452,-0.097976536,0.021443354,0.024639249,0.055306807,0.039979752,-0.1179702,-0.05120052,0.012400957,0.04489748,-0.015811147,0.015597527,0.082042284,0.052555256,0.055382337,0.00070087315,-0.019393984,0.01994989,0.11038251,-0.07126654,-0.020937268,-0.057667337,0.0013548834,-0.028954726,-0.0112794535,-0.013444409,0.08701174,-0.011877562,-0.077941716,0.039188765,-0.006396562,0.020704756,-0.050458863,0.0053959913,-0.037047394,0.07261843,-0.11010277,0.03790635,-0.055131394,0.001222146,0.00043609156,0.083624594,0.0116417315,-0.039805945,0.025670456,0.023367377,-0.030002289,0.0125302505,0.055243578,0.021209959,0.10720353,0.061538137,0.028139735,-0.0011503869,-0.033695076,0.0071400655,-0.011627619,0.018046776,0.06495565,0.038648676,0.062729195,0.055372205,0.053745285 }).Select(d => (float)d).ToArray() + }, + new() + { + Id = 1, + Name = "This is about a dog.", + VectorEmbeddings = (new double[] { 0.011346418,0.04229269,0.03587526,0.06744778,-0.04397914,-0.015584227,0.041720465,0.014429322,0.044287052,0.0043079043,0.019921772,-0.043505132,0.043102365,0.047135014,-0.08231162,0.020834573,-0.02617832,0.006700209,-0.012063026,-0.022996914,-0.019173175,0.07027755,0.015753163,-0.005731548,-0.1283017,0.022473892,0.0108027095,-0.04926471,0.036433354,0.020997025,-0.060065117,0.008349197,-0.018449424,0.035899807,-0.068035096,-0.06751048,0.03407754,0.09698124,0.08279569,0.0680067,0.050136272,-0.01190681,0.0332499,-0.039791144,-0.03364588,0.030574504,-0.09330098,-0.05924064,0.070340574,-0.025546642,-0.048177682,-0.024846006,-0.040927712,-0.02904687,-0.041816615,-0.04117641,-0.040064435,-0.009248488,-0.019651065,-0.020427428,0.028331814,0.04488185,0.039839122,0.03350426,-0.0030064413,-0.0049702036,-0.023976147,-0.023749182,-0.053586725,-0.03215982,0.06093219,0.012142569,0.0441472,-0.004723474,-0.010604439,-0.122735985,-0.0122516565,-0.0027201625,0.15101196,0.07122502,-0.056468617,-0.034811575,-0.0425355,0.068009876,0.016610937,0.033443205,0.02710159,-0.042842995,-0.08505483,-0.008643339,-0.068027355,-0.06379054,-0.029316818,0.0058006127,-0.030307215,-0.008950219,-0.047879178,-0.0913828,-0.009465094,0.053881057,0.0031493043,0.025003623,0.07674008,0.015249092,0.04850128,-0.012600311,-0.026310094,0.09720545,0.01680918,0.013852205,-0.018736323,-0.06005147,-0.020879472,0.07370358,0.09260949,0.021539116,-0.055899676,-0.05503345,-0.015129937,-0.024785034,0.08288727,-0.013179343,-0.0726965,0.020562222,0.022948798,-0.034667227,0.0526193,-1.18245034e-32,0.050559655,-0.0048819673,0.0063548665,-0.05116284,0.008868584,-0.0021307247,0.01315732,0.002422845,-0.053024683,0.043829612,-0.032446425,-0.034506686,-0.0022936491,-0.009842915,0.091232926,-0.077086814,-0.031722493,0.015017364,0.06814536,-0.0017685982,-0.017744187,0.073336154,0.019819103,-0.0644845,0.0034368993,0.005495123,-0.084544875,-0.06872212,-0.059192233,0.022209745,0.038755916,0.03318034,0.07090409,-0.04171937,-0.0981389,0.012770359,-0.009513227,-0.080934696,-0.033764116,0.05497646,0.08185176,-0.029122388,0.0384573,0.021990445,-0.0032139954,0.041740686,-0.056978893,-0.04734529,-0.03991311,0.019610234,0.0647751,0.004388865,0.10982947,-0.043376997,0.010122793,0.032659534,0.055898506,0.0195673,-0.009158063,0.123804346,0.0708504,-0.0071747694,-0.036037583,-0.049208198,0.058279518,-0.0790463,-0.12674646,-0.04109742,-0.038129613,0.07686497,-0.04693619,-0.012978095,0.06309132,-0.04087212,0.019872552,0.010217363,0.08006997,0.0026933537,0.001853705,-0.004593111,-0.048401523,0.013331191,0.036138237,-0.0344762,-0.018189447,0.00083567185,0.020371282,0.027441347,-0.054629076,0.008875433,-0.052847195,-0.030010821,-0.035492633,-0.070175104,0.054558296,6.417459e-33,-0.008761953,0.06607552,0.026014887,0.05661787,-0.011050351,0.004505302,-0.02256045,0.12539902,-0.08650192,0.061125778,-0.042015005,0.02355448,0.08238224,-0.005603566,0.09459545,0.0948628,0.017250946,0.033425502,0.040046655,-0.0628745,-0.1046111,0.065641314,0.042201586,0.0120828645,-0.04438299,0.07246842,0.02128908,-0.08484302,-0.041710783,-0.11470048,-0.053501517,-0.117971286,-0.00788975,-0.055501066,-0.05120677,0.06829499,0.031432264,-0.12799612,-0.011093621,-0.019008748,0.058515273,-0.061318677,0.037527867,0.014019596,0.060755204,0.03810258,0.0096842535,-0.04321577,0.00712279,0.046378646,-0.047010764,-0.042070884,0.04731553,-0.024275718,0.02193629,-0.018049095,-0.09782932,0.0030079312,0.041306987,0.007830905,-0.013250943,0.07706189,-0.026540434,0.093507566,-0.10470637,-0.05415237,-0.023200897,-0.039328523,0.03200482,-0.08281106,0.041430682,0.04234739,-0.03488221,-0.05283925,0.014213709,0.0029994198,0.027753718,-0.052510686,0.018441025,0.0052146562,-0.08947327,-0.05556312,-0.026160069,0.011457789,0.029503262,0.0025367353,-0.028680122,0.10300588,0.0031025852,0.015188665,-0.023412446,0.06668626,0.00421559,-0.09940255,0.020253142,-1.9181371e-8,-0.033001088,-0.013900804,-0.04732987,-0.002962324,0.06098004,0.066006675,0.03184782,-0.11235283,-0.055892687,0.019401163,0.04267367,-0.022931533,-0.07635768,0.058402903,0.013820379,0.04567165,-0.009838069,0.041856937,0.07559799,0.10492714,-0.062181033,-0.034038723,0.056748517,-0.045047298,-0.037589937,-0.036176957,0.013583488,0.07220805,-0.11159318,-0.016334815,-0.02307086,0.08323495,-0.043899763,-0.015546188,0.006285,-0.06952057,0.10265458,-0.068090186,-0.012350388,-0.001622836,0.013450635,0.09277687,-0.008574189,0.015505242,0.06175743,0.097206704,0.051632088,-0.05997204,-0.0013132367,-0.028132863,-0.09432876,0.0059271497,0.061715312,0.0057019778,-0.008623754,-0.016418574,-0.017792245,-0.066690564,0.040184673,0.051649246,-0.0033247564,0.071918964,0.02674936,0.07687916 }).Select(d => (float)d).ToArray() + }, + new() + { + Id = 2, + Name = "This is about a horse.", + VectorEmbeddings = (new double[] { 0.00066319015,0.059371606,0.019904254,0.036774546,-0.06027116,-0.016265921,0.030860845,0.055643834,0.0042737657,-0.007037373,0.006104062,-0.042017885,0.03532925,-0.01983578,-0.17182247,0.018130027,0.0006978096,0.041010544,-0.04580896,0.0066562407,-0.034156695,0.07169789,-0.003732026,0.07035341,-0.06771691,-0.045483362,-0.04587486,0.044270802,-0.039063204,0.045345873,-0.067072794,-0.036293034,0.012792531,-0.06584763,-0.15003937,0.006887971,0.03122762,0.071715415,0.060147583,0.056536376,0.1005726,-0.07508944,-0.010521994,0.04296306,0.06186383,0.048336152,-0.0026578992,-0.031771813,0.09114186,0.0298219,-0.030069979,-0.04830448,-0.06555674,0.008931567,-0.008930034,0.008409578,-0.06380391,0.031089691,-0.022144161,0.0031081801,0.027430866,0.053916246,0.03481386,0.06882019,-0.040121544,0.053177223,-0.08955376,-0.025653286,-0.061899573,-0.056186076,0.0618388,-0.041082207,0.0024167337,-0.10221955,-0.036011945,-0.02193175,-0.027966196,-0.0052382657,0.12434561,0.036994625,-0.044019878,-0.07336038,0.004784511,-0.014793327,0.05463077,0.03792995,0.052015834,-0.12259907,-0.083660625,-0.05770183,-0.05553689,-0.015968101,-0.035130754,0.015152355,0.04673879,0.02763963,0.0045751594,0.00013351876,-0.05073031,0.069344886,0.050685488,-0.006217909,0.057289913,-0.008384084,-0.010095412,0.04262858,-0.076505005,0.004814693,-0.004647454,-0.027164018,-0.052793253,-0.07996781,-0.002259928,0.05102953,0.07469597,0.047816664,-0.11623857,0.009546492,-0.07836857,-0.0061497106,0.030308023,0.0123279765,0.01727557,-0.027787533,0.0029694655,-0.048251797,0.05783699,-9.8458165e-33,-0.029556712,-0.059958287,0.020482214,-0.055502545,0.012202032,-0.019966707,0.0063604047,0.012528869,-0.023461204,0.04148049,0.010254733,-0.079121485,0.005512616,-0.050672587,0.0969294,-0.03734898,-0.016416196,-0.011727342,0.019440984,-0.0103121335,-0.004015404,0.10709091,-0.0042037196,-0.09906271,-0.020815263,-0.062631406,-0.05181335,-0.056682684,-0.07668673,0.04462328,-0.03255967,-0.01061319,0.040095136,-0.07428344,-0.08954705,-0.03655255,-0.033414897,-0.07287172,-0.046245173,0.040741507,0.043121435,-0.017704992,0.04531322,0.022490712,-0.061796345,0.12263498,0.023731327,0.04097611,-0.02944033,0.027172178,0.054726616,-0.009525099,0.08476988,0.012729447,0.026644293,0.065845735,0.01771869,0.049338885,-0.052020647,0.011609476,0.05302537,0.03191476,-0.011428133,-0.0317218,0.000072717354,-0.060584724,-0.114904806,0.020161727,0.023225058,0.059694566,-0.02951997,0.03259462,-0.04916039,-0.074076205,-0.027767904,0.0047858614,0.07736213,-0.08744972,-0.03654403,-0.01477788,-0.09468339,0.0018939535,-0.0182993,-0.036130246,0.01050344,0.033560347,0.019387867,-0.02084657,0.0009360842,0.013718448,0.046229977,0.024833633,-0.11396671,-0.10177031,0.055133626,6.0249805e-33,0.01816002,0.052471697,0.054190416,0.11878632,0.03944209,-0.046787247,0.032814603,0.058386937,-0.049842246,0.049228907,-0.066142194,0.006059039,0.059621867,-0.006291052,0.15240562,0.025465403,0.032288413,0.03136309,0.07285025,-0.041628312,-0.01245005,0.035763763,0.0012085717,-0.031110683,-0.04029571,0.07998812,-0.061727177,-0.06920058,-0.023586895,-0.06451264,-0.067086466,-0.057362735,-0.011721348,-0.010412064,-0.084145345,0.04934646,0.04814221,-0.08387328,0.018630628,-0.009486296,0.11052669,-0.03984161,0.064883254,0.024450615,0.041991018,0.017981533,0.041077986,0.063589826,-0.01104544,0.084603734,0.04332131,0.0041385363,0.07286733,-0.018887782,0.076195724,-0.021489078,-0.045993663,-0.052787732,0.0075105582,-0.0023860163,-0.010520825,0.0021552118,-0.08051806,0.061299916,-0.06347632,0.009948235,-0.06928475,-0.027916241,-0.016164059,-0.053455036,0.050054267,0.028922388,-0.028242402,-0.024950111,0.026767386,0.016913066,0.027214397,-0.027960636,0.09856684,0.0005708672,-0.06328706,-0.044772126,0.07103153,-0.014054399,0.044642996,-0.008021074,-0.07419962,0.09611253,0.044166476,0.0086504435,0.016296905,0.0574119,-0.0057896064,-0.10163813,0.042229597,-1.7167856e-8,0.008097164,0.0021034488,-0.00032095044,-0.022964565,0.014754429,0.033331983,0.028695943,-0.06141144,-0.021539602,0.026180083,0.06848516,0.035678882,-0.03490754,0.03020872,0.016107157,0.032306205,0.06105696,-0.002890444,0.0042259926,-0.030309584,-0.05571212,-0.028685763,0.0010230955,-0.061070923,-0.015631136,-0.050229665,0.009892334,0.09850854,-0.011604715,-0.01747272,0.025676185,0.0343539,-0.037994694,-0.10855406,0.0031102486,0.009522714,-0.036329266,0.032970943,0.051182617,-0.06983549,-0.06184144,0.039326083,0.032272592,0.044087004,0.06657253,0.04511283,0.081648186,0.02202941,-0.016903158,-0.070771135,-0.037015453,-0.0062212045,0.18698506,0.044452403,-0.0026717186,0.0044811214,0.010816393,0.014528357,-0.036223397,0.01895795,-0.064832956,0.02188936,0.049300164,0.033600762 }).Select(d => (float)d).ToArray() + }, + }; + var dogQueryVector = (new double[] { -0.053202216, 0.01422119, 0.007062546, 0.0685742, -0.07858203, 0.010138983, 0.10238025, -0.012096751, 0.09522599, -0.030270875, 0.002181861, -0.064782545, -0.0026875706, 0.0060957014, -0.003964779, -0.030604681, -0.047901124, -0.019261848, -0.059947517, -0.10413115, -0.08611966, 0.03632282, -0.025586247, 0.0017129881, -0.07146128, 0.061734077, 0.017160414, -0.05659205, 0.0248427, -0.07782747, -0.032485314, -0.008684083, -0.011535832, 0.038153064, -0.057013486, -0.053252906, 0.004985692, 0.032392446, 0.0725966, 0.032940567, 0.024707653, -0.083363794, -0.015673108, -0.04811024, -0.003449794, 0.004415103, -0.035913676, -0.051946636, 0.015592655, 0.0035385543, -0.010283442, 0.047748506, -0.040175628, -0.009133693, -0.03460812, -0.03693011, -0.04091714, 0.0176677, -0.00934914, -0.053623937, 0.011154383, 0.016148455, 0.013840816, 0.028249927, 0.04024405, 0.02096661, -0.014487404, -0.0016292258, -0.004891051, 0.012042645, 0.04556029, 0.0130860545, 0.070578784, -0.03086842, 0.030368855, -0.10848343, 0.05554082, -0.017487692, 0.16430159, 0.051410932, -0.027641848, -0.029989198, -0.057063058, 0.056793693, 0.050923523, 0.015136637, -0.0012497514, 0.02384801, -0.06327192, 0.028891006, -0.055418354, -0.03496716, 0.03029518, 0.026919777, -0.08353811, 0.018368296, -0.03516996, -0.08284338, -0.07195326, 0.19801475, 0.016410688, 0.0445346, -0.003741409, -0.038506165, 0.053398475, -0.0034389244, -0.04352991, 0.06336845, -0.013076868, -0.019743098, -0.045236666, 0.020782078, -0.056481004, 0.057446502, 0.055468243, 0.021229729, -0.100917056, -0.03422642, 0.02944804, -0.03325292, 0.028943142, 0.030092051, -0.051856354, 0.008190983, -0.016726157, -0.08435183, 0.011159818, -5.9255234e-33, 0.030620761, -0.085034214, 0.0028181712, -0.041073505, -0.042798948, 0.041067425, 0.029467635, 0.036486518, -0.12122617, 0.013526328, -0.01391842, 0.0312512, -0.021689802, 0.01621624, 0.11224023, -0.006686669, -0.0018879274, 0.05318519, 0.03250415, -0.03782473, -0.046973582, 0.061971873, 0.063630275, 0.050121382, -0.007621213, -0.021432782, -0.03779708, -0.08284233, -0.026234223, 0.036130365, 0.041241154, 0.014499247, 0.073483825, 0.00073006714, -0.081418164, -0.055791657, -0.04209736, -0.096603446, -0.040196676, 0.028519753, 0.12910499, 0.010470544, 0.025057316, 0.01734334, -0.02719573, -0.0049704155, 0.015811851, 0.03439927, -0.044550493, 0.020814221, 0.027571082, -0.014297911, 0.028702551, -0.021064728, 0.008865078, 0.009936881, 0.0029201612, -0.023835903, 0.012977942, 0.06633931, 0.068944834, 0.082585804, 0.008766892, -0.013999867, 0.09115506, -0.122037254, -0.045294352, -0.018009886, -0.022158505, 0.02152304, -0.03885241, -0.019468945, 0.07964807, -0.015691828, 0.06885623, -0.015452343, 0.022757484, 0.025256434, -0.03119467, -0.033447854, -0.021564618, -0.010073421, 0.0055514527, 0.048961196, -0.021559088, 0.06377866, -0.019740583, -0.030324804, 0.0062891715, 0.045206502, -0.045785706, -0.049080465, 0.087099895, 0.027371299, 0.09064848, 3.433169e-33, 0.06266184, 0.028918529, 0.000108557906, 0.09145542, -0.030282516, 0.0048763165, -0.02540525, 0.066567004, -0.034166507, 0.047780972, -0.03424499, 0.007805756, 0.10785121, 0.008996277, 0.0076608267, 0.08868162, 0.0036972803, -0.030516094, 0.02168669, -0.004358315, -0.14477515, 0.011545589, 0.018421879, -0.025913069, -0.05191015, 0.03943329, 0.037553225, -0.0147632975, -0.022263186, -0.048638437, -0.0065658195, -0.039633695, -0.041322067, -0.02844163, 0.010661134, 0.15864708, 0.04770698, -0.04730114, -0.06286664, 0.008440104, 0.059898064, 0.019403962, -0.03227739, 0.11167067, 0.016108502, 0.052688885, -0.017888643, -0.0058668335, 0.052891612, 0.018419184, -0.04730259, -0.014312523, 0.030081172, -0.07333967, -0.012648647, 0.004494484, -0.09500656, 0.018896673, -0.029087285, -0.0051991083, -0.0029317876, 0.069698535, 0.012463835, 0.1219864, -0.10485225, -0.05362739, -0.0128166545, -0.027964052, 0.05004069, -0.07638481, 0.024308309, 0.04531832, -0.029027926, 0.010168302, -0.010628256, 0.030930692, -0.046634875, 0.0045742486, 0.007714686, -0.0063424213, -0.07790265, -0.06532262, -0.047622908, 0.010272605, -0.056622025, -0.011285954, 0.0020759962, 0.06382898, -0.013343911, -0.03008575, -0.009862737, 0.054995734, -0.021704284, -0.05336612, -0.02860762, -1.3317537e-8, -0.028604865, -0.029213138, -0.04298399, -0.019619852, 0.09963344, 0.0694588, -0.030038442, -0.0401437, -0.006644881, 0.026138376, 0.044374008, -0.01637589, -0.06998592, 0.013482148, 0.04653866, -0.0153024765, -0.053351574, 0.039734483, 0.06283631, 0.07712063, -0.050968867, 0.03027798, 0.055424906, 0.0023063482, -0.051206734, -0.035924364, 0.04564326, 0.106056266, -0.08215607, 0.038128633, -0.022592563, 0.14054875, -0.07613521, -0.03006324, -0.0040755956, -0.06966433, 0.07610892, -0.07929878, 0.024970463, 0.03414342, 0.050462823, 0.15209967, -0.020093411, -0.079005316, -0.0006247459, 0.062248245, 0.026453331, -0.12163222, -0.028260367, -0.056446116, -0.09818232, -0.0074948515, 0.027907023, 0.06908376, 0.014955464, 0.005030419, -0.0131421015, -0.047915705, -0.01678274, 0.03665314, 0.1114189, 0.029845735, 0.02391984, 0.110152245 }).Select(d => (float)d).ToArray(); + + var options = new CollectionDefinition + { + Vector = new VectorOptions + { + Dimension = 384 + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + var insertResult = await collection.InsertManyAsync(items); + Assert.Equal(items.Count, insertResult.InsertedIds.Count); + var result = await collection.FindOneAsync(new FindOptions() { Sort = Builders.Sort.Vector(dogQueryVector) }, null); + Assert.Equal("This is about a dog.", result.Name); + + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + + [Fact] + public async Task QueryDocumentsWithVectorizeAsync() + { + var collectionName = "simpleObjectsWithVectorize"; + try + { + List items = new List() { + new() + { + Id = 0, + Name = "This is about a cat.", + }, + new() + { + Id = 1, + Name = "This is about a dog.", + }, + new() + { + Id = 2, + Name = "This is about a horse.", + }, + }; + var dogQueryVectorString = "dog"; + + var options = new CollectionDefinition + { + Vector = new VectorOptions + { + Metric = SimilarityMetric.Cosine, + Service = new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + var insertResult = await collection.InsertManyAsync(items); + Assert.Equal(items.Count, insertResult.InsertedIds.Count); + var result = await collection.FindManyAsync(new FindOptions() { Sort = Builders.Sort.Vectorize(dogQueryVectorString), IncludeSimilarity = true, IncludeSortVector = true }, null); + Assert.Equal("This is about a dog.", result.ToEnumerable().First().Name); + Assert.NotNull(result.ToEnumerable().First().Similarity); + Assert.NotNull(result.SortVector); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } +} + diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/appsettings.sample.json b/test/DataStax.AstraDB.DataApi.IntegrationTests/appsettings.sample.json index d3ff060..b0233e8 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/appsettings.sample.json +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/appsettings.sample.json @@ -1,5 +1,6 @@ { "TOKEN": "", "ADMINTOKEN": "", - "URL": "" + "URL": "", + "OPENAI_APIKEYNAME": "" } \ No newline at end of file