diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6554d59 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Restore dependencies + run: dotnet restore ./src/DataStax.AstraDB.DataApi/ + + - name: Build + run: dotnet build ./src/DataStax.AstraDB.DataApi/ --configuration Release --no-restore diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b1e3d61 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,52 @@ +name: Release to NuGet + +on: + push: + tags: + - "v*.*.*" # Matches stable versions like v1.2.3 + - "v*.*.*-*" # Matches pre-release versions like v2.0.1-beta + +jobs: + build-and-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Extract version from tag + id: extract_version + run: | + TAG=${{ github.ref_name }} + VERSION=${TAG#v} # Removes 'v' prefix + echo "PACKAGE_VERSION=$VERSION" >> $GITHUB_ENV + + - name: Restore dependencies + run: dotnet restore ./src/DataStax.AstraDB.DataApi/ + + - name: Build + run: dotnet build ./src/DataStax.AstraDB.DataApi/ --configuration Release --no-restore + + - name: Pack + run: dotnet pack ./src/DataStax.AstraDB.DataApi/ --configuration Release --no-build --output nupkgs -p:Version=${{ env.PACKAGE_VERSION }} + + # - name: Push to NuGet + # run: dotnet nuget push nupkgs/*.nupkg --api-key $NUGET_API_KEY --source https://api.nuget.org/v3/index.json + # env: + # NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + + - name: Dry-run NuGet push + run: | + echo "Would run: dotnet nuget push nupkgs/*.nupkg --api-key [REDACTED] --source https://api.nuget.org/v3/index.json" + + - name: Upload NuGet package as artifact + uses: actions/upload-artifact@v4 + with: + name: nuget-package + path: nupkgs/*.nupkg + retention-days: 1 diff --git a/index.md b/index.md index 2347c1a..1d1677e 100644 --- a/index.md +++ b/index.md @@ -4,7 +4,7 @@ _layout: landing # Overview -This C# Client Library simplifies using the DataStax Data API to manage and interact with AstraDB instances as well as other DataStax databases. +This C# Client Library simplifies using the DataStax Data API to manage and interact with Astra DB instances as well as other DataStax databases. # Installation diff --git a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs index e82dc32..5ee52fe 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs @@ -35,8 +35,6 @@ namespace DataStax.AstraDB.DataApi.Admin; /// public class AstraDatabasesAdmin { - private const int WAIT_IN_SECONDS = 600; - private readonly CommandOptions _adminOptions; private readonly DataApiClient _client; @@ -360,7 +358,7 @@ internal async Task CreateDatabaseAsync(DatabaseCreationOptions if (existingDb.Status == "ACTIVE") { Console.WriteLine($"Database {databaseName} already exists and is ACTIVE."); - return GetDatabaseAdmin(Guid.Parse(existingDb.Id)); + return GetDatabaseAdmin(existingDb); } throw new InvalidOperationException($"Database {databaseName} already exists but is in state: {existingDb.Status}"); @@ -403,63 +401,29 @@ internal async Task CreateDatabaseAsync(DatabaseCreationOptions private void WaitForDatabase(string databaseName) { - WaitForDatabaseAsync(databaseName, true).ResultSync(); - } - - private async Task WaitForDatabaseAsync(string databaseName) - { - await WaitForDatabaseAsync(databaseName, false).ConfigureAwait(false); + WaitForDatabaseAsync(databaseName).ResultSync(); } - internal async Task WaitForDatabaseAsync(string databaseName, bool runSynchronously) + internal async Task WaitForDatabaseAsync(string databaseName) { + const int MAX_WAIT_IN_SECONDS = 600; + const int SLEEP_SECONDS = 5; Guard.NotNullOrEmpty(databaseName, nameof(databaseName)); - if (runSynchronously) - { - Console.WriteLine($"Waiting {WAIT_IN_SECONDS} seconds synchronously before checking db status..."); - Thread.Sleep(WAIT_IN_SECONDS * 1000); - string status = GetDatabaseStatus(databaseName); - if (status != "ACTIVE") - { - throw new Exception($"Database {databaseName} is still {status} after {WAIT_IN_SECONDS} seconds."); - } + int secondsWaited = 0; - Console.WriteLine($"Database {databaseName} is ready."); - return; - } - - const int retry = 30_000; // 30 seconds - int waiting = 0; - - while (waiting < WAIT_IN_SECONDS * 1000) + while (secondsWaited < MAX_WAIT_IN_SECONDS) { string status = await GetDatabaseStatusAsync(databaseName).ConfigureAwait(false); if (status == "ACTIVE") { - Console.WriteLine($"Database {databaseName} is ready."); return; } - - Console.WriteLine($"Database {databaseName} is {status}... retrying in {retry / 1000} seconds."); - await Task.Delay(retry).ConfigureAwait(false); - waiting += retry; - } - - throw new Exception($"Database {databaseName} did not become ready within {WAIT_IN_SECONDS} seconds."); - } - - internal string GetDatabaseStatus(string databaseName) - { - Guard.NotNullOrEmpty(databaseName, nameof(databaseName)); - var db = ListDatabases().FirstOrDefault(item => databaseName.Equals(item.Info.Name)); - - if (db == null) - { - throw new Exception($"Database '{databaseName}' not found."); + await Task.Delay(SLEEP_SECONDS * 1000).ConfigureAwait(false); + secondsWaited += SLEEP_SECONDS; } - return db.Status; + throw new Exception($"Database {databaseName} did not become ready within {MAX_WAIT_IN_SECONDS} seconds."); } internal async Task GetDatabaseStatusAsync(string databaseName) @@ -638,10 +602,19 @@ internal async Task DropDatabaseAsync(Guid dbGuid, CommandOptions options, return false; } - private IDatabaseAdmin GetDatabaseAdmin(Guid dbGuid) + private DatabaseAdminAstra GetDatabaseAdmin(DatabaseInfo dbInfo) { - Guard.NotEmpty(dbGuid, nameof(dbGuid)); - return new DatabaseAdminAstra(dbGuid, _client, null); + var apiEndpoint = $"https://{dbInfo.Id}-{dbInfo.Info.Region}.apps.astra.datastax.com"; + var database = _client.GetDatabase(apiEndpoint); + return new DatabaseAdminAstra(database, _client, null); + } + + private DatabaseAdminAstra GetDatabaseAdmin(Guid dbGuid) + { + var dbInfo = GetDatabaseInfo(dbGuid); + var apiEndpoint = $"https://{dbGuid}-{dbInfo.Info.Region}.apps.astra.datastax.com"; + var database = _client.GetDatabase(apiEndpoint); + return new DatabaseAdminAstra(database, _client, null); } /// diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs index 7f03ec0..c84defc 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs @@ -59,15 +59,6 @@ public class DatabaseAdminAstra : IDatabaseAdmin private readonly DataApiClient _client; private CommandOptions[] _optionsTree => new CommandOptions[] { _client.ClientOptions, _adminOptions }; - internal DatabaseAdminAstra(Guid id, DataApiClient client, CommandOptions adminOptions) - { - Guard.NotNull(client, nameof(client)); - _client = client; - _adminOptions = adminOptions; - _database = _client.GetDatabase(id); - _id = id; - } - internal DatabaseAdminAstra(Database database, DataApiClient client, CommandOptions adminOptions) { Guard.NotNull(client, nameof(client)); @@ -172,8 +163,9 @@ internal async Task> ListKeyspaceNamesAsync(bool runSynchron } /// - /// Creates a new keyspace with the specified name. + /// Synchronous version of /// + /// /// The name of the keyspace to create. /// /// @@ -182,12 +174,13 @@ internal async Task> ListKeyspaceNamesAsync(bool runSynchron /// public void CreateKeyspace(string keyspace) { - CreateKeyspaceAsync(keyspace, false, null, true).ResultSync(); + CreateKeyspace(keyspace, false); } /// - /// Creates a new keyspace with the specified name and optionally updates the database's keyspace reference. + /// Synchronous version of /// + /// /// The name of the keyspace to create. /// Whether to set this keyspace as the active keyspace in the command options. /// @@ -197,12 +190,13 @@ public void CreateKeyspace(string keyspace) /// public void CreateKeyspace(string keyspace, bool updateDBKeyspace) { - CreateKeyspaceAsync(keyspace, updateDBKeyspace, null, true).ResultSync(); + CreateKeyspace(keyspace, updateDBKeyspace, null); } /// - /// Creates a new keyspace with the specified name using the provided command options. + /// Synchronous version of /// + /// /// The name of the keyspace to create. /// Optional settings that influence request execution. /// @@ -212,13 +206,13 @@ public void CreateKeyspace(string keyspace, bool updateDBKeyspace) /// public void CreateKeyspace(string keyspace, CommandOptions options) { - CreateKeyspaceAsync(keyspace, false, options, true).ResultSync(); + CreateKeyspace(keyspace, false, true, options); } /// - /// Creates a new keyspace with the specified name, optionally updating the database's keyspace reference, - /// and applying the given command options. + /// Synchronous version of /// + /// /// The name of the keyspace to create. /// Whether to set this keyspace as the active keyspace in the command options. /// Optional settings that influence request execution. @@ -229,28 +223,43 @@ public void CreateKeyspace(string keyspace, CommandOptions options) /// public void CreateKeyspace(string keyspace, bool updateDBKeyspace, CommandOptions options) { - CreateKeyspaceAsync(keyspace, updateDBKeyspace, options, true).ResultSync(); + CreateKeyspace(keyspace, updateDBKeyspace, true, options); } /// - /// Asynchronously creates a new keyspace with the specified name. + /// Synchronous version of /// + /// /// The name of the keyspace to create. - /// A task representing the asynchronous operation. + /// Whether to set this keyspace as the active keyspace in the command options. + /// Whether or not to wait for the keyspace to be created before returning. + /// Optional settings that influence request execution. /// /// - /// await admin.CreateKeyspaceAsync("myKeyspace"); + /// admin.CreateKeyspace("myKeyspace", true, options); /// /// - public Task CreateKeyspaceAsync(string keyspace) + public void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options) { - return CreateKeyspaceAsync(keyspace, false, null, false); + CreateKeyspaceAsync(keyspace, updateDBKeyspace, waitForCompletion, options).ResultSync(); } /// - /// Asynchronously creates a new keyspace with the specified name and optionally updates the database's keyspace reference. + /// Creates a new keyspace with the specified name. /// /// The name of the keyspace to create. + /// + /// + /// await admin.CreateKeyspaceAsync("myKeyspace"); + /// + /// + public Task CreateKeyspaceAsync(string keyspace) + { + return CreateKeyspaceAsync(keyspace, false, true, null); + } + + /// + /// The name of the keyspace to create. /// Whether to set this keyspace as the active keyspace in the command options. /// A task representing the asynchronous operation. /// @@ -260,12 +269,25 @@ public Task CreateKeyspaceAsync(string keyspace) /// public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace) { - return CreateKeyspaceAsync(keyspace, updateDBKeyspace, null, false); + return CreateKeyspaceAsync(keyspace, updateDBKeyspace, false, null); } - /// - /// Asynchronously creates a new keyspace with the specified name using the provided command options. - /// + /// + /// The name of the keyspace to create. + /// Whether to set this keyspace as the active keyspace in the command options. + /// Whether or not to wait for the keyspace to be created before returning. + /// A task representing the asynchronous operation. + /// + /// + /// await admin.CreateKeyspaceAsync("myKeyspace", true); + /// + /// + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion) + { + return CreateKeyspaceAsync(keyspace, updateDBKeyspace, waitForCompletion, null); + } + + /// /// The name of the keyspace to create. /// Optional settings that influence request execution. /// A task representing the asynchronous operation. @@ -276,15 +298,12 @@ public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace) /// public Task CreateKeyspaceAsync(string keyspace, CommandOptions options) { - return CreateKeyspaceAsync(keyspace, false, options, false); + return CreateKeyspaceAsync(keyspace, false, false, options, false); } - /// - /// Asynchronously creates a new keyspace with the specified name, optionally updating the database's keyspace reference, - /// and applying the given command options. - /// + /// /// The name of the keyspace to create. - /// Whether to set this keyspace as the active keyspace in the command options. + /// Whether or not to wait for the keyspace to be created before returning. /// Optional settings that influence request execution. /// A task representing the asynchronous operation. /// @@ -292,12 +311,28 @@ public Task CreateKeyspaceAsync(string keyspace, CommandOptions options) /// await admin.CreateKeyspaceAsync("myKeyspace", true, options); /// /// - public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, CommandOptions options) + public Task CreateKeyspaceAsync(string keyspace, bool waitForCompletion, CommandOptions options) + { + return CreateKeyspaceAsync(keyspace, false, waitForCompletion, options, false); + } + + /// + /// The name of the keyspace to create. + /// Whether to set this keyspace as the active keyspace in the command options. + /// Whether or not to wait for the keyspace to be created before returning. + /// Optional settings that influence request execution. + /// A task representing the asynchronous operation. + /// + /// + /// await admin.CreateKeyspaceAsync("myKeyspace", true, true, options); + /// + /// + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options) { - return CreateKeyspaceAsync(keyspace, updateDBKeyspace, options, false); + return CreateKeyspaceAsync(keyspace, updateDBKeyspace, waitForCompletion, options, false); } - internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, CommandOptions options, bool runSynchronously) + internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options, bool runSynchronously) { options ??= new CommandOptions(); options.IncludeKeyspaceInUrl = false; @@ -322,12 +357,24 @@ internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, { options.Keyspace = keyspace; } + + if (waitForCompletion) + { + try + { + await Wait.WaitForProcess(() => KeyspaceExistsAsync(keyspace, options, runSynchronously)).ConfigureAwait(false); + } + catch (Exception e) + { + throw new Exception($"Failed to create keyspace {keyspace} within the allotted time", e); + } + } } /// - /// Drops the keyspace with the specified name. + /// Synchronous version of /// - /// The name of the keyspace to drop. + /// /// /// /// admin.DropKeyspace("myKeyspace"); @@ -335,29 +382,27 @@ internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, /// public void DropKeyspace(string keyspace) { - DropKeyspaceAsync(keyspace, null, true).ResultSync(); + DropKeyspace(keyspace, true, null); } /// - /// Asynchronously drops the keyspace with the specified name. + /// Synchronous version of /// - /// The name of the keyspace to drop. - /// A task representing the asynchronous operation. + /// /// /// - /// await admin.DropKeyspaceAsync("myKeyspace"); + /// admin.DropKeyspace("myKeyspace", false); /// /// - public Task DropKeyspaceAsync(string keyspace) + public void DropKeyspace(string keyspace, bool waitForCompletion) { - return DropKeyspaceAsync(keyspace, null, false); + DropKeyspace(keyspace, waitForCompletion, null); } /// - /// Drops the keyspace with the specified name using the provided command options. + /// Synchronous version of /// - /// The name of the keyspace to drop. - /// Optional settings that influence request execution. + /// /// /// /// admin.DropKeyspace("myKeyspace", options); @@ -365,13 +410,49 @@ public Task DropKeyspaceAsync(string keyspace) /// public void DropKeyspace(string keyspace, CommandOptions options) { - DropKeyspaceAsync(keyspace, options, true).ResultSync(); + DropKeyspace(keyspace, true, options); } /// - /// Asynchronously drops the keyspace with the specified name using the provided command options. + /// Synchronous version of + /// + /// + public void DropKeyspace(string keyspace, bool waitForCompletion, CommandOptions options) + { + DropKeyspaceAsync(keyspace, waitForCompletion, options, true).ResultSync(); + } + + /// + /// Drops the keyspace with the specified name. /// /// The name of the keyspace to drop. + /// + /// + /// await admin.DropKeyspaceAsync("myKeyspace"); + /// + /// + /// + /// This method will wait for the keyspace to be dropped before returning. + /// If you do not want to wait for the keyspace to be dropped, use the method. + /// + public Task DropKeyspaceAsync(string keyspace) + { + return DropKeyspaceAsync(keyspace, true, null); + } + + /// + /// Whether or not to wait for the keyspace to be dropped before returning. + /// + /// + /// await admin.DropKeyspaceAsync("myKeyspace", false); + /// + /// + public Task DropKeyspaceAsync(string keyspace, bool waitForCompletion) + { + return DropKeyspaceAsync(keyspace, waitForCompletion, null); + } + + /// /// Optional settings that influence request execution. /// A task representing the asynchronous operation. /// @@ -379,12 +460,28 @@ public void DropKeyspace(string keyspace, CommandOptions options) /// await admin.DropKeyspaceAsync("myKeyspace", options); /// /// + /// + /// This method will wait for the keyspace to be dropped before returning. + /// If you do not want to wait for the keyspace to be dropped, use the method. + /// public Task DropKeyspaceAsync(string keyspace, CommandOptions options) { - return DropKeyspaceAsync(keyspace, options, false); + return DropKeyspaceAsync(keyspace, true, options, false); } - internal async Task DropKeyspaceAsync(string keyspace, CommandOptions options, bool runSynchronously) + /// + /// Whether or not to wait for the keyspace to be dropped before returning. + /// + /// + /// await admin.DropKeyspaceAsync("myKeyspace", true, options); + /// + /// + public Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, CommandOptions options) + { + return DropKeyspaceAsync(keyspace, waitForCompletion, options, false); + } + + internal async Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, CommandOptions options, bool runSynchronously) { Guard.NotNullOrEmpty(keyspace, nameof(keyspace)); @@ -394,6 +491,18 @@ internal async Task DropKeyspaceAsync(string keyspace, CommandOptions options, b await command.RunAsyncRaw(HttpMethod.Delete, runSynchronously) .ConfigureAwait(false); + + if (waitForCompletion) + { + try + { + await Wait.WaitForProcess(async () => !await KeyspaceExistsAsync(keyspace, options, runSynchronously).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (Exception e) + { + throw new Exception($"Failed to drop keyspace {keyspace} within the allotted time", e); + } + } } /// diff --git a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs index 55a65d6..4009659 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs @@ -279,81 +279,145 @@ public Task DropAsync() return _database.DropCollectionAsync(_collectionName); } + /// + /// Synchronous version of + /// + /// public T FindOne() { return FindOne(null, new DocumentFindOptions(), null); } + /// + /// Synchronous version of + /// + /// public T FindOne(CommandOptions commandOptions) { return FindOne(null, new DocumentFindOptions(), commandOptions); } + /// + /// Synchronous version of + /// + /// public T FindOne(Filter filter) { return FindOne(filter, new DocumentFindOptions(), null); } + /// + /// Synchronous version of + /// + /// public T FindOne(DocumentFindOptions findOptions) { return FindOne(null, findOptions, null); } + /// + /// Synchronous version of + /// + /// public T FindOne(DocumentFindOptions findOptions, CommandOptions commandOptions) { return FindOne(null, findOptions, commandOptions); } + /// + /// Synchronous version of + /// + /// public T FindOne(Filter filter, CommandOptions commandOptions) { return FindOne(filter, new DocumentFindOptions(), commandOptions); } + /// + /// Synchronous version of + /// + /// public T FindOne(Filter filter, DocumentFindOptions findOptions) { return FindOne(filter, findOptions, null); } + /// + /// Synchronous version of + /// + /// public T FindOne(Filter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); } + /// + /// Synchronous version of + /// + /// public TResult FindOne() { return FindOne(null, new DocumentFindOptions(), null); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(CommandOptions commandOptions) { return FindOne(null, new DocumentFindOptions(), commandOptions); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(Filter filter) { return FindOne(filter, new DocumentFindOptions(), null); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(DocumentFindOptions findOptions) { return FindOne(null, findOptions, null); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(DocumentFindOptions findOptions, CommandOptions commandOptions) { return FindOne(null, findOptions, commandOptions); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(Filter filter, CommandOptions commandOptions) { return FindOne(filter, new DocumentFindOptions(), commandOptions); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(Filter filter, DocumentFindOptions findOptions) { return FindOne(filter, findOptions, null); } + /// + /// Synchronous version of + /// + /// public TResult FindOne(Filter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); @@ -509,18 +573,18 @@ private async Task FindOneAsync(Filter filter, DocumentFind /// /// Find all documents in the collection. /// - /// The Find() methods return a object that can be used to further structure the query + /// The Find() methods return a object that can be used to further structure the query /// by adding Sort, Projection, Skip, Limit, etc. to affect the final results. /// - /// The object can be directly enumerated both synchronously and asynchronously. - /// Secondly, the results can be paged through more manually by using the method. + /// The object can be directly enumerated both synchronously and asynchronously. + /// Secondly, the results can be paged through more manually by using the method. /// /// /// /// Synchronous Enumeration: /// - /// var resultSet = collection.Find(); - /// foreach (var document in resultSet) + /// var FindEnumerator = collection.Find(); + /// foreach (var document in FindEnumerator) /// { /// // Process document /// } @@ -536,12 +600,12 @@ private async Task FindOneAsync(Filter filter, DocumentFind /// } /// /// - public ResultSet> Find() + public FindEnumerator> Find() { - return Find(null, null, null); + return Find(null, null); } - /// + /// /// /// /// @@ -551,82 +615,33 @@ public ResultSet> Find() /// var results = collection.Find(filter).Sort(sort); /// /// - public ResultSet> Find(Filter filter) + public FindEnumerator> Find(Filter filter) { - return Find(filter, null, null); + return Find(filter, null); } - /// - /// As an alternative to , this method allows for controlling the results - /// by setting properties on the parameter. - /// - /// - /// - /// - /// - /// var filter = Builders<SimpleObject>.Filter.Eq(so => so.Properties.PropertyOne, "grouptwo"); - /// var sort = Builders<SimpleObject>.Sort.Descending(o => o.Properties.PropertyTwo); - /// var inclusiveProjection = Builders<SimpleObject>.Projection - /// .Include("Properties.PropertyTwo"); - /// var findOptions = new FindOptions<SimpleObject>() - /// { - /// Sort = sort, - /// Limit = 1, - /// Skip = 2, - /// Projection = inclusiveProjection - /// }; - /// var results = collection.Find(filter, findOptions).ToList(); - /// - /// - public ResultSet> Find(DocumentFindManyOptions findOptions) - { - return Find(null, findOptions, null); - } - - /// - /// - public ResultSet> Find(CommandOptions commandOptions) - { - return Find(null, new DocumentFindManyOptions(), commandOptions); - } - - /// + /// /// - public ResultSet> Find(DocumentFindManyOptions findOptions, CommandOptions commandOptions) + public FindEnumerator> Find(CommandOptions commandOptions) { - return Find(null, findOptions, commandOptions); + return Find(null, commandOptions); } /// /// - public ResultSet> Find(Filter filter, CommandOptions commandOptions) + public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) { - return Find(filter, new DocumentFindManyOptions(), commandOptions); + return Find(filter, commandOptions); } - /// - /// - public ResultSet> Find(Filter filter, DocumentFindManyOptions findOptions) - { - return Find(filter, findOptions, null); - } - - /// - /// - public ResultSet> Find(Filter filter, DocumentFindManyOptions findOptions, CommandOptions commandOptions) - { - findOptions ??= new DocumentFindManyOptions(); - return new ResultSet>(this, filter, findOptions, commandOptions); - } - - /// + /// /// /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public ResultSet> Find() where TResult : class + public FindEnumerator> Find() where TResult : class { - return Find(null, null, null); + return Find(null, null); } /// @@ -634,19 +649,9 @@ public ResultSet> Find() where TResu /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public ResultSet> Find(Filter filter) where TResult : class + public FindEnumerator> Find(Filter filter) where TResult : class { - return Find(filter, null, null); - } - - /// - /// - /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type - /// (most commonly used when using projection to return a subset of fields) - /// - public ResultSet> Find(DocumentFindManyOptions findOptions) where TResult : class - { - return Find(null, findOptions, null); + return Find(filter, null); } /// @@ -654,45 +659,27 @@ public ResultSet> Find(DocumentFindM /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public ResultSet> Find(CommandOptions commandOptions) where TResult : class - { - return Find(null, new DocumentFindManyOptions(), commandOptions); - } - - /// - /// - public ResultSet> Find(DocumentFindManyOptions findOptions, CommandOptions commandOptions) where TResult : class - { - return Find(null, findOptions, commandOptions); - } - - /// - /// - public ResultSet> Find(Filter filter, CommandOptions commandOptions) where TResult : class - { - return Find(filter, new DocumentFindManyOptions(), commandOptions); - } - - /// - /// - public ResultSet> Find(Filter filter, DocumentFindManyOptions findOptions) where TResult : class + public FindEnumerator> Find(CommandOptions commandOptions) where TResult : class { - return Find(filter, findOptions, null); + return Find(null, commandOptions); } /// /// - public ResultSet> Find(Filter filter, DocumentFindManyOptions findOptions, CommandOptions commandOptions) where TResult : class + public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) where TResult : class { - findOptions ??= new DocumentFindManyOptions(); - return new ResultSet>(this, filter, findOptions, commandOptions); + var findOptions = new DocumentFindManyOptions() + { + Filter = filter + }; + return new FindEnumerator>(this, findOptions, commandOptions); } - internal async Task, FindStatusResult>> RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) + internal async Task, FindStatusResult>> RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) { findOptions.Filter = filter; var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnDocumentData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnDocumentData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); return response; } @@ -1044,6 +1031,55 @@ internal async Task FindOneAndReplaceAsync(Filter filter, T return response.Data.Document; } + /// + /// Finds documents in a collection through a retrieval process that uses a reranker model to combine results from a vector search and a lexical search (hybrid search) + /// + /// + public RerankSorter FindAndRerank() + { + return FindAndRerank(null); + } + + /// + /// + public RerankSorter FindAndRerank(Filter filter) + { + return FindAndRerank(filter); + } + + /// + /// + /// + public RerankSorter FindAndRerank(Filter filter, CommandOptions commandOptions) + { + return FindAndRerank(filter, commandOptions); + } + + /// + /// Finds documents in a collection through a retrieval process that uses a reranker model to combine results from a vector search and a lexical search (hybrid search) + /// + /// If you are using projection to return a subset of fields, TResult can be used to receive the projected document. + /// + public RerankSorter FindAndRerank() where TResult : class + { + return FindAndRerank(null, null); + } + + /// + /// + public RerankSorter FindAndRerank(Filter filter) where TResult : class + { + return FindAndRerank(filter, null); + } + + /// + /// + /// + public RerankSorter FindAndRerank(Filter filter, CommandOptions commandOptions) where TResult : class + { + return new RerankSorter(() => CreateCommand("findAndRerank"), filter, commandOptions); + } + /// /// Synchronous version of /// @@ -1765,83 +1801,125 @@ internal async Task UpdateManyAsync(Filter filter, UpdateBuilde /// Synchronous version of /// /// - public DocumentsCountResult CountDocuments() + public int CountDocuments() { - return CountDocumentsAsync(null, null, true).ResultSync(); + return CountDocuments(null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public DocumentsCountResult CountDocuments(CountDocumentsCommandOptions commandOptions) + /// + public int CountDocuments(int maxDocumentsToCount) { - return CountDocumentsAsync(null, commandOptions, true).ResultSync(); + return CountDocuments(null, maxDocumentsToCount); } /// - /// Count the number of documents in the collection. + /// Synchronous version of /// - /// - public Task CountDocumentsAsync() + /// + public int CountDocuments(Filter filter) { - return CountDocumentsAsync(null, null, false); + return CountDocuments(filter, MaxDocumentsToCount); } /// - /// Count the number of documents in the collection. + /// Synchronous version of /// - /// - /// - public Task CountDocumentsAsync(CountDocumentsCommandOptions commandOptions) + /// + public int CountDocuments(Filter filter, int maxDocumentsToCount) { - return CountDocumentsAsync(null, commandOptions, false); + return CountDocuments(filter, maxDocumentsToCount, null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public DocumentsCountResult CountDocuments(Filter filter) + /// + public int CountDocuments(Filter filter, int maxDocumentsToCount, CommandOptions commandOptions) { - return CountDocumentsAsync(filter, null, true).ResultSync(); + return CountDocumentsAsync(filter, maxDocumentsToCount, commandOptions, false).ResultSync(); } /// - /// Synchronous version of + /// Count the number of documents in the collection. /// - /// - public DocumentsCountResult CountDocuments(Filter filter, CountDocumentsCommandOptions commandOptions) + /// + /// + /// Count operations are expensive: for this reason, the best practice is to provide a reasonable `upperBound` + /// according to the caller expectations. Moreover, indiscriminate usage of count operations for sizeable amounts + /// of documents is discouraged in favor of alternative application-specific + /// solutions. Keep in mind that the Data API has a hard upper limit on the amount of documents it will count (1000), + /// and that an exception will be thrown by this method if this limit is encountered. + /// + /// Thrown if the number of documents to count exceeds the limit + public Task CountDocumentsAsync() { - return CountDocumentsAsync(filter, commandOptions, true).ResultSync(); + return CountDocumentsAsync(null); } - /// - /// Count the number of documents in the collection that match the provided filter. - /// + /// + /// + /// + public Task CountDocumentsAsync(int maxDocumentsToCount) + { + return CountDocumentsAsync(null, maxDocumentsToCount); + } + + /// /// /// - public Task CountDocumentsAsync(Filter filter) + public Task CountDocumentsAsync(Filter filter) { - return CountDocumentsAsync(filter, null, false); + return CountDocumentsAsync(filter, null); + } + + /// + /// + /// + public Task CountDocumentsAsync(Filter filter, int maxDocumentsToCount) + { + return CountDocumentsAsync(filter, maxDocumentsToCount, null); } /// /// - public Task CountDocumentsAsync(Filter filter, CountDocumentsCommandOptions commandOptions) + public Task CountDocumentsAsync(Filter filter, CommandOptions commandOptions) { - return CountDocumentsAsync(filter, commandOptions, false); + return CountDocumentsAsync(filter, MaxDocumentsToCount, commandOptions, false); } - internal async Task CountDocumentsAsync(Filter filter, CountDocumentsCommandOptions commandOptions, bool runSynchronously) + /// + /// + /// + /// + public Task CountDocumentsAsync(Filter filter, int maxDocumentsToCount, CommandOptions commandOptions) { + return CountDocumentsAsync(filter, maxDocumentsToCount, commandOptions, false); + } + + public const int MaxDocumentsToCount = 1000; + internal async Task CountDocumentsAsync(Filter filter, int maxDocumentsToCount, CommandOptions commandOptions, bool runSynchronously) + { + if (maxDocumentsToCount < 1 || maxDocumentsToCount > MaxDocumentsToCount) + { + throw new ArgumentException($"maxDocumentsToCount must be between 1 and {MaxDocumentsToCount}"); + } + commandOptions ??= new CommandOptions(); var findOptions = new DocumentFindOptions() { Filter = filter, }; + var command = CreateCommand("countDocuments").WithPayload(findOptions).AddCommandOptions(commandOptions); var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); - return response.Result; + if (response.Result.Count >= maxDocumentsToCount || response.Result.MoreData) + { + throw new DocumentCountExceedsMaxException(); + } + + return response.Result.Count; } /// @@ -1942,7 +2020,7 @@ internal Command CreateCommand(string name) return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _collectionName)); } - Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) + Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) { return RunFindManyAsync(filter, findOptions, commandOptions, runSynchronously); } diff --git a/src/DataStax.AstraDB.DataApi/Core/AnalyzerOptions.cs b/src/DataStax.AstraDB.DataApi/Core/AnalyzerOptions.cs new file mode 100644 index 0000000..52f75b1 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/AnalyzerOptions.cs @@ -0,0 +1,55 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Configuration for the analyzer +/// +public class AnalyzerOptions +{ + /// + /// Tokenizer configuration + /// + [JsonPropertyName("tokenizer")] + public TokenizerOptions Tokenizer { get; set; } = new(); + + /// + /// List of filters to apply + /// + [JsonIgnore] + public List Filters { get; set; } = new(); + + /// + /// List of character filters to apply + /// + [JsonPropertyName("charFilters")] + public List CharacterFilters { get; set; } = new(); + + [JsonPropertyName("filters")] + [JsonInclude] + internal List FilterOptions + { + get + { + return Filters.Select(f => new FilterOptions() { Name = f }).ToList(); + } + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/CollectionDefinition.cs b/src/DataStax.AstraDB.DataApi/Core/CollectionDefinition.cs index eb22e2f..2dd9f83 100644 --- a/src/DataStax.AstraDB.DataApi/Core/CollectionDefinition.cs +++ b/src/DataStax.AstraDB.DataApi/Core/CollectionDefinition.cs @@ -44,30 +44,48 @@ public class CollectionDefinition [JsonPropertyName("indexing")] public IndexingOptions Indexing { get; set; } + /// + /// Lexical analysis options for the collection + /// + [JsonPropertyName("lexical")] + public LexicalOptions Lexical { get; set; } + + /// + /// Reranking options for the collection + /// + [JsonPropertyName("rerank")] + public RerankOptions Rerank { get; set; } + internal static CollectionDefinition Create() + { + return CheckAddDefinitionsFromAttributes(new CollectionDefinition()); + } + + internal static CollectionDefinition CheckAddDefinitionsFromAttributes(CollectionDefinition definition) { Type type = typeof(T); PropertyInfo idProperty = null; DocumentIdAttribute idAttribute = null; - CollectionDefinition definition = new(); - - foreach (var property in type.GetProperties()) + if (definition.DefaultId == null) { - var attr = property.GetCustomAttribute(); - if (attr != null) + foreach (var property in type.GetProperties()) { - idProperty = property; - idAttribute = attr; - break; + var attr = property.GetCustomAttribute(); + if (attr != null) + { + idProperty = property; + idAttribute = attr; + break; + } } - } - if (idProperty != null) - { - if (idAttribute.DefaultIdType.HasValue) + if (idProperty != null) { - definition.DefaultId = new DefaultIdOptions() { Type = idAttribute.DefaultIdType.Value }; + if (idAttribute.DefaultIdType.HasValue) + { + definition.DefaultId = new DefaultIdOptions() { Type = idAttribute.DefaultIdType.Value }; + } } } diff --git a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs index 860d8d3..9babff0 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs @@ -202,6 +202,7 @@ internal T Deserialize(string input) deserializeOptions.Converters.Add(new DateTimeConverter()); } deserializeOptions.Converters.Add(new IpAddressConverter()); + deserializeOptions.Converters.Add(new AnalyzerOptionsConverter()); return JsonSerializer.Deserialize(input, deserializeOptions); } @@ -289,6 +290,25 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl else { response = await httpClient.SendAsync(request, linkedCts.Token).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == System.Net.HttpStatusCode.GatewayTimeout || + response.StatusCode == System.Net.HttpStatusCode.RequestTimeout) + { + throw new TimeoutException($"Request to timed out. Consider increasing the timeout settings using the CommandOptions parameter."); + } + else if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized) + { + throw new UnauthorizedAccessException("Unauthorized access. Please check your token."); + } + else + { + responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + MaybeLogDebugMessage("Response Status Code: {StatusCode}", response.StatusCode); + MaybeLogDebugMessage("Content: {Content}", responseContent); + throw new HttpRequestException($"Request to failed with status code {response.StatusCode}."); + } + } responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } diff --git a/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs b/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs index e1bf846..1cdb306 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Commands/DataApiKeywords.cs @@ -28,6 +28,8 @@ internal static class DataApiKeywords internal const string Slice = "$slice"; internal const string Similarity = "$similarity"; internal const string Vector = "$vector"; + internal const string Lexical = "$lexical"; + internal const string Hybrid = "$hybrid"; internal const string SortVector = "sortVector"; internal const string Vectorize = "$vectorize"; internal const string Binary = "$binary"; diff --git a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs index 609011a..bfe3fe7 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs @@ -14,31 +14,46 @@ * limitations under the License. */ +using DataStax.AstraDB.DataApi.Core.Results; using System; using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using DataStax.AstraDB.DataApi.Core.Results; namespace DataStax.AstraDB.DataApi.Core; /// -/// A cursor for iterating over the results of a query. +/// A cursor for iterating over the results of a query in a streaming manner. /// /// When multiple results are returned by the underlying API, they are returned in batches. -/// You can use the or methods to iterate over the batches +/// You can use the method to iterate over the batches /// and to access the current batch of results. /// -/// In most situations, using the results of a query directly as an IEnumerable or IAsyncEnumerable is recommended. -/// Use the cursor directly if you need access to the SortVectors or want to manually control iterating over the batches. +/// The and methods create a new cursor to ensure +/// iteration starts from the first batch, allowing multiple enumerations of the same query. +/// Use the cursor directly if you need access to the SortVector or want to manually control iterating over the batches. /// /// The type of the documents in the collection. -public class Cursor +public class Cursor : IDisposable, IParentCursor { - private DocumentsResult _currentBatch; - private Func, FindStatusResult>>> FetchNextBatch { get; } + private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1); + private ApiFindResult _currentBatch; + private readonly Func, FindStatusResult>>> _fetchNextBatch; + private readonly IParentCursor _parentCursor; + + + internal Cursor( + Func, FindStatusResult>>> fetchNextBatch, + IParentCursor parentCursor = null) + { + _fetchNextBatch = fetchNextBatch ?? throw new ArgumentNullException(nameof(fetchNextBatch)); + _parentCursor = parentCursor; + } + + /// + /// Gets a value indicating whether the cursor has been started or not. + /// + public bool IsStarted { get; internal set; } = false; /// /// The current batch of results. @@ -49,149 +64,128 @@ public IEnumerable Current { if (_currentBatch == null) { - throw new Exception("Cursor has not been started. Please call MoveNext()"); + throw new InvalidOperationException("Cursor has not been started. Call MoveNextAsync first."); } - return _currentBatch.Documents; + return _currentBatch.Items; } } /// - /// An array containing the sort vectors used for this query. + /// An array containing the sort vectors used for the query that created this cursor. /// - public float[] SortVectors { get; internal set; } = Array.Empty(); - - internal Cursor(Func, FindStatusResult>>> fetchNextBatch) - { - FetchNextBatch = fetchNextBatch; - } + public float[] SortVector { get; private set; } /// /// Synchronously moves the cursor to the next batch of results. /// /// True if there are more batches, false otherwise. /// - /// The asynchronous version of this method is recommended. + /// The asynchronous version is recommended to avoid potential deadlocks. /// public bool MoveNext() { - ApiResponseWithData, FindStatusResult> nextResult; - if (_currentBatch == null) + _semaphore.Wait(); + try { - nextResult = FetchNextBatch(null, true).ResultSync(); - if (nextResult.Data == null || nextResult.Data.Documents.Count == 0) - { - return false; - } + return MoveNextAsync(true).GetAwaiter().GetResult(); } - else + finally { - if (string.IsNullOrEmpty(_currentBatch.NextPageState)) - { - return false; - } - nextResult = FetchNextBatch(_currentBatch.NextPageState, true).ResultSync(); - } - if (nextResult.Status != null && nextResult.Status.SortVector != null) - { - SortVectors = SortVectors.Concat(nextResult.Status.SortVector).ToArray(); + _semaphore.Release(); } - _currentBatch = nextResult.Data; - return true; } /// - /// Moves the cursor to the next batch of results. + /// Asynchronously moves the cursor to the next batch of results. /// + /// An optional cancellation token to cancel the operation. /// True if there are more batches, false otherwise. - public async Task MoveNextAsync() + public async Task MoveNextAsync(CancellationToken cancellationToken = default) { - ApiResponseWithData, FindStatusResult> nextResult; - if (_currentBatch == null) + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - nextResult = await FetchNextBatch(null, true).ConfigureAwait(false); - if (nextResult.Data == null || nextResult.Data.Documents.Count == 0) - { - return false; - } + return await MoveNextAsync(false, cancellationToken).ConfigureAwait(false); } - else + finally { - if (string.IsNullOrEmpty(_currentBatch.NextPageState)) - { - return false; - } - nextResult = await FetchNextBatch(_currentBatch.NextPageState, true).ConfigureAwait(false); + _semaphore.Release(); + } + } + + private async Task MoveNextAsync(bool runSynchronously, CancellationToken cancellationToken = default) + { + if (_currentBatch != null && string.IsNullOrEmpty(_currentBatch.NextPageState)) + { + return false; } - if (nextResult.Status != null && nextResult.Status.SortVector != null) + + _parentCursor?.SetStarted(); + + var nextPageState = _currentBatch?.NextPageState; + var nextBatch = await _fetchNextBatch(nextPageState, runSynchronously).ConfigureAwait(false); + if (nextBatch.Data == null || nextBatch.Data.Items == null || nextBatch.Data.Items.Count == 0) { - SortVectors = SortVectors.Concat(nextResult.Status.SortVector).ToArray(); + return false; } - _currentBatch = nextResult.Data; + + var nextSortVector = nextBatch.Status?.SortVector; + _parentCursor?.SetSortVector(nextSortVector); + + _currentBatch = nextBatch.Data; return true; } -} -/// -/// Extensions for . -/// -public static class CursorExtensions -{ + public void Dispose() + { + _semaphore.Dispose(); + } + /// - /// Converts the cursor to an IAsyncEnumerable. + /// Converts the cursor to an IAsyncEnumerator, starting from the first batch. /// - /// The type of the documents in the collection. - /// The cursor to convert. /// An optional cancellation token. - /// An IAsyncEnumerable containing all of the documents in the cursor. - public static async IAsyncEnumerable ToAsyncEnumerable(this Cursor cursor, [EnumeratorCancellation] CancellationToken cancellationToken = default) + /// An IAsyncEnumerator containing all documents in the cursor. + public async IAsyncEnumerator ToAsyncEnumerator(CancellationToken cancellationToken = default) { - bool hasNext; - do + using var newCursor = new Cursor(_fetchNextBatch, this); + while (await newCursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) { - cancellationToken.ThrowIfCancellationRequested(); - hasNext = await cursor.MoveNextAsync().ConfigureAwait(false); - if (!hasNext || cursor.Current == null) - { - yield break; - } - foreach (var item in cursor.Current) + foreach (var item in newCursor.Current) { + cancellationToken.ThrowIfCancellationRequested(); yield return item; } - } while (hasNext); + } } /// - /// Converts the cursor to an IEnumerable. + /// Converts the cursor to an IEnumerable, starting from the first batch. /// - /// The type of the documents in the collection. - /// The cursor to convert. - /// An IEnumerable containing all of the documents in the cursor. - public static IEnumerable ToEnumerable(this Cursor cursor) + /// An IEnumerable containing all documents in the cursor. + public IEnumerable ToEnumerable() { - bool hasNext; - do + using var newCursor = new Cursor(_fetchNextBatch, this); + while (newCursor.MoveNext()) { - hasNext = cursor.MoveNext(); - if (!hasNext || cursor.Current == null) - { - yield break; - } - foreach (var item in cursor.Current) + foreach (var item in newCursor.Current) { yield return item; } - } while (hasNext); + } } - /// - /// Returns all of the results of the cursor as a List. - /// - /// The type of the documents in the collection. - /// The cursor to convert. - /// A List containing all of the documents in the cursor. - public static List ToList(this Cursor cursor) + void IParentCursor.SetSortVector(float[] sortVector) { - return ToEnumerable(cursor).ToList(); + if ((SortVector == null || SortVector.Length == 0) && sortVector != null && sortVector.Length > 0) + { + SortVector = sortVector; + } + } + + void IParentCursor.SetStarted() + { + IsStarted = true; } -} \ No newline at end of file + +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Database.cs b/src/DataStax.AstraDB.DataApi/Core/Database.cs index cd83681..b1e0447 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Database.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Database.cs @@ -422,17 +422,7 @@ public Task> CreateCollectionAsync(string collectionN private async Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition, DatabaseCommandOptions options, bool runSynchronously) where T : class { - if (definition == null) - { - - } - object payload = new - { - name = collectionName, - options = definition - }; - var command = CreateCommand("createCollection").WithPayload(payload).AddCommandOptions(options); - await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); + await CreateCollectionAsync(collectionName, definition, options, runSynchronously); return GetCollection(collectionName); } @@ -442,6 +432,10 @@ private async Task> CreateCollectionAsync(string coll { definition = CollectionDefinition.Create(); } + else + { + CollectionDefinition.CheckAddDefinitionsFromAttributes(definition); + } object payload = new { name = collectionName, diff --git a/src/DataStax.AstraDB.DataApi/Core/FilterOptions.cs b/src/DataStax.AstraDB.DataApi/Core/FilterOptions.cs new file mode 100644 index 0000000..82d4bf5 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/FilterOptions.cs @@ -0,0 +1,29 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Internal class used to serialize the filters in analyzer options in the format expected by the API +/// +internal class FilterOptions +{ + [JsonPropertyName("name")] + [JsonInclude] + internal string Name { get; set; } = string.Empty; +} diff --git a/src/DataStax.AstraDB.DataApi/Core/IParentCursor.cs b/src/DataStax.AstraDB.DataApi/Core/IParentCursor.cs new file mode 100644 index 0000000..0d26276 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/IParentCursor.cs @@ -0,0 +1,23 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core; + +internal interface IParentCursor +{ + void SetSortVector(float[] sortVector); + void SetStarted(); +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs b/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs new file mode 100644 index 0000000..5a21539 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs @@ -0,0 +1,38 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Lexical analysis configuration +/// +public class LexicalOptions +{ + /// + /// Configuration for the tokenizer + /// + [JsonPropertyName("analyzer")] + public AnalyzerOptions Analyzer { get; set; } + + /// + /// Whether lexical analysis is enabled + /// + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = true; +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs new file mode 100644 index 0000000..dd015d1 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs @@ -0,0 +1,50 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +internal class DocumentFindManyOptions : DocumentFindOptions, IFindManyOptions> +{ + [JsonIgnore] + public int? Skip { get => _skip; set => _skip = value; } + + [JsonIgnore] + public int? Limit { get => _limit; set => _limit = value; } + + [JsonIgnore] + internal bool? IncludeSortVector { get => _includeSortVector; set => _includeSortVector = value; } + + bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } + + IFindManyOptions> IFindManyOptions>.Clone() + { + var clone = new DocumentFindManyOptions + { + Filter = Filter != null ? Filter.Clone() : null, + PageState = PageState, + Skip = Skip, + Limit = Limit, + IncludeSortVector = IncludeSortVector, + IncludeSimilarity = IncludeSimilarity, + Projection = Projection != null ? Projection.Clone() : null, + Sort = Sort != null ? Sort.Clone() : null + }; + return clone; + } + +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindOptions.cs new file mode 100644 index 0000000..b19652a --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindOptions.cs @@ -0,0 +1,25 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class DocumentFindOptions : FindOptions> +{ + [JsonIgnore] + public override DocumentSortBuilder Sort { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs new file mode 100644 index 0000000..9087ae3 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs @@ -0,0 +1,88 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A sort builder specifically for document operations. +/// +/// The type of the document +public class DocumentSortBuilder : SortBuilder +{ + /// + /// Adds a vector sort. + /// + /// The vector to sort by. + /// The document sort builder. + public DocumentSortBuilder Vector(float[] vector) + { + Sorts.Add(Sort.Vector(vector)); + return this; + } + + /// + /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. + /// + /// The string value to be vectorized. + /// The document sort builder. + public DocumentSortBuilder Vectorize(string valueToVectorize) + { + Sorts.Add(Sort.Vectorize(valueToVectorize)); + return this; + } + + /// + public new DocumentSortBuilder Ascending(string fieldName) + { + base.Ascending(fieldName); + return this; + } + + /// + public new DocumentSortBuilder Ascending(Expression> expression) + { + base.Ascending(expression); + return this; + } + + /// + public new DocumentSortBuilder Descending(string fieldName) + { + base.Descending(fieldName); + return this; + } + + /// + public new DocumentSortBuilder Descending(Expression> expression) + { + base.Descending(expression); + return this; + } + + internal new DocumentSortBuilder Clone() + { + var clone = new DocumentSortBuilder(); + foreach (var sort in this.Sorts) + { + clone.Sorts.Add(sort.Clone()); + } + return clone; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs index 75cdcc1..6fd8b21 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs @@ -16,6 +16,7 @@ using MongoDB.Bson; using System.Collections.Generic; +using System.Linq; namespace DataStax.AstraDB.DataApi.Core.Query; @@ -29,6 +30,19 @@ public class Filter internal virtual string Name { get; } internal virtual object Value { get; } + internal virtual Filter Clone() + { + if (Value is Filter nestedFilter) + { + return new Filter(Name, nestedFilter.Clone()); + } + else if (Value is Filter[] filtersArray) + { + return new Filter(Name, filtersArray.Select(f => f.Clone()).ToArray()); + } + return new Filter(Name, Value); + } + internal Filter(string filterName, object value) { Name = filterName; diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs index 3cd2f9d..c9cc308 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs @@ -283,6 +283,32 @@ public Filter In(Expression> expression, TField[] a return new Filter(expression.GetMemberNameTree(), FilterOperator.In, array); } + /// + /// In operator -- Match where the specified array field contains the specified value. + /// + /// The name of the field for this filter. + /// The value to check for. + /// The filter + /// + /// We recommend using the method with expressions instead of strings for clarity and type safety. + /// + public Filter In(string fieldName, object value) + { + return new Filter(fieldName, FilterOperator.In, value); + } + + /// + /// In operator -- Match where the specified array field contains the specified value. + /// + /// The type of the field to check. + /// An expression that represents the field for this filter. + /// The value to check for. + /// The filter + public Filter In(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.In, value); + } + /// /// Not in operator -- Match documents where the field does not match any of the specified values. /// diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs index 1db9fe2..86d2654 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterOperator.cs @@ -30,6 +30,4 @@ public class FilterOperator public const string All = "$all"; public const string Size = "$size"; public const string Contains = "$contains"; - public const string ContainsKey = "$containsKey"; - public const string ContainsEntry = "$containsEntry"; } diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs new file mode 100644 index 0000000..3e7bf5a --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs @@ -0,0 +1,94 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +internal class FindAndRerankOptions +{ + + internal string RerankOn { get; set; } + internal bool? IncludeScores { get; set; } + internal bool? IncludeSortVector { get; set; } + internal string RerankQuery { get; set; } + internal int? Limit { get; set; } + internal Dictionary HybridLimits { get; set; } + internal Filter Filter { get; set; } + internal IProjectionBuilder Projection { get; set; } + internal List Sorts { get; set; } = new(); + + [JsonInclude] + [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + private Dictionary FilterMap => Filter?.Serialize(); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + private Dictionary SortMap => Sorts?.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + private Dictionary ProjectionMap => Projection?.Projections?.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + private Dictionary SerializableOptions + { + get + { + var options = new Dictionary() + { + { "includeScores", IncludeScores }, + { "includeSortVector", IncludeSortVector }, + { "rerankOn", RerankOn }, + { "rerankQuery", RerankQuery }, + { "limit", Limit }, + { "hybridLimits", HybridLimits } + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) + { + return null; + } + return options; + } + } + + internal FindAndRerankOptions Clone() + { + var clone = new FindAndRerankOptions + { + RerankOn = RerankOn, + IncludeScores = IncludeScores, + IncludeSortVector = IncludeSortVector, + RerankQuery = RerankQuery, + Limit = Limit, + HybridLimits = HybridLimits != null ? new Dictionary(HybridLimits) : null, + Filter = Filter?.Clone(), + Projection = Projection?.Clone(), + Sorts = Sorts?.Select(s => s.Clone()).ToList() ?? new List() + }; + return clone; + } + +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindApiOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindApiOptions.cs new file mode 100644 index 0000000..bbac2c0 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindApiOptions.cs @@ -0,0 +1,50 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// Internal class representing options for the Find API. +/// +internal class FindApiOptions +{ + [JsonInclude] + [JsonPropertyName("skip")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal int? Skip { get; set; } + + [JsonInclude] + [JsonPropertyName("limit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal int? Limit { get; set; } + + [JsonInclude] + [JsonPropertyName("includeSimilarity")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal bool? IncludeSimilarity { get; set; } + + [JsonInclude] + [JsonPropertyName("includeSortVector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal bool? IncludeSortVector { get; set; } + + [JsonInclude] + [JsonPropertyName("pageState")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal string PageState { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/ResultSet.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs similarity index 55% rename from src/DataStax.AstraDB.DataApi/Core/Query/ResultSet.cs rename to src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs index 0fdd30a..c74908c 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/ResultSet.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs @@ -15,6 +15,7 @@ */ using DataStax.AstraDB.DataApi.Core.Results; +using System; using System.Collections; using System.Collections.Generic; using System.Threading; @@ -22,44 +23,25 @@ namespace DataStax.AstraDB.DataApi.Core.Query; -public class DocumentResultSet : ResultSet> - where T : class - where TResult : class -{ - internal DocumentResultSet(IQueryRunner> queryRunner, Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions) - : base(queryRunner, filter, findOptions, commandOptions) - { - } -} - /// -/// A Fluent API for finding documents in a collection. +/// A Fluent API for finding and enumerating documents or rows. /// -/// The type of the documents in the collection. -/// The type of the result. -public class ResultSet : IAsyncEnumerable, IEnumerable +/// The type representing the document or row. +/// The type to deserialize the results to (i.e. if using ). +public class FindEnumerator : IAsyncEnumerable, IEnumerable where T : class where TResult : class where TSort : SortBuilder { - private readonly Filter _filter; private readonly IQueryRunner _queryRunner; - private IFindManyOptions _findOptions; - private CommandOptions _commandOptions; - - internal Filter Filter => _filter; - internal IQueryRunner QueryRunner => _queryRunner; + private readonly IFindManyOptions _findOptions; + private readonly CommandOptions _commandOptions; + private Cursor _cursor; - private IFindManyOptions FindOptions + internal FindEnumerator(IQueryRunner queryRunner, IFindManyOptions findOptions, CommandOptions commandOptions) { - get { return _findOptions; } - } - - internal ResultSet(IQueryRunner queryRunner, Filter filter, IFindManyOptions findOptions, CommandOptions commandOptions) - { - _filter = filter; _queryRunner = queryRunner; - _findOptions = findOptions; + _findOptions = findOptions.Clone(); _commandOptions = commandOptions; } @@ -67,7 +49,7 @@ internal ResultSet(IQueryRunner queryRunner, Filter filter, IFindMa /// Specify a Projection to apply to the results of the operation. /// /// The projection to apply. - /// The ResultSet instance to continue specifying the find options. + /// The FindEnumerator instance to continue specifying the find options. /// /// /// // Inclusive Projection, return only the nested Properties.PropertyOne field @@ -75,21 +57,19 @@ internal ResultSet(IQueryRunner queryRunner, Filter filter, IFindMa /// var projection = projectionBuilder.Include(p => p.Properties.PropertyOne); /// /// - public ResultSet Project(IProjectionBuilder projection) + public FindEnumerator Project(IProjectionBuilder projection) { - FindOptions.Projection = projection; - return this; + return UpdateOptions(options => options.Projection = projection); } /// /// Specify the maximum number of documents to return. /// /// The maximum number of documents to return. - /// The ResultSet instance to continue specifying the find options. - public ResultSet Limit(int limit) + /// The FindEnumerator instance to continue specifying the find options. + public FindEnumerator Limit(int limit) { - FindOptions.Limit = limit; - return this; + return UpdateOptions(options => options.Limit = limit); } /// @@ -97,18 +77,17 @@ public ResultSet Limit(int limit) /// Use in conjuction with to determine the order to apply before skipping. /// /// The number of documents to skip. - /// The ResultSet instance to continue specifying the find options. - public ResultSet Skip(int skip) + /// The FindEnumerator instance to continue specifying the find options. + public FindEnumerator Skip(int skip) { - FindOptions.Skip = skip; - return this; + return UpdateOptions(options => options.Skip = skip); } /// /// Specify a Sort to use when running the find. /// /// The sort to apply. - /// The ResultSet instance to continue adding options. + /// The FindEnumerator instance to continue adding options. /// /// /// // Sort by the nested Properties.PropertyOne field @@ -116,18 +95,16 @@ public ResultSet Skip(int skip) /// var sort = sortBuilder.Ascending(p => p.Properties.PropertyOne); /// /// - public ResultSet Sort(TSort sortBuilder) + public FindEnumerator Sort(TSort sortBuilder) { - FindOptions.Sort = sortBuilder; - return this; + return UpdateOptions(options => options.Sort = sortBuilder); } - /// /// Whether to include the similarity score in the result or not. /// /// Whether to include the similarity score in the result or not. - /// The ResultSet instance to continue specifying the find options. + /// The FindEnumerator instance to continue specifying the find options. /// /// You can use the attribute to map the similarity score to the result class. /// @@ -137,46 +114,53 @@ public ResultSet Sort(TSort sortBuilder) /// public double? Similarity { get; set; } /// } /// - /// var ResultSet = collection.Find<SimpleObjectWithVectorizeResult>() + /// var FindEnumerator = collection.Find<SimpleObjectWithVectorizeResult>() /// .Sort(Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString)) /// .IncludeSimilarity(true); - /// var cursor = ResultSet.ToCursor(); + /// var cursor = FindEnumerator.ToCursor(); /// var list = cursor.ToList(); /// var result = list.First(); /// var similarity = result.Similarity; /// /// - public ResultSet IncludeSimilarity(bool includeSimilarity) + public FindEnumerator IncludeSimilarity(bool includeSimilarity) { - FindOptions.IncludeSimilarity = includeSimilarity; - return this; + return UpdateOptions(options => options.IncludeSimilarity = includeSimilarity); } /// /// Whether to include the sort vector in the result or not. /// /// Whether to include the sort vector in the result or not. - /// The ResultSet instance to continue specifying the find options. + /// The FindEnumerator instance to continue specifying the find options. /// - /// To access the sort vectors, you need to use after calling on your ResultSet instance. /// - /// var ResultSet = collection.Find<SimpleObjectWithVectorizeResult>() + /// var finder = collection.Find<SimpleObjectWithVectorizeResult>() /// .Sort(Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString)) /// .IncludeSortVector(true); - /// var cursor = ResultSet.ToCursor(); - /// var sortVector = cursor.SortVectors; + /// //enumerate the results + /// var results = await finder.ToList(); + /// var sortVector = finder.GetSortVector(); /// /// - public ResultSet IncludeSortVector(bool includeSortVector) + public FindEnumerator IncludeSortVector(bool includeSortVector) { - FindOptions.IncludeSortVector = includeSortVector; - return this; + return UpdateOptions(options => options.IncludeSortVector = includeSortVector); } - internal Task, FindStatusResult>> RunAsync(string pageState = null, bool runSynchronously = false) + /// + /// Returns the sort vector created by the vectorize sort. + /// + /// The sort vector. + /// Thrown when the enumerator has not been started. + public float[] GetSortVector() { - FindOptions.PageState = pageState; - return _queryRunner.RunFindManyAsync(_filter, FindOptions, _commandOptions, runSynchronously); + var cursor = ToCursor(); + if (!cursor.IsStarted) + { + throw new InvalidOperationException("Enumerator has not been started. Enumerate the results first, call ToList(), or manually manage paging by calling ToCursor() and using MoveNextAsync() to iterate over the results."); + } + return cursor.SortVector; } /// @@ -188,8 +172,12 @@ internal Task, FindStatusResult>> R /// A cursor to iterate over the results of the find operation page by page. public Cursor ToCursor() { - var cursor = new Cursor((string pageState, bool runSynchronously) => RunAsync(pageState, runSynchronously)); - return cursor; + if (_cursor != null) + { + return _cursor; + } + _cursor = new Cursor((string pageState, bool runSynchronously) => RunAsync(pageState, runSynchronously)); + return _cursor; } /// @@ -197,14 +185,10 @@ public Cursor ToCursor() /// /// An optional cancellation token to use for the operation. /// An async enumerator - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { var cursor = ToCursor(); - await foreach (var item in cursor.ToAsyncEnumerable(cancellationToken)) - { - cancellationToken.ThrowIfCancellationRequested(); - yield return item; - } + return cursor.ToAsyncEnumerator(cancellationToken); } /// @@ -214,19 +198,7 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken canc public IEnumerator GetEnumerator() { var cursor = ToCursor(); - bool hasNext; - do - { - hasNext = cursor.MoveNext(); - if (!hasNext || cursor.Current == null) - { - yield break; - } - foreach (var item in cursor.Current) - { - yield return item; - } - } while (hasNext); + return cursor.ToEnumerable().GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() @@ -234,4 +206,16 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } + private Task, FindStatusResult>> RunAsync(string pageState = null, bool runSynchronously = false) + { + _findOptions.PageState = pageState; + return _queryRunner.RunFindManyAsync(_findOptions.Filter, _findOptions, _commandOptions, runSynchronously); + } + + private FindEnumerator UpdateOptions(Action> optionsUpdater) + { + optionsUpdater(_findOptions); + return new FindEnumerator(_queryRunner, _findOptions, _commandOptions); + } + } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs index de92785..172785b 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs @@ -20,184 +20,33 @@ namespace DataStax.AstraDB.DataApi.Core.Query; -/// -/// A set of options to be used when finding documents in a collection. -/// -/// -public class DocumentFindManyOptions : DocumentFindOptions, IFindManyOptions> -{ - - /// - /// The number of documents to skip before starting to return documents. - /// Use in conjuction with to determine the order to apply before skipping. - /// - [JsonIgnore] - public int? Skip { get => _skip; set => _skip = value; } - - /// - /// The number of documents to return. - /// - [JsonIgnore] - public int? Limit { get => _limit; set => _limit = value; } - - /// - /// Whether to include the sort vector in the result or not - /// - /// - /// - /// You can use the attribute to map the sort vector to the result class. - /// public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize - /// { - /// [DocumentMapping(DocumentMappingField.SortVector)] - /// public double? SortVector { get; set; } - /// } - /// - /// var finder = collection.Find<SimpleObjectWithVectorizeResult>( - /// new FindOptions<SimpleObjectWithVectorize>() { - /// Sort = Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString), - /// IncludeSortVector = true - /// }, null); - /// var cursor = finder.ToCursor(); - /// var list = cursor.ToList(); - /// var result = list.First(); - /// var sortVector = result.SortVector; - /// - /// - [JsonIgnore] - internal bool? IncludeSortVector { get => _includeSortVector; set => _includeSortVector = value; } - bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } -} - -/// -/// A set of options to be used when finding rows in a table. -/// -/// -public class TableFindManyOptions : TableFindOptions, IFindManyOptions> -{ - - /// - /// The number of documents to skip before starting to return documents. - /// Use in conjuction with to determine the order to apply before skipping. - /// - [JsonIgnore] - public int? Skip { get => _skip; set => _skip = value; } - - /// - /// The number of documents to return. - /// - [JsonIgnore] - public int? Limit { get => _limit; set => _limit = value; } - - /// - /// Whether to include the sort vector in the result or not - /// - /// - /// - /// You can use the attribute to map the sort vector to the result class. - /// public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize - /// { - /// [DocumentMapping(DocumentMappingField.SortVector)] - /// public double? SortVector { get; set; } - /// } - /// - /// var finder = collection.Find<SimpleObjectWithVectorizeResult>( - /// new FindOptions<SimpleObjectWithVectorize>() { - /// Sort = Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString), - /// IncludeSortVector = true - /// }, null); - /// var cursor = finder.ToCursor(); - /// var list = cursor.ToList(); - /// var result = list.First(); - /// var sortVector = result.SortVector; - /// - /// - [JsonIgnore] - internal bool? IncludeSortVector { get => _includeSortVector; set => _includeSortVector = value; } - bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } -} - -public interface IFindManyOptions : IFindOptions - where TSort : SortBuilder -{ - public int? Skip { get; set; } - public int? Limit { get; set; } - internal bool? IncludeSortVector { get; set; } -} - -public interface IFindOptions where TSort : SortBuilder -{ - internal Filter Filter { get; set; } - internal string PageState { get; set; } - public TSort Sort { get; set; } - public IProjectionBuilder Projection { get; set; } - public bool? IncludeSimilarity { get; set; } -} - -/// -/// A set of options to be used when finding documents in a collection. -/// -/// The type of the documents in the collection. public abstract class FindOptions : IFindOptions where TSort : SortBuilder { - /// - /// The builder used to define the Projection to apply when running the query. - /// - /// - /// - /// // Inclusive Projection, return only the nested Properties.PropertyOne field - /// var projectionBuilder = Builders<SimpleObject>.Projection; - /// var projection = projectionBuilder.Include(p => p.Properties.PropertyOne); - /// - /// [JsonIgnore] public IProjectionBuilder Projection { get; set; } + internal bool? IncludeSimilarity { get; set; } - /// - /// Whether to include the similarity score in the result or not - /// - /// - /// - /// You can use the attribute to map the similarity score to the result class. - /// public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize - /// { - /// [DocumentMapping(DocumentMappingField.Similarity)] - /// public double? Similarity { get; set; } - /// } - /// - /// var finder = collection.Find<SimpleObjectWithVectorizeResult>( - /// new FindOptions<SimpleObjectWithVectorize>() { - /// Sort = Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString), - /// IncludeSimilarity = true - /// }, null); - /// var cursor = finder.ToCursor(); - /// var list = cursor.ToList(); - /// var result = list.First(); - /// var similarity = result.Similarity; - /// - /// - [JsonIgnore] - public bool? IncludeSimilarity { get; set; } - - [JsonIgnore] protected bool? _includeSortVector; - [JsonIgnore] protected int? _skip; - [JsonIgnore] protected int? _limit; - [JsonIgnore] internal Filter Filter { get; set; } + [JsonIgnore] + public abstract TSort Sort { get; set; } + [JsonIgnore] internal string PageState { get; set; } string IFindOptions.PageState { get => PageState; set => PageState = value; } [JsonInclude] [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] internal Dictionary FilterMap => Filter == null ? null : Filter.Serialize(); + Filter IFindOptions.Filter { get => Filter; set => Filter = value; } [JsonInclude] @@ -205,8 +54,6 @@ public abstract class FindOptions : IFindOptions where TSort [JsonPropertyName("sort")] internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); - public abstract TSort Sort { get; set; } - [JsonInclude] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("projection")] @@ -215,90 +62,32 @@ public abstract class FindOptions : IFindOptions where TSort [JsonInclude] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("options")] - internal FindApiOptions Options + internal Dictionary Options { get { - if (IncludeSimilarity == null && - _includeSortVector == null && - PageState == null && - _skip == null && - _limit == null && - string.IsNullOrEmpty(PageState)) + var options = new Dictionary() + { + { "includeSimilarity", IncludeSimilarity }, + { "includeSortVector", _includeSortVector }, + { "pageState", PageState }, + { "skip", _skip }, + { "limit", _limit } + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) { return null; } - return new FindApiOptions - { - IncludeSimilarity = IncludeSimilarity, - IncludeSortVector = _includeSortVector, - PageState = PageState, - Skip = _skip, - Limit = _limit - }; + return options; } } -} -/// -/// A set of options to be used when finding a document in a collection. -/// -/// The type of the document in the collection. -public class DocumentFindOptions : FindOptions> -{ - /// - /// The builder used to define the sort to apply when running the query. - /// - /// - /// - /// // Sort documents by the nested Properties.PropertyOne field in ascending order - /// var sortBuilder = Builders<SimpleObject>.Sort; - /// var sort = sortBuilder.Ascending(so => so.Properties.PropertyOne); - /// - /// [JsonIgnore] - public override DocumentSortBuilder Sort { get; set; } -} - -/// -/// A set of options to be used when finding a row in a table. -/// -/// The type of the row in the table. -public class TableFindOptions : FindOptions> -{ - /// - /// The builder used to define the sort to apply when running the query. - /// - /// - /// - /// // Sort documents by the nested Properties.PropertyOne field in ascending order - /// var sortBuilder = Builders<SimpleObject>.TableSort; - /// var sort = sortBuilder.Ascending(so => so.Properties.PropertyOne); - /// - /// + TSort IFindOptions.Sort { get => Sort; set => Sort = value; } [JsonIgnore] - public override SortBuilder Sort { get; set; } -} - -internal class FindApiOptions -{ - [JsonPropertyName("skip")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? Skip { get; set; } - - [JsonPropertyName("limit")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? Limit { get; set; } - - [JsonPropertyName("includeSimilarity")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? IncludeSimilarity { get; set; } - - [JsonPropertyName("includeSortVector")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public bool? IncludeSortVector { get; set; } + IProjectionBuilder IFindOptions.Projection { get => Projection; set => Projection = value; } + [JsonIgnore] + bool? IFindOptions.IncludeSimilarity { get => IncludeSimilarity; set => IncludeSimilarity = value; } - [JsonPropertyName("pageState")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string PageState { get; set; } -} +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/IFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/IFindManyOptions.cs new file mode 100644 index 0000000..388aa29 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/IFindManyOptions.cs @@ -0,0 +1,43 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// Represents options for find operations that return multiple results. +/// +/// The type of the document/entity. +/// The type of the sort builder. +internal interface IFindManyOptions : IFindOptions + where TSort : SortBuilder +{ + /// + /// Gets or sets the number of documents to skip. + /// + internal int? Skip { get; set; } + + /// + /// Gets or sets the maximum number of documents to return. + /// + internal int? Limit { get; set; } + + /// + /// Gets or sets whether to include the sort vector in the results. + /// + internal bool? IncludeSortVector { get; set; } + + internal IFindManyOptions Clone(); +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/IFindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/IFindOptions.cs new file mode 100644 index 0000000..54a7433 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/IFindOptions.cs @@ -0,0 +1,35 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// Represents options for find operations. +/// +/// The type of the document/entity. +/// The type of the sort builder. +internal interface IFindOptions where TSort : SortBuilder +{ + internal Filter Filter { get; set; } + + internal string PageState { get; set; } + + internal TSort Sort { get; set; } + + internal IProjectionBuilder Projection { get; set; } + + internal bool? IncludeSimilarity { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs b/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs index f8517d3..b58bfba 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs @@ -5,7 +5,7 @@ namespace DataStax.AstraDB.DataApi.Core.Query; internal interface IQueryRunner where TSort : SortBuilder { - internal Task, FindStatusResult>> RunFindManyAsync( + internal Task, FindStatusResult>> RunFindManyAsync( Filter filter, IFindManyOptions findOptions, CommandOptions commandOptions, bool runSynchronously) where TProjected : class; } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs index e0332db..4329b1e 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs @@ -14,6 +14,9 @@ * limitations under the License. */ +using System; +using System.Linq; + namespace DataStax.AstraDB.DataApi.Core.Query; internal class LogicalFilter : Filter @@ -27,4 +30,17 @@ internal LogicalFilter(LogicalOperator logicalOperator, Filter filter) : base(logicalOperator.ToApiString(), filter) { } + + internal override Filter Clone() + { + if (Value is Filter nestedFilter) + { + return new LogicalFilter(Name.ToLogicalOperator(), nestedFilter.Clone()); + } + else if (Value is Filter[] filtersArray) + { + return new LogicalFilter(Name.ToLogicalOperator(), filtersArray.Select(f => f.Clone()).ToArray()); + } + return base.Clone(); + } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs index 597d207..ae3ea6b 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalOperator.cs @@ -37,4 +37,14 @@ internal static string ToApiString(this LogicalOperator value) _ => throw new ArgumentException("Invalid Logical Operator"), }; } + internal static LogicalOperator ToLogicalOperator(this string value) + { + return value switch + { + "$and" => LogicalOperator.And, + "$or" => LogicalOperator.Or, + "$not" => LogicalOperator.Not, + _ => throw new ArgumentException("Invalid API string for Logical Operator"), + }; + } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs index a65c998..0493d62 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Projection.cs @@ -50,4 +50,15 @@ internal object Value return Include; } } -} \ No newline at end of file + + internal Projection Clone() + { + return new Projection + { + FieldName = FieldName, + Include = Include, + SliceStart = SliceStart, + SliceLength = SliceLength + }; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs index 705f7a3..e2627c8 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/ProjectionBuilder.cs @@ -18,6 +18,7 @@ using DataStax.AstraDB.DataApi.Utils; using System; using System.Collections.Generic; +using System.Linq; using System.Linq.Expressions; namespace DataStax.AstraDB.DataApi.Core.Query; @@ -32,6 +33,24 @@ public class ProjectionBuilder : IProjectionBuilder List IProjectionBuilder.Projections => _projections; + IProjectionBuilder IProjectionBuilder.Clone() + { + return Clone(); + } + + internal ProjectionBuilder Clone() + { + var clone = new ProjectionBuilder(); + clone._projections.AddRange(_projections.Select(p => new Projection + { + FieldName = p.FieldName, + Include = p.Include, + SliceStart = p.SliceStart, + SliceLength = p.SliceLength + })); + return clone; + } + /// /// Create an inclusive projection by specifying a field to include. /// @@ -130,6 +149,8 @@ public ProjectionBuilder Slice(Expression> fieldExpre public interface IProjectionBuilder { internal List Projections { get; } + + internal IProjectionBuilder Clone(); } /// @@ -144,6 +165,7 @@ public abstract class ProjectionBuilderBase : IProjectionBuilder wh List IProjectionBuilder.Projections => _projections; + public abstract IProjectionBuilder Clone(); } /// @@ -251,6 +273,13 @@ public InclusiveProjectionBuilder Slice(Expression> f _projections.Add(projection); return this; } + + public override IProjectionBuilder Clone() + { + var clone = new InclusiveProjectionBuilder(); + clone._projections.AddRange(_projections.Select(p => p.Clone())); + return clone; + } } /// @@ -358,4 +387,11 @@ public ExclusiveProjectionBuilder IncludeSpecial(Expression(); + clone._projections.AddRange(_projections.Select(p => p.Clone())); + return clone; + } } diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/RerankEnumerator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/RerankEnumerator.cs new file mode 100644 index 0000000..54945fa --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/RerankEnumerator.cs @@ -0,0 +1,299 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Core.Results; +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A Fluent API for finding documents in a collection. +/// +/// The type of the documents in the collection. +/// The type of the result. +public class RerankEnumerator : IAsyncEnumerable, IEnumerable + where T : class + where TResult : class +{ + private readonly Func _commandFactory; + private readonly CommandOptions _commandOptions; + private readonly FindAndRerankOptions _findOptions; + + private volatile Task, FindStatusResult>>> _resultTask; + + internal RerankEnumerator(Func commandFactory, FindAndRerankOptions findOptions, CommandOptions commandOptions) + { + _commandFactory = commandFactory; + _commandOptions = commandOptions; + _findOptions = findOptions.Clone(); + } + + /// + /// Set the field to rerank on (defaults to the lexical sort parameter) + /// + /// + /// + public RerankEnumerator SetRerankOn(string rerankOn) + { + _findOptions.RerankOn = rerankOn; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Set the field to rerank on (defaults to the lexical sort parameter) + /// + /// + /// + public RerankEnumerator SetRerankOn(Expression> rerankOn) + { + _findOptions.RerankOn = rerankOn.GetMemberNameTree(); + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Whether to include the scores (vector similarity and reranker) in the result or not. + /// + /// Whether to include the scores in the result or not. + /// + /// + /// + /// var findAndReranker = collection.Find() + /// .Sort(Builders.Sort.Vectorize(dogQueryVectorString)) + /// .IncludeScores(true); + /// var documentsWithScores = findAndReranker.WithScoresAsync(); + /// await foreach (var document in documentsWithScores) + /// { + /// Console.WriteLine(document.Document); + /// Console.WriteLine(document.Scores); + /// } + /// + /// + public RerankEnumerator IncludeScores(bool includeScores) + { + _findOptions.IncludeScores = includeScores; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Whether to include the sort vector in the result or not. + /// + /// Whether to include the sort vector in the result or not. + /// + /// + /// To access the sort vectors, you need to use after calling on your FindAndReranker instance. + /// + /// var FindAndReranker = collection.Find() + /// .Sort(Builders.Sort.Vectorize(dogQueryVectorString)) + /// .IncludeSortVector(true); + /// var cursor = FindAndReranker.ToCursor(); + /// var sortVector = cursor.SortVector; + /// + /// + public RerankEnumerator IncludeSortVector(bool includeSortVector) + { + _findOptions.IncludeSortVector = includeSortVector; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Set the search string for the reranker to use. This must be set if you pass a vector to the Sort method. + /// If you use the hybrid sort, or pass a string to be vectorized to the Sort method, that string will + /// be used unless you pass a different string to this method. + /// + /// The rerank query to use. + /// + public RerankEnumerator SetRerankQuery(string rerankQuery) + { + _findOptions.RerankQuery = rerankQuery; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Set the limit for the number of documents to return. + /// + /// The number of documents to return. + /// + public RerankEnumerator Limit(int limit) + { + _findOptions.Limit = limit; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Customize the number of documents to return for each of the underlying searches (vector and lexical). + /// + /// + /// + /// + public RerankEnumerator SetHybridLimits(int? lexicalLimit, int? vectorLimit) + { + Dictionary hybridLimits = new(); + if (lexicalLimit != null) + { + hybridLimits.Add(DataApiKeywords.Lexical, lexicalLimit.Value); + } + if (vectorLimit != null) + { + hybridLimits.Add(DataApiKeywords.Vector, vectorLimit.Value); + } + _findOptions.HybridLimits = hybridLimits; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Specify a Projection to apply to the results of the operation. + /// + /// The projection to apply. + /// + /// + /// + /// // Inclusive Projection, return only the nested Properties.PropertyOne field + /// var projectionBuilder = Builders.Projection; + /// var projection = projectionBuilder.Include(p => p.Properties.PropertyOne); + /// + /// + public RerankEnumerator Project(IProjectionBuilder projection) + { + _findOptions.Projection = projection; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Get the sort vector for the find operation. + /// + /// + /// + /// The sort vector is only available if was called on the FindAndReranker instance. + /// This method will cause the find operation to run if it has not already been run. + /// + public async Task GetSortVectorAsync() + { + if (!_findOptions.IncludeSortVector == true) + { + throw new InvalidOperationException("IncludeSortVector must be set to true before calling GetSortVector"); + } + var response = await GetAsync().ConfigureAwait(false); + return response.Status.SortVector; + } + + /// + /// Synchronous version of + /// + /// + public float[] GetSortVector() + { + return GetSortVectorAsync().ResultSync(); + } + + /// + /// Returns an async enumerator to iterate over the results of the find operation. + /// + /// An optional cancellation token to use for the operation. + /// An async enumerator + public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + var response = await GetAsync().ConfigureAwait(false); + + foreach (var item in response.Data.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return item; + } + } + + /// + /// Returns an enumerator to iterate over the results of the find operation. + /// + /// An enumerator + public IEnumerator GetEnumerator() + { + var response = GetAsync(true).ResultSync(); + foreach (var item in response.Data.Items) + { + yield return item; + } + } + + /// + /// Returns an async enumerator to iterate over the results of the find operation, including the scores. + /// + /// An optional cancellation token to use for the operation. + /// An async enumerator of objects which include the actual document and the scores. + public async IAsyncEnumerable> WithScoresAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (_findOptions.IncludeScores != true) + { + throw new InvalidOperationException("IncludeScores must be set to true before calling WithScores"); + } + var response = await GetAsync().ConfigureAwait(false); + foreach (var result in GetRerankedResults(response, cancellationToken)) + { + yield return result; + } + } + + /// + /// Returns an enumerator to iterate over the results of the find operation, including the scores. + /// + /// An enumerator of objects which include the actual document and the scores. + public IEnumerator> WithScores() + { + if (_findOptions.IncludeScores != true) + { + throw new InvalidOperationException("IncludeScores must be set to true before calling WithScores"); + } + var response = GetAsync(true).ResultSync(); + foreach (var result in GetRerankedResults(response)) + { + yield return result; + } + } + + internal async Task, FindStatusResult>>> GetAsync(bool runSynchronously = false) + { + if (_resultTask == null) + { + var command = _commandFactory().WithPayload(_findOptions).AddCommandOptions(_commandOptions); + _resultTask = command.RunAsyncReturnDocumentData, TResult, FindStatusResult>>(runSynchronously); + } + + return await _resultTask.ConfigureAwait(false); + } + + private static IEnumerable> GetRerankedResults(ApiResponseWithData, FindStatusResult>> response, CancellationToken cancellationToken = default) + { + for (var i = 0; i < response.Data.Items.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var item = response.Data.Items[i]; + var scores = response.Status.DocumentResponses == null ? null : response.Status.DocumentResponses[i].Scores; + yield return new RerankedResult { Document = item, Scores = scores }; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/RerankSorter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/RerankSorter.cs new file mode 100644 index 0000000..d7204c2 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/RerankSorter.cs @@ -0,0 +1,72 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Commands; +using System; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class RerankSorter where T : class where TResult : class +{ + + private readonly Func _commandFactory; + private readonly CommandOptions _commandOptions; + private readonly FindAndRerankOptions _findOptions; + + internal RerankSorter(Func commandFactory, Filter filter, CommandOptions commandOptions) + { + _commandFactory = commandFactory; + _commandOptions = commandOptions; + _findOptions = new FindAndRerankOptions() { Filter = filter }; + } + + /// + /// Adds a hybrid sort using a combined string to use for lexical and vectorize parameters. + /// + /// Combined string to use for lexical and vectorize parameters. + /// The document sort builder. + public RerankEnumerator Sort(string combinedSearchString) + { + _findOptions.Sorts.Add(Query.Sort.Hybrid(combinedSearchString)); + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Adds a hybrid sort using separate parameters, one for the lexical search and the second string to vectorize for the vector ordering. + /// + /// The lexical search string. + /// The string to vectorize for the vector ordering. + /// The document sort builder. + public RerankEnumerator Sort(string lexical, string vectorize) + { + _findOptions.Sorts.Add(Query.Sort.Hybrid(lexical, vectorize)); + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } + + /// + /// Adds a hybrid sort using separate parameters, one for the lexical search and the second float array for the vector ordering. + /// + /// The lexical search string. + /// The vector parameter. + /// The document sort builder. + public RerankEnumerator Sort(string lexical, float[] vector) + { + _findOptions.Sorts.Add(Query.Sort.Hybrid(lexical, vector)); + _findOptions.RerankOn = DataApiKeywords.Lexical; + _findOptions.RerankQuery = lexical; + return new RerankEnumerator(_commandFactory, _findOptions, _commandOptions); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/RerankedResult.cs b/src/DataStax.AstraDB.DataApi/Core/Query/RerankedResult.cs new file mode 100644 index 0000000..3a46137 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/RerankedResult.cs @@ -0,0 +1,29 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +public class RerankedResult +{ + [JsonIgnore] + public T Document { get; set; } + + [JsonPropertyName("scores")] + public Dictionary Scores { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs index 8caaf64..4f156b0 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Sort.cs @@ -17,6 +17,7 @@ using DataStax.AstraDB.DataApi.Core.Commands; using DataStax.AstraDB.DataApi.Utils; using System; +using System.Collections.Generic; using System.Linq.Expressions; namespace DataStax.AstraDB.DataApi.Core.Query; @@ -26,6 +27,35 @@ internal class Sort internal string Name { get; set; } internal object Value { get; set; } + internal Sort Clone() + { + if (Value is float[] vector) + { + var vectorCopy = new float[vector.Length]; + Array.Copy(vector, vectorCopy, vector.Length); + return new Sort(Name, vectorCopy); + } + else if (Value is Dictionary dict) + { + var dictCopy = new Dictionary(); + foreach (var kvp in dict) + { + if (kvp.Value is float[] vectorValue) + { + var vectorCopy = new float[vectorValue.Length]; + Array.Copy(vectorValue, vectorCopy, vectorValue.Length); + dictCopy[kvp.Key] = vectorCopy; + } + else + { + dictCopy[kvp.Key] = kvp.Value; + } + } + return new Sort(Name, dictCopy); + } + return new Sort(Name, Value); + } + internal Sort(string sortKey, object value) { Name = sortKey; @@ -39,6 +69,12 @@ internal Sort(string sortKey, object value) internal static Sort Vector(float[] vector) => new(DataApiKeywords.Vector, vector); internal static Sort Vectorize(string valueToVectorize) => new(DataApiKeywords.Vectorize, valueToVectorize); + + internal static Sort Hybrid(string combinedSearchString) => new(DataApiKeywords.Hybrid, combinedSearchString); + + internal static Sort Hybrid(string lexical, string vectorize) => new(DataApiKeywords.Hybrid, new Dictionary { { DataApiKeywords.Lexical, lexical }, { DataApiKeywords.Vectorize, vectorize } }); + + internal static Sort Hybrid(string lexical, float[] vector) => new(DataApiKeywords.Hybrid, new Dictionary { { DataApiKeywords.Lexical, lexical }, { DataApiKeywords.Vector, vector } }); } internal class Sort : Sort diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs index 3dbe58b..281ca0b 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs @@ -27,9 +27,7 @@ namespace DataStax.AstraDB.DataApi.Core.Query; /// The type of the document public class SortBuilder { - internal readonly List _sorts = new(); - - internal List Sorts => _sorts; + internal List Sorts { get; set; } = new List(); /// /// Adds an ascending sort. @@ -41,7 +39,7 @@ public class SortBuilder /// public SortBuilder Ascending(string fieldName) { - _sorts.Add(Sort.Ascending(fieldName)); + Sorts.Add(Sort.Ascending(fieldName)); return this; } @@ -53,7 +51,7 @@ public SortBuilder Ascending(string fieldName) /// The sort builder. public SortBuilder Ascending(Expression> expression) { - _sorts.Add(Sort.Ascending(expression)); + Sorts.Add(Sort.Ascending(expression)); return this; } @@ -67,7 +65,7 @@ public SortBuilder Ascending(Expression> expression) /// public SortBuilder Descending(string fieldName) { - _sorts.Add(Sort.Descending(fieldName)); + Sorts.Add(Sort.Descending(fieldName)); return this; } @@ -79,140 +77,17 @@ public SortBuilder Descending(string fieldName) /// The sort builder. public SortBuilder Descending(Expression> expression) { - _sorts.Add(Sort.Descending(expression)); - return this; - } -} - -public class DocumentSortBuilder : SortBuilder -{ - /// - /// Adds a vector sort. - /// - /// The vector to sort by. - /// The document sort builder. - public DocumentSortBuilder Vector(float[] vector) - { - _sorts.Add(Sort.Vector(vector)); + Sorts.Add(Sort.Descending(expression)); return this; } - /// - /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. - /// - /// The string value to be vectorized. - /// The document sort builder. - public DocumentSortBuilder Vectorize(string valueToVectorize) + internal SortBuilder Clone() { - _sorts.Add(Sort.Vectorize(valueToVectorize)); - return this; - } - - /// - public new DocumentSortBuilder Ascending(string fieldName) - { - base.Ascending(fieldName); - return this; - } - - /// - public new DocumentSortBuilder Ascending(Expression> expression) - { - base.Ascending(expression); - return this; - } - - /// - public new DocumentSortBuilder Descending(string fieldName) - { - base.Descending(fieldName); - return this; - } - - /// - public new DocumentSortBuilder Descending(Expression> expression) - { - base.Descending(expression); - return this; - } -} - -public class TableSortBuilder : SortBuilder -{ - /// - /// Adds a vector sort. - /// - /// The name of the field to sort by. - /// The vector to sort by. - /// The table sort builder. - public TableSortBuilder Vector(string fieldName, float[] vector) - { - _sorts.Add(new Sort(fieldName, vector)); - return this; - } - - /// - /// Adds a vector sort. - /// - /// The expression representing the sort field. - /// The vector to sort by. - /// The table sort builder. - public TableSortBuilder Vector(Expression> expression, float[] vector) - { - _sorts.Add(new Sort(expression.GetMemberNameTree(), vector)); - return this; - } - - /// - /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. - /// - /// The name of the field to sort by. - /// The string value to be vectorized. - /// The table sort builder. - public TableSortBuilder Vectorize(string fieldName, string valueToVectorize) - { - _sorts.Add(new Sort(fieldName, valueToVectorize)); - return this; - } - - /// - /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. - /// - /// The type of the field to sort by. - /// The expression representing the sort field. - /// The string value to be vectorized. - /// The table sort builder. - public TableSortBuilder Vectorize(Expression> expression, string valueToVectorize) - { - _sorts.Add(new Sort(expression.GetMemberNameTree(), valueToVectorize)); - return this; - } - - /// - public new TableSortBuilder Ascending(string fieldName) - { - base.Ascending(fieldName); - return this; - } - - /// - public new TableSortBuilder Ascending(Expression> expression) - { - base.Ascending(expression); - return this; - } - - /// - public new TableSortBuilder Descending(string fieldName) - { - base.Descending(fieldName); - return this; - } - - /// - public new TableSortBuilder Descending(Expression> expression) - { - base.Descending(expression); - return this; + var clone = new SortBuilder(); + foreach (var sort in this.Sorts) + { + clone.Sorts.Add(sort.Clone()); + } + return clone; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs new file mode 100644 index 0000000..d9ec1de --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs @@ -0,0 +1,83 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A set of options to be used when finding rows in a table. +/// +/// +public class TableFindManyOptions : TableFindOptions, IFindManyOptions> +{ + + /// + /// The number of documents to skip before starting to return documents. + /// Use in conjuction with to determine the order to apply before skipping. + /// + [JsonIgnore] + public int? Skip { get => _skip; set => _skip = value; } + + /// + /// The number of documents to return. + /// + [JsonIgnore] + public int? Limit { get => _limit; set => _limit = value; } + + /// + /// Whether to include the sort vector in the result or not + /// + /// + /// + /// You can use the attribute to map the sort vector to the result class. + /// public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize + /// { + /// [DocumentMapping(DocumentMappingField.SortVector)] + /// public double? SortVector { get; set; } + /// } + /// + /// var finder = collection.Find<SimpleObjectWithVectorizeResult>( + /// new FindOptions<SimpleObjectWithVectorize>() { + /// Sort = Builders<SimpleObjectWithVectorize>.Sort.Vectorize(dogQueryVectorString), + /// IncludeSortVector = true + /// }, null); + /// var cursor = finder.ToCursor(); + /// var list = cursor.ToList(); + /// var result = list.First(); + /// var sortVector = result.SortVector; + /// + /// + [JsonIgnore] + internal bool? IncludeSortVector { get => _includeSortVector; set => _includeSortVector = value; } + bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } + + IFindManyOptions> IFindManyOptions>.Clone() + { + var clone = new TableFindManyOptions + { + Filter = Filter != null ? Filter.Clone() : null, + PageState = PageState, + Skip = Skip, + Limit = Limit, + IncludeSimilarity = IncludeSimilarity, + Projection = Projection != null ? Projection.Clone() : null, + Sort = Sort != null ? Sort.Clone() : null + }; + return clone; + } + +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs new file mode 100644 index 0000000..61a9a00 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs @@ -0,0 +1,32 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A set of options to be used when finding a row in a table. +/// +/// The type of the row in the table. +public class TableFindOptions : FindOptions> +{ + /// + /// The builder used to define the sort to apply when running the query. + /// + [JsonIgnore] + public override SortBuilder Sort { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableSortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableSortBuilder.cs new file mode 100644 index 0000000..60a813b --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableSortBuilder.cs @@ -0,0 +1,105 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Linq.Expressions; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A sort builder specifically for table operations. +/// +/// The type of the document +public class TableSortBuilder : SortBuilder +{ + /// + /// Adds a vector sort. + /// + /// The name of the field to sort by. + /// The vector to sort by. + /// The table sort builder. + public TableSortBuilder Vector(string fieldName, float[] vector) + { + Sorts.Add(new Sort(fieldName, vector)); + return this; + } + + /// + /// Adds a vector sort. + /// + /// The expression representing the sort field. + /// The vector to sort by. + /// The table sort builder. + public TableSortBuilder Vector(Expression> expression, float[] vector) + { + Sorts.Add(new Sort(expression.GetMemberNameTree(), vector)); + return this; + } + + /// + /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. + /// + /// The name of the field to sort by. + /// The string value to be vectorized. + /// The table sort builder. + public TableSortBuilder Vectorize(string fieldName, string valueToVectorize) + { + Sorts.Add(new Sort(fieldName, valueToVectorize)); + return this; + } + + /// + /// Adds a vector sort by specifying a string value to be vectorized using the collection's vectorizer. + /// + /// The type of the field to sort by. + /// The expression representing the sort field. + /// The string value to be vectorized. + /// The table sort builder. + public TableSortBuilder Vectorize(Expression> expression, string valueToVectorize) + { + Sorts.Add(new Sort(expression.GetMemberNameTree(), valueToVectorize)); + return this; + } + + /// + public new TableSortBuilder Ascending(string fieldName) + { + base.Ascending(fieldName); + return this; + } + + /// + public new TableSortBuilder Ascending(Expression> expression) + { + base.Ascending(expression); + return this; + } + + /// + public new TableSortBuilder Descending(string fieldName) + { + base.Descending(fieldName); + return this; + } + + /// + public new TableSortBuilder Descending(Expression> expression) + { + base.Descending(expression); + return this; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/RerankOptions.cs b/src/DataStax.AstraDB.DataApi/Core/RerankOptions.cs new file mode 100644 index 0000000..9738566 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/RerankOptions.cs @@ -0,0 +1,40 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Options for configuring document reranking in the collection +/// +public class RerankOptions +{ + /// + /// Whether reranking is enabled + /// + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = true; + + /// + /// Configuration for the reranking service + /// + [JsonPropertyName("service")] + public RerankServiceOptions Service { get; set; } + + //TODO: When implementing fluent option, have default for currently available service: + // public RerankOptions() { Service = new RerankServiceOptions() { ModelName = "nvidia/llama-3.2-nv-rerankqa-1b-v2", Provider = "nvidia" }; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/RerankServiceOptions.cs b/src/DataStax.AstraDB.DataApi/Core/RerankServiceOptions.cs new file mode 100644 index 0000000..f07b443 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/RerankServiceOptions.cs @@ -0,0 +1,37 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Configuration for the reranking service +/// +public class RerankServiceOptions +{ + /// + /// The name of the model to use for reranking + /// + [JsonPropertyName("modelName")] + public string ModelName { get; set; } + + /// + /// The provider of the reranking service + /// + [JsonPropertyName("provider")] + public string Provider { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/ApiFindResult.cs similarity index 74% rename from src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs rename to src/DataStax.AstraDB.DataApi/Core/Results/ApiFindResult.cs index 86b388e..4470758 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/ApiFindResult.cs @@ -19,17 +19,11 @@ namespace DataStax.AstraDB.DataApi.Core.Results; -/// -/// A list of documents returned from an operation. -/// -/// The type of the documents -public class DocumentsResult +internal class ApiFindResult { - /// - /// The list of documents returned. - /// + [JsonInclude] [JsonPropertyName("documents")] - public List Documents { get; set; } + internal List Items { get; set; } [JsonInclude] [JsonPropertyName("nextPageState")] diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentCountExceedsMaxException.cs b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentCountExceedsMaxException.cs new file mode 100644 index 0000000..c59ea0b --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentCountExceedsMaxException.cs @@ -0,0 +1,30 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +/// +/// Exception thrown from CountDocumentsAsync when the number of documents exceeds the maximum stated limit (or the internal limit of 1000). +/// +public class DocumentCountExceedsMaxException : Exception +{ + public DocumentCountExceedsMaxException() + { + + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs index 0f572c6..cdfbc16 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/DocumentsCountResult.cs @@ -28,4 +28,10 @@ public class DocumentsCountResult /// [JsonPropertyName("count")] public int Count { get; set; } + + /// + /// Is the count potentially larger than the count (i.e. reached the MaxDocumentsToCount) + /// + [JsonPropertyName("moreData")] + public bool MoreData { get; set; } = false; } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs index b67b321..4e377ad 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/FindStatusResult.cs @@ -14,12 +14,22 @@ * limitations under the License. */ +using System.Collections.Generic; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Results; internal class FindStatusResult { + [JsonInclude] [JsonPropertyName("sortVector")] internal float[] SortVector { get; set; } -} \ No newline at end of file + +} + +internal class FindStatusResult : FindStatusResult +{ + [JsonInclude] + [JsonPropertyName("documentResponses")] + internal List DocumentResponses { get; set; } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/TokenizerOptions.cs b/src/DataStax.AstraDB.DataApi/Core/TokenizerOptions.cs new file mode 100644 index 0000000..cfafbe8 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/TokenizerOptions.cs @@ -0,0 +1,38 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// Tokenizer configuration +/// +public class TokenizerOptions +{ + /// + /// Name of the tokenizer (e.g., "standard") + /// + [JsonPropertyName("name")] + public string Name { get; set; } + + /// + /// Additional tokenizer arguments + /// + [JsonPropertyName("args")] + public Dictionary Arguments { get; set; } = new(); +} diff --git a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs index 38014cc..add22c9 100644 --- a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs +++ b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs @@ -14,15 +14,15 @@ * limitations under the License. */ +using System; +using System.Net.Http; +using System.Threading.Tasks; using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using System.Net.Http; -using System; -using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi; @@ -32,12 +32,12 @@ namespace DataStax.AstraDB.DataApi; /// /// The DataApiClient, and the related methods for interacting with the database, accepts a set of options that can be used to affect the /// command execution. These options can be specified at any level in the call hierarchy (Client, Database, Collection, Command, etc.) -/// The most specific defined option (or it's default) will be used for each request. +/// The most specific defined option (or its default) will be used for each request. /// /// /// Once you have a instance, /// you can use it to get a instance. -/// From there you can create or connect to a +/// From there you can create or connect to a . /// /// public class DataApiClient @@ -207,238 +207,4 @@ public Database GetDatabase(string apiEndpoint, DatabaseCommandOptions dbOptions return new Database(apiEndpoint, this, dbOptions); } - /// - /// Gets an instance of a based on the database Id. - /// - /// The Guid of the database. - /// An instance of the class. - /// - /// - /// var client = new DataApiClient("token"); - /// var database = client.GetDatabase(Guid.Parse("databaseId")); - /// - /// - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabase(Guid databaseId) - { - return GetDatabase(databaseId, null as DatabaseCommandOptions); - } - - /// - /// Gets an instance of a based on the database Id, set to the provided keyspace. - /// - /// The Guid of the database. - /// The specific token to use for this database connection. - /// Optional: The keyspace to use for the database commands. - /// An instance of the class. - /// - /// - /// var client = new DataApiClient("token"); - /// var database = client.GetDatabase(Guid.Parse("databaseId"), "keyspace"); - /// - /// - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabase(Guid databaseId, string token, string keyspace = null) - { - var dbOptions = new DatabaseCommandOptions() { Keyspace = keyspace, Token = token }; - return GetDatabase(databaseId, dbOptions); - } - - /// - /// Gets an instance of a using the Guid database Id. - /// - /// - /// The Guid of the database. - /// An instance of the class. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseAsync(Guid databaseId) - { - return GetDatabaseAsync(databaseId, new DatabaseCommandOptions(), false); - } - - /// - /// Gets an instance of a based on the database Id, using the provided options. - /// - /// Any options provided in the parameter will take precedence over the options from the . - /// - /// The Guid of the database. - /// The options to use for the database. - /// An instance of the class. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabase(Guid databaseId, DatabaseCommandOptions dbOptions) - { - return GetDatabaseAsync(databaseId, dbOptions, true).ResultSync(); - } - - /// - /// Gets an instance of a based on the database Id, using the provided keyspace. - /// - /// The Guid of the database. - /// The keyspace to use for the database commands. - /// An instance of the class. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseAsync(Guid databaseId, string keyspace) - { - var dbOptions = new DatabaseCommandOptions() { Keyspace = keyspace }; - return GetDatabaseAsync(databaseId, dbOptions, true); - } - - /// - /// Gets an instance of a based on the database Id, using the provided options. - /// - /// Any options provided in the parameter will take precedence over the options from the . - /// - /// The Guid of the database. - /// The options to use for the database. - /// An instance of the class. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseAsync(Guid databaseId, DatabaseCommandOptions dbOptions) - { - return GetDatabaseAsync(databaseId, dbOptions, true); - } - - /// - /// Gets an instance of a using the database Id. - /// - /// - /// The Guid of the database. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabaseById(string databaseId) - { - return GetDatabaseById(databaseId, null as DatabaseCommandOptions); - } - - /// - /// Gets an instance of a based on the database Id, using the provided keyspace. - /// - /// The Guid of the database. - /// The keyspace to use for the database commands. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabaseById(string databaseId, string keyspace) - { - var dbOptions = new DatabaseCommandOptions() { Keyspace = keyspace }; - return GetDatabaseById(databaseId, dbOptions); - } - - /// - /// Gets an instance of a based on the database Id, using the provided options. - /// - /// Any options provided in the parameter will take precedence over the options from the . - /// - /// The Guid of the database. - /// The options to use for the database. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Database GetDatabaseById(string databaseId, DatabaseCommandOptions dbOptions) - { - return GetDatabaseByIdAsync(databaseId, dbOptions, true).ResultSync(); - } - - /// - /// Gets an instance of a using the database Id. - /// - /// - /// The Guid of the database. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseByIdAsync(string databaseId) - { - return GetDatabaseByIdAsync(databaseId, new DatabaseCommandOptions(), false); - } - - /// - /// Gets an instance of a based on the database Id, using the provided keyspace. - /// - /// The Guid of the database. - /// The keyspace to use for the database commands. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseByIdAsync(string databaseId, string keyspace) - { - var dbOptions = new DatabaseCommandOptions() { Keyspace = keyspace }; - return GetDatabaseByIdAsync(databaseId, dbOptions, false); - } - - /// - /// Gets an instance of a based on the database Id, using the provided options. - /// - /// Any options provided in the parameter will take precedence over the options from the . - /// - /// The Guid of the database. - /// The options to use for the database. - /// An instance of the class. - /// Thrown when the database Id is not a valid Guid. - /// - /// Using a Guid instead of the Database API endpoint requires an extra API call to lookup the appropriate database. - /// If you want to avoid this, use an overload that accepts the API endpoint (). - /// - public Task GetDatabaseByIdAsync(string databaseId, DatabaseCommandOptions dbOptions) - { - return GetDatabaseByIdAsync(databaseId, dbOptions, false); - } - - private Task GetDatabaseByIdAsync(string databaseId, DatabaseCommandOptions dbOptions, bool runSynchronously) - { - var parsed = Guid.TryParse(databaseId, out var guid); - if (!parsed) - { - throw new ArgumentException("Invalid database Id"); - } - return GetDatabaseAsync(guid, dbOptions, runSynchronously); - } - - private async Task GetDatabaseAsync(Guid databaseId, DatabaseCommandOptions dbOptions, bool runSynchronously) - { - DatabaseInfo dbInfo; - if (runSynchronously) - { - dbInfo = GetAstraDatabasesAdmin().GetDatabaseInfoAsync(databaseId, runSynchronously).ResultSync(); - } - else - { - dbInfo = await GetAstraDatabasesAdmin().GetDatabaseInfoAsync(databaseId).ConfigureAwait(false); - } - var apiEndpoint = $"https://{dbInfo.Id}-{dbInfo.Info.Region}.apps.astra.datastax.com"; - return GetDatabase(apiEndpoint, dbOptions); - } } diff --git a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj index 5154c77..5de6c43 100644 --- a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj +++ b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj @@ -5,9 +5,7 @@ disable disable 10.0 - DataStax.AstraDB.DataApi - 1.0.2-beta DataStax DataStax Client library for accessing the DataStax Astra DB Data API from .NET applications. diff --git a/src/DataStax.AstraDB.DataApi/README.md b/src/DataStax.AstraDB.DataApi/README.md index 395a4f1..4cf3f77 100644 --- a/src/DataStax.AstraDB.DataApi/README.md +++ b/src/DataStax.AstraDB.DataApi/README.md @@ -1,6 +1,6 @@ # Overview -This C# Client Library simplifies using the DataStax Data API to manage and interact with AstraDB instances as well as other DataStax databases. +This C# Client Library simplifies using the DataStax Data API to manage and interact with Astra DB instances as well as other DataStax databases. # Installation @@ -8,33 +8,60 @@ This C# Client Library simplifies using the DataStax Data API to manage and inte dotnet add package DataStax.AstraDB.DataApi ``` +# Upgrade to 2.0.1-beta + +If you were on a previous version of the beta, please note the following breaking change. When setting various options for a Collection or Table Find command, all of the options are chained to the Find command. The DocumentFindManyOptions parameter is no longer needed (and has been removed from the SDK) + +Instead of this: +``` + var findOptions = new DocumentFindManyOptions() + { + Sort = sort, + Limit = 1, + Skip = 2, + Projection = inclusiveProjection + }; + var results = collection.Find(filter, findOptions).ToList(); +``` + +Now do this: +``` + var results = collection.Find(filter) + .Sort(sort) + .Skip(2) + .Limit(1) + .Project(inclusiveProjection) + .ToList(); +``` + + # Quickstart ``` //instantiate a client -var client = new DataApiClient("YourTokenHere"); +var client = new DataApiClient(); //connect to a database -var database = client.GetDatabase("YourDatabaseUrlHere"); +var database = client.GetDatabase("YourDatabaseUrlHere", "YourTokenHere"); //create a new collection -var collection = await database.CreateCollectionAsync("YourCollectionNameHere"); +var collection = await database.CreateCollectionAsync("YourCollectionNameHere"); //insert a document into the collection -var documents = new List +var documents = new List { - new SimpleObject() + new User() { - Name = "Test Object 1", + Name = "Test User 1", }, - new SimpleObject() + new User() { - Name = "Test Object 2", + Name = "Test User 2", } }; var insertResult = await collection.InsertManyAsync(documents); //find a document -var filter = Builders.Filter.Eq(so => so.Name, "Test Object 1"); +var filter = Builders.Filter.Eq(x => x.Name, "Test User 1"); var results = await collection.Find(filter); ``` \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/AnalyzerOptionsConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/AnalyzerOptionsConverter.cs new file mode 100644 index 0000000..310be67 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/AnalyzerOptionsConverter.cs @@ -0,0 +1,48 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core; +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.SerDes; + + +public class AnalyzerOptionsConverter : JsonConverter +{ + public override AnalyzerOptions Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + var analyzerName = reader.GetString(); + return new AnalyzerOptions + { + Tokenizer = new TokenizerOptions { Name = analyzerName } + }; + } + else if (reader.TokenType == JsonTokenType.StartObject) + { + return JsonSerializer.Deserialize(ref reader, options); + } + throw new JsonException("Unexpected token type for AnalyzerOptions"); + } + + public override void Write(Utf8JsonWriter writer, AnalyzerOptions value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, options); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs index e77d8c3..b3be352 100644 --- a/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs +++ b/src/DataStax.AstraDB.DataApi/SerDes/DocumentConverter.cs @@ -50,6 +50,8 @@ static DocumentConverter() DocumentMappingField.Vectorize => DataApiKeywords.Vectorize, DocumentMappingField.Vector => DataApiKeywords.Vector, DocumentMappingField.Similarity => DataApiKeywords.Similarity, + DocumentMappingField.Lexical => DataApiKeywords.Lexical, + DocumentMappingField.Hybrid => DataApiKeywords.Hybrid, _ => prop.Name }; FieldMappings[prop] = jsonName; @@ -141,6 +143,12 @@ public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions } } + var docMappingAttr = prop.GetCustomAttribute(); + if (docMappingAttr != null && propValue == null) + { + continue; + } + var documentIdAttr = prop.GetCustomAttribute(); if (documentIdAttr != null && propValue == null) { diff --git a/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs index 48f8bd7..3bb47dc 100644 --- a/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs +++ b/src/DataStax.AstraDB.DataApi/SerDes/DocumentMappingField.cs @@ -26,5 +26,9 @@ public enum DocumentMappingField /// Serializes as "$vector" for vector data. Vector, /// On read operations only, deserializes the similarity result for vector comparisons - Similarity + Similarity, + /// Serializes as "$lexical" for lexical data. + Lexical, + /// Serializes as "$hybrid" for hybrid data. + Hybrid } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Tables/Table.cs b/src/DataStax.AstraDB.DataApi/Tables/Table.cs index 0050dd9..51a5522 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/Table.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/Table.cs @@ -363,45 +363,37 @@ private async Task InsertOneAsync(T row, CommandOptions c return response.Result; } - public ResultSet> Find() + public FindEnumerator> Find() { - return Find(null, null, null); + return Find(null, null); } - public ResultSet> Find(Filter filter) + public FindEnumerator> Find(Filter filter) { - return Find(filter, null, null); + return Find(filter, null); } - public ResultSet> Find(TableFindManyOptions findOptions) + public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) { - return Find(null, findOptions, null); + return Find(filter, commandOptions); } - public ResultSet> Find(Filter filter, TableFindManyOptions findOptions) + public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) where TResult : class { - return Find(filter, findOptions, null); - } - - public ResultSet> Find(Filter filter, TableFindManyOptions findOptions, CommandOptions commandOptions) - { - findOptions ??= new TableFindManyOptions(); - return new ResultSet>(this, filter, findOptions, commandOptions); - } - - public ResultSet> Find(Filter filter, TableFindManyOptions findOptions, CommandOptions commandOptions) where TResult : class - { - findOptions ??= new TableFindManyOptions(); - return new ResultSet>(this, filter, findOptions, commandOptions); + var findOptions = new TableFindManyOptions + { + Filter = filter + }; + return new FindEnumerator>(this, findOptions, commandOptions); } - internal async Task, FindStatusResult>> RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) + internal async Task, FindStatusResult>> RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) where TResult : class { findOptions.Filter = filter; commandOptions = SetRowSerializationOptions(commandOptions, false); var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnData, FindStatusResult>(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnData, FindStatusResult>(runSynchronously).ConfigureAwait(false); return response; } @@ -823,7 +815,7 @@ internal Command CreateCommand(string name) return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _tableName)); } - Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) + Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) where TProjected : class { return RunFindManyAsync(filter, findOptions, commandOptions, runSynchronously); diff --git a/src/DataStax.AstraDB.DataApi/Utils/InsertValidator.cs b/src/DataStax.AstraDB.DataApi/Utils/InsertValidator.cs index a96693d..2b0c52f 100644 --- a/src/DataStax.AstraDB.DataApi/Utils/InsertValidator.cs +++ b/src/DataStax.AstraDB.DataApi/Utils/InsertValidator.cs @@ -1,4 +1,3 @@ - /* * Copyright DataStax, Inc. * diff --git a/src/DataStax.AstraDB.DataApi/Utils/Wait.cs b/src/DataStax.AstraDB.DataApi/Utils/Wait.cs new file mode 100644 index 0000000..9240ed3 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Utils/Wait.cs @@ -0,0 +1,43 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Threading.Tasks; + +namespace DataStax.AstraDB.DataApi.Utils; + +internal class Wait +{ + internal static async Task WaitForProcess(Func> process, int maxWaitInSeconds = 600) + { + const int SLEEP_SECONDS = 5; + + int secondsWaited = 0; + + while (secondsWaited < maxWaitInSeconds) + { + var done = await process().ConfigureAwait(false); + if (done) + { + return; + } + await Task.Delay(SLEEP_SECONDS * 1000).ConfigureAwait(false); + secondsWaited += SLEEP_SECONDS; + } + + throw new Exception(); + } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs similarity index 88% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs index ddafedd..39a46d3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/AdminFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs @@ -1,11 +1,10 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using System.Text.RegularExpressions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; public class AdminFixture : IDisposable { @@ -24,7 +23,7 @@ public AdminFixture() _databaseId = GetDatabaseIdFromUrl(dbUrl) ?? throw new Exception("Database ID could not be extracted from ASTRA_DB_URL."); - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../admin_tests_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/admin_tests_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions @@ -32,6 +31,7 @@ public AdminFixture() RunMode = RunMode.Debug }; Client = new DataApiClient(token, clientOptions, logger); + Database = Client.GetDatabase(DatabaseUrl); } public void Dispose() @@ -44,6 +44,7 @@ public void Dispose() public string DatabaseName { get; private set; } public DataApiClient Client { get; private set; } public string DatabaseUrl { get; private set; } + public Database Database { get; private set; } public Database GetDatabase() { @@ -62,7 +63,7 @@ public Database GetDatabase() public DatabaseAdminAstra CreateAdmin(Database database = null) { - database ??= Client.GetDatabaseAsync(DatabaseId).GetAwaiter().GetResult(); + database ??= Database; var adminOptions = new CommandOptions { diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs similarity index 97% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs index 73592ca..a34b071 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/CollectionsFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs @@ -1,12 +1,10 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("DatabaseAndCollections")] public class DatabaseAndCollectionsCollection : ICollectionFixture @@ -35,7 +33,7 @@ public CollectionsFixture() DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../collections_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/collections_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/DatabaseFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs similarity index 92% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/DatabaseFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs index 7db78c0..a13a4e2 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/DatabaseFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Logging; using Xunit; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Database")] public class Databaseollection : ICollectionFixture @@ -32,7 +32,7 @@ public DatabaseFixture() DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../database_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/database_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/FindAndUpdateFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs similarity index 96% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/FindAndUpdateFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs index 2591fda..057c004 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/FindAndUpdateFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs @@ -1,12 +1,10 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("FindAndUpdate")] public class FindAndUpdateCollection : ICollectionFixture @@ -32,7 +30,7 @@ public FindAndUpdateFixture() var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../findandupdate_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/findandupdate_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs similarity index 96% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs index de8d393..8d7bedc 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs @@ -1,12 +1,10 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("ReplaceAndDelete")] public class ReplaceAndDeleteCollection : ICollectionFixture @@ -32,7 +30,7 @@ public ReplaceAndDeleteFixture() var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../replace_and_delete_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/replace_and_delete_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs new file mode 100644 index 0000000..935c307 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs @@ -0,0 +1,161 @@ +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.SerDes; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System.Text.Json.Serialization; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; + +[CollectionDefinition("RerankCollection")] +public class RerankCollection : ICollectionFixture +{ + +} + +public class RerankFixture : IDisposable, IAsyncLifetime +{ + private const string _queryCollectionName = "rerankQueryTests"; + + public DataApiClient Client { get; private set; } + public Database Database { get; private set; } + public string OpenAiApiKey { get; set; } + public string DatabaseUrl { get; set; } + public string Token { get; set; } + public Collection HybridSearchCollection { get; private set; } + + public RerankFixture() + { + IConfiguration configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables(prefix: "ASTRA_DB_") + .Build(); + + Token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; + DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; + OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; + + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/rerank_fixture_latest_run.log")); + ILogger logger = factory.CreateLogger("IntegrationTests"); + + var clientOptions = new CommandOptions + { + RunMode = RunMode.Debug + }; + Client = new DataApiClient(Token, clientOptions, logger); + Database = Client.GetDatabase(DatabaseUrl); + } + + public async Task InitializeAsync() + { + await CreateSearchCollection(); + var collection = Database.GetCollection(_queryCollectionName); + HybridSearchCollection = collection; + } + + public async Task DisposeAsync() + { + await Database.DropCollectionAsync(_queryCollectionName); + } + + private async Task CreateSearchCollection() + { + List items = new() { + new() + { + Name = "Cat", + HybridTest = "this animal is a cat" + }, + new() + { + Name = "NotCat", + VectorizeTest = "not cats are like cats but aren't actually cats", + LexicalTest = "this animal is not a cat" + }, + new() + { + Name = "Horse", + HybridTest = "this animal is a horse" + }, + new() + { + Name = "Cow", + LexicalTest = "this animal is a cow, it is not a cat", + VectorizeTest = "this animal is a cow, which has 4 legs like a cat does" + } + }; + + var definition = new CollectionDefinition() + { + Lexical = new LexicalOptions() + { + Analyzer = new AnalyzerOptions() + { + Tokenizer = new TokenizerOptions() + { + Name = "standard", + Arguments = new Dictionary() { } + }, + Filters = new List() + { + "lowercase", + "stop", + "porterstem", + "asciifolding" + }, + CharacterFilters = new List() { } + }, + Enabled = true + }, + Rerank = new RerankOptions() + { + Enabled = true, + Service = new RerankServiceOptions() + { + ModelName = "nvidia/llama-3.2-nv-rerankqa-1b-v2", + Provider = "nvidia" + } + }, + Vector = new VectorOptions() + { + Dimension = 1024, + Metric = SimilarityMetric.Cosine, + Service = new VectorServiceOptions() + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } + } + }; + + var collection = await Database.CreateCollectionAsync(_queryCollectionName, definition); + await collection.InsertManyAsync(items); + + HybridSearchCollection = collection; + } + + public void Dispose() + { + //nothing needed + } +} + +public class HybridSearchTestObject +{ + //TODO: Ucomment when HybridSearch supports uuids + //[DocumentId(DefaultIdType.Uuid)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Guid? Id { get; set; } + public string Name { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [DocumentMapping(DocumentMappingField.Lexical)] + public string LexicalTest { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [DocumentMapping(DocumentMappingField.Vectorize)] + public string VectorizeTest { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [DocumentMapping(DocumentMappingField.Hybrid)] + public string HybridTest { get; set; } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs similarity index 76% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs index c079879..c6ee940 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs @@ -1,13 +1,10 @@ -using DataStax.AstraDB.DataApi; -using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("TableAlter")] public class TableAlterCollection : ICollectionFixture @@ -15,7 +12,7 @@ public class TableAlterCollection : ICollectionFixture } -public class TableAlterFixture : IDisposable, IAsyncLifetime +public class TableAlterFixture { public DataApiClient Client { get; private set; } public Database Database { get; private set; } @@ -32,7 +29,7 @@ public TableAlterFixture() var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../table_Alter_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/table_Alter_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions @@ -55,21 +52,7 @@ public TableAlterFixture() } - public async Task InitializeAsync() - { - await CreateTestTable(); - } - - public async Task DisposeAsync() - { - await Database.DropTableAsync(_fixtureTableName); - } - - public Table FixtureTestTable { get; private set; } - - - private const string _fixtureTableName = "tableAlterTest"; - private async Task CreateTestTable() + public async Task> CreateTestTable(string tableName) { var startDate = DateTime.UtcNow.Date.AddDays(7); @@ -102,14 +85,10 @@ private async Task CreateTestTable() }; - var table = await Database.CreateTableAsync(_fixtureTableName); + var table = await Database.CreateTableAsync(tableName); await table.InsertManyAsync(eventRows); - FixtureTestTable = table; + return table; } - public void Dispose() - { - // nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TableIndexesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs similarity index 93% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/TableIndexesFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs index 555fc65..cc4a0c3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/TableIndexesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs @@ -1,13 +1,10 @@ -using DataStax.AstraDB.DataApi; -using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("TableIndexes")] public class TableIndexesCollection : ICollectionFixture @@ -32,7 +29,7 @@ public TableIndexesFixture() var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../table_indexes_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/table_indexes_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TablesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs similarity index 95% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/TablesFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs index 926fcdc..1b3cc5b 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/TablesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs @@ -1,14 +1,10 @@ -using DataStax.AstraDB.DataApi; -using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using System.Reflection.Metadata; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Tables")] public class TablesCollection : ICollectionFixture @@ -33,7 +29,7 @@ public TablesFixture() var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../tables_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/tables_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions @@ -119,7 +115,7 @@ await table.CreateVectorIndexAsync(new TableVectorIndex() await table.CreateIndexAsync(new TableIndex() { IndexName = "due_date_index", - Definition = new TableIndexDefinition() + Definition = new TableIndexDefinition() { Column = (b) => b.DueDate } @@ -187,7 +183,7 @@ await table.CreateVectorIndexAsync(new TableVectorIndex() await table.CreateIndexAsync(new TableIndex() { IndexName = "delete_table_due_date_index", - Definition = new TableIndexDefinition() + Definition = new TableIndexDefinition() { Column = (b) => b.DueDate } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/UpdatesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs similarity index 96% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/UpdatesFixture.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs index 441f267..db2baca 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/UpdatesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs @@ -1,12 +1,10 @@ -using DataStax.AstraDB.DataApi; using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Xunit; -using Xunit.Abstractions; -namespace DataStax.AstraDB.DataApi.IntegrationTests; +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Updates")] public class UpdatesCollection : ICollectionFixture @@ -32,7 +30,7 @@ public UpdatesFixture() var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../updates_fixture_latest_run.log")); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/updates_fixture_latest_run.log")); ILogger logger = factory.CreateLogger("IntegrationTests"); var clientOptions = new CommandOptions diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs index 6e749a6..fb35eb4 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs @@ -133,7 +133,7 @@ public class RowBook public object Author { get; set; } [ColumnPrimaryKey(2)] public int NumberOfPages { get; set; } - public DateTime DueDate { get; set; } + public DateTime? DueDate { get; set; } public HashSet Genres { get; set; } public float Rating { get; set; } } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs index cdd389b..3e67eae 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs @@ -1,11 +1,7 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Query; -using MongoDB.Bson; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -using UUIDNext; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -81,11 +77,7 @@ public async Task Book_Projection_Tests() Assert.Equal(items.Count, insertResult.InsertedIds.Count); var inclusiveProjection = Builders.Projection .Include(x => x.Author).Include(x => x.Title); - var findOptions = new DocumentFindManyOptions() - { - Projection = inclusiveProjection - }; - var allBooks = collection.Find(findOptions).ToList(); + var allBooks = collection.Find().Project(inclusiveProjection).ToList(); Assert.Equal(items.Count, allBooks.Count); var book = allBooks.Find(book => book.Title.Equals("Test Book 1")); Assert.Equal("Test Book 1", book.Title); @@ -94,11 +86,7 @@ public async Task Book_Projection_Tests() //Use property names instead of expressions inclusiveProjection = Builders.Projection .Include("number_of_pages").Include("title"); - findOptions = new DocumentFindManyOptions() - { - Projection = inclusiveProjection - }; - allBooks = collection.Find(findOptions).ToList(); + allBooks = collection.Find().Project(inclusiveProjection).ToList(); Assert.Equal(items.Count, allBooks.Count); book = allBooks.Find(book => book.Title.Equals("Test Book 1")); Assert.Equal("Test Book 1", book.Title); diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs index 87cca2a..b1b792d 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs @@ -1,9 +1,6 @@ using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Core; -using DataStax.AstraDB.DataApi.Core.Results; -using System.Diagnostics.CodeAnalysis; -using System.Reflection; -using System.Runtime.CompilerServices; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -25,17 +22,6 @@ public AdminTests(AdminFixture fixture) this.fixture = fixture; } - [Fact] - public async Task ConnectViaDbId() - { - var dbGuid = fixture.DatabaseId; - var db = await fixture.Client.GetDatabaseAsync(dbGuid); - Assert.NotNull(db); - - db = fixture.Client.GetDatabase(dbGuid); - Assert.NotNull(db); - } - [Fact] public async Task GetDatabasesList() { @@ -126,32 +112,32 @@ public async Task CheckDatabaseStatus() var status = await fixture.Client.GetAstraDatabasesAdmin().GetDatabaseStatusAsync(dbName); Assert.Equal("ACTIVE", status); - status = fixture.Client.GetAstraDatabasesAdmin().GetDatabaseStatus(dbName); + status = await fixture.Client.GetAstraDatabasesAdmin().GetDatabaseStatusAsync(dbName); Assert.Equal("ACTIVE", status); } [Fact] - public async Task DatabaseAdminAstra_GetDatabaseAdminAstra() + public void DatabaseAdminAstra_GetDatabaseAdminAstra() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); Assert.IsType(daa); } [Fact] - public async Task DatabaseAdminAstra_GetDatabase() + public void DatabaseAdminAstra_GetDatabase() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); Assert.IsType(daa.GetDatabase()); } [Fact] - public async Task DatabaseAdminAstra_GetApiEndpoint() + public void DatabaseAdminAstra_GetApiEndpoint() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); Assert.Equal(fixture.DatabaseId, AdminFixture.GetDatabaseIdFromUrl(daa.GetApiEndpoint())); @@ -160,7 +146,7 @@ public async Task DatabaseAdminAstra_GetApiEndpoint() [Fact] public async Task DatabaseAdminAstra_GetKeyspacesList() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); var names = await daa.ListKeyspaceNamesAsync(); @@ -177,7 +163,7 @@ public async Task DatabaseAdminAstra_GetKeyspacesList() [Fact] public async Task DatabaseAdminAstra_DoesKeyspaceExist() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); var keyspaceExists = await daa.KeyspaceExistsAsync("default_keyspace"); @@ -190,7 +176,7 @@ public async Task DatabaseAdminAstra_DoesKeyspaceExist() [Fact] public async Task DatabaseAdminAstra_DoesKeyspaceExist_Another() { - var database = await fixture.Client.GetDatabaseAsync(fixture.DatabaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var daa = fixture.CreateAdmin(database); var keyspaceName = "another_keyspace"; @@ -218,7 +204,7 @@ public async Task DatabaseAdminAstra_FindEmbeddingProvidersAsync() { Token = fixture.Client.ClientOptions.Token, }; - var daa = new DatabaseAdminAstra(fixture.DatabaseId, fixture.Client, adminOptions); + var daa = new DatabaseAdminAstra(fixture.Database, fixture.Client, adminOptions); var result = await daa.FindEmbeddingProvidersAsync(adminOptions, runSynchronously: false); Assert.NotNull(result); @@ -355,8 +341,7 @@ public async Task DropDatabaseByIdAsync() [Fact(Skip = AdminCollection.SkipMessage)] public async Task DatabaseAdminAstra_CreateKeyspace_ExpectedError() { - var databaseId = fixture.DatabaseId; - var database = await fixture.Client.GetDatabaseAsync(databaseId); + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var adminOptions = new CommandOptions(); var daa = new DatabaseAdminAstra(database, fixture.Client, adminOptions); @@ -375,7 +360,7 @@ public async Task DatabaseAdminAstra_CreateKeyspaceAsync() { Token = fixture.Client.ClientOptions.Token, }; - var daa = new DatabaseAdminAstra(fixture.DatabaseId, fixture.Client, adminOptions); + var daa = new DatabaseAdminAstra(fixture.Database, fixture.Client, adminOptions); await daa.CreateKeyspaceAsync(keyspaceName, adminOptions); Console.WriteLine($"DatabaseAdminAstra_CreateKeyspaceAsync > adminOptions.Keyspace: {adminOptions.Keyspace}"); @@ -392,7 +377,7 @@ public async Task DatabaseAdminAstra_CreateKeyspaceAsync_Update() { Token = fixture.Client.ClientOptions.Token, }; - var daa = new DatabaseAdminAstra(fixture.DatabaseId, fixture.Client, adminOptions); + var daa = new DatabaseAdminAstra(fixture.Database, fixture.Client, adminOptions); await daa.CreateKeyspaceAsync(keyspaceName, true, adminOptions); @@ -410,7 +395,7 @@ public async Task DatabaseAdminAstra_DropKeyspaceAsync() { Token = fixture.Client.ClientOptions.Token, }; - var daa = new DatabaseAdminAstra(fixture.DatabaseId, fixture.Client, adminOptions); + var daa = new DatabaseAdminAstra(fixture.Database, fixture.Client, adminOptions); await daa.DropKeyspaceAsync(keyspaceName, adminOptions); // todo: better test result here; for now we assume if no error, this was successful diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs index 122482b..9f6287e 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs @@ -1,11 +1,12 @@ -using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; +using MongoDB.Bson; +using Xunit; using DataStax.AstraDB.DataApi.Core.Commands; -using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.Core.Results; using DataStax.AstraDB.DataApi.SerDes; -using MongoDB.Bson; -using System.Text.Json.Serialization; using UUIDNext; -using Xunit; +using DataStax.AstraDB.DataApi.Core; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -623,7 +624,7 @@ public async Task CountDocuments_NoFilter_ReturnsCorrectCount() { var collection = fixture.SearchCollection; var count = await collection.CountDocumentsAsync(); - Assert.Equal(33, count.Count); + Assert.Equal(33, count); } [Fact] @@ -632,7 +633,34 @@ public async Task CountDocuments_Filter_ReturnsCorrectCount() var collection = fixture.SearchCollection; var filter = Builders.Filter.Eq("Properties.PropertyOne", "grouptwo"); var count = await collection.CountDocumentsAsync(filter); - Assert.Equal(3, count.Count); + Assert.Equal(3, count); + } + + [Fact] + public async Task CountDocuments_MaxCount() + { + var collectionName = "testCountObjects"; + try + { + List items = new List(); + for (var i = 0; i < 100; i++) + { + items.Add(new SimpleObject() + { + _id = i, + Name = $"Test Object {i}" + }); + } + ; + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + await collection.InsertManyAsync(items, new InsertManyOptions() { InsertInOrder = true }); + await Assert.ThrowsAsync(async () => await collection.CountDocumentsAsync(50)); + + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } } } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs index daacc13..e80bfb9 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs @@ -1,6 +1,7 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Tables; using System.Net; using System.Text; @@ -196,7 +197,7 @@ public async Task CreateCollection_WithVectorEuclidean() { Vector = new VectorOptions { - Dimension = 1536, + Dimension = 1024, Metric = SimilarityMetric.Euclidean } }; @@ -273,16 +274,12 @@ public async Task CreateCollection_WithVectorizeHeader() { Vector = new VectorOptions { - Dimension = 1536, + Dimension = 1024, Metric = SimilarityMetric.DotProduct, - Service = new VectorServiceOptions + Service = new VectorServiceOptions() { - Provider = "openai", - ModelName = "text-embedding-ada-002", - Authentication = new Dictionary - { - { "providerKey", fixture.OpenAiApiKey } - } + Provider = "nvidia", + ModelName = "NV-Embed-QA" } } }; @@ -300,16 +297,67 @@ public async Task CreateCollection_WithVectorizeSharedKey() { Vector = new VectorOptions { - Dimension = 1536, + Dimension = 1024, + Metric = SimilarityMetric.DotProduct, + Service = new VectorServiceOptions() + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } + } + }; + var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); + Assert.NotNull(collection); + Assert.Equal(collectionName, collection.CollectionName); + await fixture.Database.DropCollectionAsync(collectionName); + } + + [Fact] + public async Task CreateCollection_ForHybridSearch() + { + var collectionName = "collectionHybridSearch"; + var options = new CollectionDefinition + { + Vector = new VectorOptions + { + Dimension = 1024, Metric = SimilarityMetric.DotProduct, - Service = new VectorServiceOptions + Service = new VectorServiceOptions() + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } + }, + Lexical = new LexicalOptions + { + Analyzer = new AnalyzerOptions { - Provider = "openai", - ModelName = "text-embedding-ada-002", - Authentication = new Dictionary + Tokenizer = new TokenizerOptions { - { "providerKey", fixture.OpenAiApiKey } - } + Name = "standard", + Arguments = new Dictionary + { + { "name", "standard" } + } + }, + Filters = new List + { + "lowercase", + "stop", + "porterstem", + "asciifolding" + }, + CharacterFilters = new List() + }, + Enabled = true + }, + Rerank = new RerankOptions + { + Enabled = true, + Service = new RerankServiceOptions + { + ModelName = "nvidia/llama-3.2-nv-rerankqa-1b-v2", + Provider = "nvidia" } } }; @@ -375,80 +423,82 @@ public async Task CreateTable_Default() } } - [Fact] - public async Task CreateTable_DataTypesTest_FromObject() - { - try - { - var table = await fixture.Database.CreateTableAsync(); - Assert.NotNull(table); - var definitions = await fixture.Database.ListTablesAsync(); - var definition = definitions.FirstOrDefault(d => d.Name == "testTable"); - Assert.NotNull(definition); - Assert.Equal(4, (definition.TableDefinition.Columns["Vector"] as VectorColumn).Dimension); - - var row = new RowTestObject - { - Name = "Test", - Vector = new float[4] { 1, 2, 3, 4 }, - StringToVectorize = "TestStringToVectorize", - Text = "TestText", - Inet = IPAddress.Parse("192.168.0.1"), - Int = int.MaxValue, - TinyInt = byte.MaxValue, - SmallInt = short.MaxValue, - BigInt = long.MaxValue, - Decimal = decimal.MaxValue, - Double = double.MaxValue, - Float = float.MaxValue, - IntDictionary = new Dictionary() { { "One", 1 }, { "Two", 2 } }, - DecimalDictionary = new Dictionary() { { "One", 1.11111m }, { "Two", 2.22222m } }, - StringSet = new HashSet() { "HashSetOne", "HashSetTwo" }, - IntSet = new HashSet() { 1, 2 }, - StringList = new List() { "One", "Two" }, - ObjectList = new List() { - new Properties() { PropertyOne = "OneOne", PropertyTwo = "OneTwo" }, - new Properties() { PropertyOne = "TwoOne", PropertyTwo = "TwoTwo" } }, - Boolean = false, - Date = DateTime.Now, - UUID = Guid.NewGuid(), - Blob = Encoding.ASCII.GetBytes("Test Blob"), - Duration = Duration.Parse("12y3mo1d12h30m5s12ms7us1ns") - }; - var result = await table.InsertManyAsync(new List { row }); - Assert.Equal(1, result.InsertedCount); - var resultSet = table.Find(); - var resultList = resultSet.ToList(); - var resultRow = resultList.First(); - Assert.Equal(row.Name, resultRow.Name); - Assert.Equal(row.Vector, resultRow.Vector); - Assert.Equal(row.Text, resultRow.Text); - Assert.Equal(row.Inet, resultRow.Inet); - Assert.Equal(row.Int, resultRow.Int); - Assert.Equal(row.TinyInt, resultRow.TinyInt); - Assert.Equal(row.SmallInt, resultRow.SmallInt); - Assert.Equal(row.BigInt, resultRow.BigInt); - Assert.Equal(row.Decimal, resultRow.Decimal); - Assert.Equal(row.Double, resultRow.Double); - Assert.Equal(row.Float, resultRow.Float); - Assert.Equal(row.IntDictionary, resultRow.IntDictionary); - Assert.Equal(row.DecimalDictionary, resultRow.DecimalDictionary); - Assert.Equal(row.StringSet, resultRow.StringSet); - Assert.Equal(row.IntSet, resultRow.IntSet); - Assert.Equal(row.StringList, resultRow.StringList); - Assert.Equal(JsonSerializer.Serialize(row.ObjectList), JsonSerializer.Serialize(resultRow.ObjectList)); - Assert.Equal(row.Boolean, resultRow.Boolean); - Assert.Equal(row.Date.ToUniversalTime().ToString("MMddyyhhmmss"), resultRow.Date.ToUniversalTime().ToString("MMddyyhhmmss")); - Assert.Equal(row.UUID, resultRow.UUID); - Assert.Equal(row.Blob, resultRow.Blob); - Assert.Equal(row.Duration, resultRow.Duration); - - } - finally - { - await fixture.Database.DropTableAsync(); - } - } + //TODO re-enable this test once the api issue is fixed (https://github.com/stargate/data-api/issues/2141) + // [Fact] + // public async Task CreateTable_DataTypesTest_FromObject() + // { + // const string tableName = "tableDataTypesTestFromObject"; + // try + // { + // var table = await fixture.Database.CreateTableAsync(tableName); + // Assert.NotNull(table); + // var definitions = await fixture.Database.ListTablesAsync(); + // var definition = definitions.FirstOrDefault(d => d.Name == tableName); + // Assert.NotNull(definition); + // Assert.Equal(4, (definition.TableDefinition.Columns["Vector"] as VectorColumn).Dimension); + + // var row = new RowTestObject + // { + // Name = "Test", + // Vector = new float[4] { 1, 2, 3, 4 }, + // StringToVectorize = "TestStringToVectorize", + // Text = "TestText", + // Inet = IPAddress.Parse("192.168.0.1"), + // Int = int.MaxValue, + // TinyInt = byte.MaxValue, + // SmallInt = short.MaxValue, + // BigInt = long.MaxValue, + // Decimal = decimal.MaxValue, + // Double = double.MaxValue, + // Float = float.MaxValue, + // IntDictionary = new Dictionary() { { "One", 1 }, { "Two", 2 } }, + // DecimalDictionary = new Dictionary() { { "One", 1.11111m }, { "Two", 2.22222m } }, + // StringSet = new HashSet() { "HashSetOne", "HashSetTwo" }, + // IntSet = new HashSet() { 1, 2 }, + // StringList = new List() { "One", "Two" }, + // ObjectList = new List() { + // new Properties() { PropertyOne = "OneOne", PropertyTwo = "OneTwo" }, + // new Properties() { PropertyOne = "TwoOne", PropertyTwo = "TwoTwo" } }, + // Boolean = false, + // Date = DateTime.Now, + // UUID = Guid.NewGuid(), + // Blob = Encoding.ASCII.GetBytes("Test Blob"), + // Duration = Duration.Parse("12y3mo1d12h30m5s12ms7us1ns") + // }; + // var result = await table.InsertManyAsync(new List { row }); + // Assert.Equal(1, result.InsertedCount); + // var resultSet = table.Find(); + // var resultList = resultSet.ToList(); + // var resultRow = resultList.First(); + // Assert.Equal(row.Name, resultRow.Name); + // Assert.Equal(row.Vector, resultRow.Vector); + // Assert.Equal(row.Text, resultRow.Text); + // Assert.Equal(row.Inet, resultRow.Inet); + // Assert.Equal(row.Int, resultRow.Int); + // Assert.Equal(row.TinyInt, resultRow.TinyInt); + // Assert.Equal(row.SmallInt, resultRow.SmallInt); + // Assert.Equal(row.BigInt, resultRow.BigInt); + // Assert.Equal(row.Decimal, resultRow.Decimal); + // Assert.Equal(row.Double, resultRow.Double); + // Assert.Equal(row.Float, resultRow.Float); + // Assert.Equal(row.IntDictionary, resultRow.IntDictionary); + // Assert.Equal(row.DecimalDictionary, resultRow.DecimalDictionary); + // Assert.Equal(row.StringSet, resultRow.StringSet); + // Assert.Equal(row.IntSet, resultRow.IntSet); + // Assert.Equal(row.StringList, resultRow.StringList); + // Assert.Equal(JsonSerializer.Serialize(row.ObjectList), JsonSerializer.Serialize(resultRow.ObjectList)); + // Assert.Equal(row.Boolean, resultRow.Boolean); + // Assert.Equal(row.Date.ToUniversalTime().ToString("MMddyyhhmmss"), resultRow.Date.ToUniversalTime().ToString("MMddyyhhmmss")); + // Assert.Equal(row.UUID, resultRow.UUID); + // Assert.Equal(row.Blob, resultRow.Blob); + // Assert.Equal(row.Duration, resultRow.Duration); + + // } + // finally + // { + // await fixture.Database.DropTableAsync(tableName); + // } + // } [Fact] public async Task CreateTable_DataTypesTest_FromDefinition() diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs index 0a93393..406cbf0 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs @@ -1,5 +1,5 @@ - using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Core; using Xunit; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs index 5b31661..d2ffa30 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs @@ -1,4 +1,5 @@ using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Core; using Xunit; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs similarity index 99% rename from test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteTests.cs rename to test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs index 1b4a868..2f3e938 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/ReplaceAndDeleteTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs @@ -1,5 +1,6 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -555,7 +556,7 @@ public async Task DeleteMany_Tests() Assert.Equal(30, result.DeletedCount); result = await collection.DeleteAllAsync(); Assert.Equal(-1, result.DeletedCount); - Assert.Equal(0, (await collection.CountDocumentsAsync()).Count); + Assert.Equal(0, await collection.CountDocumentsAsync()); } finally { diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs new file mode 100644 index 0000000..bc1a1dc --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs @@ -0,0 +1,149 @@ +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +[Collection("RerankCollection")] +public class RerankTests : IClassFixture +{ + private readonly RerankFixture _fixture; + private readonly Collection _collection; + + public RerankTests(RerankFixture fixture) + { + _fixture = fixture; + _collection = fixture.HybridSearchCollection; + } + + [Fact] + public async Task FindAndRerank_BasicSearch_ReturnsResults_WithAsyncEnumeration() + { + var searchResultsList = new List(); + var results = _collection.FindAndRerank() + .Sort("cat"); + await foreach (var result in results) + { + searchResultsList.Add(result); + } + Assert.NotNull(searchResultsList); + Assert.NotEmpty(searchResultsList); + Assert.Contains(searchResultsList, r => r.Name == "Cat"); + } + + [Fact] + public void FindAndRerank_BasicVectorAndLexicalSearch_ReturnsResults() + { + var lexicalQuery = "cat"; + var vectorQuery = new float[] { -0.032348633f, 0.021102905f, -0.0020217896f, -0.038208008f, -0.0070228577f, 0.019866943f, -0.023010254f, -0.0057525635f, 0.029373169f, -0.042388916f, 0.021881104f, 0.018707275f, 0.004951477f, 0.021560669f, 0.018615723f, 0.01210022f, -0.028274536f, -0.0317688f, 0.0015859604f, 0.012504578f, 0.020477295f, 0.038482666f, 0.04727173f, 0.03869629f, 0.012176514f, 0.055480957f, 0.0018234253f, -0.023666382f, 0.046020508f, 0.0418396f, -0.04284668f, -0.07733154f, 0.020080566f, -0.015701294f, 0.012641907f, -0.026535034f, 0.021026611f, -0.01361084f, -0.042053223f, 0.0051002502f, 0.047058105f, -0.016952515f, -0.0041046143f, -0.008255005f, 0.006538391f, 0.00793457f, -0.03668213f, -0.03842163f, 0.027923584f, -0.033325195f, 0.044677734f, -0.037750244f, 0.043792725f, -0.008148193f, -4.6658516E-4f, 0.027404785f, 0.062805176f, 0.06713867f, -0.03552246f, -0.031311035f, -0.0049743652f, -0.09277344f, 0.023544312f, -8.029938E-4f, 6.3562393E-4f, 0.05480957f, -0.029632568f, -0.005596161f, 0.016052246f, -0.05142212f, 0.033813477f, -0.011146545f, -0.009048462f, 0.0027103424f, 0.006793976f, -0.04067993f, -0.00774765f, 0.0340271f, 0.054718018f, 0.046081543f, 0.038146973f, 0.023483276f, -0.007472992f, 0.009315491f, 0.0289917f, 0.006587982f, 0.034973145f, -0.07128906f, 0.0053596497f, 0.007133484f, 0.0069618225f, -0.020263672f, 0.04147339f, -0.019165039f, -0.00818634f, -0.022369385f, -0.018096924f, 0.004878998f, -0.017028809f, -0.020935059f, 0.014099121f, -0.036621094f, 0.05444336f, 0.031585693f, -0.020233154f, 0.035217285f, -0.01222229f, 0.008430481f, -8.9502335E-4f, 0.0041618347f, -0.023269653f, -0.007369995f, 0.023269653f, 0.045135498f, 0.015670776f, -0.036956787f, -0.022903442f, -0.015266418f, 0.015609741f, 0.0690918f, -0.016220093f, -0.047088623f, 7.8344345E-4f, -0.0013170242f, 4.0888786E-4f, 0.042388916f, 0.011291504f, -8.72612E-4f, 0.004714966f, 0.024536133f, -0.046905518f, -0.0385437f, 0.014793396f, 0.0023384094f, 0.04888916f, -0.051239014f, -0.1303711f, -0.045898438f, 0.03756714f, -0.02935791f, -0.031402588f, -0.07470703f, -0.02079773f, 0.022918701f, 0.064697266f, 0.054870605f, -0.0030708313f, 0.0690918f, 0.006389618f, 0.035980225f, -0.030685425f, -0.019683838f, 0.007911682f, 0.01083374f, 0.020141602f, -0.050201416f, 0.03982544f, -0.014724731f, 0.02507019f, 0.013755798f, 0.014076233f, -0.0013895035f, -0.028320312f, -0.010955811f, -0.034423828f, -0.022827148f, 0.037506104f, -0.002040863f, 0.003982544f, 0.062561035f, 0.0345459f, -0.0022239685f, 0.035339355f, 0.04937744f, 0.029769897f, -0.039733887f, 0.005794525f, -0.0135650635f, -0.021774292f, 0.026870728f, -0.025039673f, -0.009223938f, 0.024597168f, 0.0289917f, 0.011695862f, 0.045837402f, -0.047668457f, -0.081604004f, 0.041259766f, -0.040100098f, -0.046905518f, -0.014923096f, 0.040161133f, -0.048797607f, -0.0068626404f, -0.043395996f, 0.03579712f, 0.011192322f, -0.012359619f, -0.023483276f, -0.0418396f, 0.02748108f, 0.014137268f, -0.0025177002f, 0.02432251f, 0.039093018f, 0.018661499f, -0.02243042f, 0.006576538f, -0.012031555f, -0.045654297f, -0.04159546f, -0.013946533f, -0.007472992f, 0.008872986f, 0.006576538f, 0.024795532f, -0.0031147003f, -0.040771484f, 0.034729004f, 0.01473999f, 0.062805176f, -0.009857178f, -0.022994995f, -0.013000488f, 0.008071899f, -0.012161255f, -0.014442444f, -0.040496826f, -0.032104492f, -0.05404663f, -0.0118637085f, 0.038208008f, -0.023361206f, -0.038238525f, 0.0102005005f, 0.0054969788f, -0.009132385f, 0.03378296f, 0.024536133f, -0.017410278f, 0.0062828064f, 0.011413574f, 0.0027256012f, -0.004283905f, 6.723404E-4f, 0.045806885f, 0.020645142f, 0.01876831f, -0.04196167f, -0.030532837f, 0.02279663f, -0.018127441f, -0.026489258f, 0.051513672f, -0.0385437f, 0.03729248f, -0.03668213f, 0.0059890747f, 0.025344849f, -0.04046631f, 0.044952393f, 0.027374268f, -0.0046691895f, -0.010978699f, 0.0018043518f, -0.02444458f, -0.030532837f, 0.06842041f, -0.025024414f, 0.01335907f, 0.018218994f, -0.038482666f, 0.018463135f, 0.009262085f, 0.028015137f, -0.019104004f, 0.0057411194f, 0.02758789f, -0.017501831f, -0.042755127f, -0.05218506f, -0.014190674f, 0.04324341f, -0.023452759f, 0.022476196f, -0.047332764f, -0.008415222f, 0.0345459f, 0.0058021545f, 0.027038574f, -0.0077209473f, -0.0067596436f, -0.031097412f, 0.039978027f, -0.03866577f, 0.0066337585f, 0.005908966f, 0.046783447f, 0.010917664f, 7.9774857E-4f, 0.005039215f, 0.006134033f, -0.056488037f, -0.014984131f, -0.009284973f, 0.012283325f, -0.0059547424f, -0.011070251f, 0.062438965f, -0.0010519028f, -0.022201538f, -0.0027637482f, -0.029525757f, -0.028533936f, 0.015403748f, 0.046020508f, 0.009262085f, 0.013923645f, 0.017745972f, 0.061462402f, 0.048217773f, -0.068359375f, 0.023956299f, -0.010009766f, -0.017333984f, 0.029281616f, 0.01171875f, -0.006111145f, 0.035827637f, -0.05444336f, 0.028869629f, -0.028457642f, 0.012672424f, -0.007965088f, -0.029205322f, 0.016830444f, 0.037902832f, -0.02407837f, -0.017807007f, 0.031341553f, -0.0067367554f, 0.011680603f, -0.023910522f, -0.01171875f, 0.015571594f, -0.0047302246f, 0.0039405823f, -0.021438599f, 0.013870239f, -0.028961182f, 0.030334473f, -0.005268097f, -0.039855957f, -0.009223938f, 0.01574707f, 0.030975342f, 0.032104492f, -0.031402588f, 0.026657104f, 0.06604004f, -0.012687683f, -0.050048828f, -0.072021484f, 0.0020809174f, 0.05609131f, 0.002796173f, 0.0019798279f, -0.0012483597f, 0.037109375f, 0.035003662f, 0.03225708f, -0.015975952f, 0.005596161f, -0.025344849f, -0.0059776306f, 0.0037078857f, 0.021835327f, -0.006412506f, 0.018661499f, 0.011886597f, 0.009757996f, 0.019714355f, 0.010444641f, -0.06781006f, -0.056610107f, 0.028274536f, 0.018051147f, 0.0049819946f, -0.045532227f, -1.7368793E-4f, 0.04763794f, -0.028793335f, -0.044799805f, 0.0036792755f, -0.022247314f, 0.030532837f, -0.02861023f, 0.010040283f, 0.015792847f, 0.015136719f, 0.084106445f, -0.021026611f, -0.026657104f, 0.004096985f, 0.05618286f, -0.028915405f, -0.03314209f, -2.6583672E-4f, 0.032958984f, 0.041900635f, 0.032409668f, 0.028656006f, 7.092953E-5f, 0.009605408f, -0.0053138733f, 0.044677734f, -0.018859863f, 0.016601562f, 0.016143799f, -0.054718018f, -0.010719299f, -0.028457642f, 0.010551453f, -0.032165527f, -0.0036716461f, 0.008834839f, 0.0287323f, -0.010536194f, 0.022201538f, -0.0065994263f, 0.053619385f, -0.035705566f, -0.04034424f, -0.03933716f, -0.006664276f, -0.034362793f, 0.036865234f, 0.006111145f, 0.027038574f, 0.059326172f, -0.005012512f, 0.0034713745f, -0.055419922f, -0.040802002f, 0.017501831f, 0.011711121f, -0.06378174f, 0.0038585663f, -0.006793976f, 0.0067825317f, 0.0075416565f, -0.010269165f, 0.0069007874f, -0.1381836f, 0.015174866f, -0.030639648f, 0.054718018f, -0.042388916f, 0.008590698f, -0.04864502f, -0.017501831f, -0.014442444f, 0.009010315f, 0.0073394775f, -0.06744385f, -0.004360199f, -0.0034923553f, 0.005886078f, 0.025924683f, 0.010429382f, -0.0039711f, -0.0098724365f, -0.013214111f, -0.0013942719f, 0.031829834f, 7.9488754E-4f, -0.02407837f, 3.8933754E-4f, -6.122589E-4f, 0.0027122498f, 0.041534424f, 0.045288086f, -0.004348755f, -0.048980713f, 0.018920898f, -0.008811951f, 0.0021820068f, 0.018035889f, 0.009437561f, 0.05618286f, -0.052124023f, -0.020645142f, -0.034851074f, -0.02784729f, -0.0010900497f, -0.009483337f, -0.0013189316f, -0.012878418f, 0.025604248f, 0.035980225f, 0.0041503906f, -0.024261475f, -0.014831543f, -0.005622864f, 0.03668213f, -0.048217773f, 0.015609741f, -0.013885498f, 0.0118637085f, 0.04269409f, -0.06512451f, 0.014190674f, -0.013175964f, -0.016860962f, -0.0062675476f, 0.008270264f, 0.047821045f, 0.020629883f, -0.0065193176f, 6.213188E-4f, -0.04348755f, -0.04977417f, -0.0033779144f, -0.017578125f, -0.021469116f, -0.0044021606f, -0.014343262f, -0.033050537f, -0.017959595f, -0.038848877f, -0.0018701553f, -0.034698486f, 0.024658203f, -0.014701843f, -0.00907135f, 0.0042915344f, -0.08880615f, -0.025985718f, 0.046875f, 0.027511597f, 0.013664246f, -0.03213501f, 0.101135254f, 0.038330078f, 0.013519287f, -0.044036865f, -0.010826111f, -0.035614014f, 0.026306152f, -0.0040016174f, 0.0048179626f, 0.008850098f, -0.008255005f, -0.031921387f, -0.0121154785f, 0.015213013f, 0.013076782f, 0.013381958f, 0.009338379f, -0.0143585205f, 0.014305115f, 0.040222168f, -0.038879395f, -0.019363403f, 0.010734558f, 0.03213501f, -0.043182373f, -0.0069732666f, -0.018585205f, -0.028564453f, 0.036621094f, -0.041870117f, 0.01525116f, 0.015808105f, 0.0038928986f, 0.022994995f, 0.06689453f, -0.0079956055f, -0.0029888153f, 0.0440979f, 0.008834839f, -0.0063591003f, 0.023880005f, -0.020584106f, -0.0463562f, -0.0036201477f, 0.011680603f, -0.013076782f, -0.028121948f, 0.028274536f, 0.009483337f, 0.0053901672f, 0.019439697f, -0.054718018f, 0.020477295f, -0.0074424744f, -0.04269409f, 0.018798828f, -0.0035705566f, 0.016174316f, -0.0030021667f, 0.007987976f, -0.025161743f, -0.027038574f, 0.017852783f, -0.018356323f, 0.016098022f, -0.010009766f, 0.01838684f, 0.07531738f, -0.006801605f, -0.004257202f, 0.015571594f, 0.037750244f, 0.04751587f, -0.03967285f, 0.03491211f, 0.009918213f, 0.035217285f, 0.007068634f, 0.0059165955f, -0.007850647f, 0.047088623f, -0.010536194f, -0.054901123f, -0.015930176f, 0.0071525574f, -0.05758667f, 0.017532349f, -0.034820557f, -0.024414062f, 0.013252258f, -0.03677368f, -0.0028648376f, -0.017868042f, -0.007987976f, -0.030792236f, 0.048034668f, 0.0067214966f, 0.045684814f, -0.0051879883f, -0.028152466f, 0.04815674f, 0.013923645f, 0.044799805f, 6.23703E-4f, -0.089538574f, -0.004173279f, -0.02947998f, -0.015319824f, 0.08105469f, -0.0028629303f, 0.011909485f, -0.011245728f, -0.022705078f, 0.060333252f, -0.012939453f, -0.0040245056f, 0.0118637085f, 0.055541992f, -0.01689148f, 0.030792236f, -0.0076828003f, 0.056549072f, 0.20996094f, 0.014930725f, 0.004497528f, -0.07965088f, 5.559921E-4f, -0.032165527f, 0.023544312f, -0.024627686f, -0.008140564f, -0.006313324f, 0.0074157715f, 0.051940918f, 0.029327393f, 0.016525269f, 0.026321411f, 0.087646484f, -0.016021729f, -0.022949219f, 0.01802063f, 0.010513306f, 0.016571045f, 0.058746338f, -0.0074272156f, -0.028961182f, 0.0063056946f, -0.018463135f, 0.01676941f, -0.019119263f, 0.009338379f, 0.027938843f, -0.07611084f, 0.014305115f, -0.05328369f, 0.034606934f, 0.009315491f, -0.00680542f, 0.009262085f, 0.021347046f, -0.04296875f, 0.026428223f, 0.012413025f, -0.011665344f, -0.0129470825f, -0.0059814453f, -0.021347046f, -0.023788452f, -0.04449463f, -0.005569458f, -0.027328491f, 0.04736328f, -0.051940918f, 0.036712646f, 0.034210205f, -0.025024414f, 0.031982422f, -0.0031108856f, 0.0046463013f, -0.04562378f, -0.036315918f, -0.10223389f, 0.019515991f, 0.012481689f, 0.03866577f, -0.01776123f, -0.01512146f, -2.515316E-5f, 0.0017700195f, 0.039245605f, 0.036956787f, -0.006088257f, 0.05218506f, 0.018829346f, -0.019256592f, -0.03050232f, 0.022659302f, 0.010932922f, 0.034362793f, 0.028274536f, -0.038238525f, -0.011421204f, -0.047729492f, 0.008384705f, 0.026397705f, -0.0012645721f, -0.040863037f, 0.016220093f, 0.036102295f, -0.018249512f, 0.023010254f, 0.012809753f, -0.02571106f, 0.018707275f, -0.019989014f, 0.0118637085f, -0.0109939575f, 0.04095459f, -0.00844574f, 0.013458252f, -0.022277832f, 0.041748047f, 0.051208496f, 0.008987427f, -0.027038574f, 0.056243896f, 0.048339844f, -0.016693115f, 0.034210205f, -0.040771484f, -0.037841797f, 0.008392334f, -0.0034732819f, -0.015670776f, -0.01121521f, -0.077697754f, -0.031173706f, -0.04095459f, -0.029449463f, -0.026428223f, 0.021102905f, -0.02520752f, 0.027450562f, -0.04095459f, -0.0024967194f, -0.057373047f, -0.0038585663f, 0.0039711f, 0.014526367f, -0.053527832f, -0.0030097961f, -0.04763794f, 0.012466431f, 0.016555786f, 0.02281189f, -0.0031032562f, 0.04107666f, 0.0146102905f, -9.717941E-4f, 0.030166626f, -0.04977417f, -0.019073486f, -0.010795593f, -0.030151367f, -0.02204895f, -0.011169434f, -0.05038452f, 0.027923584f, 0.05569458f, -0.0047073364f, 0.027252197f, -0.00111866f, -0.091308594f, -0.0073509216f, -0.02583313f, -0.021377563f, -0.019104004f, 0.0032577515f, -0.025970459f, -0.0146102905f, -0.038482666f, 0.005405426f, 0.05001831f, 0.0071144104f, 0.025604248f, -0.005302429f, 0.0087509155f, 0.003326416f, 0.03086853f, 0.022369385f, 0.023330688f, -0.01322937f, 0.009284973f, 0.06903076f, 0.023910522f, 0.02709961f, -0.05444336f, -0.002943039f, 0.02180481f, -0.029663086f, 0.04763794f, -0.025436401f, -0.017532349f, 0.062805176f, 0.00497818f, 0.030883789f, -0.046020508f, -0.025131226f, 0.0063819885f, 0.011726379f, 0.013641357f, -0.0069732666f, 0.0071792603f, -0.0129852295f, 0.024353027f, 0.014724731f, 0.02015686f, 0.020874023f, -0.009552002f, -0.002729416f, 0.0057258606f, -0.033569336f, 0.05999756f, -0.03414917f, 0.003276825f, 0.029846191f, 0.049591064f, -0.019241333f, 0.045135498f, 0.04547119f, -0.012565613f, 0.03201294f, 0.019012451f, -0.03668213f, -0.031341553f, 0.012420654f, 0.043762207f, 0.03050232f, 0.007095337f, -0.008636475f, 0.009414673f, -0.015991211f, -0.062438965f, 0.0413208f, -0.009811401f, 0.008613586f, 4.4178963E-4f, 0.024734497f, -0.004047394f, 0.0055770874f, -0.02758789f, -0.0012454987f, 0.0413208f, -0.008331299f, -0.05014038f, -6.637573E-4f, -0.0047912598f, 0.006980896f, 0.010871887f, 0.0036201477f, 0.026489258f, -0.02973938f, -0.0345459f, -0.02507019f, 0.0063819885f, -0.0017623901f, -0.025970459f, -0.037902832f, 0.0357666f, 0.08380127f, -0.026000977f, 0.014152527f, 0.0032482147f, -0.013473511f, 0.0049819946f, -0.014213562f, 0.022201538f, -0.02784729f, 0.019592285f, 0.0019550323f, -0.011741638f, -0.020858765f, -0.022918701f, 0.02368164f, -0.023773193f, -0.008674622f, -0.011001587f, 0.035736084f, -0.010154724f, -0.035064697f, -0.03753662f, -0.053985596f, -0.01436615f, -8.7070465E-4f, 4.887581E-4f, -0.051605225f, 0.019607544f, 0.036132812f, -0.032928467f, -0.018920898f, -0.024658203f, 0.022827148f, 0.06378174f, 0.064575195f, -0.028656006f, 0.037475586f, 0.046722412f, 0.024871826f, 0.0047073364f, -0.009902954f, -0.03086853f, 2.8252602E-4f, -0.012207031f, 0.00831604f, -0.04421997f, 0.02406311f, -0.005908966f, -0.011940002f, 0.015563965f, 0.010757446f, -0.013648987f, 0.019821167f, 0.041656494f, 0.048461914f, 0.035705566f, -0.004135132f, 0.020980835f, -0.0033550262f, 0.01096344f, 0.016067505f, 0.0178833f, 0.0012598038f, 0.0046691895f, 0.046875f, -0.02381897f, 0.004501343f, 0.026657104f, -0.004798889f, 0.05871582f, 0.0025024414f, 0.017562866f, -0.0017290115f, -0.0077209473f, 0.04348755f, -0.011116028f, 0.007499695f, 0.039031982f, 0.014457703f, -0.010559082f, 0.009048462f, 0.03765869f, -0.012161255f, 0.00283432f, -0.012458801f, 0.05368042f, 0.011909485f, -0.023773193f, -0.0033168793f, -0.009674072f, 0.023117065f, -0.0058174133f, -0.0065994263f, -0.019195557f, 0.021026611f, -0.01574707f, 7.543564E-4f, -0.005077362f, -0.035217285f, -0.028320312f, 0.029403687f, 0.0029773712f, -0.015098572f, 0.034698486f, -0.0030441284f, 0.031051636f, -0.04232788f, 0.011833191f, 4.8828125E-4f, -0.014801025f, -8.404255E-5f, -0.027862549f }; + var results = _collection.FindAndRerank() + .Sort(lexicalQuery, vectorQuery); + var searchResultsList = results.ToList(); + Assert.NotNull(searchResultsList); + Assert.NotEmpty(searchResultsList); + Assert.Contains(searchResultsList, r => r.Name == "Cat"); + } + + [Fact] + public void FindAndRerank_BasicLexicalAndVectorizeSearch_ReturnsResults() + { + var results = _collection.FindAndRerank() + .Sort(lexical: "not a cat", vectorize: "cat") + .IncludeSortVector(true); + var searchResultsList = results.ToList(); + var sortVector = results.GetSortVectorAsync(); + Assert.NotNull(searchResultsList); + Assert.NotEmpty(searchResultsList); + Assert.Contains(searchResultsList, r => r.Name == "Cat"); + } + + [Fact] + public void FindAndRerank_WithLimit_ReturnsLimitedResults() + { + var query = "animal"; + var limit = 2; + + var results = _collection.FindAndRerank() + .Sort(query) + .Limit(limit) + .ToList(); + + Assert.True(results.Count <= limit); + } + + [Fact] + public void FindAndRerank_WithFilter_ReturnsFilteredResults() + { + var results = new List(); + var filter = Builders.Filter.Eq("Name", "Horse"); + var query = _collection.FindAndRerank(filter) + .Sort("cat") + .ToList(); + + Assert.NotNull(results); + Assert.All(results, r => Assert.Equal("Horse", r.Name)); + } + + [Fact] + public async Task FindAndRerank_WithIncludeScores_ReturnsScores() + { + var results = _collection.FindAndRerank() + .Sort("cat") + .IncludeScores(true); + + await foreach (var result in results.WithScoresAsync()) + { + Assert.NotNull(result.Scores); + Assert.NotEmpty(result.Scores); + } + } + + [Fact] + public async Task FindAndRerank_WithExclusiveProjection_ExcludesProperties() + { + var results = new List(); + await foreach (var result in _collection.FindAndRerank() + .Sort("cat") + .Project(Builders.Projection.Exclude("Name"))) + { + Assert.Null(result.Name); + } + } + + [Fact] + public async Task FindAndRerank_WithIncludeSortVector_ReturnsSortVector() + { + var results = _collection.FindAndRerank() + .Sort("cat") + .IncludeSortVector(true); + var sortVector = await results.GetSortVectorAsync(); + Assert.NotNull(sortVector); + Assert.NotEmpty(sortVector); + } + + [Fact] + public void FindAndRerank_WithAllOptions_ReturnsSuccessfully() + { + var lexicalQuery = "cat"; + var rerankQuery = "cow"; + var results = _collection.FindAndRerank() + .Sort(lexical: lexicalQuery, vector: new float[] { -0.032348633f, 0.021102905f, -0.0020217896f, -0.038208008f, -0.0070228577f, 0.019866943f, -0.023010254f, -0.0057525635f, 0.029373169f, -0.042388916f, 0.021881104f, 0.018707275f, 0.004951477f, 0.021560669f, 0.018615723f, 0.01210022f, -0.028274536f, -0.0317688f, 0.0015859604f, 0.012504578f, 0.020477295f, 0.038482666f, 0.04727173f, 0.03869629f, 0.012176514f, 0.055480957f, 0.0018234253f, -0.023666382f, 0.046020508f, 0.0418396f, -0.04284668f, -0.07733154f, 0.020080566f, -0.015701294f, 0.012641907f, -0.026535034f, 0.021026611f, -0.01361084f, -0.042053223f, 0.0051002502f, 0.047058105f, -0.016952515f, -0.0041046143f, -0.008255005f, 0.006538391f, 0.00793457f, -0.03668213f, -0.03842163f, 0.027923584f, -0.033325195f, 0.044677734f, -0.037750244f, 0.043792725f, -0.008148193f, -4.6658516E-4f, 0.027404785f, 0.062805176f, 0.06713867f, -0.03552246f, -0.031311035f, -0.0049743652f, -0.09277344f, 0.023544312f, -8.029938E-4f, 6.3562393E-4f, 0.05480957f, -0.029632568f, -0.005596161f, 0.016052246f, -0.05142212f, 0.033813477f, -0.011146545f, -0.009048462f, 0.0027103424f, 0.006793976f, -0.04067993f, -0.00774765f, 0.0340271f, 0.054718018f, 0.046081543f, 0.038146973f, 0.023483276f, -0.007472992f, 0.009315491f, 0.0289917f, 0.006587982f, 0.034973145f, -0.07128906f, 0.0053596497f, 0.007133484f, 0.0069618225f, -0.020263672f, 0.04147339f, -0.019165039f, -0.00818634f, -0.022369385f, -0.018096924f, 0.004878998f, -0.017028809f, -0.020935059f, 0.014099121f, -0.036621094f, 0.05444336f, 0.031585693f, -0.020233154f, 0.035217285f, -0.01222229f, 0.008430481f, -8.9502335E-4f, 0.0041618347f, -0.023269653f, -0.007369995f, 0.023269653f, 0.045135498f, 0.015670776f, -0.036956787f, -0.022903442f, -0.015266418f, 0.015609741f, 0.0690918f, -0.016220093f, -0.047088623f, 7.8344345E-4f, -0.0013170242f, 4.0888786E-4f, 0.042388916f, 0.011291504f, -8.72612E-4f, 0.004714966f, 0.024536133f, -0.046905518f, -0.0385437f, 0.014793396f, 0.0023384094f, 0.04888916f, -0.051239014f, -0.1303711f, -0.045898438f, 0.03756714f, -0.02935791f, -0.031402588f, -0.07470703f, -0.02079773f, 0.022918701f, 0.064697266f, 0.054870605f, -0.0030708313f, 0.0690918f, 0.006389618f, 0.035980225f, -0.030685425f, -0.019683838f, 0.007911682f, 0.01083374f, 0.020141602f, -0.050201416f, 0.03982544f, -0.014724731f, 0.02507019f, 0.013755798f, 0.014076233f, -0.0013895035f, -0.028320312f, -0.010955811f, -0.034423828f, -0.022827148f, 0.037506104f, -0.002040863f, 0.003982544f, 0.062561035f, 0.0345459f, -0.0022239685f, 0.035339355f, 0.04937744f, 0.029769897f, -0.039733887f, 0.005794525f, -0.0135650635f, -0.021774292f, 0.026870728f, -0.025039673f, -0.009223938f, 0.024597168f, 0.0289917f, 0.011695862f, 0.045837402f, -0.047668457f, -0.081604004f, 0.041259766f, -0.040100098f, -0.046905518f, -0.014923096f, 0.040161133f, -0.048797607f, -0.0068626404f, -0.043395996f, 0.03579712f, 0.011192322f, -0.012359619f, -0.023483276f, -0.0418396f, 0.02748108f, 0.014137268f, -0.0025177002f, 0.02432251f, 0.039093018f, 0.018661499f, -0.02243042f, 0.006576538f, -0.012031555f, -0.045654297f, -0.04159546f, -0.013946533f, -0.007472992f, 0.008872986f, 0.006576538f, 0.024795532f, -0.0031147003f, -0.040771484f, 0.034729004f, 0.01473999f, 0.062805176f, -0.009857178f, -0.022994995f, -0.013000488f, 0.008071899f, -0.012161255f, -0.014442444f, -0.040496826f, -0.032104492f, -0.05404663f, -0.0118637085f, 0.038208008f, -0.023361206f, -0.038238525f, 0.0102005005f, 0.0054969788f, -0.009132385f, 0.03378296f, 0.024536133f, -0.017410278f, 0.0062828064f, 0.011413574f, 0.0027256012f, -0.004283905f, 6.723404E-4f, 0.045806885f, 0.020645142f, 0.01876831f, -0.04196167f, -0.030532837f, 0.02279663f, -0.018127441f, -0.026489258f, 0.051513672f, -0.0385437f, 0.03729248f, -0.03668213f, 0.0059890747f, 0.025344849f, -0.04046631f, 0.044952393f, 0.027374268f, -0.0046691895f, -0.010978699f, 0.0018043518f, -0.02444458f, -0.030532837f, 0.06842041f, -0.025024414f, 0.01335907f, 0.018218994f, -0.038482666f, 0.018463135f, 0.009262085f, 0.028015137f, -0.019104004f, 0.0057411194f, 0.02758789f, -0.017501831f, -0.042755127f, -0.05218506f, -0.014190674f, 0.04324341f, -0.023452759f, 0.022476196f, -0.047332764f, -0.008415222f, 0.0345459f, 0.0058021545f, 0.027038574f, -0.0077209473f, -0.0067596436f, -0.031097412f, 0.039978027f, -0.03866577f, 0.0066337585f, 0.005908966f, 0.046783447f, 0.010917664f, 7.9774857E-4f, 0.005039215f, 0.006134033f, -0.056488037f, -0.014984131f, -0.009284973f, 0.012283325f, -0.0059547424f, -0.011070251f, 0.062438965f, -0.0010519028f, -0.022201538f, -0.0027637482f, -0.029525757f, -0.028533936f, 0.015403748f, 0.046020508f, 0.009262085f, 0.013923645f, 0.017745972f, 0.061462402f, 0.048217773f, -0.068359375f, 0.023956299f, -0.010009766f, -0.017333984f, 0.029281616f, 0.01171875f, -0.006111145f, 0.035827637f, -0.05444336f, 0.028869629f, -0.028457642f, 0.012672424f, -0.007965088f, -0.029205322f, 0.016830444f, 0.037902832f, -0.02407837f, -0.017807007f, 0.031341553f, -0.0067367554f, 0.011680603f, -0.023910522f, -0.01171875f, 0.015571594f, -0.0047302246f, 0.0039405823f, -0.021438599f, 0.013870239f, -0.028961182f, 0.030334473f, -0.005268097f, -0.039855957f, -0.009223938f, 0.01574707f, 0.030975342f, 0.032104492f, -0.031402588f, 0.026657104f, 0.06604004f, -0.012687683f, -0.050048828f, -0.072021484f, 0.0020809174f, 0.05609131f, 0.002796173f, 0.0019798279f, -0.0012483597f, 0.037109375f, 0.035003662f, 0.03225708f, -0.015975952f, 0.005596161f, -0.025344849f, -0.0059776306f, 0.0037078857f, 0.021835327f, -0.006412506f, 0.018661499f, 0.011886597f, 0.009757996f, 0.019714355f, 0.010444641f, -0.06781006f, -0.056610107f, 0.028274536f, 0.018051147f, 0.0049819946f, -0.045532227f, -1.7368793E-4f, 0.04763794f, -0.028793335f, -0.044799805f, 0.0036792755f, -0.022247314f, 0.030532837f, -0.02861023f, 0.010040283f, 0.015792847f, 0.015136719f, 0.084106445f, -0.021026611f, -0.026657104f, 0.004096985f, 0.05618286f, -0.028915405f, -0.03314209f, -2.6583672E-4f, 0.032958984f, 0.041900635f, 0.032409668f, 0.028656006f, 7.092953E-5f, 0.009605408f, -0.0053138733f, 0.044677734f, -0.018859863f, 0.016601562f, 0.016143799f, -0.054718018f, -0.010719299f, -0.028457642f, 0.010551453f, -0.032165527f, -0.0036716461f, 0.008834839f, 0.0287323f, -0.010536194f, 0.022201538f, -0.0065994263f, 0.053619385f, -0.035705566f, -0.04034424f, -0.03933716f, -0.006664276f, -0.034362793f, 0.036865234f, 0.006111145f, 0.027038574f, 0.059326172f, -0.005012512f, 0.0034713745f, -0.055419922f, -0.040802002f, 0.017501831f, 0.011711121f, -0.06378174f, 0.0038585663f, -0.006793976f, 0.0067825317f, 0.0075416565f, -0.010269165f, 0.0069007874f, -0.1381836f, 0.015174866f, -0.030639648f, 0.054718018f, -0.042388916f, 0.008590698f, -0.04864502f, -0.017501831f, -0.014442444f, 0.009010315f, 0.0073394775f, -0.06744385f, -0.004360199f, -0.0034923553f, 0.005886078f, 0.025924683f, 0.010429382f, -0.0039711f, -0.0098724365f, -0.013214111f, -0.0013942719f, 0.031829834f, 7.9488754E-4f, -0.02407837f, 3.8933754E-4f, -6.122589E-4f, 0.0027122498f, 0.041534424f, 0.045288086f, -0.004348755f, -0.048980713f, 0.018920898f, -0.008811951f, 0.0021820068f, 0.018035889f, 0.009437561f, 0.05618286f, -0.052124023f, -0.020645142f, -0.034851074f, -0.02784729f, -0.0010900497f, -0.009483337f, -0.0013189316f, -0.012878418f, 0.025604248f, 0.035980225f, 0.0041503906f, -0.024261475f, -0.014831543f, -0.005622864f, 0.03668213f, -0.048217773f, 0.015609741f, -0.013885498f, 0.0118637085f, 0.04269409f, -0.06512451f, 0.014190674f, -0.013175964f, -0.016860962f, -0.0062675476f, 0.008270264f, 0.047821045f, 0.020629883f, -0.0065193176f, 6.213188E-4f, -0.04348755f, -0.04977417f, -0.0033779144f, -0.017578125f, -0.021469116f, -0.0044021606f, -0.014343262f, -0.033050537f, -0.017959595f, -0.038848877f, -0.0018701553f, -0.034698486f, 0.024658203f, -0.014701843f, -0.00907135f, 0.0042915344f, -0.08880615f, -0.025985718f, 0.046875f, 0.027511597f, 0.013664246f, -0.03213501f, 0.101135254f, 0.038330078f, 0.013519287f, -0.044036865f, -0.010826111f, -0.035614014f, 0.026306152f, -0.0040016174f, 0.0048179626f, 0.008850098f, -0.008255005f, -0.031921387f, -0.0121154785f, 0.015213013f, 0.013076782f, 0.013381958f, 0.009338379f, -0.0143585205f, 0.014305115f, 0.040222168f, -0.038879395f, -0.019363403f, 0.010734558f, 0.03213501f, -0.043182373f, -0.0069732666f, -0.018585205f, -0.028564453f, 0.036621094f, -0.041870117f, 0.01525116f, 0.015808105f, 0.0038928986f, 0.022994995f, 0.06689453f, -0.0079956055f, -0.0029888153f, 0.0440979f, 0.008834839f, -0.0063591003f, 0.023880005f, -0.020584106f, -0.0463562f, -0.0036201477f, 0.011680603f, -0.013076782f, -0.028121948f, 0.028274536f, 0.009483337f, 0.0053901672f, 0.019439697f, -0.054718018f, 0.020477295f, -0.0074424744f, -0.04269409f, 0.018798828f, -0.0035705566f, 0.016174316f, -0.0030021667f, 0.007987976f, -0.025161743f, -0.027038574f, 0.017852783f, -0.018356323f, 0.016098022f, -0.010009766f, 0.01838684f, 0.07531738f, -0.006801605f, -0.004257202f, 0.015571594f, 0.037750244f, 0.04751587f, -0.03967285f, 0.03491211f, 0.009918213f, 0.035217285f, 0.007068634f, 0.0059165955f, -0.007850647f, 0.047088623f, -0.010536194f, -0.054901123f, -0.015930176f, 0.0071525574f, -0.05758667f, 0.017532349f, -0.034820557f, -0.024414062f, 0.013252258f, -0.03677368f, -0.0028648376f, -0.017868042f, -0.007987976f, -0.030792236f, 0.048034668f, 0.0067214966f, 0.045684814f, -0.0051879883f, -0.028152466f, 0.04815674f, 0.013923645f, 0.044799805f, 6.23703E-4f, -0.089538574f, -0.004173279f, -0.02947998f, -0.015319824f, 0.08105469f, -0.0028629303f, 0.011909485f, -0.011245728f, -0.022705078f, 0.060333252f, -0.012939453f, -0.0040245056f, 0.0118637085f, 0.055541992f, -0.01689148f, 0.030792236f, -0.0076828003f, 0.056549072f, 0.20996094f, 0.014930725f, 0.004497528f, -0.07965088f, 5.559921E-4f, -0.032165527f, 0.023544312f, -0.024627686f, -0.008140564f, -0.006313324f, 0.0074157715f, 0.051940918f, 0.029327393f, 0.016525269f, 0.026321411f, 0.087646484f, -0.016021729f, -0.022949219f, 0.01802063f, 0.010513306f, 0.016571045f, 0.058746338f, -0.0074272156f, -0.028961182f, 0.0063056946f, -0.018463135f, 0.01676941f, -0.019119263f, 0.009338379f, 0.027938843f, -0.07611084f, 0.014305115f, -0.05328369f, 0.034606934f, 0.009315491f, -0.00680542f, 0.009262085f, 0.021347046f, -0.04296875f, 0.026428223f, 0.012413025f, -0.011665344f, -0.0129470825f, -0.0059814453f, -0.021347046f, -0.023788452f, -0.04449463f, -0.005569458f, -0.027328491f, 0.04736328f, -0.051940918f, 0.036712646f, 0.034210205f, -0.025024414f, 0.031982422f, -0.0031108856f, 0.0046463013f, -0.04562378f, -0.036315918f, -0.10223389f, 0.019515991f, 0.012481689f, 0.03866577f, -0.01776123f, -0.01512146f, -2.515316E-5f, 0.0017700195f, 0.039245605f, 0.036956787f, -0.006088257f, 0.05218506f, 0.018829346f, -0.019256592f, -0.03050232f, 0.022659302f, 0.010932922f, 0.034362793f, 0.028274536f, -0.038238525f, -0.011421204f, -0.047729492f, 0.008384705f, 0.026397705f, -0.0012645721f, -0.040863037f, 0.016220093f, 0.036102295f, -0.018249512f, 0.023010254f, 0.012809753f, -0.02571106f, 0.018707275f, -0.019989014f, 0.0118637085f, -0.0109939575f, 0.04095459f, -0.00844574f, 0.013458252f, -0.022277832f, 0.041748047f, 0.051208496f, 0.008987427f, -0.027038574f, 0.056243896f, 0.048339844f, -0.016693115f, 0.034210205f, -0.040771484f, -0.037841797f, 0.008392334f, -0.0034732819f, -0.015670776f, -0.01121521f, -0.077697754f, -0.031173706f, -0.04095459f, -0.029449463f, -0.026428223f, 0.021102905f, -0.02520752f, 0.027450562f, -0.04095459f, -0.0024967194f, -0.057373047f, -0.0038585663f, 0.0039711f, 0.014526367f, -0.053527832f, -0.0030097961f, -0.04763794f, 0.012466431f, 0.016555786f, 0.02281189f, -0.0031032562f, 0.04107666f, 0.0146102905f, -9.717941E-4f, 0.030166626f, -0.04977417f, -0.019073486f, -0.010795593f, -0.030151367f, -0.02204895f, -0.011169434f, -0.05038452f, 0.027923584f, 0.05569458f, -0.0047073364f, 0.027252197f, -0.00111866f, -0.091308594f, -0.0073509216f, -0.02583313f, -0.021377563f, -0.019104004f, 0.0032577515f, -0.025970459f, -0.0146102905f, -0.038482666f, 0.005405426f, 0.05001831f, 0.0071144104f, 0.025604248f, -0.005302429f, 0.0087509155f, 0.003326416f, 0.03086853f, 0.022369385f, 0.023330688f, -0.01322937f, 0.009284973f, 0.06903076f, 0.023910522f, 0.02709961f, -0.05444336f, -0.002943039f, 0.02180481f, -0.029663086f, 0.04763794f, -0.025436401f, -0.017532349f, 0.062805176f, 0.00497818f, 0.030883789f, -0.046020508f, -0.025131226f, 0.0063819885f, 0.011726379f, 0.013641357f, -0.0069732666f, 0.0071792603f, -0.0129852295f, 0.024353027f, 0.014724731f, 0.02015686f, 0.020874023f, -0.009552002f, -0.002729416f, 0.0057258606f, -0.033569336f, 0.05999756f, -0.03414917f, 0.003276825f, 0.029846191f, 0.049591064f, -0.019241333f, 0.045135498f, 0.04547119f, -0.012565613f, 0.03201294f, 0.019012451f, -0.03668213f, -0.031341553f, 0.012420654f, 0.043762207f, 0.03050232f, 0.007095337f, -0.008636475f, 0.009414673f, -0.015991211f, -0.062438965f, 0.0413208f, -0.009811401f, 0.008613586f, 4.4178963E-4f, 0.024734497f, -0.004047394f, 0.0055770874f, -0.02758789f, -0.0012454987f, 0.0413208f, -0.008331299f, -0.05014038f, -6.637573E-4f, -0.0047912598f, 0.006980896f, 0.010871887f, 0.0036201477f, 0.026489258f, -0.02973938f, -0.0345459f, -0.02507019f, 0.0063819885f, -0.0017623901f, -0.025970459f, -0.037902832f, 0.0357666f, 0.08380127f, -0.026000977f, 0.014152527f, 0.0032482147f, -0.013473511f, 0.0049819946f, -0.014213562f, 0.022201538f, -0.02784729f, 0.019592285f, 0.0019550323f, -0.011741638f, -0.020858765f, -0.022918701f, 0.02368164f, -0.023773193f, -0.008674622f, -0.011001587f, 0.035736084f, -0.010154724f, -0.035064697f, -0.03753662f, -0.053985596f, -0.01436615f, -8.7070465E-4f, 4.887581E-4f, -0.051605225f, 0.019607544f, 0.036132812f, -0.032928467f, -0.018920898f, -0.024658203f, 0.022827148f, 0.06378174f, 0.064575195f, -0.028656006f, 0.037475586f, 0.046722412f, 0.024871826f, 0.0047073364f, -0.009902954f, -0.03086853f, 2.8252602E-4f, -0.012207031f, 0.00831604f, -0.04421997f, 0.02406311f, -0.005908966f, -0.011940002f, 0.015563965f, 0.010757446f, -0.013648987f, 0.019821167f, 0.041656494f, 0.048461914f, 0.035705566f, -0.004135132f, 0.020980835f, -0.0033550262f, 0.01096344f, 0.016067505f, 0.0178833f, 0.0012598038f, 0.0046691895f, 0.046875f, -0.02381897f, 0.004501343f, 0.026657104f, -0.004798889f, 0.05871582f, 0.0025024414f, 0.017562866f, -0.0017290115f, -0.0077209473f, 0.04348755f, -0.011116028f, 0.007499695f, 0.039031982f, 0.014457703f, -0.010559082f, 0.009048462f, 0.03765869f, -0.012161255f, 0.00283432f, -0.012458801f, 0.05368042f, 0.011909485f, -0.023773193f, -0.0033168793f, -0.009674072f, 0.023117065f, -0.0058174133f, -0.0065994263f, -0.019195557f, 0.021026611f, -0.01574707f, 7.543564E-4f, -0.005077362f, -0.035217285f, -0.028320312f, 0.029403687f, 0.0029773712f, -0.015098572f, 0.034698486f, -0.0030441284f, 0.031051636f, -0.04232788f, 0.011833191f, 4.8828125E-4f, -0.014801025f, -8.404255E-5f, -0.027862549f }) + .IncludeScores(true) + .IncludeSortVector(true) + .SetRerankOn("Name") + .Project(Builders.Projection.Include(x => x.Name)) + .SetRerankQuery(rerankQuery); + var resultList = results.ToList(); + Assert.NotNull(resultList); + Assert.NotEmpty(resultList); + Assert.Contains(resultList, r => r.Name == "Cat"); + } + +} + diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs index 673a440..ec15d9f 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs @@ -1,10 +1,8 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using MongoDB.Bson; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; using UUIDNext; using Xunit; @@ -360,14 +358,12 @@ public void NotFluent_RunsAsync_ReturnsExpectedResults() var sort = Builders.Sort.Descending(o => o.Properties.PropertyTwo); var inclusiveProjection = Builders.Projection .Include("Properties.PropertyTwo"); - var findOptions = new DocumentFindManyOptions() - { - Sort = sort, - Limit = 1, - Skip = 2, - Projection = inclusiveProjection - }; - var results = collection.Find(filter, findOptions).ToList(); + var results = collection.Find(filter) + .Sort(sort) + .Skip(2) + .Limit(1) + .Project(inclusiveProjection) + .ToList(); var expectedArray = new[] { "alligator" }; var actualArray = results.Select(o => o.Properties.PropertyTwo).ToArray(); Assert.True(!expectedArray.Except(actualArray).Any() && !actualArray.Except(expectedArray).Any()); @@ -667,9 +663,8 @@ public async Task QueryDocumentsWithVectorsAsync() var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); var insertResult = await collection.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIds.Count); - var result = collection.Find(new DocumentFindManyOptions() { Sort = Builders.Sort.Vector(dogQueryVector) }, null); + var result = collection.Find().Sort(Builders.Sort.Vector(dogQueryVector)); Assert.Equal("This is about a dog.", result.First().Name); - } finally { @@ -701,6 +696,14 @@ public async Task QueryDocumentsWithVectorizeAsync() }, }; var dogQueryVectorString = "dog"; + for (var i = 3; i < 100; i++) + { + items.Add(new SimpleObjectWithVectorize + { + Id = i, + Name = $"This is about a random object {i}." + }); + } var options = new CollectionDefinition { @@ -717,13 +720,17 @@ public async Task QueryDocumentsWithVectorizeAsync() var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); var insertResult = await collection.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIds.Count); - var finder = collection.Find(new DocumentFindManyOptions() { Sort = Builders.Sort.Vectorize(dogQueryVectorString), IncludeSimilarity = true, IncludeSortVector = true }, null); - var cursor = finder.ToCursor(); - var list = cursor.ToList(); + var finder = collection.Find() + .Sort(Builders.Sort.Vectorize(dogQueryVectorString)) + .IncludeSimilarity(true) + .IncludeSortVector(true); + var list = finder.ToList(); var result = list.First(); Assert.Equal("This is about a dog.", result.Name); Assert.NotNull(result.Similarity); - Assert.NotNull(cursor.SortVectors); + var sortVector = finder.GetSortVector(); + Assert.NotNull(sortVector); + Assert.NotEmpty(sortVector); } finally { @@ -773,12 +780,12 @@ public async Task QueryDocumentsWithVectorize_Fluent_Async() Assert.Equal(items.Count, insertResult.InsertedIds.Count); var finder = collection.Find().Sort( Builders.Sort.Vectorize(dogQueryVectorString)).IncludeSimilarity(true).IncludeSortVector(true); - var cursor = finder.ToCursor(); - var list = cursor.ToList(); - var result = list.First(); + var result = finder.First(); Assert.Equal("This is about a dog.", result.Name); Assert.NotNull(result.Similarity); - Assert.NotNull(cursor.SortVectors); + var sortVector = finder.GetSortVector(); + Assert.NotNull(sortVector); + Assert.NotEmpty(sortVector); } finally { diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs index 8e35c4b..f33c88a 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs @@ -1,12 +1,11 @@ -using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; using DataStax.AstraDB.DataApi.Core.Query; using DataStax.AstraDB.DataApi.Core.Results; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.SerDes; using DataStax.AstraDB.DataApi.Tables; -using System.Runtime.CompilerServices; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -156,6 +155,7 @@ public void CompoundKeySerializationTest() } //{"data":{"documents":[{"_id":"3a0cdac3-679b-435a-8cda-c3679bf35a6b","title":"Test Book 1","author":"Test Author 1","number_of_pages":100} + [Fact] public void BookDeserializationTest() { @@ -171,4 +171,24 @@ public void BookDeserializationTest() Assert.Equal(100, deserialized.NumberOfPages); } + [Fact] + public void HybridSearchResponseDeserializationTest() + { + var serializationTestString = "{\"data\":{\"documents\":[{\"_id\":\"f9bf6e20-6efd-421d-bf6e-206efdb21d49\",\"Name\":\"Cat\"},{\"_id\":\"a02bb811-98e6-416f-abb8-1198e6816fa5\",\"Name\":\"Cat\"},{\"_id\":\"03479720-4ba5-4928-8797-204ba5392893\",\"Name\":\"NotCat\"},{\"_id\":\"54726b36-8202-4f72-b26b-368202ef7209\",\"Name\":\"NotCat\"},{\"_id\":\"6d8ea948-7ac1-49f6-8ea9-487ac149f6d7\",\"Name\":\"Cow\"},{\"_id\":\"aeee0998-9941-4015-ae09-989941e0158d\",\"Name\":\"Cow\"},{\"_id\":\"817fbc98-13d0-4aca-bfbc-9813d0dacae9\",\"Name\":\"Horse\"},{\"_id\":\"8be82ac3-ff2c-4f2b-a82a-c3ff2cbf2bd7\",\"Name\":\"Horse\"}],\"nextPageState\":null},\"status\":{\"documentResponses\":[{\"scores\":{\"$rerank\":1.7070312,\"$vector\":0.75348675,\"$vectorRank\":1,\"$bm25Rank\":1,\"$rrf\":0.032786883}},{\"scores\":{\"$rerank\":1.7070312,\"$vector\":0.75348675,\"$vectorRank\":2,\"$bm25Rank\":2,\"$rrf\":0.032258064}},{\"scores\":{\"$rerank\":1.2802734,\"$vector\":0.71900904,\"$vectorRank\":5,\"$bm25Rank\":3,\"$rrf\":0.031257633}},{\"scores\":{\"$rerank\":1.2802734,\"$vector\":0.71900904,\"$vectorRank\":6,\"$bm25Rank\":4,\"$rrf\":0.030776516}},{\"scores\":{\"$rerank\":-3.4140625,\"$vector\":0.741625,\"$vectorRank\":3,\"$bm25Rank\":5,\"$rrf\":0.031257633}},{\"scores\":{\"$rerank\":-3.4140625,\"$vector\":0.741625,\"$vectorRank\":4,\"$bm25Rank\":6,\"$rrf\":0.030776516}},{\"scores\":{\"$rerank\":-9.1015625,\"$vector\":0.69422597,\"$vectorRank\":7,\"$bm25Rank\":null,\"$rrf\":0.014925373}},{\"scores\":{\"$rerank\":-9.1015625,\"$vector\":0.69422597,\"$vectorRank\":8,\"$bm25Rank\":null,\"$rrf\":0.014705882}}]}}"; + var commandOptions = new List + { + new CommandOptions() + { + OutputConverter = new DocumentConverter() + } + }; + var command = new Command("deserializationTest", new DataApiClient(), commandOptions.ToArray(), null); + var deserialized = command.Deserialize, FindStatusResult>>>(serializationTestString); + Assert.NotNull(deserialized); + Assert.NotNull(deserialized.Data); + Assert.NotNull(deserialized.Status); + Assert.NotNull(deserialized.Status.DocumentResponses); + Assert.NotEmpty(deserialized.Status.DocumentResponses.First().Scores); + } + } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs index e4d942f..01947ee 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs @@ -1,4 +1,5 @@ using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Tables; using Xunit; @@ -18,166 +19,230 @@ public TableAlterTests(TableAlterFixture fixture) [Fact] public async Task AlterTableAddColumns() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - var newColumns = new Dictionary + const string tableName = "addColumnsTest"; + try { - ["is_archived"] = new AlterTableColumnDefinition { Type = "boolean" }, - ["review_notes"] = new AlterTableColumnDefinition { Type = "text" } - }; + var table = await fixture.CreateTestTable(tableName); + + var newColumns = new Dictionary + { + ["is_archived"] = new AlterTableColumnDefinition { Type = "boolean" }, + ["review_notes"] = new AlterTableColumnDefinition { Type = "text" } + }; - await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); - //throws error on dupe - var ex = await Assert.ThrowsAsync(() => - table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false)); + //throws error on dupe + var ex = await Assert.ThrowsAsync(() => + table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false)); - Assert.Contains("unique", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("unique", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableAddColumnsMapSet() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - var newColumns = new Dictionary + const string tableName = "addColumnsMapSet"; + try { - ["column_test_map"] = new AlterTableColumnDefinition - { - Type = "map", - KeyType = "text", - ValueType = "text" - }, - ["column_test_set"] = new AlterTableColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + var newColumns = new Dictionary { - Type = "text", - ValueType = "text" - } + ["column_test_map"] = new AlterTableColumnDefinition + { + Type = "map", + KeyType = "text", + ValueType = "text" + }, + ["column_test_set"] = new AlterTableColumnDefinition + { + Type = "text", + ValueType = "text" + } - }; + }; - await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } // Requires a pre-configured embedding provider on the Astra backend. [Fact] public async Task AlterTableAddVectorColumnsWithEmbedding() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + const string tableName = "alterTableAddVectorColumnsWithEmbedding"; + try { - ["plot_synopsis"] = new AlterTableVectorColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary { - //VectorDimension = 1536, - VectorDimension = null, - Service = new VectorServiceOptions + ["plot_synopsis"] = new AlterTableVectorColumnDefinition { - Provider = "nvidia", - ModelName = "NV-Embed-QA" + //VectorDimension = 1536, + VectorDimension = null, + Service = new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } } - } - }), null, false); + }), null, false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableAddVectorColumnsNoConfig() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + const string tableName = "alterTableAddVectorColumnsNoConfig"; + try { - ["plot_synopsis_no_config"] = new AlterTableVectorColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary { - VectorDimension = 2 - } - }), null, false); + ["plot_synopsis_no_config"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableDropColumn() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - var newColumns = new Dictionary + const string tableName = "alterTableDropColumn"; + try { - ["is_archived_drop"] = new AlterTableColumnDefinition { Type = "boolean" } - }; + var table = await fixture.CreateTestTable(tableName); - await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + var newColumns = new Dictionary + { + ["is_archived_drop"] = new AlterTableColumnDefinition { Type = "boolean" } + }; + + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); - await table.AlterAsync(new AlterTableDropColumns(new[] { "is_archived_drop" }), null, runSynchronously: false); + await table.AlterAsync(new AlterTableDropColumns(new[] { "is_archived_drop" }), null, runSynchronously: false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableDropVectorColumns() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + const string tableName = "alterTableDropVectorColumns"; + try { - ["plot_synopsis_drop"] = new AlterTableVectorColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary { - VectorDimension = 2 - } - }), null, false); + ["plot_synopsis_drop"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); - var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis_drop" }); - await table.AlterAsync(dropColumn, null, runSynchronously: false); + var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis_drop" }); + await table.AlterAsync(dropColumn, null, runSynchronously: false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableAddVectorize() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + const string tableName = "alterTableAddVectorize"; + try { - ["plot_synopsis_vectorize"] = new AlterTableVectorColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary { - VectorDimension = 1024 - } - }), null, false); + ["plot_synopsis_vectorize"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 1024 + } + }), null, false); - await table.AlterAsync(new AlterTableAddVectorize(new Dictionary - { - ["plot_synopsis_vectorize"] = new VectorServiceOptions + await table.AlterAsync(new AlterTableAddVectorize(new Dictionary { - Provider = "nvidia", - ModelName = "NV-Embed-QA" - } + ["plot_synopsis_vectorize"] = new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } - }), null, false); + }), null, false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } [Fact] public async Task AlterTableOperationDropVectorize() { - var table = fixture.Database.GetTable("tableAlterTest", null); - - await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + const string tableName = "alterTableOperationDropVectorize"; + try { - ["plot_synopsis_vectorize_drop"] = new AlterTableVectorColumnDefinition + var table = await fixture.CreateTestTable(tableName); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary { - VectorDimension = 1024 - } - }), null, false); + ["plot_synopsis_vectorize_drop"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 1024 + } + }), null, false); - await table.AlterAsync(new AlterTableAddVectorize(new Dictionary - { - ["plot_synopsis_vectorize_drop"] = new VectorServiceOptions + await table.AlterAsync(new AlterTableAddVectorize(new Dictionary { - Provider = "nvidia", - ModelName = "NV-Embed-QA" - } + ["plot_synopsis_vectorize_drop"] = new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + } - }), null, false); + }), null, false); - var dropVectorize = new AlterTableDropVectorize(new[] { "plot_synopsis_vectorize_drop" }); - await table.AlterAsync(dropVectorize, null, runSynchronously: false); + var dropVectorize = new AlterTableDropVectorize(new[] { "plot_synopsis_vectorize_drop" }); + await table.AlterAsync(dropVectorize, null, runSynchronously: false); - var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis_vectorize_drop" }); - await table.AlterAsync(dropColumn, null, runSynchronously: false); + var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis_vectorize_drop" }); + await table.AlterAsync(dropColumn, null, runSynchronously: false); + } + finally + { + await fixture.Database.DropTableAsync(tableName); + } } } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs index 7fb92be..1f56f79 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs @@ -1,6 +1,7 @@ +using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Tables; -using System.Data; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs index d247439..589a728 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs @@ -1,11 +1,8 @@ - - using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Tables; using Microsoft.VisualBasic; -using System.Linq.Expressions; -using System.Threading.Tasks; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -170,9 +167,9 @@ public void FindMany_RetrieveAll() public void FindMany_Vectorize() { var table = fixture.SearchTable; - var sort = Builders.TableSort; - var filter = sort.Vectorize(b => b.Author, "Walter Dray"); - var results = table.Find(null, new TableFindManyOptions() { Sort = filter }).ToList(); + var sorter = Builders.TableSort; + var sort = sorter.Vectorize(b => b.Author, "Walter Dray"); + var results = table.Find().Sort(sort).ToList(); Assert.Equal("Desert Peace", results.First().Title); } @@ -184,9 +181,9 @@ public async Task FindOne_Vectorize() var filter = sort.Vectorize(b => b.Author, "Walter Dray"); var result = await table.FindOneAsync(null, new TableFindOptions() { Sort = filter, IncludeSimilarity = true }); - //TODO: not currently finding expected result - //Assert.Equal("Desert Peace", result.Title); - Assert.NotEqual(0, result.Similarity); + Assert.Equal("Desert Peace", result.Title); + //TODO: similarity not being returned currently via API + //Assert.NotEqual(0, result.Similarity); } [Fact] @@ -205,14 +202,14 @@ public async Task FindOne_Sort() public void FindOne_Sort_Skip_Exclude() { var table = fixture.SearchTable; - var sort = Builders.TableSort; - var filter = sort.Ascending(b => b.Title); - var projection = Builders.Projection.Exclude(b => b.Author); - var results = table.Find(new TableFindManyOptions() { Sort = filter, Projection = projection }).Skip(2).Limit(5); + var sorter = Builders.TableSort; + var sort = sorter.Ascending(b => b.Title); + var projection = Builders.Projection.Exclude(b => b.DueDate); + var results = table.Find().Sort(sort).Project(projection).Skip(2).Limit(5); Assert.Equal(5, results.Count()); //TODO: not working on API side yet? //Assert.Equal("Title 2", results.First().Title); - Assert.Null(results.First().Author); + Assert.Null(results.First().DueDate); Assert.NotEqual(default(int), results.First().NumberOfPages); } @@ -343,7 +340,7 @@ await table.CreateIndexAsync(new TableIndex() var filter = Builders.Filter .Eq(so => so.Title, "Title 1"); var findResult = table.Find(filter).ToList(); - Assert.Equal(1, findResult.Count); + Assert.Single(findResult); var result = await table.DeleteManyAsync(filter); var deletedResult = table.Find(filter).ToList(); Assert.Empty(deletedResult); diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs index 086c9b6..5bc05cc 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs @@ -1,4 +1,5 @@ using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Core; using Xunit; diff --git a/test/DataStax.AstraDB.DataApi.UnitTests/DataStax.AstraDB.DataAPI.UnitTests.sln b/test/DataStax.AstraDB.DataApi.UnitTests/DataStax.AstraDB.DataAPI.UnitTests.sln new file mode 100644 index 0000000..ef19faa --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.UnitTests/DataStax.AstraDB.DataAPI.UnitTests.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStax.AstraDB.DataApi.UnitTests", "DataStax.AstraDB.DataApi.UnitTests.csproj", "{9C72D42B-E75F-50CC-063C-AA57C623CDB5}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9C72D42B-E75F-50CC-063C-AA57C623CDB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C72D42B-E75F-50CC-063C-AA57C623CDB5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C72D42B-E75F-50CC-063C-AA57C623CDB5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C72D42B-E75F-50CC-063C-AA57C623CDB5}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {74079B0A-F82B-4150-8C01-CE4659A683BB} + EndGlobalSection +EndGlobal diff --git a/test/DataStax.AstraDB.DataApi.UnitTests/FindAndRerankOptionsTests.cs b/test/DataStax.AstraDB.DataApi.UnitTests/FindAndRerankOptionsTests.cs new file mode 100644 index 0000000..e2b8351 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.UnitTests/FindAndRerankOptionsTests.cs @@ -0,0 +1,194 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Query; +using Xunit; +using System.Collections.Generic; +using System.Linq; + +namespace DataStax.AstraDB.DataApi.UnitTests.Core.Query; + +public class TestDocument { } + +public class FindAndRerankOptionsTests +{ + [Fact] + public void Clone_WithAllPropertiesSet_ShouldCreateExactCopy() + { + // Arrange + var original = new FindAndRerankOptions + { + RerankOn = "score_field", + IncludeScores = true, + IncludeSortVector = true, + RerankQuery = "test query", + Limit = 10, + HybridLimits = new Dictionary { { "vector", 5 }, { "text", 15 } }, + Filter = Filter.Eq("field1", "value1") & Filter.Neq("field2", 42), + Projection = new ProjectionBuilder().Include("field1").Exclude("field2"), + Sorts = new List + { + Sort.Ascending("field1"), + Sort.Descending("field2"), + Sort.Vector(new float[] { 1.0f, 2.0f, 3.0f }) + } + }; + + // Act + var clone = original.Clone(); + + // Assert - Verify all properties are equal + Assert.Equal(original.RerankOn, clone.RerankOn); + Assert.Equal(original.IncludeScores, clone.IncludeScores); + Assert.Equal(original.IncludeSortVector, clone.IncludeSortVector); + Assert.Equal(original.RerankQuery, clone.RerankQuery); + Assert.Equal(original.Limit, clone.Limit); + + // Verify HybridLimits is a deep copy + Assert.NotNull(clone.HybridLimits); + Assert.NotSame(original.HybridLimits, clone.HybridLimits); + Assert.Equal(original.HybridLimits.Count, clone.HybridLimits.Count); + foreach (var kvp in original.HybridLimits) + { + Assert.True(clone.HybridLimits.ContainsKey(kvp.Key)); + Assert.Equal(kvp.Value, clone.HybridLimits[kvp.Key]); + } + + // Verify Filter is a deep copy + Assert.NotNull(clone.Filter); + Assert.NotSame(original.Filter, clone.Filter); + + // Verify Projection is a deep copy + Assert.NotNull(clone.Projection); + Assert.NotSame(original.Projection, clone.Projection); + Assert.Equal(original.Projection.Projections.Count, clone.Projection.Projections.Count); + + // Verify Sorts is a deep copy + Assert.NotNull(clone.Sorts); + Assert.NotSame(original.Sorts, clone.Sorts); + Assert.Equal(original.Sorts.Count, clone.Sorts.Count); + for (int i = 0; i < original.Sorts.Count; i++) + { + Assert.Equal(original.Sorts[i].Name, clone.Sorts[i].Name); + if (original.Sorts[i].Value is float[] originalVector) + { + var cloneVector = (float[])clone.Sorts[i].Value; + Assert.Equal(originalVector, cloneVector); + Assert.NotSame(originalVector, cloneVector); + } + else + { + Assert.Equal(original.Sorts[i].Value, clone.Sorts[i].Value); + } + } + } + + [Fact] + public void Clone_WithNullProperties_ShouldHandleNullsCorrectly() + { + // Arrange + var original = new FindAndRerankOptions + { + // Only set required properties, leave others null + RerankOn = null, + IncludeScores = null, + IncludeSortVector = null, + RerankQuery = null, + Limit = null, + HybridLimits = null, + Filter = null, + Projection = null, + Sorts = null + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.Null(clone.RerankOn); + Assert.Null(clone.IncludeScores); + Assert.Null(clone.IncludeSortVector); + Assert.Null(clone.RerankQuery); + Assert.Null(clone.Limit); + Assert.Null(clone.HybridLimits); + Assert.Null(clone.Filter); + Assert.Null(clone.Projection); + Assert.NotNull(clone.Sorts); // Should be an empty list, not null + Assert.Empty(clone.Sorts); + } + + [Fact] + public void Clone_WithEmptyCollections_ShouldCreateEmptyCollections() + { + // Arrange + var original = new FindAndRerankOptions + { + HybridLimits = new Dictionary(), + Sorts = new List() + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotNull(clone.HybridLimits); + Assert.Empty(clone.HybridLimits); + Assert.NotNull(clone.Sorts); + Assert.Empty(clone.Sorts); + } + + [Fact] + public void Clone_WithComplexFilter_ShouldDeepCloneFilter() + { + // Arrange + var filter = Filter.Eq("field1", "value1") | + (Filter.Gt("field2", 10) & Filter.Lt("field2", 20)); + + var original = new FindAndRerankOptions + { + Filter = filter + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotNull(clone.Filter); + Assert.NotSame(original.Filter, clone.Filter); + // Note: More detailed filter structure validation would require exposing more of the filter internals + } + + [Fact] + public void Clone_WithVectorSort_ShouldDeepCloneVector() + { + // Arrange + var vector = new float[] { 1.0f, 2.0f, 3.0f }; + var original = new FindAndRerankOptions + { + Sorts = new List { Sort.Vector(vector) } + }; + + // Act + var clone = original.Clone(); + + // Assert + var originalVector = (float[])original.Sorts[0].Value; + var cloneVector = (float[])clone.Sorts[0].Value; + + Assert.Equal(originalVector, cloneVector); + Assert.NotSame(originalVector, cloneVector); + } +}