diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1e3d61..262f48d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,14 +35,10 @@ jobs: - name: Pack run: dotnet pack ./src/DataStax.AstraDB.DataApi/ --configuration Release --no-build --output nupkgs -p:Version=${{ env.PACKAGE_VERSION }} - # - name: Push to NuGet - # run: dotnet nuget push nupkgs/*.nupkg --api-key $NUGET_API_KEY --source https://api.nuget.org/v3/index.json - # env: - # NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} - - - name: Dry-run NuGet push - run: | - echo "Would run: dotnet nuget push nupkgs/*.nupkg --api-key [REDACTED] --source https://api.nuget.org/v3/index.json" + - name: Push to NuGet + run: dotnet nuget push nupkgs/*.nupkg --api-key $NUGET_API_KEY --source https://api.nuget.org/v3/index.json + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} - name: Upload NuGet package as artifact uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index f3f09f7..1339186 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ appsettings.json _site api api-docs +test/DataStax.AstraDB.DataAPI.IntegrationTests/TestResults +test/DataStax.AstraDB.DataAPI.IntegrationTests/coveragereport diff --git a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs index 5ee52fe..6b047fa 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/AstraDatabasesAdmin.cs @@ -20,7 +20,6 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; -using System.Threading; using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Admin; @@ -58,7 +57,7 @@ internal AstraDatabasesAdmin(DataApiClient client, CommandOptions adminOptions) /// public List ListDatabaseNames() { - return ListDatabases().Select(db => db.Info.Name).ToList(); + return ListDatabases().Select(db => db.Name).ToList(); } /// @@ -87,7 +86,7 @@ public Task> ListDatabaseNamesAsync() /// public List ListDatabaseNames(CommandOptions options) { - return ListDatabases(options).Select(db => db.Info.Name).ToList(); + return ListDatabases(options).Select(db => db.Name).ToList(); } /// @@ -103,7 +102,7 @@ public List ListDatabaseNames(CommandOptions options) public async Task> ListDatabaseNamesAsync(CommandOptions options) { var databases = await ListDatabasesAsync(options).ConfigureAwait(false); - return databases.Select(db => db.Info.Name).ToList(); + return databases.Select(db => db.Name).ToList(); } /// @@ -164,14 +163,15 @@ public Task> ListDatabasesAsync(CommandOptions options) return ListDatabasesAsync(options, false); } - internal Task> ListDatabasesAsync(CommandOptions options, bool runSynchronously) + internal async Task> ListDatabasesAsync(CommandOptions options, bool runSynchronously) { var command = CreateCommand() .AddUrlPath("databases") + .WithTimeoutManager(new DatabaseAdminTimeoutManager()) .AddCommandOptions(options); - var response = command.RunAsyncRaw>(HttpMethod.Get, runSynchronously); - return response; + var rawResults = await command.RunAsyncRaw>(HttpMethod.Get, runSynchronously).ConfigureAwait(false); + return rawResults?.Select(db => new DatabaseInfo(db)).ToList() ?? new List(); } /// @@ -351,11 +351,11 @@ internal async Task CreateDatabaseAsync(DatabaseCreationOptions List dbList = await ListDatabasesAsync(commandOptions, runSynchronously).ConfigureAwait(false); - DatabaseInfo existingDb = dbList.FirstOrDefault(item => databaseName.Equals(item.Info.Name)); + DatabaseInfo existingDb = dbList.FirstOrDefault(item => databaseName.Equals(item.Name)); if (existingDb != null) { - if (existingDb.Status == "ACTIVE") + if (existingDb.Status == AstraDatabaseStatus.ACTIVE) { Console.WriteLine($"Database {databaseName} already exists and is ACTIVE."); return GetDatabaseAdmin(existingDb); @@ -367,6 +367,7 @@ internal async Task CreateDatabaseAsync(DatabaseCreationOptions Command command = CreateCommand() .AddUrlPath("databases") .WithPayload(creationOptions) + .WithTimeoutManager(new DatabaseAdminTimeoutManager()) .AddCommandOptions(commandOptions); Guid newDbId = Guid.Empty; @@ -396,7 +397,7 @@ internal async Task CreateDatabaseAsync(DatabaseCreationOptions } } - return GetDatabaseAdmin(newDbId); + return await GetDatabaseAdminAsync(newDbId); } private void WaitForDatabase(string databaseName) @@ -414,8 +415,8 @@ internal async Task WaitForDatabaseAsync(string databaseName) while (secondsWaited < MAX_WAIT_IN_SECONDS) { - string status = await GetDatabaseStatusAsync(databaseName).ConfigureAwait(false); - if (status == "ACTIVE") + var status = await GetDatabaseStatusAsync(databaseName).ConfigureAwait(false); + if (status == AstraDatabaseStatus.ACTIVE) { return; } @@ -426,11 +427,11 @@ internal async Task WaitForDatabaseAsync(string databaseName) throw new Exception($"Database {databaseName} did not become ready within {MAX_WAIT_IN_SECONDS} seconds."); } - internal async Task GetDatabaseStatusAsync(string databaseName) + internal async Task GetDatabaseStatusAsync(string databaseName) { Guard.NotNullOrEmpty(databaseName, nameof(databaseName)); var dbList = await ListDatabasesAsync(); - var db = dbList.FirstOrDefault(item => databaseName.Equals(item.Info.Name)); + var db = dbList.FirstOrDefault(item => databaseName.Equals(item.Name)); if (db == null) { @@ -569,7 +570,7 @@ internal async Task DropDatabaseAsync(string databaseName, CommandOptions Guard.NotNullOrEmpty(databaseName, nameof(databaseName)); var dbList = await ListDatabasesAsync(options, runSynchronously).ConfigureAwait(false); - var dbInfo = dbList.FirstOrDefault(item => item.Info.Name.Equals(databaseName)); + var dbInfo = dbList.FirstOrDefault(item => item.Name.Equals(databaseName)); if (dbInfo == null) { return false; @@ -593,6 +594,7 @@ internal async Task DropDatabaseAsync(Guid dbGuid, CommandOptions options, .AddUrlPath("databases") .AddUrlPath(dbGuid.ToString()) .AddUrlPath("terminate") + .WithTimeoutManager(new DatabaseAdminTimeoutManager()) .AddCommandOptions(options); await command.RunAsyncRaw(runSynchronously).ConfigureAwait(false); @@ -602,21 +604,6 @@ internal async Task DropDatabaseAsync(Guid dbGuid, CommandOptions options, return false; } - private DatabaseAdminAstra GetDatabaseAdmin(DatabaseInfo dbInfo) - { - var apiEndpoint = $"https://{dbInfo.Id}-{dbInfo.Info.Region}.apps.astra.datastax.com"; - var database = _client.GetDatabase(apiEndpoint); - return new DatabaseAdminAstra(database, _client, null); - } - - private DatabaseAdminAstra GetDatabaseAdmin(Guid dbGuid) - { - var dbInfo = GetDatabaseInfo(dbGuid); - var apiEndpoint = $"https://{dbGuid}-{dbInfo.Info.Region}.apps.astra.datastax.com"; - var database = _client.GetDatabase(apiEndpoint); - return new DatabaseAdminAstra(database, _client, null); - } - /// /// Retrieves database information for the specified GUID. /// @@ -684,18 +671,35 @@ internal async Task GetDatabaseInfoAsync(Guid dbGuid, bool runSync return await GetDatabaseInfoAsync(dbGuid, null, runSynchronously).ConfigureAwait(false); } - internal Task GetDatabaseInfoAsync(Guid dbGuid, CommandOptions options, bool runSynchronously) + internal async Task GetDatabaseInfoAsync(Guid dbGuid, CommandOptions options, bool runSynchronously) { Guard.NotEmpty(dbGuid, nameof(dbGuid)); var command = CreateCommand() .AddUrlPath("databases") .AddUrlPath(dbGuid.ToString()) + .WithTimeoutManager(new DatabaseAdminTimeoutManager()) .AddCommandOptions(options); - var response = command.RunAsyncRaw(HttpMethod.Get, runSynchronously); + var rawInfo = await command.RunAsyncRaw(HttpMethod.Get, runSynchronously); + var response = new DatabaseInfo(rawInfo); return response; } + private DatabaseAdminAstra GetDatabaseAdmin(DatabaseInfo dbInfo) + { + var apiEndpoint = $"https://{dbInfo.Id}-{dbInfo.Region}.apps.astra.datastax.com"; + var database = _client.GetDatabase(apiEndpoint); + return new DatabaseAdminAstra(database, _client, null); + } + + private async Task GetDatabaseAdminAsync(Guid dbGuid) + { + var dbInfo = await GetDatabaseInfoAsync(dbGuid).ConfigureAwait(false); + var apiEndpoint = $"https://{dbGuid}-{dbInfo.Region}.apps.astra.datastax.com"; + var database = _client.GetDatabase(apiEndpoint); + return new DatabaseAdminAstra(database, _client, null); + } + private Command CreateCommand() { return new Command(_client, OptionsTree, new AdminCommandUrlBuilder()); diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs index c84defc..224c5f4 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminAstra.cs @@ -25,22 +25,6 @@ namespace DataStax.AstraDB.DataApi.Admin { - /// - /// Provides metadata about an embedding provider registered in Astra, - /// including its name, description, version, and supported model identifiers. - /// - /// - /// - /// var provider = new EmbeddingProviderInfo { Name = "OpenAI", Description = "OpenAI API", Version = "1.0.0", SupportedModels = new List<string> { "text-embedding-ada-002" } }; - /// - /// - public class EmbeddingProviderInfo - { - public string Name { get; set; } - public string Description { get; set; } - public string Version { get; set; } - public List SupportedModels { get; set; } - } /// /// Provides administrative operations for an Astra database, including keyspace management @@ -104,12 +88,12 @@ public string GetApiEndpoint() /// A collection of keyspace names. /// /// - /// IEnumerable<string> keyspaces = admin.ListKeyspaceNames(); + /// IEnumerable<string> keyspaces = admin.ListKeyspaces(); /// /// - public IEnumerable ListKeyspaceNames() + public IEnumerable ListKeyspaces() { - return ListKeyspaceNamesAsync(true, null).ResultSync(); + return ListKeyspacesAsync(true, null).ResultSync(); } /// @@ -118,12 +102,12 @@ public IEnumerable ListKeyspaceNames() /// A task that resolves to a collection of keyspace names. /// /// - /// var keyspaces = await admin.ListKeyspaceNamesAsync(); + /// var keyspaces = await admin.ListKeyspacesAsync(); /// /// - public Task> ListKeyspaceNamesAsync() + public Task> ListKeyspacesAsync() { - return ListKeyspaceNamesAsync(false, null); + return ListKeyspacesAsync(false, null); } /// @@ -133,12 +117,12 @@ public Task> ListKeyspaceNamesAsync() /// A collection of keyspace names. /// /// - /// var keyspaces = admin.ListKeyspaceNames(options); + /// var keyspaces = admin.ListKeyspaces(options); /// /// - public IEnumerable ListKeyspaceNames(CommandOptions options) + public IEnumerable ListKeyspaces(CommandOptions options) { - return ListKeyspaceNamesAsync(true, options).ResultSync(); + return ListKeyspacesAsync(true, options).ResultSync(); } /// @@ -148,30 +132,24 @@ public IEnumerable ListKeyspaceNames(CommandOptions options) /// A task that resolves to a collection of keyspace names. /// /// - /// var keyspaces = await admin.ListKeyspaceNamesAsync(options); + /// var keyspaces = await admin.ListKeyspacesAsync(options); /// /// - public Task> ListKeyspaceNamesAsync(CommandOptions options) + public Task> ListKeyspacesAsync(CommandOptions options) { - return ListKeyspaceNamesAsync(false, options); + return ListKeyspacesAsync(false, options); } - internal async Task> ListKeyspaceNamesAsync(bool runSynchronously, CommandOptions options) + internal async Task> ListKeyspacesAsync(bool runSynchronously, CommandOptions options) { var databaseInfo = await _client.GetAstraDatabasesAdmin().GetDatabaseInfoAsync(_id, options, runSynchronously); - return databaseInfo.Info.Keyspaces; + return databaseInfo.Keyspaces; } /// /// Synchronous version of /// /// - /// The name of the keyspace to create. - /// - /// - /// admin.CreateKeyspace("myKeyspace"); - /// - /// public void CreateKeyspace(string keyspace) { CreateKeyspace(keyspace, false); @@ -181,13 +159,6 @@ public void CreateKeyspace(string keyspace) /// Synchronous version of /// /// - /// The name of the keyspace to create. - /// Whether to set this keyspace as the active keyspace in the command options. - /// - /// - /// admin.CreateKeyspace("myKeyspace", true); - /// - /// public void CreateKeyspace(string keyspace, bool updateDBKeyspace) { CreateKeyspace(keyspace, updateDBKeyspace, null); @@ -197,30 +168,24 @@ public void CreateKeyspace(string keyspace, bool updateDBKeyspace) /// Synchronous version of /// /// - /// The name of the keyspace to create. - /// Optional settings that influence request execution. - /// - /// - /// admin.CreateKeyspace("myKeyspace", options); - /// - /// public void CreateKeyspace(string keyspace, CommandOptions options) { CreateKeyspace(keyspace, false, true, options); } + /// + /// Synchronous version of + /// + /// + public void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion) + { + CreateKeyspace(keyspace, updateDBKeyspace, waitForCompletion, null); + } + /// /// Synchronous version of /// /// - /// The name of the keyspace to create. - /// Whether to set this keyspace as the active keyspace in the command options. - /// Optional settings that influence request execution. - /// - /// - /// admin.CreateKeyspace("myKeyspace", true, options); - /// - /// public void CreateKeyspace(string keyspace, bool updateDBKeyspace, CommandOptions options) { CreateKeyspace(keyspace, updateDBKeyspace, true, options); @@ -230,15 +195,6 @@ public void CreateKeyspace(string keyspace, bool updateDBKeyspace, CommandOption /// Synchronous version of /// /// - /// The name of the keyspace to create. - /// Whether to set this keyspace as the active keyspace in the command options. - /// Whether or not to wait for the keyspace to be created before returning. - /// Optional settings that influence request execution. - /// - /// - /// admin.CreateKeyspace("myKeyspace", true, options); - /// - /// public void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options) { CreateKeyspaceAsync(keyspace, updateDBKeyspace, waitForCompletion, options).ResultSync(); @@ -261,7 +217,6 @@ public Task CreateKeyspaceAsync(string keyspace) /// /// The name of the keyspace to create. /// Whether to set this keyspace as the active keyspace in the command options. - /// A task representing the asynchronous operation. /// /// /// await admin.CreateKeyspaceAsync("myKeyspace", true); @@ -276,7 +231,6 @@ public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace) /// The name of the keyspace to create. /// Whether to set this keyspace as the active keyspace in the command options. /// Whether or not to wait for the keyspace to be created before returning. - /// A task representing the asynchronous operation. /// /// /// await admin.CreateKeyspaceAsync("myKeyspace", true); @@ -290,7 +244,6 @@ public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool wai /// /// The name of the keyspace to create. /// Optional settings that influence request execution. - /// A task representing the asynchronous operation. /// /// /// await admin.CreateKeyspaceAsync("myKeyspace", options); @@ -305,7 +258,6 @@ public Task CreateKeyspaceAsync(string keyspace, CommandOptions options) /// The name of the keyspace to create. /// Whether or not to wait for the keyspace to be created before returning. /// Optional settings that influence request execution. - /// A task representing the asynchronous operation. /// /// /// await admin.CreateKeyspaceAsync("myKeyspace", true, options); @@ -321,7 +273,6 @@ public Task CreateKeyspaceAsync(string keyspace, bool waitForCompletion, Command /// Whether to set this keyspace as the active keyspace in the command options. /// Whether or not to wait for the keyspace to be created before returning. /// Optional settings that influence request execution. - /// A task representing the asynchronous operation. /// /// /// await admin.CreateKeyspaceAsync("myKeyspace", true, true, options); @@ -338,7 +289,7 @@ internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, options.IncludeKeyspaceInUrl = false; Guard.NotNullOrEmpty(keyspace, nameof(keyspace)); - bool exists = await KeyspaceExistsAsync(keyspace, options, runSynchronously).ConfigureAwait(false); + bool exists = await DoesKeyspaceExistAsync(keyspace, options, runSynchronously).ConfigureAwait(false); if (exists) { throw new InvalidOperationException($"Keyspace {keyspace} already exists"); @@ -349,6 +300,7 @@ internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, .AddUrlPath(_id.ToString()) .AddUrlPath("keyspaces") .AddUrlPath(keyspace) + .WithTimeoutManager(new KeyspaceAdminTimeoutManager()) .AddCommandOptions(options); await command.RunAsyncRaw(HttpMethod.Post, runSynchronously).ConfigureAwait(false); @@ -362,7 +314,7 @@ internal async Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, { try { - await Wait.WaitForProcess(() => KeyspaceExistsAsync(keyspace, options, runSynchronously)).ConfigureAwait(false); + await Wait.WaitForProcess(() => DoesKeyspaceExistAsync(keyspace, options, runSynchronously)).ConfigureAwait(false); } catch (Exception e) { @@ -454,7 +406,7 @@ public Task DropKeyspaceAsync(string keyspace, bool waitForCompletion) /// /// Optional settings that influence request execution. - /// A task representing the asynchronous operation. + /// /// /// await admin.DropKeyspaceAsync("myKeyspace", options); @@ -487,6 +439,7 @@ internal async Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, C var command = CreateCommandAdmin() .AddUrlPath($"databases/{_id}/keyspaces/{keyspace}") + .WithTimeoutManager(new KeyspaceAdminTimeoutManager()) .AddCommandOptions(options); await command.RunAsyncRaw(HttpMethod.Delete, runSynchronously) @@ -496,7 +449,7 @@ internal async Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, C { try { - await Wait.WaitForProcess(async () => !await KeyspaceExistsAsync(keyspace, options, runSynchronously).ConfigureAwait(false)).ConfigureAwait(false); + await Wait.WaitForProcess(async () => !await DoesKeyspaceExistAsync(keyspace, options, runSynchronously).ConfigureAwait(false)).ConfigureAwait(false); } catch (Exception e) { @@ -512,12 +465,12 @@ internal async Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, C /// true if the keyspace exists; otherwise, false. /// /// - /// bool exists = admin.KeyspaceExists("myKeyspace"); + /// bool exists = admin.DoesKeyspaceExist("myKeyspace"); /// /// - public bool KeyspaceExists(string keyspace) + public bool DoesKeyspaceExist(string keyspace) { - return KeyspaceExistsAsync(keyspace, null, true).ResultSync(); + return DoesKeyspaceExistAsync(keyspace, null, true).ResultSync(); } /// @@ -527,18 +480,18 @@ public bool KeyspaceExists(string keyspace) /// A task that resolves to true if the keyspace exists; otherwise, false. /// /// - /// bool exists = await admin.KeyspaceExistsAsync("myKeyspace"); + /// bool exists = await admin.DoesKeyspaceExistAsync("myKeyspace"); /// /// - public Task KeyspaceExistsAsync(string keyspace) + public Task DoesKeyspaceExistAsync(string keyspace) { - return KeyspaceExistsAsync(keyspace, null, false); + return DoesKeyspaceExistAsync(keyspace, null, false); } - internal async Task KeyspaceExistsAsync(string keyspace, CommandOptions options, bool runSynchronously) + internal async Task DoesKeyspaceExistAsync(string keyspace, CommandOptions options, bool runSynchronously) { Guard.NotNullOrEmpty(keyspace, nameof(keyspace)); - var keyspaces = await ListKeyspaceNamesAsync(runSynchronously, options).ConfigureAwait(false); + var keyspaces = await ListKeyspacesAsync(runSynchronously, options).ConfigureAwait(false); return keyspaces.Contains(keyspace); } @@ -605,48 +558,18 @@ public Task FindEmbeddingProvidersAsync(CommandOpt return FindEmbeddingProvidersAsync(options, false); } - /// - /// Represents the raw response returned from the embedding provider discovery request. - /// - public class FindEmbeddingProvidersResponse - { - public Status status { get; set; } - } - - /// - /// Contains the status payload of the embedding provider discovery response, - /// including a dictionary of discovered embedding providers. - /// - public class Status - { - public Dictionary embeddingProviders { get; set; } - } internal async Task FindEmbeddingProvidersAsync(CommandOptions options, bool runSynchronously) { var command = CreateCommandEmbedding() .AddCommandOptions(options) + .WithTimeoutManager(new DatabaseAdminTimeoutManager()) .WithPayload(new { findEmbeddingProviders = new { } }); var response = await command - .RunAsyncRaw(HttpMethod.Post, runSynchronously) + .RunAsyncReturnStatus(runSynchronously) .ConfigureAwait(false); - var result = new FindEmbeddingProvidersResult(); - - if (response?.status?.embeddingProviders is Dictionary providers) - { - foreach (var kvp in providers) - { - result.EmbeddingProviders[kvp.Key] = kvp.Value; - } - } - - return result; - } - - private Command CreateCommandDb() - { - return new Command(_database.Client, _optionsTree, new DatabaseCommandUrlBuilder(_database, null)); + return response.Result; } private Command CreateCommandAdmin() @@ -658,6 +581,6 @@ private Command CreateCommandEmbedding() { return new Command(_database.Client, _optionsTree, new EmbeddingCommandUrlBuilder(_database)); } - } -} + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminOther.cs b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminOther.cs index c3f8bc5..eb4c8fc 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminOther.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/DatabaseAdminOther.cs @@ -14,12 +14,12 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Results; using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Admin { @@ -44,17 +44,18 @@ public DatabaseAdminOther(Guid id) { _id = id; } - public IEnumerable ListKeyspaceNames() + + public IEnumerable ListKeyspaces() { throw new NotImplementedException(); } - public FindEmbeddingProvidersResult FindEmbeddingProviders() + public Task> ListKeyspacesAsync() { throw new NotImplementedException(); } - public Task> ListKeyspaceNamesAsync() + public FindEmbeddingProvidersResult FindEmbeddingProviders() { throw new NotImplementedException(); } @@ -104,7 +105,82 @@ public Task CreateKeyspaceAsync(string keyspace) throw new NotImplementedException(); } - public bool KeyspaceExists(string keyspace) + public bool DoesKeyspaceExist(string keyspace) + { + throw new NotImplementedException(); + } + + public void CreateKeyspace(string keyspace, CommandOptions options) + { + throw new NotImplementedException(); + } + + public void CreateKeyspace(string keyspace, bool updateDBKeyspace, CommandOptions options) + { + throw new NotImplementedException(); + } + + public void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion) + { + throw new NotImplementedException(); + } + + public void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task CreateKeyspaceAsync(string keyspace, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace) + { + throw new NotImplementedException(); + } + + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion) + { + throw new NotImplementedException(); + } + + public Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options) + { + throw new NotImplementedException(); + } + + public void DropKeyspace(string keyspace, bool waitForCompletion) + { + throw new NotImplementedException(); + } + + public void DropKeyspace(string keyspace, bool waitForCompletion, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task DropKeyspaceAsync(string keyspace, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task DropKeyspaceAsync(string keyspace, bool waitForCompletion) + { + throw new NotImplementedException(); + } + + public Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, CommandOptions options) + { + throw new NotImplementedException(); + } + + public Task DoesKeyspaceExistAsync(string keyspace) { throw new NotImplementedException(); } diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs b/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs index e404726..fba1f37 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/DatabaseCreationOptions.cs @@ -25,10 +25,10 @@ public class DatabaseCreationOptions public string Name { get; set; } [JsonPropertyName("cloudProvider")] - public CloudProviderType CloudProvider { get; set; } = CloudProviderType.GCP; + public CloudProviderType CloudProvider { get; set; } [JsonPropertyName("region")] - public string Region { get; set; } = "us-east1"; + public string Region { get; set; } [JsonPropertyName("keyspace")] public string Keyspace { get; set; } = Database.DefaultKeyspace; @@ -38,4 +38,7 @@ public class DatabaseCreationOptions [JsonPropertyName("tier")] public string Tier { get; set; } = "serverless"; + + [JsonPropertyName("dbType")] + public string DatabaseType { get; set; } = "vector"; } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Admin/IDatabaseAdmin.cs b/src/DataStax.AstraDB.DataApi/Admin/IDatabaseAdmin.cs index 0220e10..fea84ef 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/IDatabaseAdmin.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/IDatabaseAdmin.cs @@ -23,16 +23,30 @@ namespace DataStax.AstraDB.DataApi.Admin; public interface IDatabaseAdmin { - IEnumerable ListKeyspaceNames(); + IEnumerable ListKeyspaces(); + Task> ListKeyspacesAsync(); FindEmbeddingProvidersResult FindEmbeddingProviders(); - Task> ListKeyspaceNamesAsync(); - Database GetDatabase(); + void CreateKeyspace(string keyspace); + void CreateKeyspace(string keyspace, CommandOptions options); + void CreateKeyspace(string keyspace, bool updateDBKeyspace); + void CreateKeyspace(string keyspace, bool updateDBKeyspace, CommandOptions options); + void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion); + void CreateKeyspace(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options); + Task CreateKeyspaceAsync(string keyspace); + Task CreateKeyspaceAsync(string keyspace, CommandOptions options); + Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace); + Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, CommandOptions options); + Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion); + Task CreateKeyspaceAsync(string keyspace, bool updateDBKeyspace, bool waitForCompletion, CommandOptions options); void DropKeyspace(string keyspace); void DropKeyspace(string keyspace, CommandOptions options); + void DropKeyspace(string keyspace, bool waitForCompletion); + void DropKeyspace(string keyspace, bool waitForCompletion, CommandOptions options); Task DropKeyspaceAsync(string keyspace); - void CreateKeyspace(string keyspace, bool updateDBKeyspace); - void CreateKeyspace(string keyspace); - Task CreateKeyspaceAsync(string keyspace); - bool KeyspaceExists(string keyspace); + Task DropKeyspaceAsync(string keyspace, CommandOptions options); + Task DropKeyspaceAsync(string keyspace, bool waitForCompletion); + Task DropKeyspaceAsync(string keyspace, bool waitForCompletion, CommandOptions options); + bool DoesKeyspaceExist(string keyspace); + Task DoesKeyspaceExistAsync(string keyspace); } diff --git a/src/DataStax.AstraDB.DataApi/Admin/DatabaseInfo.cs b/src/DataStax.AstraDB.DataApi/Admin/RawDatabaseInfo.cs similarity index 92% rename from src/DataStax.AstraDB.DataApi/Admin/DatabaseInfo.cs rename to src/DataStax.AstraDB.DataApi/Admin/RawDatabaseInfo.cs index 6a24698..676b57c 100644 --- a/src/DataStax.AstraDB.DataApi/Admin/DatabaseInfo.cs +++ b/src/DataStax.AstraDB.DataApi/Admin/RawDatabaseInfo.cs @@ -20,13 +20,13 @@ namespace DataStax.AstraDB.DataApi.Admin; -public class DatabaseInfo +public class RawDatabaseInfo { [JsonPropertyName("availableActions")] public List AvailableActions { get; set; } [JsonPropertyName("cost")] - public DatabaseCost Cost { get; set; } + public RawDatabaseCost Cost { get; set; } [JsonPropertyName("cqlshUrl")] public string CqlshUrl { get; set; } @@ -47,13 +47,13 @@ public class DatabaseInfo public string Id { get; set; } [JsonPropertyName("info")] - public DatabaseDetailsInfo Info { get; set; } + public RawDatabaseDetailsInfo Info { get; set; } [JsonPropertyName("lastUsageTime")] public DateTime LastUsageTime { get; set; } [JsonPropertyName("metrics")] - public DatabaseMetrics Metrics { get; set; } + public RawDatabaseMetrics Metrics { get; set; } [JsonPropertyName("observedStatus")] public string ObservedStatus { get; set; } @@ -68,13 +68,13 @@ public class DatabaseInfo public string Status { get; set; } [JsonPropertyName("storage")] - public DatabaseStorage Storage { get; set; } + public RawDatabaseStorage Storage { get; set; } [JsonPropertyName("terminationTime")] public DateTime TerminationTime { get; set; } } -public class DatabaseCost +public class RawDatabaseCost { [JsonPropertyName("costPerDayCents")] public decimal CostPerDayCents { get; set; } @@ -122,7 +122,7 @@ public class DatabaseCost public decimal CostPerWrittenGbCents { get; set; } } -public class DatabaseDetailsInfo +public class RawDatabaseDetailsInfo { [JsonPropertyName("capacityUnits")] public int CapacityUnits { get; set; } @@ -131,7 +131,7 @@ public class DatabaseDetailsInfo public string CloudProvider { get; set; } [JsonPropertyName("datacenters")] - public List Datacenters { get; set; } + public List Datacenters { get; set; } [JsonPropertyName("dbType")] public string DbType { get; set; } @@ -152,7 +152,7 @@ public class DatabaseDetailsInfo public string Tier { get; set; } } -public class DatacenterInfo +public class RawDatacenterInfo { [JsonPropertyName("capacityUnits")] public int CapacityUnits { get; set; } @@ -173,7 +173,7 @@ public class DatacenterInfo public string Name { get; set; } } -public class DatabaseMetrics +public class RawDatabaseMetrics { [JsonPropertyName("errorsTotalCount")] public long ErrorsTotalCount { get; set; } @@ -188,7 +188,7 @@ public class DatabaseMetrics public long WriteRequestsTotalCount { get; set; } } -public class DatabaseStorage +public class RawDatabaseStorage { [JsonPropertyName("displayStorage")] public int DisplayStorage { get; set; } diff --git a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs index 4009659..4604db5 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs @@ -127,7 +127,7 @@ private async Task> InsertOneAsync(T document, Co var outputConverter = typeof(TId) == typeof(object) ? new IdListConverter() : null; commandOptions.SetConvertersIfNull(new DocumentConverter(), outputConverter); var command = CreateCommand("insertOne").WithPayload(payload).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); return new CollectionInsertOneResult { InsertedId = response.Result.InsertedIds[0] }; } @@ -172,6 +172,9 @@ public CollectionInsertManyResult InsertMany(List documents, InsertManyO /// /// The list of documents to insert. /// + /// Thrown if the documents list is null or empty. + /// Thrown if an error occurs during the bulk operation, + /// with partial results returned in the property. public Task> InsertManyAsync(List documents) { return InsertManyAsync(documents, new CommandOptions()); @@ -213,36 +216,51 @@ private async Task> InsertManyAsync(List docu InsertValidator.Validate(doc); } - var start = DateTime.Now; - var result = new CollectionInsertManyResult(); var tasks = new List(); var semaphore = new SemaphoreSlim(insertOptions.Concurrency); + var (timeout, cts) = BulkOperationHelper.InitTimeout(GetOptionsTree(), ref commandOptions); - var chunks = documents.CreateBatch(insertOptions.ChunkSize); - - foreach (var chunk in chunks) + using (cts) { - tasks.Add(Task.Run(async () => + var bulkOperationTimeoutToken = cts.Token; + try { - await semaphore.WaitAsync(); - try + var chunks = documents.CreateBatch(insertOptions.ChunkSize); + + foreach (var chunk in chunks) { - var runResult = await RunInsertManyAsync(chunk, insertOptions.InsertInOrder, commandOptions, runSynchronously).ConfigureAwait(false); - lock (result.InsertedIds) + tasks.Add(Task.Run(async () => { - result.InsertedIds.AddRange(runResult.InsertedIds); - } - } - finally - { - semaphore.Release(); + await semaphore.WaitAsync(bulkOperationTimeoutToken); + try + { + var runResult = await RunInsertManyAsync(chunk, insertOptions.InsertInOrder, commandOptions, runSynchronously).ConfigureAwait(false); + lock (result.InsertedIds) + { + result.InsertedIds.AddRange(runResult.InsertedIds); + } + } + finally + { + semaphore.Release(); + } + }, bulkOperationTimeoutToken)); } - })); - } - await Task.WhenAll(tasks); - return result; + await Task.WhenAll(tasks).WithCancellation(bulkOperationTimeoutToken); + return result; + } + catch (OperationCanceledException) + { + var innerException = new TimeoutException($"Bulk operation timed out after {timeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter."); + throw new BulkOperationException>(innerException, result); + } + catch (Exception ex) + { + throw new BulkOperationException>(ex, result); + } + } } private async Task> RunInsertManyAsync(IEnumerable documents, bool insertOrdered, CommandOptions commandOptions, bool runSynchronously) @@ -259,8 +277,8 @@ private async Task> RunInsertManyAsync(IEnumerab var outputConverter = typeof(TId) == typeof(object) ? new IdListConverter() : null; commandOptions.SetConvertersIfNull(new DocumentConverter(), outputConverter); var command = CreateCommand("insertMany").WithPayload(payload).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); - return new CollectionInsertManyResult { InsertedIds = response.Result.InsertedIds.ToList() }; + var response = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); + return response.Result; } /// @@ -571,7 +589,7 @@ private async Task FindOneAsync(Filter filter, DocumentFind } /// - /// Find all documents in the collection. + /// Find documents in the collection. /// /// The Find() methods return a object that can be used to further structure the query /// by adding Sort, Projection, Skip, Limit, etc. to affect the final results. @@ -600,6 +618,12 @@ private async Task FindOneAsync(Filter filter, DocumentFind /// } /// /// + /// + /// Timeouts passed in the ( + /// and ) will be used for each batched request to the API, + /// however settings are ignored due to the nature of Enueration. + /// If you need to enforce a timeout for the entire operation, you can pass a to GetAsyncEnumerator. + /// public FindEnumerator> Find() { return Find(null, null); @@ -716,7 +740,7 @@ public T FindOneAndUpdate(Filter filter, UpdateBuilder update, FindOneAndU /// /// /// - /// + /// Updated document or null /// /// /// var updater = Builders<SimpleObject>.Update; @@ -733,6 +757,9 @@ public Task FindOneAndUpdateAsync(Filter filter, UpdateBuilder update) } /// + /// + /// Use the parameter on to specify whether the original or updated document should be returned. + /// /// Set Sort, Projection, Upsert options public Task FindOneAndUpdateAsync(Filter filter, UpdateBuilder update, FindOneAndUpdateOptions updateOptions) { @@ -839,7 +866,7 @@ public T FindOneAndReplace(T replacement, ReplaceOptions replaceOptions, Comm /// Find a document and replace it with the provided replacement /// /// - /// + /// The replaced document, or null if not found public Task FindOneAndReplaceAsync(T replacement) { return FindOneAndReplaceAsync(replacement, new ReplaceOptions(), null); @@ -847,6 +874,9 @@ public Task FindOneAndReplaceAsync(T replacement) /// /// + /// + /// Use the parameter on to specify whether the original or updated document should be returned. + /// public Task FindOneAndReplaceAsync(T replacement, ReplaceOptions replaceOptions) { return FindOneAndReplaceAsync(replacement, replaceOptions, null); @@ -1027,7 +1057,7 @@ internal async Task FindOneAndReplaceAsync(Filter filter, T replaceOptions.Filter = filter; replaceOptions.Replacement = replacement; var command = CreateCommand("findOneAndReplace").WithPayload(replaceOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnDocumentData, T, ReplaceResult>(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnDocumentData, T, UpdateResult>(runSynchronously).ConfigureAwait(false); return response.Data.Document; } @@ -1084,7 +1114,7 @@ public RerankSorter FindAndRerank(Filter filter, Command /// Synchronous version of /// /// - public ReplaceResult ReplaceOne(Filter filter, T replacement) + public UpdateResult ReplaceOne(Filter filter, T replacement) { return ReplaceOne(filter, replacement, new ReplaceOptions(), null); } @@ -1093,7 +1123,7 @@ public ReplaceResult ReplaceOne(Filter filter, T replacement) /// Synchronous version of /// /// - public ReplaceResult ReplaceOne(Filter filter, T replacement, ReplaceOptions replaceOptions) + public UpdateResult ReplaceOne(Filter filter, T replacement, ReplaceOptions replaceOptions) { return ReplaceOne(filter, replacement, replaceOptions, null); } @@ -1102,7 +1132,7 @@ public ReplaceResult ReplaceOne(Filter filter, T replacement, ReplaceOptions< /// Synchronous version of /// /// - public ReplaceResult ReplaceOne(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions) + public UpdateResult ReplaceOne(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions) { var response = ReplaceOneAsync(filter, replacement, replaceOptions, commandOptions, true).ResultSync(); return response; @@ -1115,32 +1145,32 @@ public ReplaceResult ReplaceOne(Filter filter, T replacement, ReplaceOptions< /// /// /// - public Task ReplaceOneAsync(Filter filter, T replacement) + public Task ReplaceOneAsync(Filter filter, T replacement) { return ReplaceOneAsync(filter, replacement, new ReplaceOptions(), null); } /// /// - public Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions) + public Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions) { return ReplaceOneAsync(filter, replacement, replaceOptions, null); } /// /// - public Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions) + public Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions) { return ReplaceOneAsync(filter, replacement, replaceOptions, commandOptions, false); } - internal async Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions, bool runSynchronously) + internal async Task ReplaceOneAsync(Filter filter, T replacement, ReplaceOptions replaceOptions, CommandOptions commandOptions, bool runSynchronously) { replaceOptions.Filter = filter; replaceOptions.Replacement = replacement; replaceOptions.Projection = new ExclusiveProjectionBuilder().Exclude("*"); var command = CreateCommand("findOneAndReplace").WithPayload(replaceOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); return response.Result; } @@ -1291,7 +1321,7 @@ public TResult FindOneAndDelete(Filter filter, FindOneAndDeleteOptio /// /// Find a document and delete it from the collection /// - /// The deleted document + /// The deleted document, or null if not found public Task FindOneAndDeleteAsync() { return FindOneAndDeleteAsync(null, new FindOneAndDeleteOptions(), null); @@ -1526,24 +1556,6 @@ internal async Task DeleteOneAsync(Filter filter, DeleteOptions return response.Result; } - /// - /// Synchronous version of - /// - /// - public DeleteResult DeleteAll() - { - return DeleteAll(null); - } - - /// - /// Synchronous version of - /// - /// - public DeleteResult DeleteAll(CommandOptions commandOptions) - { - return DeleteMany(null, commandOptions); - } - /// /// Synchronous version of /// @@ -1563,27 +1575,20 @@ public DeleteResult DeleteMany(Filter filter, CommandOptions commandOptions) return response; } - /// - /// Delete all documents from the collection. - /// - /// - public Task DeleteAllAsync() - { - return DeleteManyAsync(null, null); - } - - /// - /// - public Task DeleteAllAsync(CommandOptions commandOptions) - { - return DeleteManyAsync(null, commandOptions); - } - /// /// Delete all documents matching the filter from the collection. /// /// /// + /// + /// Deleting all documents in a collection is not recommended for large collections. However, if needed + /// you can pass null as the filter to delete all documents in the collection. + /// + /// + /// var deleteResult = await collection.DeleteManyAsync(null); + /// + /// + /// public Task DeleteManyAsync(Filter filter) { return DeleteManyAsync(filter, null); @@ -1603,14 +1608,33 @@ internal async Task DeleteManyAsync(Filter filter, CommandOptio Filter = filter }; - var keepProcessing = true; var deleteResult = new DeleteResult(); - while (keepProcessing) + var (timeout, cts) = BulkOperationHelper.InitTimeout(GetOptionsTree(), ref commandOptions); + + using (cts) { - var command = CreateCommand("deleteMany").WithPayload(deleteOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); - deleteResult.DeletedCount += response.Result.DeletedCount; - keepProcessing = response.Result.MoreData; + var bulkOperationTimeoutToken = cts.Token; + var keepProcessing = true; + + try + { + while (keepProcessing) + { + var command = CreateCommand("deleteMany").WithPayload(deleteOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + deleteResult.DeletedCount += response.Result.DeletedCount; + keepProcessing = response.Result.MoreData; + } + } + catch (OperationCanceledException) + { + var innerException = new TimeoutException($"Bulk operation timed out after {timeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter."); + throw new BulkOperationException(innerException, deleteResult); + } + catch (Exception ex) + { + throw new BulkOperationException(ex, deleteResult); + } } return deleteResult; @@ -1779,19 +1803,38 @@ internal async Task UpdateManyAsync(Filter filter, UpdateBuilde var keepProcessing = true; var updateResult = new UpdateResult(); string nextPageState = null; - while (keepProcessing) + + var (timeout, cts) = BulkOperationHelper.InitTimeout(GetOptionsTree(), ref commandOptions); + + using (cts) { - updateOptions ??= new UpdateManyOptions(); - updateOptions.NextPageState = nextPageState; - var command = CreateCommand("updateMany").WithPayload(updateOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); - updateResult.MatchedCount += response.Result.MatchedCount; - updateResult.ModifiedCount += response.Result.ModifiedCount; - updateResult.UpsertedId = response.Result.UpsertedId; - nextPageState = response.Result.NextPageState; - if (string.IsNullOrEmpty(nextPageState)) + var bulkOperationTimeoutToken = cts.Token; + try { - keepProcessing = false; + while (keepProcessing) + { + updateOptions ??= new UpdateManyOptions(); + updateOptions.NextPageState = nextPageState; + var command = CreateCommand("updateMany").WithPayload(updateOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + updateResult.MatchedCount += response.Result.MatchedCount; + updateResult.ModifiedCount += response.Result.ModifiedCount; + updateResult.UpsertedId = response.Result.UpsertedId; + nextPageState = response.Result.NextPageState; + if (string.IsNullOrEmpty(nextPageState)) + { + keepProcessing = false; + } + } + } + catch (OperationCanceledException) + { + var innerException = new TimeoutException($"Bulk operation timed out after {timeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter."); + throw new BulkOperationException(innerException, updateResult); + } + catch (Exception ex) + { + throw new BulkOperationException(ex, updateResult); } } return updateResult; @@ -1926,7 +1969,7 @@ internal async Task CountDocumentsAsync(Filter filter, int maxDocumentsT /// Synchronous version of /// /// - public EstimatedDocumentsCountResult EstimateDocumentCount() + public int EstimateDocumentCount() { return EstimateDocumentCountAsync(null, true).ResultSync(); } @@ -1935,7 +1978,7 @@ public EstimatedDocumentsCountResult EstimateDocumentCount() /// Synchronous version of /// /// - public EstimatedDocumentsCountResult EstimateDocumentCount(CommandOptions commandOptions) + public int EstimateDocumentCount(CommandOptions commandOptions) { return EstimateDocumentCountAsync(commandOptions, true).ResultSync(); } @@ -1944,7 +1987,7 @@ public EstimatedDocumentsCountResult EstimateDocumentCount(CommandOptions comman /// Estimate the number of documents in the collection. /// /// - public Task EstimateDocumentCountAsync() + public Task EstimateDocumentCountAsync() { return EstimateDocumentCountAsync(null, false); } @@ -1954,16 +1997,16 @@ public Task EstimateDocumentCountAsync() /// /// /// - public Task EstimateDocumentCountAsync(CommandOptions commandOptions) + public Task EstimateDocumentCountAsync(CommandOptions commandOptions) { return EstimateDocumentCountAsync(commandOptions, false); } - internal async Task EstimateDocumentCountAsync(CommandOptions commandOptions, bool runSynchronously) + internal async Task EstimateDocumentCountAsync(CommandOptions commandOptions, bool runSynchronously) { var command = CreateCommand("estimatedDocumentCount").WithPayload(new { }).AddCommandOptions(commandOptions); var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); - return response.Result; + return response.Result.Count; } /// @@ -2014,9 +2057,15 @@ public T CheckDeserialization(string json, CommandOptions commandOptions) return command.Deserialize(json); } + private List GetOptionsTree() + { + var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }); + return optionsTree.ToList(); + } + internal Command CreateCommand(string name) { - var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }).ToArray(); + var optionsTree = GetOptionsTree().ToArray(); return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _collectionName)); } diff --git a/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs index c3195b6..36f7318 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/CollectionInsertManyResult.cs @@ -15,6 +15,7 @@ */ using System.Collections.Generic; +using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Collections; @@ -23,7 +24,8 @@ public class CollectionInsertManyResult /// /// A list of the Ids of the inserted documents /// - public List InsertedIds { get; internal set; } = new List(); + [JsonPropertyName("insertedIds")] + public List InsertedIds { get; set; } = new List(); /// /// The number of documents that were inserted /// diff --git a/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs b/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs index 6a32100..9b570a4 100644 --- a/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/CommandOptions.cs @@ -63,11 +63,12 @@ public class CommandOptions public HttpClientOptions HttpClientOptions { get; set; } /// - /// Connection and request timeout options - /// - /// Defaults to ConnectTimeoutMillis: 5000, RequestTimeoutMillis: 30000 + /// Specify various timeout options to use for the command execution. /// - public TimeoutOptions TimeoutOptions { get; set; } + /// + /// See for information. + /// + public TimeoutOptions TimeoutOptions { get; set; } = new TimeoutOptions(); /// /// API version to connect to @@ -81,6 +82,8 @@ public class CommandOptions /// public CancellationToken? CancellationToken { get; set; } + internal CancellationToken? BulkOperationCancellationToken { get; set; } + internal void SetConvertersIfNull(JsonConverter inputConverter, JsonConverter outputConverter) { InputConverter ??= inputConverter; @@ -104,7 +107,22 @@ internal static CommandOptions Merge(params CommandOptions[] arr) RunMode = list.Select(o => o.RunMode).Merge(), Destination = list.Select(o => o.Destination).Merge(), HttpClientOptions = list.Select(o => o.HttpClientOptions).Merge(), - TimeoutOptions = list.Select(o => o.TimeoutOptions).Merge(), + TimeoutOptions = new TimeoutOptions + { + ConnectionTimeout = list.Select(o => o.TimeoutOptions?.ConnectionTimeout).Merge(), + RequestTimeout = list.Select(o => o.TimeoutOptions?.RequestTimeout).Merge(), + BulkOperationTimeout = list.Select(o => o.TimeoutOptions?.BulkOperationTimeout).Merge(), + Defaults = new TimeoutDefaults() + { + RequestTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.RequestTimeout).Merge() ?? TimeoutDefaults.DefaultRequestTimeout, + ConnectionTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.ConnectionTimeout).Merge() ?? TimeoutDefaults.DefaultConnectionTimeout, + BulkOperationTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.BulkOperationTimeout).Merge() ?? TimeoutDefaults.DefaultBulkOperationTimeout, + CollectionAdminTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.CollectionAdminTimeout).Merge() ?? TimeoutDefaults.DefaultCollectionAdminTimeout, + TableAdminTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.TableAdminTimeout).Merge() ?? TimeoutDefaults.DefaultTableAdminTimeout, + DatabaseAdminTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.DatabaseAdminTimeout).Merge() ?? TimeoutDefaults.DefaultDatabaseAdminTimeout, + KeyspaceAdminTimeout = list.Select(o => o.TimeoutOptions?.Defaults?.KeyspaceAdminTimeout).Merge() ?? TimeoutDefaults.DefaultKeyspaceAdminTimeout, + } + }, ApiVersion = list.Select(o => o.ApiVersion).Merge(), CancellationToken = list.Select(o => o.CancellationToken).Merge(), Keyspace = list.Select(o => o.Keyspace).Merge(), @@ -113,6 +131,7 @@ internal static CommandOptions Merge(params CommandOptions[] arr) IncludeKeyspaceInUrl = FirstNonNull(x => x.IncludeKeyspaceInUrl) ?? Defaults().IncludeKeyspaceInUrl, SerializeGuidAsDollarUuid = FirstNonNull(x => x.SerializeGuidAsDollarUuid) ?? Defaults().SerializeGuidAsDollarUuid, SerializeDateAsDollarDate = FirstNonNull(x => x.SerializeDateAsDollarDate) ?? Defaults().SerializeDateAsDollarDate, + BulkOperationCancellationToken = list.Select(o => o.BulkOperationCancellationToken).Merge() }; return options; } @@ -134,8 +153,13 @@ public static CommandOptions Defaults() IncludeKeyspaceInUrl = true, SerializeGuidAsDollarUuid = true, SerializeDateAsDollarDate = true, + TimeoutOptions = new TimeoutOptions + { + ConnectionTimeout = TimeoutDefaults.DefaultConnectionTimeout, + RequestTimeout = TimeoutDefaults.DefaultRequestTimeout, + BulkOperationTimeout = TimeoutDefaults.DefaultBulkOperationTimeout, + Defaults = new TimeoutDefaults() + } }; } } - - diff --git a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs index 9babff0..60eba05 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Commands/Command.cs @@ -40,6 +40,7 @@ internal class Command internal object Payload { get; set; } internal string UrlPostfix { get; set; } + internal TimeoutManager TimeoutManager { get; set; } = new TimeoutManager(); internal readonly struct EmptyResult { } @@ -75,6 +76,12 @@ internal Command WithPayload(object document) return this; } + internal Command WithTimeoutManager(TimeoutManager timeoutManager) + { + TimeoutManager = timeoutManager; + return this; + } + internal Command AddUrlPath(string path) { _urlPaths.Add(path); @@ -228,9 +235,10 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl { AllowAutoRedirect = commandOptions.HttpClientOptions.FollowRedirects }; - if (commandOptions.TimeoutOptions != null && commandOptions.TimeoutOptions.ConnectTimeoutMillis != 0) + var connectTimeout = TimeoutManager.GetConnectionTimeout(commandOptions); + if (connectTimeout.TotalMilliseconds != 0) { - handler.ConnectTimeout = TimeSpan.FromMilliseconds(commandOptions.TimeoutOptions.ConnectTimeoutMillis); + handler.ConnectTimeout = connectTimeout; } httpClient = new HttpClient(handler); @@ -263,13 +271,25 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl HttpResponseMessage response = null; var ctsForTimeout = new CancellationTokenSource(); - if (commandOptions.TimeoutOptions != null && commandOptions.TimeoutOptions.RequestTimeoutMillis != 0) + var requestTimeout = TimeoutManager.GetRequestTimeout(commandOptions); + if (requestTimeout.Milliseconds > 0) { - ctsForTimeout.CancelAfter(TimeSpan.FromMilliseconds(commandOptions.TimeoutOptions.RequestTimeoutMillis)); + ctsForTimeout.CancelAfter(requestTimeout); } var cancellationTokenForTimeout = ctsForTimeout.Token; - using (var linkedCts = commandOptions.CancellationToken == null ? ctsForTimeout : CancellationTokenSource.CreateLinkedTokenSource(commandOptions.CancellationToken.Value, cancellationTokenForTimeout)) + List cancellationTokens = new(); + cancellationTokens.Add(cancellationTokenForTimeout); + if (commandOptions.CancellationToken != null) + { + cancellationTokens.Add(commandOptions.CancellationToken.Value); + } + if (commandOptions.BulkOperationCancellationToken != null) + { + cancellationTokens.Add(commandOptions.BulkOperationCancellationToken.Value); + } + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationTokens.ToArray()); + try { if (runSynchronously) { @@ -295,7 +315,7 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl if (response.StatusCode == System.Net.HttpStatusCode.GatewayTimeout || response.StatusCode == System.Net.HttpStatusCode.RequestTimeout) { - throw new TimeoutException($"Request to timed out. Consider increasing the timeout settings using the CommandOptions parameter."); + throw new TimeoutException($"Request to timed out. Consider increasing the RequestTimeout settings using the CommandOptions parameter."); } else if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized) { @@ -311,31 +331,51 @@ private async Task RunCommandAsync(HttpMethod method, bool runSynchronousl } responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } - - if (_responseHandler != null) + } + catch (TaskCanceledException ex) + { + if (commandOptions.BulkOperationCancellationToken != null && commandOptions.BulkOperationCancellationToken.Value.IsCancellationRequested) { - if (runSynchronously) - { - _responseHandler(response).ResultSync(); - } - else - { - await _responseHandler(response); - } + throw new TimeoutException($"Bulk operation timed out after {TimeoutManager.GetBulkOperationTimeout(commandOptions).TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter.", ex); } - - MaybeLogDebugMessage("Response Status Code: {StatusCode}", response.StatusCode); - MaybeLogDebugMessage("Content: {Content}", responseContent); - - MaybeLogDebugMessage("Raw Response: {Response}", response); - - if (string.IsNullOrEmpty(responseContent)) + if (cancellationTokenForTimeout.IsCancellationRequested) { - return default; + throw new TimeoutException($"HTTP request timed out after {requestTimeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.RequestTimeout parameter.", ex); } + throw; // Other cancellation sources (e.g., commandOptions.CancellationToken) + } + catch (HttpRequestException ex) + { + _logger.LogError(ex, "HTTP request failed."); + throw; + } + finally + { + httpClient.Dispose(); + } + if (_responseHandler != null) + { + if (runSynchronously) + { + _responseHandler(response).ResultSync(); + } + else + { + await _responseHandler(response); + } + } - return Deserialize(responseContent); + MaybeLogDebugMessage("Response Status Code: {StatusCode}", response.StatusCode); + MaybeLogDebugMessage("Content: {Content}", responseContent); + + MaybeLogDebugMessage("Raw Response: {Response}", response); + + if (string.IsNullOrEmpty(responseContent)) + { + return default; } + + return Deserialize(responseContent); } private void MaybeLogDebugMessage(string message, params object[] args) diff --git a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs index bfe3fe7..78048ca 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs @@ -38,12 +38,12 @@ public class Cursor : IDisposable, IParentCursor { private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1); private ApiFindResult _currentBatch; - private readonly Func, FindStatusResult>>> _fetchNextBatch; + private readonly Func, FindStatusResult>>> _fetchNextBatch; private readonly IParentCursor _parentCursor; internal Cursor( - Func, FindStatusResult>>> fetchNextBatch, + Func, FindStatusResult>>> fetchNextBatch, IParentCursor parentCursor = null) { _fetchNextBatch = fetchNextBatch ?? throw new ArgumentNullException(nameof(fetchNextBatch)); @@ -123,7 +123,7 @@ private async Task MoveNextAsync(bool runSynchronously, CancellationToken _parentCursor?.SetStarted(); var nextPageState = _currentBatch?.NextPageState; - var nextBatch = await _fetchNextBatch(nextPageState, runSynchronously).ConfigureAwait(false); + var nextBatch = await _fetchNextBatch(nextPageState, cancellationToken, runSynchronously).ConfigureAwait(false); if (nextBatch.Data == null || nextBatch.Data.Items == null || nextBatch.Data.Items.Count == 0) { return false; diff --git a/src/DataStax.AstraDB.DataApi/Core/Database.cs b/src/DataStax.AstraDB.DataApi/Core/Database.cs index b1e0447..a90a132 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Database.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Database.cs @@ -31,10 +31,10 @@ namespace DataStax.AstraDB.DataApi.Core; /// Entrypoint for the interactions with a specific database such as creating/deleting collections/tables, /// connecting to collections/tables, and executing arbitrary commands. /// -/// Note that creating an instance of a Db doesn't trigger actual database creation; the database must have already existed beforehand. If you need to create a new database, use the AstraAdmin class. +/// Note that creating an instance of a Database doesn't trigger actual database creation; the database must have already existed beforehand. If you need to create a new database, use the AstraAdmin class. /// /// -/// The Database class has a concept of a "working keyspace", which is the keyspace used for all operations. This can be overridden in each method call via an overload with the parameter, +/// The Database class has a concept of a "current keyspace", which is the keyspace used for all operations. This can be overridden in each method call via an overload with the parameter, /// or when creating the instance (see ). /// If unset, the default keyspace will be used. /// @@ -101,7 +101,7 @@ internal Database(string apiEndpoint, DataApiClient client, DatabaseCommandOptio } /// - /// Set the active keyspace to use for all subsequent operations (can be overridden in each method call via an overload with the parameter) + /// Set the current keyspace to use for all subsequent operations (can be overridden in each method call via an overload with the parameter) /// /// public void UseKeyspace(string keyspace) @@ -225,7 +225,10 @@ private async Task ListCollectionsAsync(bool includeDetails, DatabaseComma { options = new { explain = includeDetails } }; - var command = CreateCommand("findCollections").WithPayload(payload).AddCommandOptions(commandOptions); + var command = CreateCommand("findCollections") + .WithPayload(payload) + .WithTimeoutManager(new CollectionAdminTimeoutManager()) + .AddCommandOptions(commandOptions); var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); return response.Result; } @@ -290,7 +293,7 @@ public Task> CreateCollectionAsync(string collectionName, D } /// - /// Specify options to use when creating the collection (id, vector, and indexing). + /// Specify options to use when creating the collection. public Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition) { return CreateCollectionAsync(collectionName, definition, null); @@ -352,7 +355,7 @@ public Task> CreateCollectionAsync(string collectionName) where } /// - /// Specify options to use when creating the collection (id, vector, and indexing). + /// Specify options to use when creating the collection. public Task> CreateCollectionAsync(string collectionName, CollectionDefinition definition) where T : class { return CreateCollectionAsync(collectionName, definition, null); @@ -441,7 +444,10 @@ private async Task> CreateCollectionAsync(string coll name = collectionName, options = definition }; - var command = CreateCommand("createCollection").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("createCollection") + .WithPayload(payload) + .WithTimeoutManager(new CollectionAdminTimeoutManager()) + .AddCommandOptions(options); await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); return GetCollection(collectionName); } @@ -450,9 +456,6 @@ private async Task> CreateCollectionAsync(string coll /// Returns an instance of that can be used to perform database management operations for this database /// /// - /// - /// If this database is not an Astra database, use instead, passing the appropriate destination to the Destination parameter on . - /// public IDatabaseAdmin GetAdmin() { return GetAdmin(null); @@ -569,7 +572,10 @@ private async Task DropCollectionAsync(string collectionName, DatabaseCommandOpt { name = collectionName }; - var command = CreateCommand("deleteCollection").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("deleteCollection") + .WithPayload(payload) + .WithTimeoutManager(new CollectionAdminTimeoutManager()) + .AddCommandOptions(options); await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); } @@ -617,7 +623,10 @@ private async Task> CreateTableAsync(string tableName, TableDe Name = tableName, Definition = definition }; - var command = CreateCommand("createTable").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("createTable") + .WithPayload(payload) + .WithTimeoutManager(new TableAdminTimeoutManager()) + .AddCommandOptions(options); await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); return GetTable(tableName, options); } @@ -830,7 +839,10 @@ private async Task DropTableAsync(string tableName, bool onlyIfExists, DatabaseC ifExists = onlyIfExists } }; - var command = CreateCommand("dropTable").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("dropTable") + .WithPayload(payload) + .WithTimeoutManager(new TableAdminTimeoutManager()) + .AddCommandOptions(options); await command.RunAsyncReturnDictionary(runSynchronously).ConfigureAwait(false); } @@ -881,7 +893,10 @@ private async Task> ListTablesAsync(DatabaseCommandOption explain = includeDetails } }; - var command = CreateCommand("listTables").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("listTables") + .WithPayload(payload) + .WithTimeoutManager(new TableAdminTimeoutManager()) + .AddCommandOptions(options); var result = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); return result.Result.Tables; } @@ -933,7 +948,10 @@ private async Task> ListTableNamesAsync(DatabaseCommandOptio explain = includeDetails } }; - var command = CreateCommand("listTables").WithPayload(payload).AddCommandOptions(options); + var command = CreateCommand("listTables") + .WithPayload(payload) + .WithTimeoutManager(new TableAdminTimeoutManager()) + .AddCommandOptions(options); var result = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); return result.Result.Tables; } @@ -983,7 +1001,10 @@ private async Task DropTableIndexAsync(string indexName, DropIndexCommandOptions ifExists = commandOptions?.SkipIfNotExists ?? false, } }; - var command = CreateCommand("dropIndex").WithPayload(payload).AddCommandOptions(commandOptions); + var command = CreateCommand("dropIndex") + .WithPayload(payload) + .WithTimeoutManager(new TableAdminTimeoutManager()) + .AddCommandOptions(commandOptions); await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); } diff --git a/src/DataStax.AstraDB.DataApi/Core/DatabaseInfo.cs b/src/DataStax.AstraDB.DataApi/Core/DatabaseInfo.cs new file mode 100644 index 0000000..3917880 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/DatabaseInfo.cs @@ -0,0 +1,93 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Admin; +using System; +using System.Collections.Generic; + +namespace DataStax.AstraDB.DataApi.Core; + +/// +/// The metadata information for a database. +/// +public class DatabaseInfo +{ + internal DatabaseInfo(RawDatabaseInfo rawInfo) + { + Id = rawInfo.Id; + Name = rawInfo.Info.Name; + OrgId = rawInfo.OrgId; + OwnerId = rawInfo.OwnerId; + Status = Enum.TryParse(rawInfo.Status, true, out var status) ? status : AstraDatabaseStatus.ERROR; + CloudProvider = Enum.TryParse(rawInfo.Info.CloudProvider, true, out var cloudProvider) ? cloudProvider : AstraDatabaseCloudProvider.AWS; + CreatedAt = rawInfo.CreationTime; + LastUsed = rawInfo.LastUsageTime; + Keyspaces = rawInfo.Info.Keyspaces; + Region = rawInfo.Info.Region; + RawDetails = rawInfo; + } + + public string Id { get; set; } + public string Name { get; set; } + public string OrgId { get; set; } + public string OwnerId { get; set; } + public AstraDatabaseStatus Status { get; set; } + public AstraDatabaseCloudProvider CloudProvider { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime LastUsed { get; set; } + public List Keyspaces { get; set; } = new(); + public string Region { get; set; } + public string Environment { get; set; } = "prod"; + public RawDatabaseInfo RawDetails { get; set; } +} + +public class AstraDatabaseRegionInfo +{ + public string ApiEndpoint { get; set; } + public DateTime CreatedAt { get; set; } + public string Name { get; set; } +} + +public enum AstraDatabaseCloudProvider +{ + AWS, + GCP, + AZURE +} + +public enum AstraDatabaseStatus +{ + ACTIVE, + ERROR, + DECOMMISSIONING, + DEGRADED, + HIBERNATED, + HIBERNATING, + INITIALIZING, + MAINTENANCE, + PARKED, + PARKING, + PENDING, + PREPARED, + PREPARING, + RESIZING, + RESUMING, + TERMINATED, + TERMINATING, + UNKNOWN, + UNPARKING, + SYNCHRONIZING +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs b/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs index 5a21539..085eb68 100644 --- a/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/LexicalOptions.cs @@ -14,7 +14,6 @@ * limitations under the License. */ -using System.Collections.Generic; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core; diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs index 9087ae3..98f4af3 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentSortBuilder.cs @@ -14,7 +14,6 @@ * limitations under the License. */ -using DataStax.AstraDB.DataApi.Utils; using System; using System.Linq.Expressions; diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs index 6fd8b21..036f25c 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/Filter.cs @@ -14,17 +14,20 @@ * limitations under the License. */ +using DataStax.AstraDB.DataApi.SerDes; using MongoDB.Bson; using System.Collections.Generic; using System.Linq; +using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Query; /// -/// Filters are used target specific documents in a collection. This class is not used directly, +/// Filters are used to target specific documents in a collection. This class is not used directly, /// you can create filters using the class. /// /// Type of document in the collection +[JsonConverter(typeof(FilterConverterFactory))] public class Filter { internal virtual string Name { get; } diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs index c9cc308..34d3757 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs @@ -294,7 +294,7 @@ public Filter In(Expression> expression, TField[] a /// public Filter In(string fieldName, object value) { - return new Filter(fieldName, FilterOperator.In, value); + return new Filter(fieldName, FilterOperator.In, new object[] { value }); } /// @@ -306,7 +306,7 @@ public Filter In(string fieldName, object value) /// The filter public Filter In(Expression> expression, TField value) { - return new Filter(expression.GetMemberNameTree(), FilterOperator.In, value); + return new Filter(expression.GetMemberNameTree(), FilterOperator.In, new object[] { value }); } /// @@ -348,6 +348,29 @@ public Filter Nin(Expression> expression, TField[] return new Filter(expression.GetMemberNameTree(), FilterOperator.NotIn, array); } + /// + /// Not in operator -- Match documents where the array field does not match the specified value. + /// + /// The type of the field to check + /// An expression that represents the field for this filter + /// The value to not match + /// The filter + public Filter Nin(Expression> expression, TField value) + { + return new Filter(expression.GetMemberNameTree(), FilterOperator.NotIn, new object[] { value }); + } + + /// + /// Not in operator -- Match documents where the array field does not match the specified value. + /// + /// The type of the field to check + /// The value to not match + /// The filter + public Filter Nin(string field, object value) + { + return new Filter(field, FilterOperator.NotIn, new object[] { value }); + } + /// /// Exists operator -- Match documents where the field exists. /// @@ -463,6 +486,12 @@ public Filter CompoundKey(PrimaryKeyFilter[] partitionColumns, Filter[] cl public class PrimaryKeyFilter { + public PrimaryKeyFilter(string columnName, object value) + { + ColumnName = columnName; + Value = value; + } + public string ColumnName { get; set; } public object Value { get; set; } } @@ -470,8 +499,7 @@ public class PrimaryKeyFilter public class PrimaryKeyFilter : PrimaryKeyFilter { public PrimaryKeyFilter(Expression> columnExpression, TValue value) + : base(columnExpression.GetMemberNameTree(), value) { - ColumnName = columnExpression.GetMemberNameTree(); - Value = value; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs index 3e7bf5a..ae92519 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindAndRerankOptions.cs @@ -16,7 +16,6 @@ using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Query; diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs index c74908c..d50a4e6 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs @@ -176,7 +176,7 @@ public Cursor ToCursor() { return _cursor; } - _cursor = new Cursor((string pageState, bool runSynchronously) => RunAsync(pageState, runSynchronously)); + _cursor = new Cursor((string pageState, CancellationToken cancellationToken, bool runSynchronously) => RunAsync(pageState, cancellationToken, runSynchronously)); return _cursor; } @@ -185,6 +185,12 @@ public Cursor ToCursor() /// /// An optional cancellation token to use for the operation. /// An async enumerator + /// + /// Timeouts passed in the ( + /// and ) will be used for each batched request to the API. + /// If you need to enforce a timeout for the entire operation, you can pass a to this method. + /// settings are ignored for this operation. + /// public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { var cursor = ToCursor(); @@ -206,9 +212,13 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } - private Task, FindStatusResult>> RunAsync(string pageState = null, bool runSynchronously = false) + private Task, FindStatusResult>> RunAsync(string pageState = null, CancellationToken cancellationToken = default, bool runSynchronously = false) { _findOptions.PageState = pageState; + if (cancellationToken != default && _commandOptions.BulkOperationCancellationToken == null) + { + _commandOptions.BulkOperationCancellationToken = cancellationToken; + } return _queryRunner.RunFindManyAsync(_findOptions.Filter, _findOptions, _commandOptions, runSynchronously); } diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs index 4329b1e..866cce5 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/LogicalFilter.cs @@ -14,7 +14,6 @@ * limitations under the License. */ -using System; using System.Linq; namespace DataStax.AstraDB.DataApi.Core.Query; diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs index 281ca0b..0d0b96e 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/SortBuilder.cs @@ -14,7 +14,6 @@ * limitations under the License. */ -using DataStax.AstraDB.DataApi.Utils; using System; using System.Collections.Generic; using System.Linq.Expressions; diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/BulkOperationException.cs b/src/DataStax.AstraDB.DataApi/Core/Results/BulkOperationException.cs new file mode 100644 index 0000000..34dc3d4 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Results/BulkOperationException.cs @@ -0,0 +1,37 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace DataStax.AstraDB.DataApi.Core.Results; + +/// +/// Exception thrown from bulk operaions (e.g. Collection.InsertManyAsync, Collection.UpdateManyAsync, etc.) when an error occurs. +/// If the operation was partially successful, the property will contain the results of the operation that succeeded. +/// +public class BulkOperationException : Exception +{ + public BulkOperationException(string message, T partialResult) : base(message) + { + PartialResult = partialResult; + } + public BulkOperationException(Exception causingException, T partialResult) : base(causingException.Message, causingException) + { + PartialResult = partialResult; + } + + public T PartialResult { get; set; } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs b/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs index 5852ebe..c6d69aa 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/EmbeddingProvider.cs @@ -58,14 +58,10 @@ public class Parameter public string Type { get; set; } public bool Required { get; set; } public string DefaultValue { get; set; } - public Validation Validation { get; set; } + public Dictionary> Validation { get; set; } public string Help { get; set; } public string DisplayName { get; set; } public string Hint { get; set; } } - public class Validation - { - public List NumericRange { get; set; } - } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs index f736c84..df2f57c 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/EstimatedDocumentsCountResult.cs @@ -18,14 +18,9 @@ namespace DataStax.AstraDB.DataApi.Core.Results; -/// -/// The result of an estimated count operation. -/// -public class EstimatedDocumentsCountResult +internal class EstimatedDocumentsCountResult { - /// - /// The estimated number of documents in the collection. - /// + [JsonInclude] [JsonPropertyName("count")] - public int Count { get; set; } + internal int Count { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs index 17ca003..7761710 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/FindEmbeddingProvidersResult.cs @@ -15,6 +15,7 @@ */ using System.Collections.Generic; +using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Results; @@ -26,5 +27,6 @@ public class FindEmbeddingProvidersResult /// /// A dictionary of embedding provider names to details. /// + [JsonPropertyName("embeddingProviders")] public Dictionary EmbeddingProviders { get; set; } = new(); } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs b/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs deleted file mode 100644 index 94a893d..0000000 --- a/src/DataStax.AstraDB.DataApi/Core/Results/InsertDocumentsCommandResponse.cs +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace DataStax.AstraDB.DataApi.Core.Results; - -public class InsertDocumentsCommandResponse -{ - [JsonPropertyName("insertedIds")] - public List InsertedIds { get; set; } -} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionNamesResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionNamesResult.cs index e1b36ad..fd68161 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionNamesResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionNamesResult.cs @@ -19,14 +19,9 @@ namespace DataStax.AstraDB.DataApi.Core.Results; -/// -/// The result object for an operation returning a list of collection names. -/// -public class ListCollectionNamesResult +internal class ListCollectionNamesResult { - /// - /// A list of collection names. - /// + [JsonInclude] [JsonPropertyName("collections")] public List CollectionNames { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs index fd03ac4..b5b9368 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Results/ListCollectionsResult.cs @@ -18,14 +18,9 @@ namespace DataStax.AstraDB.DataApi.Core.Results; -/// -/// The result object for an operation returning a list of collections. -/// -public class ListCollectionsResult +internal class ListCollectionsResult { - /// - /// An array of describing the collections. - /// + [JsonInclude] [JsonPropertyName("collections")] public CollectionInfo[] Collections { get; set; } } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Results/ReplaceResult.cs b/src/DataStax.AstraDB.DataApi/Core/Results/ReplaceResult.cs deleted file mode 100644 index 3d6b046..0000000 --- a/src/DataStax.AstraDB.DataApi/Core/Results/ReplaceResult.cs +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Text.Json.Serialization; - -namespace DataStax.AstraDB.DataApi.Core.Results; - -/// -/// The results of a replace operation -/// -public class ReplaceResult -{ - /// - /// If an insert was performed, the id of the inserted document - /// - [JsonPropertyName("upsertedId")] - public object UpsertedId { get; set; } - - /// - /// The number of documents that matched the query - /// - [JsonPropertyName("matchedCount")] - public int MatchedCount { get; set; } - - /// - /// The number of documents that were modified - /// - [JsonPropertyName("modifiedCount")] - public int ModifiedCount { get; set; } -} diff --git a/src/DataStax.AstraDB.DataApi/Core/TimeoutManager.cs b/src/DataStax.AstraDB.DataApi/Core/TimeoutManager.cs new file mode 100644 index 0000000..bf9bc4c --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/TimeoutManager.cs @@ -0,0 +1,95 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core; +using System; + +internal class TimeoutManager +{ + internal TimeSpan GetRequestTimeout(CommandOptions commandOptions) + { + if (commandOptions.TimeoutOptions?.RequestTimeout != null) + { + return commandOptions.TimeoutOptions.RequestTimeout.Value; + } + return GetDefaultRequestTimeout(commandOptions); + } + + internal TimeSpan GetConnectionTimeout(CommandOptions commandOptions) + { + if (commandOptions.TimeoutOptions?.ConnectionTimeout != null) + { + return commandOptions.TimeoutOptions.ConnectionTimeout.Value; + } + return GetDefaultConnectionTimeout(commandOptions); + } + + internal TimeSpan GetBulkOperationTimeout(CommandOptions commandOptions) + { + if (commandOptions.TimeoutOptions?.BulkOperationTimeout != null) + { + return commandOptions.TimeoutOptions.BulkOperationTimeout.Value; + } + return GetDefaultBulkOperationTimeout(commandOptions); + } + + virtual internal TimeSpan GetDefaultRequestTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.RequestTimeout; + } + + virtual internal TimeSpan GetDefaultConnectionTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.ConnectionTimeout; + } + + virtual internal TimeSpan GetDefaultBulkOperationTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.BulkOperationTimeout; + } +} + +internal class CollectionAdminTimeoutManager : TimeoutManager +{ + override internal TimeSpan GetDefaultRequestTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.CollectionAdminTimeout; + } +} + +internal class TableAdminTimeoutManager : TimeoutManager +{ + override internal TimeSpan GetDefaultRequestTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.TableAdminTimeout; + } +} + +internal class DatabaseAdminTimeoutManager : TimeoutManager +{ + override internal TimeSpan GetDefaultRequestTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.DatabaseAdminTimeout; + } +} + +internal class KeyspaceAdminTimeoutManager : TimeoutManager +{ + override internal TimeSpan GetDefaultRequestTimeout(CommandOptions commandOptions) + { + return commandOptions.TimeoutOptions.Defaults.KeyspaceAdminTimeout; + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/TimeoutOptions.cs b/src/DataStax.AstraDB.DataApi/Core/TimeoutOptions.cs index 2a1af33..3b7ba30 100644 --- a/src/DataStax.AstraDB.DataApi/Core/TimeoutOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/TimeoutOptions.cs @@ -14,10 +14,133 @@ * limitations under the License. */ +using System; + namespace DataStax.AstraDB.DataApi.Core; +/// +/// While timeouts can be set on a per-request basis (via the ), +/// when the timeouts are not specified, the default timeouts will be used. These can also be overridden +/// by setting the property at any level in the command +/// hierarchy (e.g. , , ). +/// +/// +/// +/// The following example shows how to override the default timeouts at the client level. +/// Let's change the defaults for each request, as well as those for collection administration operations, +/// but leave the rest of the defaults unchanged. +/// var client = new DataApiClient(new CommandOptions +/// { +/// TimeoutOptions = new TimeoutOptions +/// { +/// Defaults = new TimeoutDefaults +/// { +/// RequestTimeout = TimeSpan.FromSeconds(45), +/// CollectionAdminTimeout = TimeSpan.FromMinutes(3), +/// } +/// } +/// }); +/// +/// +/// +/// The following example shows how to override the default timeouts at the collection level. +/// Let's change the defaults for each request, as well as those for collection administration operations, +/// and connections to the API, but leave the rest of the defaults unchanged. +/// var collection = client.GetDatabase("mydb").GetCollection("myCollection", new DatabaseCommandOptions +/// { +/// TimeoutOptions = new TimeoutOptions +/// { +/// Defaults = new TimeoutDefaults +/// { +/// ConnectionTimeout = TimeSpan.FromSeconds(3), +/// RequestTimeout = TimeSpan.FromMinutes(1), +/// CollectionAdminTimeout = TimeSpan.FromMinutes(5), +/// } +/// } +/// }); +/// +public class TimeoutDefaults +{ + /// + /// 10 seconds. + /// + public static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(10); + /// + /// 5 seconds. + /// + public static readonly TimeSpan DefaultConnectionTimeout = TimeSpan.FromSeconds(5); + /// + /// 30 seconds. + /// + public static readonly TimeSpan DefaultBulkOperationTimeout = TimeSpan.FromSeconds(30); + /// + /// 60 seconds. + /// + public static readonly TimeSpan DefaultCollectionAdminTimeout = TimeSpan.FromSeconds(60); + /// + /// 30 seconds. + /// + public static readonly TimeSpan DefaultTableAdminTimeout = TimeSpan.FromSeconds(30); + /// + /// 10 minutes. + /// + public static readonly TimeSpan DefaultDatabaseAdminTimeout = TimeSpan.FromMinutes(10); + /// + /// 60 seconds. + /// + public static readonly TimeSpan DefaultKeyspaceAdminTimeout = TimeSpan.FromSeconds(60); + + /// + /// The default timeout for a single request to the API. + /// + public TimeSpan RequestTimeout { get; set; } + /// + /// The default timeout for establishing a connection to the API. + /// + public TimeSpan ConnectionTimeout { get; set; } + /// + /// The default timeout for bulk operations that involve multiple requests to the API (i.e. InsertMany) + /// + public TimeSpan BulkOperationTimeout { get; set; } + /// + /// The default timeout for collection administration operations, such as creating or deleting collections. + /// + public TimeSpan CollectionAdminTimeout { get; set; } + /// + /// The default timeout for table administration operations, such as creating or deleting tables. + /// + public TimeSpan TableAdminTimeout { get; set; } + /// + /// The default timeout for database administration operations, such as creating or deleting databases. + /// + public TimeSpan DatabaseAdminTimeout { get; set; } + /// + /// The default timeout for keyspace administration operations, such as creating or deleting keyspaces. + /// + public TimeSpan KeyspaceAdminTimeout { get; set; } +} + +/// +/// An options class that allows you to override the timeouts for the interactions with the Data API. +/// public class TimeoutOptions { - public int ConnectTimeoutMillis { get; set; } = 5000; - public int RequestTimeoutMillis { get; set; } = 30000; -} \ No newline at end of file + /// + /// The timeout for establishing a connection to the API. + /// + public TimeSpan? ConnectionTimeout { get; set; } + /// + /// The timeout for individual requests to the API. + /// + public TimeSpan? RequestTimeout { get; set; } + /// + /// The timeout for bulk operations that involve multiple requests to the API. + /// + public TimeSpan? BulkOperationTimeout { get; set; } + /// + /// Defaults used for operations for which timeouts are not specified. + /// See for more details. + /// + public TimeoutDefaults Defaults { get; set; } +} + diff --git a/src/DataStax.AstraDB.DataApi/Core/UpdateBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/UpdateBuilder.cs index 4fde35d..8329f74 100644 --- a/src/DataStax.AstraDB.DataApi/Core/UpdateBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/UpdateBuilder.cs @@ -14,11 +14,11 @@ * limitations under the License. */ +using DataStax.AstraDB.DataApi.Utils; using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; -using DataStax.AstraDB.DataApi.Utils; namespace DataStax.AstraDB.DataApi.Core; @@ -43,17 +43,6 @@ internal Dictionary Serialize() return result; } - /// - /// Noop, here for mongo compatibility - /// - /// - /// - public UpdateBuilder Combine(params UpdateBuilder[] updates) - { - //noop, here for mongo compatibility - return this; - } - /// /// Set the value of a field to the current date and time. /// diff --git a/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs b/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs index d47f532..be67544 100644 --- a/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/VectorOptions.cs @@ -40,4 +40,14 @@ public class VectorOptions /// [JsonPropertyName("service")] public VectorServiceOptions Service { get; set; } + + /// + /// Configures the index with the fastest settings for a given source of embeddings vectors. + /// + /// As of time of writing, example sourceModels include 'openai-v3-large', 'cohere-v3', 'bert', and a handful of others. + /// + /// If no source model if provided, this setting will default to 'other'. + /// + [JsonPropertyName("sourceModel")] + public string SourceModel { get; set; } } diff --git a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs index add22c9..65e66c5 100644 --- a/src/DataStax.AstraDB.DataApi/DataAPIClient.cs +++ b/src/DataStax.AstraDB.DataApi/DataAPIClient.cs @@ -14,15 +14,13 @@ * limitations under the License. */ -using System; -using System.Net.Http; -using System.Threading.Tasks; using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using System.Net.Http; namespace DataStax.AstraDB.DataApi; diff --git a/src/DataStax.AstraDB.DataApi/SerDes/FilterConverter.cs b/src/DataStax.AstraDB.DataApi/SerDes/FilterConverter.cs new file mode 100644 index 0000000..85d38b8 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/SerDes/FilterConverter.cs @@ -0,0 +1,59 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Query; +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.SerDes; + +public class FilterConverter : JsonConverter> +{ + public override Filter Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Deserialization logic (if needed) + throw new NotImplementedException("Deserialization not implemented"); + } + + public override void Write(Utf8JsonWriter writer, Filter value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + writer.WritePropertyName(value.Name); + JsonSerializer.Serialize(writer, value.Value, value.Value?.GetType() ?? typeof(object), options); + writer.WriteEndObject(); + } +} + +public class FilterConverterFactory : JsonConverterFactory +{ + public override bool CanConvert(Type typeToConvert) + { + // Check if the type is Filter (or a derived type) + return typeToConvert.IsGenericType && typeToConvert.GetGenericTypeDefinition() == typeof(Filter<>); + } + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + // Get the generic type T from Filter + Type genericType = typeToConvert.GetGenericArguments()[0]; + + // Create an instance of FilterConverter for the specific T + Type converterType = typeof(FilterConverter<>).MakeGenericType(genericType); + return (JsonConverter)Activator.CreateInstance(converterType); + } +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Tables/Row.cs b/src/DataStax.AstraDB.DataApi/Tables/Row.cs index 8f912cf..7df65f5 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/Row.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/Row.cs @@ -14,10 +14,15 @@ * limitations under the License. */ -namespace DataStax.AstraDB.DataApi.Tables; +using System.Collections.Generic; +namespace DataStax.AstraDB.DataApi.Tables; -public abstract class Row +/// +/// Represents a row in a table as a dictionary of column names and their corresponding values +/// (as untyped objects ). +/// +public class Row : Dictionary { } diff --git a/src/DataStax.AstraDB.DataApi/Tables/Table.cs b/src/DataStax.AstraDB.DataApi/Tables/Table.cs index 51a5522..c82a665 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/Table.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/Table.cs @@ -28,6 +28,11 @@ namespace DataStax.AstraDB.DataApi.Tables; + +/// +/// This is the main entry point for interacting with a table in the Astra DB Data API. +/// +/// The type to use for rows in the table (when not specified, defaults to public class Table : IQueryRunner> where T : class { private readonly string _tableName; @@ -245,31 +250,61 @@ public TableInsertManyResult InsertMany(IEnumerable rows) return InsertMany(rows, null as CommandOptions); } + /// + /// Synchronous version of + /// + /// public TableInsertManyResult InsertMany(IEnumerable rows, CommandOptions commandOptions) { return InsertManyAsync(rows, null, commandOptions, true).ResultSync(); } + /// + /// Synchronous version of + /// + /// public TableInsertManyResult InsertMany(IEnumerable rows, InsertManyOptions insertOptions) { return InsertManyAsync(rows, insertOptions, null, true).ResultSync(); } + /// + /// Insert multiple rows into the table. + /// + /// + /// + /// + /// If you need to control concurrency, chunk size, or whether the insert is ordered or not, use the overload. + /// To additionally control timesouts, use the overload. + /// + /// Thrown if the rows collection is null or empty. + /// Thrown if an error occurs during the bulk operation, with partial results returned in the property. public Task InsertManyAsync(IEnumerable rows) { return InsertManyAsync(rows, null, null, false); } + /// + /// public Task InsertManyAsync(IEnumerable rows, CommandOptions commandOptions) { return InsertManyAsync(rows, null, commandOptions, false); } + /// + /// public Task InsertManyAsync(IEnumerable rows, InsertManyOptions insertOptions) { return InsertManyAsync(rows, insertOptions, null, false); } + /// + /// + public Task InsertManyAsync(IEnumerable rows, InsertManyOptions insertOptions, CommandOptions commandOptions) + { + return InsertManyAsync(rows, insertOptions, commandOptions, false); + } + private async Task InsertManyAsync(IEnumerable rows, InsertManyOptions insertOptions, CommandOptions commandOptions, bool runSynchronously) { Guard.NotNullOrEmpty(rows, nameof(rows)); @@ -280,38 +315,54 @@ private async Task InsertManyAsync(IEnumerable rows, I throw new ArgumentException("Cannot run ordered insert_many concurrently."); } - var start = DateTime.Now; - var result = new TableInsertManyResult(); var tasks = new List(); var semaphore = new SemaphoreSlim(insertOptions.Concurrency); + var (timeout, cts) = BulkOperationHelper.InitTimeout(GetOptionsTree(), ref commandOptions); - var chunks = rows.CreateBatch(insertOptions.ChunkSize); - - foreach (var chunk in chunks) + using (cts) { - tasks.Add(Task.Run(async () => + var bulkOperationTimeoutToken = cts.Token; + try { - await semaphore.WaitAsync(); - try + var chunks = rows.CreateBatch(insertOptions.ChunkSize); + + foreach (var chunk in chunks) { - var runResult = await RunInsertManyAsync(chunk, insertOptions.InsertInOrder, insertOptions.ReturnDocumentResponses, commandOptions, runSynchronously).ConfigureAwait(false); - lock (result.InsertedIds) + tasks.Add(Task.Run(async () => { - result.PrimaryKeys = runResult.PrimaryKeys; - result.InsertedIds.AddRange(runResult.InsertedIds); - result.DocumentResponses.AddRange(runResult.DocumentResponses); - } - } - finally - { - semaphore.Release(); + await semaphore.WaitAsync(bulkOperationTimeoutToken); + try + { + var runResult = await RunInsertManyAsync(chunk, insertOptions.InsertInOrder, insertOptions.ReturnDocumentResponses, commandOptions, runSynchronously).ConfigureAwait(false); + lock (result.InsertedIds) + { + result.PrimaryKeys = runResult.PrimaryKeys; + result.InsertedIds.AddRange(runResult.InsertedIds); + result.DocumentResponses.AddRange(runResult.DocumentResponses); + } + } + finally + { + semaphore.Release(); + } + }, bulkOperationTimeoutToken)); } - })); - } - await Task.WhenAll(tasks); - return result; + await Task.WhenAll(tasks).WithCancellation(bulkOperationTimeoutToken); + } + catch (OperationCanceledException) + { + var innerException = new TimeoutException($"InsertMany operation timed out after {timeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter."); + throw new BulkOperationException(innerException, result); + } + catch (Exception ex) + { + throw new BulkOperationException(ex, result); + } + + return result; + } } private async Task RunInsertManyAsync(IEnumerable rows, bool insertOrdered, bool returnDocumentResponses, CommandOptions commandOptions, bool runSynchronously) @@ -331,21 +382,36 @@ private async Task RunInsertManyAsync(IEnumerable rows return response.Result; } + /// + /// Synchronous version of + /// + /// public TableInsertManyResult InsertOne(T row) { return InsertOne(row, null); } + /// + /// Synchronous version of + /// + /// public TableInsertManyResult InsertOne(T row, CommandOptions commandOptions) { return InsertOneAsync(row, commandOptions, true).ResultSync(); } + /// + /// Insert a single row into the table. + /// + /// + /// public Task InsertOneAsync(T row) { return InsertOneAsync(row, null); } + /// + /// public Task InsertOneAsync(T row, CommandOptions commandOptions) { return InsertOneAsync(row, commandOptions, false); @@ -363,21 +429,80 @@ private async Task InsertOneAsync(T row, CommandOptions c return response.Result; } + /// + /// Find rows in the table. + /// + /// The Find() methods return a object that can be used to further structure the query + /// by adding Sort, Projection, Skip, Limit, etc. to affect the final results. + /// + /// The object can be directly enumerated both synchronously and asynchronously. + /// Secondarily, the results can be paged through manually by using the results of . + /// + /// + /// + /// Synchronous Enumeration: + /// + /// var FindEnumerator = table.Find(); + /// foreach (var row in FindEnumerator) + /// { + /// // Process row + /// } + /// + /// + /// + /// Asynchronous Enumeration: + /// + /// var results = table.Find(); + /// await foreach (var row in results) + /// { + /// // Process row + /// } + /// + /// + /// + /// Timeouts passed in the ( + /// and ) will be used for each batched request to the API, + /// however settings are ignored due to the nature of Enueration. + /// If you need to enforce a timeout for the entire operation, you can pass a to GetAsyncEnumerator. + /// public FindEnumerator> Find() { return Find(null, null); } + /// + /// The filter(s) to apply to the query. + /// + /// + /// + /// var filterBuilder = Builders.Filter; + /// var filter = filterBuilder.Gt(x => x.NumberOfPages, 430); + /// var matchingBooks = table.Find(filter).ToList(); + /// await foreach (var bookRow in matchingBooks) + /// { + /// //handle each row + /// } + /// + /// public FindEnumerator> Find(Filter filter) { return Find(filter, null); } + /// + /// public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) { return Find(filter, commandOptions); } + /// + /// + /// + /// + /// This overload of Find() allows you to specify a different result class type + /// which the resultant rows will be deserialized into. This is generally used along with .Project() to limit the fields returned + /// public FindEnumerator> Find(Filter filter, CommandOptions commandOptions) where TResult : class { var findOptions = new TableFindManyOptions @@ -481,6 +606,10 @@ private CommandOptions SetRowSerializationOptions(CommandOptions comman commandOptions ??= new CommandOptions(); commandOptions.SerializeGuidAsDollarUuid = false; commandOptions.SerializeDateAsDollarDate = false; + if (typeof(TResult) == typeof(Row)) + { + return commandOptions; + } if (isInsert) { commandOptions.InputConverter = new RowConverter(); @@ -730,12 +859,30 @@ internal async Task DeleteManyAsync(Filter filter, CommandOptio var keepProcessing = true; var deleteResult = new DeleteResult(); - while (keepProcessing) + var (timeout, cts) = BulkOperationHelper.InitTimeout(GetOptionsTree(), ref commandOptions); + + using (cts) { - var command = CreateCommand("deleteMany").WithPayload(deleteOptions).AddCommandOptions(commandOptions); - var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); - deleteResult.DeletedCount += response.Result.DeletedCount; - keepProcessing = response.Result.MoreData; + var bulkOperationTimeoutToken = cts.Token; + try + { + while (keepProcessing) + { + var command = CreateCommand("deleteMany").WithPayload(deleteOptions).AddCommandOptions(commandOptions); + var response = await command.RunAsyncReturnStatus(runSynchronously).ConfigureAwait(false); + deleteResult.DeletedCount += response.Result.DeletedCount; + keepProcessing = response.Result.MoreData; + } + } + catch (OperationCanceledException) + { + var innerException = new TimeoutException($"DeleteMany operation timed out after {timeout.TotalSeconds} seconds. Consider increasing the timeout using the CommandOptions.TimeoutOptions.BulkOperationTimeout parameter."); + throw new BulkOperationException(innerException, deleteResult); + } + catch (Exception ex) + { + throw new BulkOperationException(ex, deleteResult); + } } return deleteResult; @@ -809,9 +956,15 @@ internal async Task> AlterAsync(IAlterTableOperation ope return result.Result; } + private List GetOptionsTree() + { + var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }); + return optionsTree.ToList(); + } + internal Command CreateCommand(string name) { - var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }).ToArray(); + var optionsTree = GetOptionsTree().ToArray(); return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _tableName)); } diff --git a/src/DataStax.AstraDB.DataApi/Tables/TableDefinition.cs b/src/DataStax.AstraDB.DataApi/Tables/TableDefinition.cs index 1ec6cc8..c27a4a9 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/TableDefinition.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/TableDefinition.cs @@ -379,19 +379,49 @@ internal class ColumnTypeConstants public static class TableDefinitionExtensions { - //NOTE: not thinking that this is a good idea because certain column types need additional properties - // public static TableDefinition AddColumn(this TableDefinition tableDefinition, string columnName, ColumnTypes columnType) - // { - // tableDefinition.Columns.Add(columnName, new { type = columnType }); - // return tableDefinition; - // } - public static TableDefinition AddPrimaryKey(this TableDefinition tableDefinition, string keyName) + + public static TableDefinition AddSinglePrimaryKey(this TableDefinition tableDefinition, string keyName) { return tableDefinition.AddCompoundPrimaryKey(keyName, 1); } + public static TableDefinition AddCompositePrimaryKey(this TableDefinition tableDefinition, string[] keyNames) + { + if (keyNames == null || keyNames.Length == 0) + { + throw new ArgumentException("Key names cannot be null or empty.", nameof(keyNames)); + } + + var primaryKey = tableDefinition.PrimaryKey ?? new PrimaryKeyDefinition(); + for (int i = 0; i < keyNames.Length; i++) + { + primaryKey.KeyList.Add(i + 1, keyNames[i]); + } + tableDefinition.PrimaryKey = primaryKey; + return tableDefinition; + } + + public static TableDefinition AddCompoundPrimaryKey(this TableDefinition tableDefinition, string[] keyNames, PrimaryKeySort[] partitionSorts) + { + if (keyNames == null || keyNames.Length == 0) + { + throw new ArgumentException("Key names cannot be null or empty.", nameof(keyNames)); + } + + var primaryKey = tableDefinition.PrimaryKey ?? new PrimaryKeyDefinition(); + for (int i = 0; i < keyNames.Length; i++) + { + primaryKey.KeyList.Add(i + 1, keyNames[i]); + } + for (int i = 0; i < partitionSorts.Length; i++) + { + primaryKey.SortList.Add(i + 1, partitionSorts[i]); + } + tableDefinition.PrimaryKey = primaryKey; + return tableDefinition; + } - public static TableDefinition AddCompoundPrimaryKey(this TableDefinition tableDefinition, string keyName, int order) + internal static TableDefinition AddCompoundPrimaryKey(this TableDefinition tableDefinition, string keyName, int order) { var primaryKey = tableDefinition.PrimaryKey ?? new PrimaryKeyDefinition(); primaryKey.KeyList.Add(order, keyName); @@ -399,7 +429,7 @@ public static TableDefinition AddCompoundPrimaryKey(this TableDefinition tableDe return tableDefinition; } - public static TableDefinition AddCompoundPrimaryKeySort(this TableDefinition tableDefinition, string keyName, int keyOrder, SortDirection direction) + internal static TableDefinition AddCompoundPrimaryKeySort(this TableDefinition tableDefinition, string keyName, int keyOrder, SortDirection direction) { var primaryKey = tableDefinition.PrimaryKey ?? new PrimaryKeyDefinition(); primaryKey.SortList.Add(keyOrder, new PrimaryKeySort(keyName, direction)); diff --git a/src/DataStax.AstraDB.DataApi/Tables/TableInsertManyResult.cs b/src/DataStax.AstraDB.DataApi/Tables/TableInsertManyResult.cs index f56aed2..8775688 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/TableInsertManyResult.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/TableInsertManyResult.cs @@ -47,49 +47,4 @@ public class TableInsertManyResult /// [JsonIgnore] public int InsertedCount => InsertedIds.Count != 0 ? InsertedIds.Count : DocumentResponses.Count; -} - -/* -{ - "status": { - "primaryKeySchema": { - "email": { - "type": "ascii" - }, - "graduation_year": { - "type": "int" - } - }, - "insertedIds": [ - [ - "tal@example.com", - 2024 - ], - [ - "sami@example.com", - 2024 - ], - [ - "kiran@example.com", - 2024 - ] - ] - } -} - -// document response -{ - "status": { - "primaryKeySchema": { - "email": { - "type": "ascii" - } - }, - "documentResponses": [ - {"_id":["tal@example.com"], "status":"OK"}, - {"_id":["sami@example.com"], "status":"OK"}, - {"_id":["kirin@example.com"], "status":"OK"} - ] - } -} -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Utils/BulkOperationHelper.cs b/src/DataStax.AstraDB.DataApi/Utils/BulkOperationHelper.cs new file mode 100644 index 0000000..8ee78c4 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Utils/BulkOperationHelper.cs @@ -0,0 +1,64 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace DataStax.AstraDB.DataApi.Utils; + +/// +/// Helper class for setting up bulk operation timeouts and cancellation tokens. +/// +internal static class BulkOperationHelper +{ + /// + /// Sets up bulk operation timeout and cancellation token for the given command options. + /// + /// The tree of command options to merge. + /// The command options to configure with timeout. + /// A tuple containing the timeout duration and the cancellation token source. + internal static (TimeSpan timeout, CancellationTokenSource cts) InitTimeout( + List optionsTree, + ref CommandOptions commandOptions) + { + if (commandOptions != null) + { + optionsTree.Add(commandOptions); + } + var mergedOptions = CommandOptions.Merge(optionsTree.ToArray()); + var timeout = new TimeoutManager().GetBulkOperationTimeout(mergedOptions); + var cts = new CancellationTokenSource(timeout); + var bulkOperationTimeoutToken = cts.Token; + + if (commandOptions == null) + { + commandOptions = new CommandOptions + { + BulkOperationCancellationToken = bulkOperationTimeoutToken + }; + } + else + { + commandOptions.BulkOperationCancellationToken = bulkOperationTimeoutToken; + } + + return (timeout, cts); + } + +} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs b/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs index 2acf4d7..ba02862 100644 --- a/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs +++ b/src/DataStax.AstraDB.DataApi/Utils/Extensions.cs @@ -23,6 +23,8 @@ using System.Reflection; using System.Text; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; namespace DataStax.AstraDB.DataApi.Utils; @@ -103,4 +105,14 @@ private static void BuildPropertyName(MemberExpression memberExpression, StringB sb.Append(name); } + internal static async Task WithCancellation(this Task task, CancellationToken cancellationToken) + { + var tcs = new TaskCompletionSource(); + using (cancellationToken.Register(s => ((TaskCompletionSource)s).TrySetCanceled(), tcs)) + { + var completedTask = await Task.WhenAny(task, tcs.Task); + await completedTask; + } + } + } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/AssemblyInfo.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/AssemblyInfo.cs new file mode 100644 index 0000000..bc8df96 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/AssemblyInfo.cs @@ -0,0 +1 @@ +[assembly: Xunit.AssemblyFixture(typeof(DataStax.AstraDB.DataApi.IntegrationTests.Fixtures.AssemblyFixture))] diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj b/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj index ea94ae1..4454d78 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/DataStax.AstraDB.DataApi.IntegrationTests.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs index 39a46d3..cf35cc6 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AdminFixture.cs @@ -1,54 +1,31 @@ using DataStax.AstraDB.DataApi.Admin; using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using System.Text.RegularExpressions; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; -public class AdminFixture : IDisposable +public class AdminFixture : BaseFixture { - public AdminFixture() - { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["ADMINTOKEN"] ?? configuration["AstraDB:AdminToken"]; - var dbUrl = configuration["URL"]; - DatabaseUrl = dbUrl; - DatabaseName = configuration["DATABASE_NAME"]; - - _databaseId = GetDatabaseIdFromUrl(dbUrl) ?? throw new Exception("Database ID could not be extracted from ASTRA_DB_URL."); - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/admin_tests_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); + public string DatabaseName { get; set; } + public Guid DatabaseId { get; set; } - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); + public AdminFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "admin") + { + DatabaseName = assemblyFixture.DatabaseName; + DatabaseId = GetDatabaseIdFromUrl(assemblyFixture.DatabaseUrl).Value; } - public void Dispose() + public DatabaseAdminAstra CreateAdmin(Database database = null) { - // ... clean up test data from the database ... - } + database ??= Database; - private readonly Guid _databaseId; - public Guid DatabaseId => _databaseId; - public string DatabaseName { get; private set; } - public DataApiClient Client { get; private set; } - public string DatabaseUrl { get; private set; } - public Database Database { get; private set; } + var adminOptions = new CommandOptions + { + Token = Client.ClientOptions.Token, + Environment = DBEnvironment.Production // or default + }; - public Database GetDatabase() - { - return Client.GetDatabase(DatabaseUrl); + return new DatabaseAdminAstra(database, Client, adminOptions); } public static Guid? GetDatabaseIdFromUrl(string url) @@ -61,17 +38,4 @@ public Database GetDatabase() return match.Success ? Guid.Parse(match.Value) : null; } - public DatabaseAdminAstra CreateAdmin(Database database = null) - { - database ??= Database; - - var adminOptions = new CommandOptions - { - Token = Client.ClientOptions.Token, - Environment = DBEnvironment.Production // or default - }; - - return new DatabaseAdminAstra(database, Client, adminOptions); - } - } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AssemblyFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AssemblyFixture.cs new file mode 100644 index 0000000..df482fd --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/AssemblyFixture.cs @@ -0,0 +1,47 @@ +using DataStax.AstraDB.DataApi.Admin; +using DataStax.AstraDB.DataApi.Core; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; + +public class AssemblyFixture +{ + public string Token { get; private set; } + public string AdminToken { get; private set; } + public string OpenAiApiKey { get; private set; } + + public string DatabaseUrl { get; private set; } + public string DatabaseName { get; private set; } + + public AssemblyFixture() + { + IConfiguration configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables(prefix: "ASTRA_DB_") + .Build(); + + Token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; + AdminToken = configuration["ADMINTOKEN"] ?? configuration["AstraDB:AdminToken"]; + OpenAiApiKey = configuration["OPENAI_APIKEYNAME"] ?? configuration["AstraDB:OpenAiApiKey"]; + DatabaseName = configuration["DATABASE_NAME"] ?? configuration["AstraDB:DatabaseName"]; + DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:Url"]; + } + + public DataApiClient CreateApiClient(string fixtureName, bool useToken = true) + { + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger($"../../../_logs/{fixtureName}_fixture_latest_run.log")); + ILogger logger = factory.CreateLogger(fixtureName); + + var clientOptions = new CommandOptions + { + RunMode = RunMode.Debug + }; + + return new DataApiClient(useToken ? Token : null, clientOptions, logger); + } + +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/BaseFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/BaseFixture.cs new file mode 100644 index 0000000..693b673 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/BaseFixture.cs @@ -0,0 +1,25 @@ +using DataStax.AstraDB.DataApi.Core; + +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; + +public class BaseFixture +{ + private readonly AssemblyFixture _assemblyFixture; + + public DataApiClient Client { get; set; } + public Database Database { get; set; } + public DataApiClient ClientWithoutToken { get; set; } + public string DatabaseUrl { get; set; } + public string Token { get; set; } + + public BaseFixture(AssemblyFixture assemblyFixture, string fixtureName) + { + _assemblyFixture = assemblyFixture; + Client = _assemblyFixture.CreateApiClient(fixtureName); + DatabaseUrl = _assemblyFixture.DatabaseUrl; + Database = Client.GetDatabase(DatabaseUrl); + ClientWithoutToken = _assemblyFixture.CreateApiClient(fixtureName, false); + Token = _assemblyFixture.Token; + } + +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs index a34b071..a7fd708 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionsFixture.cs @@ -1,69 +1,34 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("DatabaseAndCollections")] -public class DatabaseAndCollectionsCollection : ICollectionFixture +public class DatabaseAndCollectionsCollection : ICollectionFixture, ICollectionFixture { - } -public class CollectionsFixture : IDisposable, IAsyncLifetime +public class CollectionsFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public DataApiClient ClientWithoutToken { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - public string DatabaseUrl { get; set; } - public string Token { get; set; } - - public CollectionsFixture() + public CollectionsFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "collections") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - Token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/collections_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(Token, clientOptions, logger); - var noTokenClientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - ClientWithoutToken = new DataApiClient(null, noTokenClientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); } - public async Task InitializeAsync() + public Collection SearchCollection { get; private set; } + + public async ValueTask InitializeAsync() { await CreateSearchCollection(); var collection = Database.GetCollection(_queryCollectionName); SearchCollection = collection; } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropCollectionAsync(_queryCollectionName); } - public Collection SearchCollection { get; private set; } - - private const string _queryCollectionName = "simpleObjectsQueryTests"; private async Task CreateSearchCollection() { @@ -186,8 +151,4 @@ private async Task CreateSearchCollection() SearchCollection = collection; } - public void Dispose() - { - //nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs index a13a4e2..e96f0e6 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/DatabaseFixture.cs @@ -1,51 +1,17 @@ using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Database")] -public class Databaseollection : ICollectionFixture +public class DatabaseCollection : ICollectionFixture, ICollectionFixture { } -public class DatabaseFixture +public class DatabaseFixture : BaseFixture { - public DataApiClient Client { get; private set; } - public DataApiClient ClientWithoutToken { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - public string DatabaseUrl { get; set; } - public string Token { get; set; } - - public DatabaseFixture() + public DatabaseFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "database") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - Token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/database_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(Token, clientOptions, logger); - var clientWithoutTokenOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - ClientWithoutToken = new DataApiClient(null, clientWithoutTokenOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); } - } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs index 057c004..1713eb2 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/FindAndUpdateFixture.cs @@ -1,61 +1,34 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("FindAndUpdate")] -public class FindAndUpdateCollection : ICollectionFixture +public class FindAndUpdateCollection : ICollectionFixture, ICollectionFixture { - } -public class FindAndUpdateFixture : IDisposable, IAsyncLifetime +public class FindAndUpdateFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - - public FindAndUpdateFixture() + public FindAndUpdateFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "findAndUpdate") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/findandupdate_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(databaseUrl); } - public async Task InitializeAsync() + public Collection UpdatesCollection { get; private set; } + + public async ValueTask InitializeAsync() { await CreateUpdatesCollection(); var collection = Database.GetCollection(_queryCollectionName); UpdatesCollection = collection; } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropCollectionAsync(_queryCollectionName); } - public Collection UpdatesCollection { get; private set; } - - private const string _queryCollectionName = "findAndUpdateCollection"; private async Task CreateUpdatesCollection() { @@ -151,8 +124,4 @@ internal async Task> CreateUpdatesCollection(string col return collection; } - public void Dispose() - { - //nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs index 8d7bedc..5d3e39d 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/ReplaceAndDeleteFixture.cs @@ -1,61 +1,34 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("ReplaceAndDelete")] -public class ReplaceAndDeleteCollection : ICollectionFixture +public class ReplaceAndDeleteCollection : ICollectionFixture, ICollectionFixture { - } -public class ReplaceAndDeleteFixture : IDisposable, IAsyncLifetime +public class ReplaceAndDeleteFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - - public ReplaceAndDeleteFixture() + public ReplaceAndDeleteFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "replaceAndDelete") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/replace_and_delete_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(databaseUrl); } - public async Task InitializeAsync() + public Collection ReplaceCollection { get; private set; } + + public async ValueTask InitializeAsync() { await CreateReplaceCollection(); var collection = Database.GetCollection(_queryCollectionName); ReplaceCollection = collection; } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropCollectionAsync(_queryCollectionName); } - public Collection ReplaceCollection { get; private set; } - - private const string _queryCollectionName = "replaceCollection"; private async Task CreateReplaceCollection() { @@ -151,8 +124,4 @@ internal async Task> CreateReplaceCollection(string col return collection; } - public void Dispose() - { - //nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs index 935c307..0cae55e 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/RerankFixture.cs @@ -1,61 +1,35 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.SerDes; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using System.Text.Json.Serialization; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("RerankCollection")] -public class RerankCollection : ICollectionFixture +public class RerankCollection : ICollectionFixture, ICollectionFixture { } -public class RerankFixture : IDisposable, IAsyncLifetime +public class RerankFixture : BaseFixture, IAsyncLifetime { private const string _queryCollectionName = "rerankQueryTests"; - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - public string DatabaseUrl { get; set; } - public string Token { get; set; } public Collection HybridSearchCollection { get; private set; } - public RerankFixture() + public RerankFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "rerank") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - Token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/rerank_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(Token, clientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); } - public async Task InitializeAsync() + public async ValueTask InitializeAsync() { await CreateSearchCollection(); var collection = Database.GetCollection(_queryCollectionName); HybridSearchCollection = collection; } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropCollectionAsync(_queryCollectionName); } @@ -136,10 +110,6 @@ private async Task CreateSearchCollection() HybridSearchCollection = collection; } - public void Dispose() - { - //nothing needed - } } public class HybridSearchTestObject diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs index c6ee940..fb5ebbc 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableAlterFixture.cs @@ -1,47 +1,22 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("TableAlter")] -public class TableAlterCollection : ICollectionFixture +public class TableAlterCollection : ICollectionFixture, ICollectionFixture { } -public class TableAlterFixture +public class TableAlterFixture : BaseFixture { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string DatabaseUrl { get; set; } - - public TableAlterFixture() + public TableAlterFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "tableAlter") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/table_Alter_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); - try { - var keyspaces = Database.GetAdmin().ListKeyspaceNames(); + var keyspaces = Database.GetAdmin().ListKeyspaces(); Console.WriteLine($"[Fixture] Connected. Keyspaces found: {keyspaces.Count()}"); } catch (Exception ex) @@ -49,7 +24,6 @@ public TableAlterFixture() Console.WriteLine($"[Fixture] Connection failed: {ex.Message}"); throw; } - } public async Task> CreateTestTable(string tableName) diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs index cc4a0c3..82e141a 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TableIndexesFixture.cs @@ -1,47 +1,21 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("TableIndexes")] -public class TableIndexesCollection : ICollectionFixture +public class TableIndexesCollection : ICollectionFixture, ICollectionFixture { - } -public class TableIndexesFixture : IDisposable, IAsyncLifetime +public class TableIndexesFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string DatabaseUrl { get; set; } - - public TableIndexesFixture() + public TableIndexesFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "tableIndexes") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/table_indexes_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); - try { - var keyspaces = Database.GetAdmin().ListKeyspaceNames(); + var keyspaces = Database.GetAdmin().ListKeyspaces(); Console.WriteLine($"[Fixture] Connected. Keyspaces found: {keyspaces.Count()}"); } catch (Exception ex) @@ -49,22 +23,20 @@ public TableIndexesFixture() Console.WriteLine($"[Fixture] Connection failed: {ex.Message}"); throw; } - } - public async Task InitializeAsync() + public Table FixtureTestTable { get; private set; } + + public async ValueTask InitializeAsync() { await CreateTestTable(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropTableAsync(_fixtureTableName); } - public Table FixtureTestTable { get; private set; } - - private const string _fixtureTableName = "tableIndexesTest"; private async Task CreateTestTable() { @@ -105,8 +77,4 @@ private async Task CreateTestTable() FixtureTestTable = table; } - public void Dispose() - { - // nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs index 1b3cc5b..562e67c 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/TablesFixture.cs @@ -1,61 +1,48 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Tables; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Tables")] -public class TablesCollection : ICollectionFixture +public class TablesCollection : ICollectionFixture, ICollectionFixture { - } -public class TablesFixture : IDisposable, IAsyncLifetime +public class TablesFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string DatabaseUrl { get; set; } - - public TablesFixture() + public TablesFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "tables") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/tables_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(DatabaseUrl); } - public async Task InitializeAsync() + public Table SearchTable { get; private set; } + public Table DeleteTable { get; private set; } + public Table UntypedTableSinglePrimaryKey { get; private set; } + public Table UntypedTableCompoundPrimaryKey { get; private set; } + public Table UntypedTableCompositePrimaryKey { get; private set; } + + public async ValueTask InitializeAsync() { await CreateSearchTable(); await CreateDeleteTable(); + await CreateTestTablesNotTyped(); } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropTableAsync(_queryTableName); await Database.DropTableAsync(_deleteTableName); + await Database.DropTableAsync(_untypedSinglePkTableName); + await Database.DropTableAsync(_untypedCompositePkTableName); + await Database.DropTableAsync(_untypedCompoundPkTableName); } - public Table SearchTable { get; private set; } - public Table DeleteTable { get; private set; } - private const string _queryTableName = "tableQueryTests"; + private const string _untypedSinglePkTableName = "tableNotTyped_SinglePrimaryKey"; + private const string _untypedCompoundPkTableName = "tableNotTyped_CompoundPrimaryKey"; + private const string _untypedCompositePkTableName = "tableNotTyped_CompositePrimaryKey"; + private const string _deleteTableName = "tableDeleteTests"; + private async Task CreateSearchTable() { try @@ -129,7 +116,130 @@ await table.CreateIndexAsync(new TableIndex() } } - private const string _deleteTableName = "tableDeleteTests"; + private async Task CreateTestTablesNotTyped() + { + // Create a table with a single primary key + var createDefinition = new TableDefinition() + .AddIntColumn("Id") + .AddTextColumn("IdTwo") + .AddTextColumn("Name") + .AddTextColumn("SortOneAscending") + .AddTextColumn("SortTwoDescending") + .AddVectorizeColumn("Vectorize", 1024, new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + }) + .AddVectorColumn("Vector", 384) + .AddSinglePrimaryKey("Id"); + + UntypedTableSinglePrimaryKey = await Database.CreateTableAsync(_untypedSinglePkTableName, createDefinition); + await UntypedTableSinglePrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "vectorize_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vectorize", + } + }); + await UntypedTableSinglePrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "vector_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vector", + } + }); + + // Create a table with a composite primary key + createDefinition = new TableDefinition() + .AddIntColumn("Id") + .AddTextColumn("IdTwo") + .AddTextColumn("Name") + .AddTextColumn("SortOneAscending") + .AddTextColumn("SortTwoDescending") + .AddVectorizeColumn("Vectorize", 1024, new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + }) + .AddVectorColumn("Vector", 384) + .AddCompositePrimaryKey(new[] { "Id", "IdTwo" }); + UntypedTableCompositePrimaryKey = await Database.CreateTableAsync(_untypedCompositePkTableName, createDefinition); + await UntypedTableCompositePrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "composite_vectorize_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vectorize", + } + }); + await UntypedTableCompositePrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "composite_vector_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vector", + } + }); + + // Create a table with a compound primary key + createDefinition = new TableDefinition() + .AddIntColumn("Id") + .AddTextColumn("IdTwo") + .AddTextColumn("Name") + .AddTextColumn("SortOneAscending") + .AddTextColumn("SortTwoDescending") + .AddVectorizeColumn("Vectorize", 1024, new VectorServiceOptions + { + Provider = "nvidia", + ModelName = "NV-Embed-QA" + }) + .AddVectorColumn("Vector", 384) + .AddCompoundPrimaryKey(new[] { "Id", "IdTwo" }, new[] + { + new PrimaryKeySort("SortOneAscending", SortDirection.Ascending), + new PrimaryKeySort("SortTwoDescending", SortDirection.Descending) + }); + UntypedTableCompoundPrimaryKey = await Database.CreateTableAsync(_untypedCompoundPkTableName, createDefinition); + await UntypedTableCompoundPrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "compound_vectorize_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vectorize", + } + }); + await UntypedTableCompoundPrimaryKey.CreateVectorIndexAsync(new TableVectorIndex() + { + IndexName = "compound_vector_index", + Definition = new TableVectorIndexDefinition() + { + ColumnName = "Vector", + } + }); + + // Populate untyped tables with sample data + List rows = new List(); + for (var i = 0; i < 50; i++) + { + var rowSingle = new Row + { + {"Id", i}, + {"IdTwo", "IdTwo_" + i}, + {"Name", "Name_" + i}, + {"SortOneAscending", "SortOne_" + i}, + {"SortTwoDescending", "SortTwo_" + (50 - i)}, + {"Vectorize", "String To Vectorize " + i}, + {"Vector", new double[] { -0.053202216, 0.01422119, 0.007062546, 0.0685742, -0.07858203, 0.010138983, 0.10238025, -0.012096751, 0.09522599, -0.030270875, 0.002181861, -0.064782545, -0.0026875706, 0.0060957014, -0.003964779, -0.030604681, -0.047901124, -0.019261848, -0.059947517, -0.10413115, -0.08611966, 0.03632282, -0.025586247, 0.0017129881, -0.07146128, 0.061734077, 0.017160414, -0.05659205, 0.0248427, -0.07782747, -0.032485314, -0.008684083, -0.011535832, 0.038153064, -0.057013486, -0.053252906, 0.004985692, 0.032392446, 0.0725966, 0.032940567, 0.024707653, -0.083363794, -0.015673108, -0.04811024, -0.003449794, 0.004415103, -0.035913676, -0.051946636, 0.015592655, 0.0035385543, -0.010283442, 0.047748506, -0.040175628, -0.009133693, -0.03460812, -0.03693011, -0.04091714, 0.0176677, -0.00934914, -0.053623937, 0.011154383, 0.016148455, 0.013840816, 0.028249927, 0.04024405, 0.02096661, -0.014487404, -0.0016292258, -0.004891051, 0.012042645, 0.04556029, 0.0130860545, 0.070578784, -0.03086842, 0.030368855, -0.10848343, 0.05554082, -0.017487692, 0.16430159, 0.051410932, -0.027641848, -0.029989198, -0.057063058, 0.056793693, 0.050923523, 0.015136637, -0.0012497514, 0.02384801, -0.06327192, 0.028891006, -0.055418354, -0.03496716, 0.03029518, 0.026919777, -0.08353811, 0.018368296, -0.03516996, -0.08284338, -0.07195326, 0.19801475, 0.016410688, 0.0445346, -0.003741409, -0.038506165, 0.053398475, -0.0034389244, -0.04352991, 0.06336845, -0.013076868, -0.019743098, -0.045236666, 0.020782078, -0.056481004, 0.057446502, 0.055468243, 0.021229729, -0.100917056, -0.03422642, 0.02944804, -0.03325292, 0.028943142, 0.030092051, -0.051856354, 0.008190983, -0.016726157, -0.08435183, 0.011159818, -5.9255234e-33, 0.030620761, -0.085034214, 0.0028181712, -0.041073505, -0.042798948, 0.041067425, 0.029467635, 0.036486518, -0.12122617, 0.013526328, -0.01391842, 0.0312512, -0.021689802, 0.01621624, 0.11224023, -0.006686669, -0.0018879274, 0.05318519, 0.03250415, -0.03782473, -0.046973582, 0.061971873, 0.063630275, 0.050121382, -0.007621213, -0.021432782, -0.03779708, -0.08284233, -0.026234223, 0.036130365, 0.041241154, 0.014499247, 0.073483825, 0.00073006714, -0.081418164, -0.055791657, -0.04209736, -0.096603446, -0.040196676, 0.028519753, 0.12910499, 0.010470544, 0.025057316, 0.01734334, -0.02719573, -0.0049704155, 0.015811851, 0.03439927, -0.044550493, 0.020814221, 0.027571082, -0.014297911, 0.028702551, -0.021064728, 0.008865078, 0.009936881, 0.0029201612, -0.023835903, 0.012977942, 0.06633931, 0.068944834, 0.082585804, 0.008766892, -0.013999867, 0.09115506, -0.122037254, -0.045294352, -0.018009886, -0.022158505, 0.02152304, -0.03885241, -0.019468945, 0.07964807, -0.015691828, 0.06885623, -0.015452343, 0.022757484, 0.025256434, -0.03119467, -0.033447854, -0.021564618, -0.010073421, 0.0055514527, 0.048961196, -0.021559088, 0.06377866, -0.019740583, -0.030324804, 0.0062891715, 0.045206502, -0.045785706, -0.049080465, 0.087099895, 0.027371299, 0.09064848, 3.433169e-33, 0.06266184, 0.028918529, 0.000108557906, 0.09145542, -0.030282516, 0.0048763165, -0.02540525, 0.066567004, -0.034166507, 0.047780972, -0.03424499, 0.007805756, 0.10785121, 0.008996277, 0.0076608267, 0.08868162, 0.0036972803, -0.030516094, 0.02168669, -0.004358315, -0.14477515, 0.011545589, 0.018421879, -0.025913069, -0.05191015, 0.03943329, 0.037553225, -0.0147632975, -0.022263186, -0.048638437, -0.0065658195, -0.039633695, -0.041322067, -0.02844163, 0.010661134, 0.15864708, 0.04770698, -0.04730114, -0.06286664, 0.008440104, 0.059898064, 0.019403962, -0.03227739, 0.11167067, 0.016108502, 0.052688885, -0.017888643, -0.0058668335, 0.052891612, 0.018419184, -0.04730259, -0.014312523, 0.030081172, -0.07333967, -0.012648647, 0.004494484, -0.09500656, 0.018896673, -0.029087285, -0.0051991083, -0.0029317876, 0.069698535, 0.012463835, 0.1219864, -0.10485225, -0.05362739, -0.0128166545, -0.027964052, 0.05004069, -0.07638481, 0.024308309, 0.04531832, -0.029027926, 0.010168302, -0.010628256, 0.030930692, -0.046634875, 0.0045742486, 0.007714686, -0.0063424213, -0.07790265, -0.06532262, -0.047622908, 0.010272605, -0.056622025, -0.011285954, 0.0020759962, 0.06382898, -0.013343911, -0.03008575, -0.009862737, 0.054995734, -0.021704284, -0.05336612, -0.02860762, -1.3317537e-8, -0.028604865, -0.029213138, -0.04298399, -0.019619852, 0.09963344, 0.0694588, -0.030038442, -0.0401437, -0.006644881, 0.026138376, 0.044374008, -0.01637589, -0.06998592, 0.013482148, 0.04653866, -0.0153024765, -0.053351574, 0.039734483, 0.06283631, 0.07712063, -0.050968867, 0.03027798, 0.055424906, 0.0023063482, -0.051206734, -0.035924364, 0.04564326, 0.106056266, -0.08215607, 0.038128633, -0.022592563, 0.14054875, -0.07613521, -0.03006324, -0.0040755956, -0.06966433, 0.07610892, -0.07929878, 0.024970463, 0.03414342, 0.050462823, 0.15209967, -0.020093411, -0.079005316, -0.0006247459, 0.062248245, 0.026453331, -0.12163222, -0.028260367, -0.056446116, -0.09818232, -0.0074948515, 0.027907023, 0.06908376, 0.014955464, 0.005030419, -0.0131421015, -0.047915705, -0.01678274, 0.03665314, 0.1114189, 0.029845735, 0.02391984, 0.110152245 }} + }; + rows.Add(rowSingle); + } + await UntypedTableSinglePrimaryKey.InsertManyAsync(rows); + await UntypedTableCompositePrimaryKey.InsertManyAsync(rows); + await UntypedTableCompoundPrimaryKey.InsertManyAsync(rows); + } + private async Task CreateDeleteTable() { var rows = new List(); @@ -192,8 +302,4 @@ await table.CreateIndexAsync(new TableIndex() DeleteTable = table; } - public void Dispose() - { - //nothing needed - } -} \ No newline at end of file +} diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs index db2baca..be3abe0 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/UpdatesFixture.cs @@ -1,61 +1,34 @@ using DataStax.AstraDB.DataApi.Collections; using DataStax.AstraDB.DataApi.Core; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; [CollectionDefinition("Updates")] -public class UpdatesCollection : ICollectionFixture +public class UpdatesCollection : ICollectionFixture, ICollectionFixture { - } -public class UpdatesFixture : IDisposable, IAsyncLifetime +public class UpdatesFixture : BaseFixture, IAsyncLifetime { - public DataApiClient Client { get; private set; } - public Database Database { get; private set; } - public string OpenAiApiKey { get; set; } - - public UpdatesFixture() + public UpdatesFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "updates") { - IConfiguration configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true) - .AddEnvironmentVariables(prefix: "ASTRA_DB_") - .Build(); - - var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; - var databaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; - OpenAiApiKey = configuration["OPENAI_APIKEYNAME"]; - - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../_logs/updates_fixture_latest_run.log")); - ILogger logger = factory.CreateLogger("IntegrationTests"); - - var clientOptions = new CommandOptions - { - RunMode = RunMode.Debug - }; - Client = new DataApiClient(token, clientOptions, logger); - Database = Client.GetDatabase(databaseUrl); } - public async Task InitializeAsync() + public Collection UpdatesCollection { get; private set; } + + public async ValueTask InitializeAsync() { await CreateUpdatesCollection(); var collection = Database.GetCollection(_queryCollectionName); UpdatesCollection = collection; } - public async Task DisposeAsync() + public async ValueTask DisposeAsync() { await Database.DropCollectionAsync(_queryCollectionName); } - public Collection UpdatesCollection { get; private set; } - - private const string _queryCollectionName = "updatesCollection"; private async Task CreateUpdatesCollection() { @@ -151,8 +124,4 @@ internal async Task> CreateUpdatesCollection(string col return collection; } - public void Dispose() - { - //nothing needed - } } \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs index 3e67eae..f04b687 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs @@ -11,7 +11,7 @@ public class AdditionalCollectionTests { DatabaseFixture fixture; - public AdditionalCollectionTests(DatabaseFixture fixture) + public AdditionalCollectionTests(AssemblyFixture assemblyFixture, DatabaseFixture fixture) { this.fixture = fixture; } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs index b1b792d..629c2e3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdminTests.cs @@ -6,7 +6,7 @@ namespace DataStax.AstraDB.DataApi.IntegrationTests; [CollectionDefinition("Admin Collection")] -public class AdminCollection : ICollectionFixture +public class AdminCollection : ICollectionFixture, ICollectionFixture { public const string SkipMessage = "Please read 'How to run these skipped tests'"; } @@ -17,7 +17,7 @@ public class AdminTests { AdminFixture fixture; - public AdminTests(AdminFixture fixture) + public AdminTests(AssemblyFixture assemblyFixture, AdminFixture fixture) { this.fixture = fixture; } @@ -110,26 +110,24 @@ public async Task CheckDatabaseStatus() var dbName = fixture.DatabaseName; var status = await fixture.Client.GetAstraDatabasesAdmin().GetDatabaseStatusAsync(dbName); - Assert.Equal("ACTIVE", status); + Assert.Equal(AstraDatabaseStatus.ACTIVE, status); status = await fixture.Client.GetAstraDatabasesAdmin().GetDatabaseStatusAsync(dbName); - Assert.Equal("ACTIVE", status); + Assert.Equal(AstraDatabaseStatus.ACTIVE, status); } [Fact] public void DatabaseAdminAstra_GetDatabaseAdminAstra() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.Client.GetAstraDatabasesAdmin(); - Assert.IsType(daa); + Assert.IsType(daa); } [Fact] public void DatabaseAdminAstra_GetDatabase() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.CreateAdmin(fixture.Database); Assert.IsType(daa.GetDatabase()); } @@ -137,8 +135,7 @@ public void DatabaseAdminAstra_GetDatabase() [Fact] public void DatabaseAdminAstra_GetApiEndpoint() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.CreateAdmin(fixture.Database); Assert.Equal(fixture.DatabaseId, AdminFixture.GetDatabaseIdFromUrl(daa.GetApiEndpoint())); } @@ -146,13 +143,12 @@ public void DatabaseAdminAstra_GetApiEndpoint() [Fact] public async Task DatabaseAdminAstra_GetKeyspacesList() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.CreateAdmin(fixture.Database); - var names = await daa.ListKeyspaceNamesAsync(); + var names = await daa.ListKeyspacesAsync(); Assert.NotNull(names); - names = daa.ListKeyspaceNames(); + names = daa.ListKeyspaces(); Assert.NotNull(names); var list = names.ToList(); @@ -163,31 +159,29 @@ public async Task DatabaseAdminAstra_GetKeyspacesList() [Fact] public async Task DatabaseAdminAstra_DoesKeyspaceExist() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.CreateAdmin(fixture.Database); - var keyspaceExists = await daa.KeyspaceExistsAsync("default_keyspace"); + var keyspaceExists = await daa.DoesKeyspaceExistAsync("default_keyspace"); Assert.True(keyspaceExists); - keyspaceExists = daa.KeyspaceExists("default_keyspace"); + keyspaceExists = await daa.DoesKeyspaceExistAsync("default_keyspace"); Assert.True(keyspaceExists); } [Fact] public async Task DatabaseAdminAstra_DoesKeyspaceExist_Another() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); - var daa = fixture.CreateAdmin(database); + var daa = fixture.CreateAdmin(fixture.Database); var keyspaceName = "another_keyspace"; try { - var keyspaceExists = await daa.KeyspaceExistsAsync(keyspaceName); + var keyspaceExists = await daa.DoesKeyspaceExistAsync(keyspaceName); if (!keyspaceExists) { await daa.CreateKeyspaceAsync(keyspaceName); - Thread.Sleep(30 * 1000); //wait for keyspace to be created - keyspaceExists = await daa.KeyspaceExistsAsync(keyspaceName); + await Task.Delay(30 * 1000, TestContext.Current.CancellationToken); //wait for keyspace to be created + keyspaceExists = await daa.DoesKeyspaceExistAsync(keyspaceName); } Assert.True(keyspaceExists); } @@ -341,9 +335,8 @@ public async Task DropDatabaseByIdAsync() [Fact(Skip = AdminCollection.SkipMessage)] public async Task DatabaseAdminAstra_CreateKeyspace_ExpectedError() { - var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); var adminOptions = new CommandOptions(); - var daa = new DatabaseAdminAstra(database, fixture.Client, adminOptions); + var daa = fixture.CreateAdmin(fixture.Database); var ex = await Assert.ThrowsAsync( () => daa.CreateKeyspaceAsync("default_keyspace") diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs index 9f6287e..583287b 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionTests.cs @@ -1,12 +1,13 @@ using DataStax.AstraDB.DataApi.Collections; -using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; -using MongoDB.Bson; -using Xunit; +using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; using DataStax.AstraDB.DataApi.Core.Results; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.SerDes; +using MongoDB.Bson; +using System.Text; using UUIDNext; -using DataStax.AstraDB.DataApi.Core; +using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -15,7 +16,7 @@ public class CollectionTests { CollectionsFixture fixture; - public CollectionTests(CollectionsFixture fixture) + public CollectionTests(AssemblyFixture assemblyFixture, CollectionsFixture fixture) { this.fixture = fixture; } @@ -200,6 +201,14 @@ public class DefaultIdUUIDObjectId public string Name { get; set; } } + public class TestByteArray + { + [DocumentId] + public Guid? Id { get; set; } + public string Name { get; set; } + public byte[] Data { get; set; } + } + [Fact] public async Task DefaultId_UUIDV4_FromObject() { @@ -663,5 +672,31 @@ public async Task CountDocuments_MaxCount() } } + [Fact] + public async Task ByteArray_AsDollarBinary() + { + var collectionName = "byteArrayAsDollarBinary"; + try + { + var collection = await fixture.Database.CreateCollectionAsync(collectionName); + var newObject = new TestByteArray() + { + Name = "Test Object 1", + Data = Encoding.UTF8.GetBytes("Test Data") + }; + + var result = await collection.InsertOneAsync(newObject); + + var added = collection.Find(Builders.Filter.Eq(t => t.Name, "Test Object 1")).FirstOrDefault(); + Assert.Equal("Test Object 1", added.Name); + Assert.NotNull(added.Data); + Assert.Equal("Test Data", Encoding.UTF8.GetString(added.Data)); + } + finally + { + await fixture.Database.DropCollectionAsync(collectionName); + } + } + } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs index e80bfb9..8010b56 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/DatabaseTests.cs @@ -15,7 +15,7 @@ public class DatabaseTests { DatabaseFixture fixture; - public DatabaseTests(DatabaseFixture fixture) + public DatabaseTests(AssemblyFixture assemblyFixture, DatabaseFixture fixture) { this.fixture = fixture; } @@ -38,7 +38,7 @@ public async Task KeyspaceTests() //create keyspace await admin.CreateKeyspaceAsync(keyspaceName); - Thread.Sleep(30 * 1000); + await Task.Delay(30 * 1000, TestContext.Current.CancellationToken); //passed-in dboptions should override keyspace and creation should work await fixture.Database.CreateCollectionAsync(collectionName, dbOptions); @@ -56,6 +56,68 @@ public async Task KeyspaceTests() } } + [Fact] + public async Task Create_And_Drop_Keyspace_DoesNotWaitForCompletion() + { + var keyspaceName = "dropAndDoNotWaitKeyspace"; + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); + var admin = database.GetAdmin(); + + try + { + await admin.CreateKeyspaceAsync(keyspaceName, true, false); + var keyspaceExists = await admin.DoesKeyspaceExistAsync(keyspaceName); + Assert.False(keyspaceExists, $"Keyspace '{keyspaceName}' should still be being created."); + + var maxAttempts = 30; + while (!keyspaceExists && maxAttempts > 0) + { + maxAttempts--; + // Wait for the keyspace to be created + await Task.Delay(1000, TestContext.Current.CancellationToken); + keyspaceExists = await admin.DoesKeyspaceExistAsync(keyspaceName); + } + Assert.True(keyspaceExists, $"Keyspace '{keyspaceName}' should exist now."); + + await admin.DropKeyspaceAsync(keyspaceName, false); + keyspaceExists = await admin.DoesKeyspaceExistAsync(keyspaceName); + Assert.True(keyspaceExists, $"Keyspace '{keyspaceName}' should still be being dropped."); + } + catch (Exception) + { + await admin.DropKeyspaceAsync(keyspaceName, true); + } + } + + [Fact] + public async Task Create_And_Drop_Keyspace_WaitsForCompletionWhenRequested() + { + var keyspaceName = "dropAndWaitKeyspace"; + var database = fixture.Client.GetDatabase(fixture.DatabaseUrl); + var admin = database.GetAdmin(); + + try + { + await admin.CreateKeyspaceAsync(keyspaceName, true, true); + var keyspaceExists = await admin.DoesKeyspaceExistAsync(keyspaceName); + Assert.True(keyspaceExists, $"Keyspace '{keyspaceName}' should exist after creation."); + await admin.DropKeyspaceAsync(keyspaceName, true); + keyspaceExists = await admin.DoesKeyspaceExistAsync(keyspaceName); + Assert.False(keyspaceExists, $"Keyspace '{keyspaceName}' should not exist after drop."); + } + finally + { + try + { + await admin.DropKeyspaceAsync(keyspaceName, true); + } + catch (Exception) + { + // Ignore any exceptions during drop, as the keyspace may not exist + } + } + } + [Fact] public async Task PassTokenToDatabase() { @@ -514,7 +576,7 @@ public async Task CreateTable_DataTypesTest_FromDefinition() Provider = "nvidia", ModelName = "NV-Embed-QA" }) - .AddPrimaryKey("Name"); + .AddSinglePrimaryKey("Name"); var table = await fixture.Database.CreateTableAsync(tableName, createDefinition); Assert.NotNull(table); @@ -609,7 +671,7 @@ public async Task CreateTable_CompoundPrimaryKey_FromDefinition() { return; } - await Task.Delay(waitInSeconds * 1000); + await Task.Delay(waitInSeconds * 1000, TestContext.Current.CancellationToken); tryNumber++; } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs index 406cbf0..c7cf215 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ExamplesTests.cs @@ -10,7 +10,7 @@ public class ExamplesTests { DatabaseFixture fixture; - public ExamplesTests(DatabaseFixture fixture) + public ExamplesTests(AssemblyFixture assemblyFixture, DatabaseFixture fixture) { this.fixture = fixture; } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs index d2ffa30..c6787b2 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/FindAndUpdateTests.cs @@ -1,6 +1,6 @@ using DataStax.AstraDB.DataApi.Collections; -using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -10,7 +10,7 @@ public class FindAndUpdateTests { FindAndUpdateFixture fixture; - public FindAndUpdateTests(FindAndUpdateFixture fixture) + public FindAndUpdateTests(AssemblyFixture assemblyFixture, FindAndUpdateFixture fixture) { this.fixture = fixture; } @@ -22,11 +22,9 @@ public async Task FindAndUpdateOne_DefaultReturnsOriginalDocument() var filter = Builders.Filter .Eq(so => so.Name, "Cat"); var updater = Builders.Update; - var combinedUpdate = updater.Combine( - updater.Set(so => so.Properties.PropertyTwo, "CatUpdated"), - updater.Unset("Properties.PropertyOne") - ); - var result = await collection.FindOneAndUpdateAsync(filter, combinedUpdate); + var update = updater.Set(so => so.Properties.PropertyTwo, "CatUpdated") + .Unset("Properties.PropertyOne"); + var result = await collection.FindOneAndUpdateAsync(filter, update); Assert.Equal("cat", result.Properties.PropertyTwo); Assert.NotNull(result.Properties.PropertyOne); } @@ -38,12 +36,10 @@ public async Task FindAndUpdateOne_ReturnsUpdatedDocument() var filter = Builders.Filter .Eq(so => so.Name, "Animal5"); var updater = Builders.Update; - var combinedUpdate = updater.Combine( - updater.Set(so => so.Properties.PropertyTwo, "Animal5Updated"), - updater.Unset("Properties.PropertyOne") - ); + var update = updater.Set(so => so.Properties.PropertyTwo, "Animal5Updated") + .Unset("Properties.PropertyOne"); var options = new FindOneAndUpdateOptions { ReturnDocument = ReturnDocumentDirective.After }; - var result = await collection.FindOneAndUpdateAsync(filter, combinedUpdate, options); + var result = await collection.FindOneAndUpdateAsync(filter, update, options); Assert.Equal("Animal5Updated", result.Properties.PropertyTwo); Assert.Null(result.Properties.PropertyOne); } @@ -55,13 +51,11 @@ public async Task FindAndUpdateOne_WithInclusiveProjection() var filter = Builders.Filter .Eq(so => so.Name, "Horse"); var updater = Builders.Update; - var combinedUpdate = updater.Combine( - updater.Set(so => so.Properties.PropertyTwo, "HorseUpdated") - ); + var update = updater.Set(so => so.Properties.PropertyTwo, "HorseUpdated"); var inclusiveProjection = Builders.Projection .Include("Properties.PropertyTwo"); var options = new FindOneAndUpdateOptions { ReturnDocument = ReturnDocumentDirective.After, Projection = inclusiveProjection }; - var result = await collection.FindOneAndUpdateAsync(filter, combinedUpdate, options); + var result = await collection.FindOneAndUpdateAsync(filter, update, options); Assert.Equal("HorseUpdated", result.Properties.PropertyTwo); Assert.Null(result.Properties.PropertyOne); Assert.True(string.IsNullOrEmpty(result.Name)); diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs index 2f3e938..3cd6cc3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/ReplaceAndDeleteTests.cs @@ -10,7 +10,7 @@ public class ReplaceAndDeleteTests { ReplaceAndDeleteFixture fixture; - public ReplaceAndDeleteTests(ReplaceAndDeleteFixture fixture) + public ReplaceAndDeleteTests(AssemblyFixture assemblyFixture, ReplaceAndDeleteFixture fixture) { this.fixture = fixture; } @@ -249,6 +249,16 @@ public void FindAndDeleteOne_ReturnsFoundDocument_WhenRunSynchronously() Assert.Equal("Animal8", result.Name); } + [Fact] + public void FindAndDeleteOne_ReturnsNull_WhenDocumentNotFound() + { + var collection = fixture.ReplaceCollection; + var filter = Builders.Filter + .Eq(so => so.Name, "ThisDocumentDoesNotExist"); + var result = collection.FindOneAndDelete(filter); + Assert.Null(result); + } + [Fact] public async Task FindAndDeleteOne_WithSimpleSort() { @@ -554,7 +564,7 @@ public async Task DeleteMany_Tests() | filters.Eq(so => so.Properties.PropertyOne, "group2"); var result = await collection.DeleteManyAsync(filter); Assert.Equal(30, result.DeletedCount); - result = await collection.DeleteAllAsync(); + result = await collection.DeleteManyAsync(null); Assert.Equal(-1, result.DeletedCount); Assert.Equal(0, await collection.CountDocumentsAsync()); } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs index bc1a1dc..a8973db 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/RerankTests.cs @@ -16,7 +16,7 @@ public class RerankTests : IClassFixture private readonly RerankFixture _fixture; private readonly Collection _collection; - public RerankTests(RerankFixture fixture) + public RerankTests(AssemblyFixture assemblyFixture, RerankFixture fixture) { _fixture = fixture; _collection = fixture.HybridSearchCollection; @@ -97,7 +97,7 @@ public async Task FindAndRerank_WithIncludeScores_ReturnsScores() .Sort("cat") .IncludeScores(true); - await foreach (var result in results.WithScoresAsync()) + await foreach (var result in results.WithScoresAsync(TestContext.Current.CancellationToken)) { Assert.NotNull(result.Scores); Assert.NotEmpty(result.Scores); diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs index ec15d9f..7a41baa 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs @@ -13,7 +13,7 @@ public class SearchTests { CollectionsFixture fixture; - public SearchTests(CollectionsFixture fixture) + public SearchTests(AssemblyFixture assemblyFixture, CollectionsFixture fixture) { this.fixture = fixture; } @@ -573,6 +573,46 @@ public void InArray_WithArrays() Assert.Equal(2, results.Count); } + [Fact] + public void In_SingleValue() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.In(so => so.Properties.StringArrayProperty, "cat1"); + var results = collection.Find(filter).ToList(); + Assert.Single(results); + } + + [Fact] + public void In_SingleValue_StringFieldName() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.In("Properties.StringArrayProperty", "cat1"); + var results = collection.Find(filter).ToList(); + Assert.Single(results); + } + + [Fact] + public void NotIn_SingleValue() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Nin(so => so.Properties.StringArrayProperty, "cat1"); + var results = collection.Find(filter).ToList(); + Assert.Equal(32, results.Count); + } + + [Fact] + public void NotIn_SingleValue_StringFieldName() + { + var collection = fixture.SearchCollection; + var builder = Builders.Filter; + var filter = builder.Nin("Properties.StringArrayProperty", "cat1"); + var results = collection.Find(filter).ToList(); + Assert.Equal(32, results.Count); + } + [Fact] public async Task PropertyExists() { diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs index f33c88a..fa00be3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SerializationTests.cs @@ -14,18 +14,16 @@ namespace DataStax.AstraDB.DataApi.IntegrationTests; public class SerializationTests { AdminFixture fixture; - readonly Database _database; - public SerializationTests(AdminFixture fixture) + public SerializationTests(AssemblyFixture assemblyFixture, AdminFixture fixture) { this.fixture = fixture; - _database = fixture.GetDatabase(); } [Fact] public void TestTypeSerializationDeserialization() { - var collection = _database.GetCollection("serializationTest"); + var collection = fixture.Database.GetCollection("serializationTest"); var testObject = new SerializationTest() { TestId = 1, @@ -57,7 +55,7 @@ public void TestTypeSerializationDeserialization() [Fact] public void TestTypeSerializationDeserialization_WithDocumentSerializer() { - var collection = _database.GetCollection("serializationTest"); + var collection = fixture.Database.GetCollection("serializationTest"); var testObject = new SerializationTest() { TestId = 1, @@ -92,7 +90,7 @@ public void TestTypeSerializationDeserialization_WithDocumentSerializer() public void TestSpecific() { string serializationTestString = "{\"_id\":19,\"Name\":\"Animal19\",\"Properties\":{\"PropertyOne\":\"groupthree\",\"PropertyTwo\":\"animal19\",\"IntProperty\":20,\"StringArrayProperty\":[\"animal19\",\"animal119\",\"animal219\"],\"BoolProperty\":true,\"DateTimeProperty\":\"2019-05-19T00:00:00\",\"DateTimeOffsetProperty\":\"0001-01-01T00:00:00+00:00\"}}"; - var collection = _database.GetCollection("serializationTest2"); + var collection = fixture.Database.GetCollection("serializationTest2"); var commandOptions = new CommandOptions() { OutputConverter = new DocumentConverter() @@ -104,7 +102,7 @@ public void TestSpecific() public void IdList_Guid() { string serializationTestString = "{\"insertedIds\":[{\"$uuid\":\"315c2015-e404-432c-9c20-15e404532ceb\"}]}"; - var collection = _database.GetCollection>("serializationTest"); + var collection = fixture.Database.GetCollection>("serializationTest"); var commandOptions = new CommandOptions() { OutputConverter = new IdListConverter() @@ -116,7 +114,7 @@ public void IdList_Guid() public void IdList_ObjectId() { string serializationTestString = "{\"insertedIds\":[{\"$objectId\":\"67eaab273cc8411120638d65\"}]}"; - var collection = _database.GetCollection>("serializationTest"); + var collection = fixture.Database.GetCollection>("serializationTest"); var commandOptions = new CommandOptions() { OutputConverter = new IdListConverter() @@ -135,32 +133,11 @@ public void TableInsertManyResult() Assert.Equal("Test", deserialized.InsertedIds.First().First().ToString()); } - [Fact] - public void CompoundKeySerializationTest() - { - var filterBuilder = Builders.Filter; - var filter = filterBuilder.CompoundKey( - new[] { - new PrimaryKeyFilter(x => x.KeyOne, "KeyOne3"), - new PrimaryKeyFilter(x => x.KeyTwo, "KeyTwo3") - }, - new[] { - filterBuilder.Eq(x => x.SortOneAscending,"SortOneAscending3"), - filterBuilder.Eq(x => x.SortTwoDescending, "SortTwoDescending3") - }); - var commandOptions = Array.Empty(); - var command = new Command("deserializationTest", new DataApiClient(), commandOptions, null); - var serialized = command.Serialize(filter); - Console.WriteLine(serialized); - } - - //{"data":{"documents":[{"_id":"3a0cdac3-679b-435a-8cda-c3679bf35a6b","title":"Test Book 1","author":"Test Author 1","number_of_pages":100} - [Fact] public void BookDeserializationTest() { var serializationTestString = "{\"_id\":\"3a0cdac3-679b-435a-8cda-c3679bf35a6b\",\"title\":\"Test Book 1\",\"author\":\"Test Author 1\",\"number_of_pages\":100}"; - var collection = _database.GetCollection("bookTestTable"); + var collection = fixture.Database.GetCollection("bookTestTable"); var commandOptions = new CommandOptions() { OutputConverter = new DocumentConverter() diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs index 01947ee..bc4f945 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs @@ -11,7 +11,7 @@ public class TableAlterTests { private readonly TableAlterFixture fixture; - public TableAlterTests(TableAlterFixture fixture) + public TableAlterTests(AssemblyFixture assemblyFixture, TableAlterFixture fixture) { this.fixture = fixture; } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs index 1f56f79..07e7511 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableIndexesTests.cs @@ -11,7 +11,7 @@ public class TableIndexesTests { private readonly TableIndexesFixture fixture; - public TableIndexesTests(TableIndexesFixture fixture) + public TableIndexesTests(AssemblyFixture assemblyFixture, TableIndexesFixture fixture) { this.fixture = fixture; } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs index 589a728..5d548ef 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs @@ -12,7 +12,7 @@ public class TableTests { TablesFixture fixture; - public TableTests(TablesFixture fixture) + public TableTests(AssemblyFixture assemblyFixture, TablesFixture fixture) { this.fixture = fixture; } @@ -190,10 +190,10 @@ public async Task FindOne_Vectorize() public async Task FindOne_Sort() { var table = fixture.SearchTable; - var sort = Builders.TableSort; - var filter = sort.Descending(b => b.Title); + var sorter = Builders.TableSort; + var sort = sorter.Descending(b => b.Title); var projection = Builders.Projection.Include(b => b.Title); - var result = await table.FindOneAsync(null, new TableFindOptions() { Sort = filter, Projection = projection }); + var result = await table.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); Assert.Equal("Title 99", result.Title); Assert.Null(result.Author); } @@ -463,5 +463,153 @@ public async Task Delete_All() await fixture.Database.DropTableAsync(tableName); } } + + // same tests on untyped tables + [Fact] + public void LogicalAnd_MongoStyle_Untyped() + { + var builder = Builders.Filter; + var filter = builder.Gte("Id", 10); + //TODO: AND not working yet via API + //var filter = builder.Gte("Id", 10) & builder.Gt("IdTwo", "IdTwo_20"); + + var results = fixture.UntypedTableSinglePrimaryKey.Find(filter).ToList(); + Assert.Equal(40, results.Count); + results = fixture.UntypedTableCompositePrimaryKey.Find(filter).ToList(); + Assert.Equal(40, results.Count); + results = fixture.UntypedTableCompoundPrimaryKey.Find(filter).ToList(); + Assert.Equal(40, results.Count); + + } + + [Fact] + public void LogicalAnd_AstraStyle_Untyped() + { + var builder = Builders.Filter; + //TODO: AND not working yet via API + //var filter = builder.And(builder.Gt("Id", 10), builder.Eq("IdTwo", "IdTwo_20")); + var filter = builder.Gt("Id", 20); + + var results = fixture.UntypedTableSinglePrimaryKey.Find(filter).ToList(); + Assert.Equal(29, results.Count); + results = fixture.UntypedTableCompositePrimaryKey.Find(filter).ToList(); + Assert.Equal(29, results.Count); + results = fixture.UntypedTableCompoundPrimaryKey.Find(filter).ToList(); + Assert.Equal(29, results.Count); + } + + [Fact] + public void FindMany_RetrieveAll_Untyped() + { + var results = fixture.UntypedTableSinglePrimaryKey.Find().ToList(); + Assert.Equal(50, results.Count); + results = fixture.UntypedTableCompositePrimaryKey.Find().ToList(); + Assert.Equal(50, results.Count); + results = fixture.UntypedTableCompoundPrimaryKey.Find().ToList(); + Assert.Equal(50, results.Count); + } + + [Fact] + public void FindMany_Vectorize_Untyped() + { + var sorter = Builders.TableSort; + var sort = sorter.Vectorize("Vectorize", "String To Vectorize 12"); + var results = fixture.UntypedTableSinglePrimaryKey.Find().Sort(sort).ToList(); + Assert.Equal(50, results.Count); + Assert.Equal("Name_12", results.First()["Name"].ToString()); + results = fixture.UntypedTableCompositePrimaryKey.Find().Sort(sort).ToList(); + Assert.Equal("Name_12", results.First()["Name"].ToString()); + Assert.Equal(50, results.Count); + results = fixture.UntypedTableCompoundPrimaryKey.Find().Sort(sort).ToList(); + Assert.Equal("Name_12", results.First()["Name"].ToString()); + Assert.Equal(50, results.Count); + } + + [Fact] + public async Task FindOne_Vectorize_Untyped() + { + var sorter = Builders.TableSort; + var sort = sorter.Vectorize("Vectorize", "String To Vectorize 22"); + var results = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(null, + new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + Assert.Equal("Name_22", results["Name"].ToString()); + results = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(null, + new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + Assert.Equal("Name_22", results["Name"].ToString()); + results = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(null, + new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + Assert.Equal("Name_22", results["Name"].ToString()); + } + + [Fact] + public async Task FindOne_Sort_Untyped() + { + var sorter = Builders.TableSort; + var sort = sorter.Descending("Name"); + var projection = Builders.Projection.Include("Name"); + var result = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + Assert.Equal("Name_9", result["Name"].ToString()); + Assert.False(result.ContainsKey("SortOneAscending")); + result = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + Assert.Equal("Name_9", result["Name"].ToString()); + Assert.False(result.ContainsKey("SortOneAscending")); + result = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + Assert.Equal("Name_9", result["Name"].ToString()); + Assert.False(result.ContainsKey("SortOneAscending")); + } + + [Fact] + public void FindOne_Sort_Skip_Exclude_Untyped() + { + var sorter = Builders.TableSort; + var sort = sorter.Descending("Name"); + var projection = Builders.Projection.Exclude("SortOneAscending"); + var results = fixture.UntypedTableSinglePrimaryKey.Find().Sort(sort).Project(projection).Skip(2).Limit(5).ToList(); + Assert.Equal(5, results.Count()); + Assert.Equal("Name_7", results.First()["Name"].ToString()); + Assert.False(results.First().ContainsKey("SortOneAscending")); + results = fixture.UntypedTableCompositePrimaryKey.Find().Sort(sort).Project(projection).Skip(2).Limit(5).ToList(); + Assert.Equal(5, results.Count()); + Assert.Equal("Name_7", results.First()["Name"].ToString()); + Assert.False(results.First().ContainsKey("SortOneAscending")); + results = fixture.UntypedTableCompoundPrimaryKey.Find().Sort(sort).Project(projection).Skip(2).Limit(5).ToList(); + Assert.Equal(5, results.Count()); + Assert.Equal("Name_7", results.First()["Name"].ToString()); + Assert.False(results.First().ContainsKey("SortOneAscending")); + } + + [Fact] + public async Task Update_Test_Untyped() + { + var filter = Builders.Filter.Eq("Id", 3); + var update = Builders.Update.Set("Name", "Name_3_Updated"); + var result = await fixture.UntypedTableSinglePrimaryKey.UpdateOneAsync(filter, update); + Assert.Equal(1, result.ModifiedCount); + var updatedDocument = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(filter); + Assert.Equal("Name_3_Updated", updatedDocument["Name"].ToString()); + + filter = Builders.Filter.CompositeKey( + new PrimaryKeyFilter("Id", 3), + new PrimaryKeyFilter("IdTwo", "IdTwo_3")); + result = await fixture.UntypedTableCompositePrimaryKey.UpdateOneAsync(filter, update); + Assert.Equal(1, result.ModifiedCount); + updatedDocument = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(filter); + Assert.Equal("Name_3_Updated", updatedDocument["Name"].ToString()); + + filter = Builders.Filter.CompoundKey( + new[] { + new PrimaryKeyFilter("Id", 3), + new PrimaryKeyFilter("IdTwo", "IdTwo_3"), + }, + new[] { + Builders.Filter.Eq("SortOneAscending", "SortOneAscending3"), + Builders.Filter.Eq("SortTwoDescending", "SortTwoDescending47") + }); + result = await fixture.UntypedTableCompoundPrimaryKey.UpdateOneAsync(filter, update); + Assert.Equal(1, result.ModifiedCount); + updatedDocument = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(filter); + Assert.Equal("Name_3_Updated", updatedDocument["Name"].ToString()); + } + } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs index 5bc05cc..0404609 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/UpdateTests.cs @@ -1,6 +1,6 @@ using DataStax.AstraDB.DataApi.Collections; -using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; using Xunit; namespace DataStax.AstraDB.DataApi.IntegrationTests; @@ -10,7 +10,7 @@ public class UpdateTests { UpdatesFixture fixture; - public UpdateTests(UpdatesFixture fixture) + public UpdateTests(AssemblyFixture assemblyFixture, UpdatesFixture fixture) { this.fixture = fixture; } @@ -22,11 +22,9 @@ public async Task UpdateOneAsync() var filter = Builders.Filter .Eq(so => so.Name, "Cat"); var updater = Builders.Update; - var combinedUpdate = updater.Combine( - updater.Set(so => so.Properties.PropertyTwo, "CatUpdated"), - updater.Unset("Properties.PropertyOne") - ); - var result = await collection.UpdateOneAsync(filter, combinedUpdate); + var update = updater.Set(so => so.Properties.PropertyTwo, "CatUpdated") + .Unset("Properties.PropertyOne"); + var result = await collection.UpdateOneAsync(filter, update); Assert.Equal(1, result.ModifiedCount); var updatedDocument = await collection.FindOneAsync(filter); Assert.Equal("CatUpdated", updatedDocument.Properties.PropertyTwo); diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/readme.md b/test/DataStax.AstraDB.DataApi.IntegrationTests/readme.md index b4f4269..ccf7ad4 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/readme.md +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/readme.md @@ -4,4 +4,11 @@ Set the following environment variables before running: ``` export ASTRA_DB_TOKEN="your_token_here" export ASTRA_DB_URL="your_db_url_here" -``` \ No newline at end of file +``` + +dotnet test --collect:"XPlat Code Coverage" + +reportgenerator +-reports:"Path\To\TestProject\TestResults\{guid}\coverage.cobertura.xml" +-targetdir:"coveragereport" +-reporttypes:Html \ No newline at end of file