From 7f54f18d941587af579fd14b20ca075626758ec8 Mon Sep 17 00:00:00 2001 From: jeffrey-elliott Date: Thu, 15 May 2025 10:12:37 -0700 Subject: [PATCH] added alter table operations --- src/DataStax.AstraDB.DataApi/Tables/Table.cs | 54 +++++ .../Tables/TableAlter.cs | 229 ++++++++++++++++++ .../TableAlterFixture.cs | 115 +++++++++ .../Tests/TableAlterTests.cs | 204 ++++++++++++++++ 4 files changed, 602 insertions(+) create mode 100644 src/DataStax.AstraDB.DataApi/Tables/TableAlter.cs create mode 100644 test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs create mode 100644 test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs diff --git a/src/DataStax.AstraDB.DataApi/Tables/Table.cs b/src/DataStax.AstraDB.DataApi/Tables/Table.cs index 7648d1d..6f9cea9 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/Table.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/Table.cs @@ -696,6 +696,60 @@ internal async Task DeleteManyAsync(Filter filter, CommandOptio return deleteResult; } + /// + /// This is a synchronous version of . + /// + /// + public Dictionary Alter(IAlterTableOperation operation) + { + return Alter(operation, null); + } + + /// + /// This is a synchronous version of . + /// + /// + public Dictionary Alter(IAlterTableOperation operation, CommandOptions commandOptions) + { + var response = AlterAsync(operation, commandOptions, true).ResultSync(); + return response.Result; + } + + /// + /// Alters a table using the specified operation. + /// + /// The alteration operation to apply. + /// The status result of the alterTable command. + public async Task> AlterAsync(IAlterTableOperation operation) + { + var response = await AlterAsync(operation, null, false); + return response.Result; + } + + /// + /// Options to customize the command execution. + public async Task> AlterAsync(IAlterTableOperation operation, CommandOptions commandOptions) + { + var response = await AlterAsync(operation, commandOptions, false); + return response.Result; + } + + internal async Task>> AlterAsync(IAlterTableOperation operation, CommandOptions commandOptions, bool runSynchronously) + { + var payload = new + { + operation = operation.ToJsonFragment() + }; + + var command = CreateCommand("alterTable") + .WithPayload(payload) + .AddCommandOptions(commandOptions); + + var result = await command.RunAsyncReturnStatus>(runSynchronously).ConfigureAwait(false); + + return result; + } + internal Command CreateCommand(string name) { var optionsTree = _commandOptions == null ? _database.OptionsTree : _database.OptionsTree.Concat(new[] { _commandOptions }).ToArray(); diff --git a/src/DataStax.AstraDB.DataApi/Tables/TableAlter.cs b/src/DataStax.AstraDB.DataApi/Tables/TableAlter.cs new file mode 100644 index 0000000..ca382e3 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Tables/TableAlter.cs @@ -0,0 +1,229 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Collections.Generic; +using System.Text.Json.Serialization; +using DataStax.AstraDB.DataApi.Core; + +namespace DataStax.AstraDB.DataApi.Tables; + +/// +/// Represents an alter table operation that can be converted to a JSON fragment for transmission. +/// +public interface IAlterTableOperation +{ + /// + /// Converts the operation to its JSON representation. + /// + /// A serializable object representing the operation. + object ToJsonFragment(); +} + +/// +/// Represents an operation to add new columns to a table. +/// +public class AlterTableAddColumns : IAlterTableOperation +{ + /// + /// Gets the columns to be added. + /// + public Dictionary Columns { get; } + + /// + /// Initializes a new instance with the specified columns. + /// + /// The columns to add. + public AlterTableAddColumns(Dictionary columns) + { + Columns = columns; + } + + /// + public object ToJsonFragment() => new + { + add = new + { + columns = Columns + } + }; +} + +/// +/// Describes the definition of a single column in an alter operation. +/// +public class AlterTableColumnDefinition +{ + /// + /// Gets or sets the type of the column. + /// + [JsonPropertyName("type")] + public string Type { get; set; } + + /// + /// Gets or sets the key type if the column is a map or similar structure. + /// + [JsonPropertyName("keyType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string KeyType { get; set; } + + /// + /// Gets or sets the value type if the column is a map or similar structure. + /// + [JsonPropertyName("valueType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string ValueType { get; set; } +} + +/// +/// Represents an operation to add vector columns to a table. +/// +public class AlterTableAddVectorColumns : IAlterTableOperation +{ + /// + /// Gets the vector columns to be added. + /// + public Dictionary Columns { get; } + + /// + /// Initializes a new instance with the specified vector columns. + /// + /// The vector columns to add. + public AlterTableAddVectorColumns(Dictionary columns) + { + Columns = columns; + } + + /// + public object ToJsonFragment() => new + { + add = new + { + columns = Columns + } + }; +} + +/// +/// Describes the definition of a vector column. +/// +public class AlterTableVectorColumnDefinition +{ + /// + /// Gets the type of the column. Always returns "vector". + /// + [JsonPropertyName("type")] + public string Type => "vector"; + + /// + /// Gets or sets the vector dimension. + /// + [JsonPropertyName("dimension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? VectorDimension { get; set; } + + /// + /// Gets or sets the vector service options. + /// + [JsonPropertyName("service")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public VectorServiceOptions Service { get; set; } +} + +/// +/// Represents an operation to drop columns from a table. +/// +public class AlterTableDropColumns : IAlterTableOperation +{ + /// + /// Gets the list of column names to drop. + /// + public List Columns { get; } + + /// + /// Initializes a new instance with the specified column names. + /// + /// The column names to drop. + public AlterTableDropColumns(IEnumerable columns) + { + Columns = new List(columns); + } + + /// + public object ToJsonFragment() => new + { + drop = new + { + columns = Columns + } + }; +} + +/// +/// Represents an operation to add vectorization services to specific columns. +/// +public class AlterTableAddVectorize : IAlterTableOperation +{ + /// + /// Gets the columns and associated vector service options. + /// + public Dictionary Columns { get; } + + /// + /// Initializes a new instance with the specified vectorization settings. + /// + /// The columns and their vector services. + public AlterTableAddVectorize(Dictionary columns) + { + Columns = columns; + } + + /// + public object ToJsonFragment() => new + { + addVectorize = new + { + columns = Columns + } + }; +} + +/// +/// Represents an operation to remove vectorization from specific columns. +/// +public class AlterTableDropVectorize : IAlterTableOperation +{ + /// + /// Gets the list of column names to remove vectorization from. + /// + public List Columns { get; } + + /// + /// Initializes a new instance with the specified column names. + /// + /// The columns to remove vectorization from. + public AlterTableDropVectorize(IEnumerable columns) + { + Columns = new List(columns); + } + + /// + public object ToJsonFragment() => new + { + dropVectorize = new + { + columns = Columns + } + }; +} diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs new file mode 100644 index 0000000..c079879 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/TableAlterFixture.cs @@ -0,0 +1,115 @@ +using DataStax.AstraDB.DataApi; +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Tables; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Xunit; +using Xunit.Abstractions; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +[CollectionDefinition("TableAlter")] +public class TableAlterCollection : ICollectionFixture +{ + +} + +public class TableAlterFixture : IDisposable, IAsyncLifetime +{ + public DataApiClient Client { get; private set; } + public Database Database { get; private set; } + public string DatabaseUrl { get; set; } + + public TableAlterFixture() + { + IConfiguration configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables(prefix: "ASTRA_DB_") + .Build(); + + var token = configuration["TOKEN"] ?? configuration["AstraDB:Token"]; + DatabaseUrl = configuration["URL"] ?? configuration["AstraDB:DatabaseUrl"]; + + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddFileLogger("../../../table_Alter_fixture_latest_run.log")); + ILogger logger = factory.CreateLogger("IntegrationTests"); + + var clientOptions = new CommandOptions + { + RunMode = RunMode.Debug + }; + Client = new DataApiClient(token, clientOptions, logger); + Database = Client.GetDatabase(DatabaseUrl); + + try + { + var keyspaces = Database.GetAdmin().ListKeyspaceNames(); + Console.WriteLine($"[Fixture] Connected. Keyspaces found: {keyspaces.Count()}"); + } + catch (Exception ex) + { + Console.WriteLine($"[Fixture] Connection failed: {ex.Message}"); + throw; + } + + } + + public async Task InitializeAsync() + { + await CreateTestTable(); + } + + public async Task DisposeAsync() + { + await Database.DropTableAsync(_fixtureTableName); + } + + public Table FixtureTestTable { get; private set; } + + + private const string _fixtureTableName = "tableAlterTest"; + private async Task CreateTestTable() + { + var startDate = DateTime.UtcNow.Date.AddDays(7); + + var eventRows = new List + { + new() + { + EventDate = startDate, + Id = Guid.NewGuid(), + Title = "Board Meeting", + Location = "East Wing", + Category = "administrative" + }, + new() + { + EventDate = startDate.AddDays(1), + Id = Guid.NewGuid(), + Title = "Fire Drill", + Location = "Building A", + Category = "safety" + }, + new() + { + EventDate = startDate.AddDays(2), + Id = Guid.NewGuid(), + Title = "Team Lunch", + Location = "Cafeteria", + Category = "social" + } + }; + + + var table = await Database.CreateTableAsync(_fixtureTableName); + await table.InsertManyAsync(eventRows); + + FixtureTestTable = table; + } + + public void Dispose() + { + // nothing needed + } +} \ No newline at end of file diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs new file mode 100644 index 0000000..4d4986a --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableAlterTests.cs @@ -0,0 +1,204 @@ +using DataStax.AstraDB.DataApi.Tables; +using DataStax.AstraDB.DataApi.Core; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +// Tests in this collection should be run individually; changes are applied to the underlying fixture. +[Collection("TableAlter")] +public class TableAlterTests +{ + private readonly TableAlterFixture fixture; + + public TableAlterTests(TableAlterFixture fixture) + { + this.fixture = fixture; + } + + [Fact] + public async Task AlterTableAddColumns() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + var newColumns = new Dictionary + { + ["is_archived"] = new AlterTableColumnDefinition { Type = "boolean" }, + ["review_notes"] = new AlterTableColumnDefinition { Type = "text" } + }; + + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + } + + [Fact] + public async Task AlterTableAddColumnsDupe() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + var newColumns = new Dictionary + { + ["is_archived"] = new AlterTableColumnDefinition { Type = "boolean" }, + ["review_notes"] = new AlterTableColumnDefinition { Type = "text" } + }; + + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + + var ex = await Assert.ThrowsAsync(() => + table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false)); + + Assert.Contains("unique", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task AlterTableAddColumnsMapSet() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + var newColumns = new Dictionary + { + ["column_test_map"] = new AlterTableColumnDefinition + { + Type = "map", + KeyType = "text", + ValueType = "text" + }, + ["column_test_set"] = new AlterTableColumnDefinition + { + Type = "text", + ValueType = "text" + } + + }; + + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + } + + // Requires a pre-configured embedding provider on the Astra backend. + [Fact] + public async Task AlterTableAddVectorColumnsWithEmbedding() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + { + ["plot_synopsis"] = new AlterTableVectorColumnDefinition + { + //VectorDimension = 1536, + VectorDimension = null, + Service = new VectorServiceOptions + { + Provider = "openai", + //ModelName = "text-embedding-ada-002", + ModelName = "text-embedding-3-small", + Authentication = new Dictionary { ["providerKey"] = "OPEN_AI_TEST_SCOPE" }, + Parameters = new Dictionary { ["organizationId"] = "An optional organization ID" }, + } + } + }), null, false); + } + + [Fact] + public async Task AlterTableAddVectorColumnsNoConfig() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + { + ["plot_synopsis"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); + } + + [Fact] + public async Task AlterTableDropColumn() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + var newColumns = new Dictionary + { + ["is_archived"] = new AlterTableColumnDefinition { Type = "boolean" }, + ["review_notes"] = new AlterTableColumnDefinition { Type = "text" } + }; + + await table.AlterAsync(new AlterTableAddColumns(newColumns), null, runSynchronously: false); + + await table.AlterAsync(new AlterTableDropColumns(new[] { "is_archived" }), null, runSynchronously: false); + } + + [Fact] + public async Task AlterTableDropVectorColumns() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + { + ["plot_synopsis"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); + + var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis" }); + await table.AlterAsync(dropColumn, null, runSynchronously: false); + } + + [Fact] + public async Task AlterTableAddVectorize() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + { + ["plot_synopsis"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); + + await table.AlterAsync(new AlterTableAddVectorize(new Dictionary + { + ["plot_synopsis"] = new VectorServiceOptions + { + Provider = "openai", + ModelName = "text-embedding-3-small", + Authentication = new Dictionary { ["providerKey"] = "OPEN_AI_TEST_SCOPE" }, + Parameters = new Dictionary { ["organizationId"] = "An optional organization ID" }, + } + + }), null, false); + } + + [Fact] + public async Task AlterTableOperationDropVectorize() + { + var table = fixture.Database.GetTable("tableAlterTest", null); + + await table.AlterAsync(new AlterTableAddVectorColumns(new Dictionary + { + ["plot_synopsis"] = new AlterTableVectorColumnDefinition + { + VectorDimension = 2 + } + }), null, false); + + await table.AlterAsync(new AlterTableAddVectorize(new Dictionary + { + ["plot_synopsis"] = new VectorServiceOptions + { + Provider = "openai", + ModelName = "text-embedding-3-small", + Authentication = new Dictionary { ["providerKey"] = "OPEN_AI_TEST_SCOPE" }, + Parameters = new Dictionary { ["organizationId"] = "An optional organization ID" }, + } + + }), null, false); + + var dropVectorize = new AlterTableDropVectorize(new[] { "plot_synopsis" }); + await table.AlterAsync(dropVectorize, null, runSynchronously: false); + + var dropColumn = new AlterTableDropColumns(new[] { "plot_synopsis" }); + await table.AlterAsync(dropColumn, null, runSynchronously: false); + } + +}