diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 74df001..90cfc24 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -31,6 +31,9 @@ jobs: with: dotnet-version: 10.x + - name: Install dotnet-ef + run: dotnet tool install --global dotnet-ef --version 10.0.5 + - name: Test with Coverage run: dotnet test test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj --configuration Release --verbosity normal --logger GitHubActions /clp:ErrorsOnly /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:SkipAutoProps=true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b2e63c1..791fe80 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,6 +21,9 @@ jobs: with: dotnet-version: 10.0.x + - name: Install dotnet-ef + run: dotnet tool install --global dotnet-ef --version 10.0.5 + - name: Restore run: dotnet restore diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..6353860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +v0.2.0 +--- +* **Table engine configuration** via fluent API: MergeTree, ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree, VersionedCollapsingMergeTree, GraphiteMergeTree, plus simple engines (Log, TinyLog, StripeLog, Memory). +* **Engine clauses**: ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, SETTINGS — all configurable per-entity. +* **Column-level DDL features**: CODEC, TTL, COMMENT, DEFAULT values. +* **Data-skipping indexes**: configurable type, granularity, and parameters. +* **Migrations support**: `dotnet ef migrations add` / `database update` with full DDL generation (CREATE TABLE, ALTER TABLE ADD/DROP/MODIFY/RENAME COLUMN, RENAME TABLE, CREATE/DROP DATABASE). +* **Model validation**: engine parameter columns checked for existence and correct store types (Int8 for sign, UInt8 for isDeleted). Foreign key warnings. +* **Default engine convention**: MergeTree with ORDER BY derived from primary key when no explicit engine is configured. +* Lambda-based overloads for engine configuration (e.g. `HasReplacingMergeTreeEngine(e => e.Version)`). +* `ListToArrayConverter` handles null → empty array for ClickHouse `Array(T)` columns. +* Nullable wrapping correctly skips container types (Array, Map, Tuple, Variant, Dynamic, Json). + +v0.1.0 +--- +Initial preview release. + +* **LINQ query translation**: Where, OrderBy, Take, Skip, Select, First, Single, Any, Count, Sum, Min, Max, Average, Distinct, GroupBy (with DISTINCT and predicate overloads), LongCount. +* **60+ Math/MathF method translations**: Abs, Floor, Ceiling, Round, Truncate, Pow, Sqrt, Exp, Log, trig functions, etc. +* **String method translations**: Contains, StartsWith, EndsWith, IndexOf, Replace, Substring, Trim, ToLower, ToUpper, Length. +* **INSERT support**: `SaveChanges()` / `SaveChangesAsync()` via the driver's native `InsertBinaryAsync` (RowBinary with GZip compression). `BulkInsertAsync()` for high-throughput bulk loads. UPDATE/DELETE throw `NotSupportedException`. +* **Type support**: `String`, `Bool`, `Int8`–`Int64`, `UInt8`–`UInt64`, `Float32`/`Float64`, `Decimal(P,S)` (32/64/128/256), `Date`/`Date32`, `DateTime`, `DateTime64`, `FixedString(N)`, `UUID`, `BFloat16`, Nullable(T)/LowCardinality(T) unwrapping, Enum8/Enum16, IPv4/IPv6, BigInteger (Int128/Int256/UInt128/UInt256), Array(T), Map(K,V), Tuple(T1,...), Time/Time64, Variant(T1,...,TN), Dynamic, Json (JsonNode + string), geographic types (Point, Ring, Polygon, MultiPolygon, Geometry). diff --git a/ClickHouse.EntityFrameworkCore.sln b/ClickHouse.EntityFrameworkCore.sln index e966daf..468f5c9 100644 --- a/ClickHouse.EntityFrameworkCore.sln +++ b/ClickHouse.EntityFrameworkCore.sln @@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EFCore.ClickHouse.Tests", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EFCore.ClickHouse.FunctionalTests", "test\EFCore.ClickHouse.FunctionalTests\EFCore.ClickHouse.FunctionalTests.csproj", "{FC3416A6-643A-43C1-8D6F-E0308181979E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EFCore.ClickHouse.DesignSmoke", "test\EFCore.ClickHouse.DesignSmoke\EFCore.ClickHouse.DesignSmoke.csproj", "{4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,6 +61,18 @@ Global {FC3416A6-643A-43C1-8D6F-E0308181979E}.Release|x64.Build.0 = Release|Any CPU {FC3416A6-643A-43C1-8D6F-E0308181979E}.Release|x86.ActiveCfg = Release|Any CPU {FC3416A6-643A-43C1-8D6F-E0308181979E}.Release|x86.Build.0 = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|x64.ActiveCfg = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|x64.Build.0 = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|x86.ActiveCfg = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Debug|x86.Build.0 = Debug|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|Any CPU.Build.0 = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|x64.ActiveCfg = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|x64.Build.0 = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|x86.ActiveCfg = Release|Any CPU + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -67,5 +81,6 @@ Global {A3E072A2-61A5-4274-9720-316FE98B876B} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {79990B71-0CA9-495D-9988-1FE3BC5EB9A8} = {0C88DD14-F956-CE84-757C-A364CCF449FC} {FC3416A6-643A-43C1-8D6F-E0308181979E} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {4A1A592D-DB71-4CD4-A4F2-3BFBAEED0BB4} = {0C88DD14-F956-CE84-757C-A364CCF449FC} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 05106f1..427bad8 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ public class PageView ## Current Status -This provider is in early development. It supports **read-only queries** and **inserts** — you can map entities to existing ClickHouse tables, query them with LINQ, and write data via `SaveChanges`. +This provider is in active development. It supports **LINQ queries**, **inserts**, **table engine configuration**, and **migrations** — you can define ClickHouse tables with engine-specific settings, create them via `dotnet ef migrations` or `EnsureCreated`, query with LINQ, and write data via `SaveChanges`. ### LINQ Queries @@ -188,11 +188,69 @@ entity.Property(e => e.Payload).HasColumnType("Json"); - **NULL semantics** — ClickHouse's JSON type returns `{}` (empty object) for NULL values rather than SQL NULL. A row inserted with `Data = null` will read back as an empty `JsonNode`, not `null`. - **Integer precision** — ClickHouse JSON stores all integers as `Int64` unless the path is typed otherwise. When reading via `JsonNode`, use `GetValue()` rather than `GetValue()`. +### Table Engine Configuration + +Configure ClickHouse table engines, ordering, partitioning, and more via EF Core's fluent API: + +```csharp +modelBuilder.Entity(b => +{ + b.HasKey(e => e.Id); + b.Property(e => e.Temperature).HasCodec("Delta, ZSTD"); + b.Property(e => e.Location).HasColumnComment("Installation site"); + b.HasIndex(e => e.Timestamp) + .HasSkippingIndexType("minmax") + .HasGranularity(4); + b.ToTable("sensor_readings", t => t + .HasReplacingMergeTreeEngine("Version") + .WithOrderBy("Id", "Timestamp") + .WithPartitionBy("toYYYYMM(Timestamp)") + .WithPrimaryKey("Id") + .WithTtl("Timestamp + INTERVAL 1 YEAR") + .WithSetting("index_granularity", "4096")); +}); +``` + +**Supported engines:** `MergeTree`, `ReplacingMergeTree`, `SummingMergeTree`, `AggregatingMergeTree`, `CollapsingMergeTree`, `VersionedCollapsingMergeTree`, `GraphiteMergeTree`, `Log`, `TinyLog`, `StripeLog`, `Memory` + +**Column-level DDL:** `.HasCodec("Delta, ZSTD")`, `.HasColumnTtl("expr")`, `.HasColumnComment("text")` + +**Data-skipping indices:** `.HasSkippingIndexType("minmax")`, `.HasGranularity(4)`, `.HasSkippingIndexParams("100")` + +**Engine settings:** `.WithSetting("index_granularity", "4096")` — any ClickHouse setting as a key-value pair + +**Default behavior:** If no engine is configured, the provider defaults to `MergeTree` with the EF primary key as `ORDER BY`. + +### Migrations + +The provider supports `dotnet ef migrations` for creating and applying migrations: + +```bash +dotnet ef migrations add InitialCreate +dotnet ef database update +``` + +`EnsureCreated()` / `EnsureDeleted()` also work for quick setup without migrations. + +**Supported migration operations:** +- CREATE TABLE with full ENGINE clause (all engine types, ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, SETTINGS, codecs, comments, data-skipping indices) +- ADD COLUMN, DROP COLUMN, MODIFY COLUMN, RENAME COLUMN, RENAME TABLE +- DROP TABLE, CREATE/DROP INDEX (data-skipping) +- Custom `ClickHouseCreateDatabaseOperation` / `ClickHouseDropDatabaseOperation` + +**ClickHouse limitations reflected in migrations:** +- ALTER TABLE cannot change engine, ORDER BY, PARTITION BY, or other structural metadata — the provider throws `NotSupportedException` with a clear message +- Foreign keys, unique constraints, and sequences throw `NotSupportedException` +- Primary key add/drop is a no-op (ClickHouse PK is structural, not a constraint) +- Idempotent scripts (`--idempotent`) are not supported (ClickHouse has no conditional SQL blocks) +- Transactions are suppressed (ClickHouse does not support them) + ### Not Yet Implemented - UPDATE / DELETE (ClickHouse mutations are async, not OLTP-compatible) -- Migrations - JOINs, subqueries, set operations +- Reverse engineering / scaffolding (`dotnet ef dbcontext scaffold`) +- JSON path query translation ## Building diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 57d29b6..3748da6 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,8 +1,12 @@ -v0.1.0 +v0.2.0 --- -Initial preview release. - -* Read-only functionality. -* Limited type support: `String`, `Bool`, `Int8`/`Int16`/`Int32`/`Int64`, `UInt8`/`UInt16`/`UInt32`/`UInt64`, `Float32`/`Float64`, `Decimal(P, S)`, `Date`/`Date32`, `DateTime`, `DateTime64(P, 'TZ')`, `FixedString(N)`, `UUID`. -* Basic aggregations: `Where`, `OrderBy`, `Take`, `Skip`, `Select`, `First`, `Single`, `Any`, `Count`, `Sum`, `Min`, `Max`, `Average`, `Distinct`, `GroupBy`. -* String methods: `Contains`, `StartsWith`, `EndsWith`, `IndexOf`, `Replace`, `Substring`, `Trim`, `ToLower`, `ToUpper`, `Length`. \ No newline at end of file +* **Table engine configuration** via fluent API: MergeTree, ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree, VersionedCollapsingMergeTree, GraphiteMergeTree, plus simple engines (Log, TinyLog, StripeLog, Memory). +* **Engine clauses**: ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, SETTINGS — all configurable per-entity. +* **Column-level DDL features**: CODEC, TTL, COMMENT, DEFAULT values. +* **Data-skipping indexes**: configurable type, granularity, and parameters. +* **Migrations support**: `dotnet ef migrations add` / `database update` with full DDL generation (CREATE TABLE, ALTER TABLE ADD/DROP/MODIFY/RENAME COLUMN, RENAME TABLE, CREATE/DROP DATABASE). +* **Model validation**: engine parameter columns checked for existence and correct store types (Int8 for sign, UInt8 for isDeleted). Foreign key warnings. +* **Default engine convention**: MergeTree with ORDER BY derived from primary key when no explicit engine is configured. +* Lambda-based overloads for engine configuration (e.g. `HasReplacingMergeTreeEngine(e => e.Version)`). +* `ListToArrayConverter` handles null → empty array for ClickHouse `Array(T)` columns. +* Nullable wrapping correctly skips container types (Array, Map, Tuple, Variant, Dynamic, Json). diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs new file mode 100644 index 0000000..e955688 --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseAnnotationCodeGenerator.cs @@ -0,0 +1,46 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseAnnotationCodeGenerator : AnnotationCodeGenerator +{ + public ClickHouseAnnotationCodeGenerator(AnnotationCodeGeneratorDependencies dependencies) + : base(dependencies) + { + } + + protected override bool IsHandledByConvention(IModel model, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(model, annotation); + } + + protected override bool IsHandledByConvention(IEntityType entityType, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(entityType, annotation); + } + + protected override bool IsHandledByConvention(IProperty property, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(property, annotation); + } + + protected override bool IsHandledByConvention(IIndex index, IAnnotation annotation) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + return false; + + return base.IsHandledByConvention(index, annotation); + } +} diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs new file mode 100644 index 0000000..e2c591f --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseCodeGenerator.cs @@ -0,0 +1,30 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Scaffolding; +using System.Reflection; + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseCodeGenerator : ProviderCodeGenerator +{ + private static readonly MethodInfo UseClickHouseMethodInfo + = typeof(ClickHouseDbContextOptionsBuilderExtensions).GetRuntimeMethod( + nameof(ClickHouseDbContextOptionsBuilderExtensions.UseClickHouse), + [typeof(DbContextOptionsBuilder), typeof(string), typeof(Action)])!; + + public ClickHouseCodeGenerator(ProviderCodeGeneratorDependencies dependencies) + : base(dependencies) + { + } + + public override MethodCallCodeFragment GenerateUseProvider( + string connectionString, + MethodCallCodeFragment? providerOptions) + => new( + UseClickHouseMethodInfo, + providerOptions is null + ? [connectionString] + : [connectionString, new NestedClosureCodeFragment("x", providerOptions)]); +} diff --git a/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs b/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs new file mode 100644 index 0000000..5b85cd8 --- /dev/null +++ b/src/EFCore.ClickHouse/Design/Internal/ClickHouseDesignTimeServices.cs @@ -0,0 +1,22 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Scaffolding; +using Microsoft.Extensions.DependencyInjection; + +[assembly: DesignTimeProviderServices( + "ClickHouse.EntityFrameworkCore.Design.Internal.ClickHouseDesignTimeServices")] + +namespace ClickHouse.EntityFrameworkCore.Design.Internal; + +public class ClickHouseDesignTimeServices : IDesignTimeServices +{ + public void ConfigureDesignTimeServices(IServiceCollection serviceCollection) + { + serviceCollection.AddEntityFrameworkClickHouse(); + + new EntityFrameworkRelationalDesignServicesBuilder(serviceCollection) + .TryAdd() + .TryAdd() + .TryAddCoreServices(); + } +} diff --git a/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj b/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj index f1c819f..1d4da58 100644 --- a/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj +++ b/src/EFCore.ClickHouse/EFCore.ClickHouse.csproj @@ -25,6 +25,7 @@ + diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs new file mode 100644 index 0000000..1506752 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeBuilderExtensions.cs @@ -0,0 +1,142 @@ +using System.Linq.Expressions; +using System.Reflection; +using ClickHouse.EntityFrameworkCore.Metadata.Builders; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseEntityTypeBuilderExtensions +{ + public static ClickHouseMergeTreeEngineBuilder HasMergeTreeEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder)); + } + + public static ClickHouseReplacingMergeTreeEngineBuilder HasReplacingMergeTreeEngine( + this TableBuilder tableBuilder, string? version = null, string? isDeleted = null) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), version, isDeleted); + } + + public static ClickHouseSummingMergeTreeEngineBuilder HasSummingMergeTreeEngine( + this TableBuilder tableBuilder, params string[] columns) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), columns); + } + + public static ClickHouseAggregatingMergeTreeEngineBuilder HasAggregatingMergeTreeEngine( + this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder)); + } + + public static ClickHouseCollapsingMergeTreeEngineBuilder HasCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, string sign) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(sign); + return new(GetEntityType(tableBuilder), sign); + } + + public static ClickHouseVersionedCollapsingMergeTreeEngineBuilder HasVersionedCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, string sign, string version) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(sign); + ArgumentException.ThrowIfNullOrWhiteSpace(version); + return new(GetEntityType(tableBuilder), sign, version); + } + + public static ClickHouseGraphiteMergeTreeEngineBuilder HasGraphiteMergeTreeEngine( + this TableBuilder tableBuilder, string configSection) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + ArgumentException.ThrowIfNullOrWhiteSpace(configSection); + return new(GetEntityType(tableBuilder), configSection); + } + + public static ClickHouseSimpleEngineBuilder HasTinyLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.TinyLog); + } + + public static ClickHouseSimpleEngineBuilder HasStripeLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.StripeLog); + } + + public static ClickHouseSimpleEngineBuilder HasLogEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.Log); + } + + public static ClickHouseSimpleEngineBuilder HasMemoryEngine(this TableBuilder tableBuilder) + { + ArgumentNullException.ThrowIfNull(tableBuilder); + return new(GetEntityType(tableBuilder), ClickHouseAnnotationNames.Memory); + } + + // ── Generic (lambda-based) overloads for TableBuilder ───────── + + public static ClickHouseReplacingMergeTreeEngineBuilder HasReplacingMergeTreeEngine( + this TableBuilder tableBuilder, + Expression>? version = null, + Expression>? isDeleted = null) + where TEntity : class + => tableBuilder.HasReplacingMergeTreeEngine( + GetPropertyName(version), GetPropertyName(isDeleted)); + + public static ClickHouseSummingMergeTreeEngineBuilder HasSummingMergeTreeEngine( + this TableBuilder tableBuilder, + params Expression>[] columns) + where TEntity : class + => tableBuilder.HasSummingMergeTreeEngine( + columns.Select(GetPropertyName).ToArray()!); + + public static ClickHouseCollapsingMergeTreeEngineBuilder HasCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, + Expression> sign) + where TEntity : class + => tableBuilder.HasCollapsingMergeTreeEngine(GetPropertyName(sign)!); + + public static ClickHouseVersionedCollapsingMergeTreeEngineBuilder HasVersionedCollapsingMergeTreeEngine( + this TableBuilder tableBuilder, + Expression> sign, + Expression> version) + where TEntity : class + => tableBuilder.HasVersionedCollapsingMergeTreeEngine( + GetPropertyName(sign)!, GetPropertyName(version)!); + + // ── Helpers ────────────────────────────────────────────────────────────── + + private static IMutableEntityType GetEntityType(TableBuilder tableBuilder) + => (IMutableEntityType)tableBuilder.Metadata; + + private static string? GetPropertyName(Expression>? expression) + { + if (expression is null) + return null; + + var body = expression.Body; + + // Unwrap Convert() that the compiler adds for value types boxed to object + if (body is UnaryExpression { NodeType: ExpressionType.Convert } unary) + body = unary.Operand; + + if (body is MemberExpression { Member: PropertyInfo property }) + return property.Name; + + throw new ArgumentException( + $"Expression '{expression}' does not refer to a property. " + + "Use a simple property access like 'e => e.MyProperty'."); + } +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs new file mode 100644 index 0000000..aef61c4 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseEntityTypeExtensions.cs @@ -0,0 +1,132 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseEntityTypeExtensions +{ + // Engine + + public static string? GetEngine(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.Engine]; + + public static void SetEngine(this IMutableEntityType entityType, string? engine) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.Engine, engine); + + // ORDER BY + + public static string[]? GetOrderBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.OrderBy]; + + public static void SetOrderBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.OrderBy, + columns is { Length: > 0 } ? columns : null); + + // PARTITION BY + + public static string[]? GetPartitionBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.PartitionBy]; + + public static void SetPartitionBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.PartitionBy, + columns is { Length: > 0 } ? columns : null); + + // PRIMARY KEY (ClickHouse structural key, distinct from EF's HasKey) + + public static string[]? GetClickHousePrimaryKey(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.PrimaryKey]; + + public static void SetClickHousePrimaryKey(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.PrimaryKey, + columns is { Length: > 0 } ? columns : null); + + // SAMPLE BY + + public static string[]? GetSampleBy(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.SampleBy]; + + public static void SetSampleBy(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SampleBy, + columns is { Length: > 0 } ? columns : null); + + // TTL (table-level) + + public static string? GetTtl(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.Ttl]; + + public static void SetTtl(this IMutableEntityType entityType, string? ttlExpression) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.Ttl, ttlExpression); + + // ReplacingMergeTree + + public static string? GetReplacingMergeTreeVersion(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.ReplacingMergeTreeVersion]; + + public static void SetReplacingMergeTreeVersion(this IMutableEntityType entityType, string? version) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, version); + + public static string? GetReplacingMergeTreeIsDeleted(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted]; + + public static void SetReplacingMergeTreeIsDeleted(this IMutableEntityType entityType, string? isDeleted) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted, isDeleted); + + // SummingMergeTree + + public static string[]? GetSummingMergeTreeColumns(this IReadOnlyEntityType entityType) + => (string[]?)entityType[ClickHouseAnnotationNames.SummingMergeTreeColumns]; + + public static void SetSummingMergeTreeColumns(this IMutableEntityType entityType, string[]? columns) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns, + columns is { Length: > 0 } ? columns : null); + + // CollapsingMergeTree + + public static string? GetCollapsingMergeTreeSign(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.CollapsingMergeTreeSign]; + + public static void SetCollapsingMergeTreeSign(this IMutableEntityType entityType, string? sign) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign, sign); + + // VersionedCollapsingMergeTree + + public static string? GetVersionedCollapsingMergeTreeSign(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign]; + + public static void SetVersionedCollapsingMergeTreeSign(this IMutableEntityType entityType, string? sign) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign, sign); + + public static string? GetVersionedCollapsingMergeTreeVersion(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion]; + + public static void SetVersionedCollapsingMergeTreeVersion(this IMutableEntityType entityType, string? version) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion, version); + + // GraphiteMergeTree + + public static string? GetGraphiteMergeTreeConfigSection(this IReadOnlyEntityType entityType) + => (string?)entityType[ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection]; + + public static void SetGraphiteMergeTreeConfigSection(this IMutableEntityType entityType, string? configSection) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection, configSection); + + // Settings (prefix-based key-value) + + public static Dictionary GetSettings(this IReadOnlyEntityType entityType) + { + var settings = new Dictionary(); + foreach (var annotation in entityType.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.SettingPrefix, StringComparison.Ordinal) + && annotation.Value is string value) + { + var key = annotation.Name[ClickHouseAnnotationNames.SettingPrefix.Length..]; + settings[key] = value; + } + } + return settings; + } + + public static void SetSetting(this IMutableEntityType entityType, string settingName, string? value) + => entityType.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SettingPrefix + settingName, value); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs new file mode 100644 index 0000000..123bc32 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexBuilderExtensions.cs @@ -0,0 +1,25 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseIndexBuilderExtensions +{ + public static IndexBuilder HasSkippingIndexType(this IndexBuilder indexBuilder, string type) + { + indexBuilder.Metadata.SetSkippingIndexType(type); + return indexBuilder; + } + + public static IndexBuilder HasGranularity(this IndexBuilder indexBuilder, int granularity) + { + indexBuilder.Metadata.SetGranularity(granularity); + return indexBuilder; + } + + public static IndexBuilder HasSkippingIndexParams(this IndexBuilder indexBuilder, string parameters) + { + indexBuilder.Metadata.SetSkippingIndexParams(parameters); + return indexBuilder; + } +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs new file mode 100644 index 0000000..855a607 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseIndexExtensions.cs @@ -0,0 +1,31 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHouseIndexExtensions +{ + // Skipping index type (minmax, set, bloom_filter, etc.) + + public static string? GetSkippingIndexType(this IReadOnlyIndex index) + => (string?)index[ClickHouseAnnotationNames.SkippingIndexType]; + + public static void SetSkippingIndexType(this IMutableIndex index, string? type) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexType, type); + + // Granularity + + public static int? GetGranularity(this IReadOnlyIndex index) + => (int?)index[ClickHouseAnnotationNames.SkippingIndexGranularity]; + + public static void SetGranularity(this IMutableIndex index, int? granularity) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexGranularity, granularity); + + // Skipping index params (e.g., "100" for set(100), "0.01" for bloom_filter(0.01)) + + public static string? GetSkippingIndexParams(this IReadOnlyIndex index) + => (string?)index[ClickHouseAnnotationNames.SkippingIndexParams]; + + public static void SetSkippingIndexParams(this IMutableIndex index, string? parameters) + => index.SetOrRemoveAnnotation(ClickHouseAnnotationNames.SkippingIndexParams, parameters); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs new file mode 100644 index 0000000..7e62e51 --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyBuilderExtensions.cs @@ -0,0 +1,46 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHousePropertyBuilderExtensions +{ + public static PropertyBuilder HasCodec(this PropertyBuilder propertyBuilder, string codec) + { + propertyBuilder.Metadata.SetCodec(codec); + return propertyBuilder; + } + + public static PropertyBuilder HasCodec( + this PropertyBuilder propertyBuilder, string codec) + { + propertyBuilder.Metadata.SetCodec(codec); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnTtl(this PropertyBuilder propertyBuilder, string ttlExpression) + { + propertyBuilder.Metadata.SetColumnTtl(ttlExpression); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnTtl( + this PropertyBuilder propertyBuilder, string ttlExpression) + { + propertyBuilder.Metadata.SetColumnTtl(ttlExpression); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnComment(this PropertyBuilder propertyBuilder, string comment) + { + propertyBuilder.Metadata.SetColumnComment(comment); + return propertyBuilder; + } + + public static PropertyBuilder HasColumnComment( + this PropertyBuilder propertyBuilder, string comment) + { + propertyBuilder.Metadata.SetColumnComment(comment); + return propertyBuilder; + } +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs new file mode 100644 index 0000000..eae4efc --- /dev/null +++ b/src/EFCore.ClickHouse/Extensions/ClickHousePropertyExtensions.cs @@ -0,0 +1,31 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Extensions; + +public static class ClickHousePropertyExtensions +{ + // Codec + + public static string? GetCodec(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnCodec]; + + public static void SetCodec(this IMutableProperty property, string? codec) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnCodec, codec); + + // Column TTL + + public static string? GetColumnTtl(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnTtl]; + + public static void SetColumnTtl(this IMutableProperty property, string? ttlExpression) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnTtl, ttlExpression); + + // Column Comment + + public static string? GetColumnComment(this IReadOnlyProperty property) + => (string?)property[ClickHouseAnnotationNames.ColumnComment]; + + public static void SetColumnComment(this IMutableProperty property, string? comment) + => property.SetOrRemoveAnnotation(ClickHouseAnnotationNames.ColumnComment, comment); +} diff --git a/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs b/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs index 4d9528c..e0f4861 100644 --- a/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs +++ b/src/EFCore.ClickHouse/Extensions/ClickHouseServiceCollectionExtensions.cs @@ -7,8 +7,11 @@ using ClickHouse.EntityFrameworkCore.Query.ExpressionTranslators.Internal; using ClickHouse.EntityFrameworkCore.Query.Internal; using ClickHouse.EntityFrameworkCore.Storage.Internal; +using ClickHouse.EntityFrameworkCore.Migrations; +using ClickHouse.EntityFrameworkCore.Migrations.Internal; using ClickHouse.EntityFrameworkCore.Update.Internal; using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Metadata.Conventions.Infrastructure; @@ -35,6 +38,9 @@ public static IServiceCollection AddEntityFrameworkClickHouse(this IServiceColle .TryAdd(p => p.GetRequiredService()) .TryAdd() .TryAdd() + .TryAdd() + .TryAdd() + .TryAdd() .TryAdd() .TryAdd() .TryAdd() diff --git a/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs b/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs index cdea2d2..c50f2ac 100644 --- a/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs +++ b/src/EFCore.ClickHouse/Infrastructure/Internal/ClickHouseModelValidator.cs @@ -1,7 +1,10 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.Logging; namespace ClickHouse.EntityFrameworkCore.Infrastructure.Internal; @@ -20,6 +23,7 @@ public override void Validate(IModel model, IDiagnosticsLogger logger) + { + foreach (var entityType in model.GetEntityTypes()) + { + if (entityType.IsOwned() || entityType.GetTableName() is null) + continue; + + var engine = entityType.GetEngine(); + if (engine is null) + continue; + + // Log engines should not have ORDER BY/PARTITION BY + if (engine is ClickHouseAnnotationNames.TinyLog + or ClickHouseAnnotationNames.StripeLog + or ClickHouseAnnotationNames.Log + or ClickHouseAnnotationNames.Memory) + { + if (entityType.GetOrderBy() is not null) + { + logger.Logger.Log(LogLevel.Warning, + "Entity type '{EntityType}' uses the '{Engine}' engine which does not support ORDER BY.", + entityType.DisplayName(), engine); + } + } + + // Validate CollapsingMergeTree sign column: must exist and be Int8 + if (engine is ClickHouseAnnotationNames.CollapsingMergeTree) + { + var sign = entityType.GetCollapsingMergeTreeSign(); + ValidateColumnReference(entityType, sign, "CollapsingMergeTree", "sign"); + ValidateColumnStoreType(entityType, sign, "Int8", + "CollapsingMergeTree", "sign"); + } + + // Validate VersionedCollapsingMergeTree: sign must be Int8, version must exist + if (engine is ClickHouseAnnotationNames.VersionedCollapsingMergeTree) + { + var sign = entityType.GetVersionedCollapsingMergeTreeSign(); + ValidateColumnReference(entityType, sign, "VersionedCollapsingMergeTree", "sign"); + ValidateColumnStoreType(entityType, sign, "Int8", + "VersionedCollapsingMergeTree", "sign"); + + ValidateColumnReference(entityType, entityType.GetVersionedCollapsingMergeTreeVersion(), + "VersionedCollapsingMergeTree", "version"); + } + + // Validate ReplacingMergeTree: version must exist, isDeleted must be UInt8 + if (engine is ClickHouseAnnotationNames.ReplacingMergeTree) + { + ValidateColumnReference(entityType, entityType.GetReplacingMergeTreeVersion(), + "ReplacingMergeTree", "version"); + + var isDeleted = entityType.GetReplacingMergeTreeIsDeleted(); + ValidateColumnReference(entityType, isDeleted, "ReplacingMergeTree", "isDeleted"); + ValidateColumnStoreType(entityType, isDeleted, "UInt8", + "ReplacingMergeTree", "isDeleted"); + } + + // Validate SummingMergeTree columns exist + if (engine is ClickHouseAnnotationNames.SummingMergeTree) + { + var columns = entityType.GetSummingMergeTreeColumns(); + if (columns is not null) + { + foreach (var col in columns) + { + ValidateColumnReference(entityType, col, "SummingMergeTree", "sum column"); + } + } + } + } + } + + private static void ValidateColumnReference( + IEntityType entityType, string? columnName, string engineName, string parameterName) + { + if (columnName is not null && !HasPropertyWithColumn(entityType, columnName)) + { + throw new InvalidOperationException( + $"Entity type '{entityType.DisplayName()}' uses {engineName} with {parameterName} column " + + $"'{columnName}' which does not match any mapped property."); + } + } + + private static void ValidateColumnStoreType( + IEntityType entityType, string? columnName, string requiredStoreType, + string engineName, string parameterName) + { + if (columnName is null) + return; + + var property = FindPropertyByColumn(entityType, columnName); + if (property is null) + return; // existence check is handled by ValidateColumnReference + + var storeType = (property.FindTypeMapping() as RelationalTypeMapping)?.StoreType + ?? property.GetColumnType(); + if (storeType is not null + && !storeType.Equals(requiredStoreType, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Entity type '{entityType.DisplayName()}' uses {engineName} with {parameterName} column " + + $"'{columnName}' which must have store type {requiredStoreType}, " + + $"but the resolved type is '{storeType}'."); + } + } + + private static IProperty? FindPropertyByColumn(IEntityType entityType, string columnName) + => entityType.GetProperties().FirstOrDefault(p => + string.Equals(p.GetColumnName(), columnName, StringComparison.Ordinal) + || string.Equals(p.Name, columnName, StringComparison.Ordinal)); + + private static bool HasPropertyWithColumn(IEntityType entityType, string columnName) + => entityType.GetProperties().Any(p => + string.Equals(p.GetColumnName(), columnName, StringComparison.Ordinal) + || string.Equals(p.Name, columnName, StringComparison.Ordinal)); } diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..b0c0b9e --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseAggregatingMergeTreeEngineBuilder.cs @@ -0,0 +1,48 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseAggregatingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseAggregatingMergeTreeEngineBuilder(IMutableEntityType entityType) + : base(entityType, ClickHouseAnnotationNames.AggregatingMergeTree) + { + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseAggregatingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..ec06535 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseCollapsingMergeTreeEngineBuilder.cs @@ -0,0 +1,50 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseCollapsingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseCollapsingMergeTreeEngineBuilder(IMutableEntityType entityType, string sign) + : base(entityType, ClickHouseAnnotationNames.CollapsingMergeTree) + { + entityType.SetCollapsingMergeTreeSign(sign); + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseCollapsingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs new file mode 100644 index 0000000..5b57490 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseEngineBuilder.cs @@ -0,0 +1,61 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public abstract class ClickHouseEngineBuilder +{ + protected IMutableEntityType EntityType { get; } + + protected ClickHouseEngineBuilder(IMutableEntityType entityType, string engineName) + { + ArgumentNullException.ThrowIfNull(entityType); + ArgumentException.ThrowIfNullOrWhiteSpace(engineName); + + EntityType = entityType; + entityType.SetEngine(engineName); + } + + public ClickHouseEngineBuilder WithOrderBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetOrderBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithPartitionBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetPartitionBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithPrimaryKey(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetClickHousePrimaryKey(columns); + return this; + } + + public ClickHouseEngineBuilder WithSampleBy(params string[] columns) + { + ArgumentNullException.ThrowIfNull(columns); + EntityType.SetSampleBy(columns); + return this; + } + + public ClickHouseEngineBuilder WithTtl(string ttlExpression) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ttlExpression); + EntityType.SetTtl(ttlExpression); + return this; + } + + public ClickHouseEngineBuilder WithSetting(string key, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(value); + EntityType.SetSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..3b37778 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseGraphiteMergeTreeEngineBuilder.cs @@ -0,0 +1,50 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseGraphiteMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseGraphiteMergeTreeEngineBuilder(IMutableEntityType entityType, string configSection) + : base(entityType, ClickHouseAnnotationNames.GraphiteMergeTree) + { + entityType.SetGraphiteMergeTreeConfigSection(configSection); + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseGraphiteMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..160b965 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseMergeTreeEngineBuilder.cs @@ -0,0 +1,48 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseMergeTreeEngineBuilder(IMutableEntityType entityType) + : base(entityType, ClickHouseAnnotationNames.MergeTree) + { + } + + public new ClickHouseMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..5ca8fb9 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseReplacingMergeTreeEngineBuilder.cs @@ -0,0 +1,54 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseReplacingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseReplacingMergeTreeEngineBuilder( + IMutableEntityType entityType, string? version = null, string? isDeleted = null) + : base(entityType, ClickHouseAnnotationNames.ReplacingMergeTree) + { + if (version is not null) + entityType.SetReplacingMergeTreeVersion(version); + if (isDeleted is not null) + entityType.SetReplacingMergeTreeIsDeleted(isDeleted); + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseReplacingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs new file mode 100644 index 0000000..f1eb0a9 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSimpleEngineBuilder.cs @@ -0,0 +1,40 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +/// +/// Engine builder for simple engines (TinyLog, StripeLog, Log, Memory) that do not support +/// ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, or SETTINGS. +/// +public class ClickHouseSimpleEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseSimpleEngineBuilder(IMutableEntityType entityType, string engineName) + : base(entityType, engineName) + { + } + + public new ClickHouseSimpleEngineBuilder WithOrderBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support ORDER BY."); + + public new ClickHouseSimpleEngineBuilder WithPartitionBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support PARTITION BY."); + + public new ClickHouseSimpleEngineBuilder WithPrimaryKey(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support PRIMARY KEY."); + + public new ClickHouseSimpleEngineBuilder WithSampleBy(params string[] columns) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support SAMPLE BY."); + + public new ClickHouseSimpleEngineBuilder WithTtl(string ttlExpression) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support TTL."); + + public new ClickHouseSimpleEngineBuilder WithSetting(string key, string value) + => throw new InvalidOperationException( + $"The '{EntityType.GetEngine()}' engine does not support SETTINGS."); +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..cbe3d42 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseSummingMergeTreeEngineBuilder.cs @@ -0,0 +1,51 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseSummingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseSummingMergeTreeEngineBuilder(IMutableEntityType entityType, params string[] columns) + : base(entityType, ClickHouseAnnotationNames.SummingMergeTree) + { + if (columns.Length > 0) + entityType.SetSummingMergeTreeColumns(columns); + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseSummingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs new file mode 100644 index 0000000..fd3b5c3 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Builders/ClickHouseVersionedCollapsingMergeTreeEngineBuilder.cs @@ -0,0 +1,52 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Builders; + +public class ClickHouseVersionedCollapsingMergeTreeEngineBuilder : ClickHouseEngineBuilder +{ + public ClickHouseVersionedCollapsingMergeTreeEngineBuilder( + IMutableEntityType entityType, string sign, string version) + : base(entityType, ClickHouseAnnotationNames.VersionedCollapsingMergeTree) + { + entityType.SetVersionedCollapsingMergeTreeSign(sign); + entityType.SetVersionedCollapsingMergeTreeVersion(version); + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithOrderBy(params string[] columns) + { + base.WithOrderBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithPartitionBy(params string[] columns) + { + base.WithPartitionBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithPrimaryKey(params string[] columns) + { + base.WithPrimaryKey(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithSampleBy(params string[] columns) + { + base.WithSampleBy(columns); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithTtl(string ttlExpression) + { + base.WithTtl(ttlExpression); + return this; + } + + public new ClickHouseVersionedCollapsingMergeTreeEngineBuilder WithSetting(string key, string value) + { + base.WithSetting(key, value); + return this; + } +} diff --git a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs index f6bbf75..af8ec2a 100644 --- a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs +++ b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseConventionSetBuilder.cs @@ -15,8 +15,32 @@ public ClickHouseConventionSetBuilder( public override ConventionSet CreateConventionSet() { var conventionSet = base.CreateConventionSet(); - // ClickHouse doesn't support auto-increment. - // Remove ValueGenerationConvention if needed, or handle in model validator. + + // ClickHouse doesn't support foreign keys — remove all FK-related conventions + // to prevent EF from creating implicit indexes for FK columns. + RemoveForeignKeyIndexConvention(conventionSet.EntityTypeBaseTypeChangedConventions); + conventionSet.ForeignKeyAddedConventions.Clear(); + conventionSet.ForeignKeyAnnotationChangedConventions.Clear(); + conventionSet.ForeignKeyDependentRequirednessChangedConventions.Clear(); + conventionSet.ForeignKeyOwnershipChangedConventions.Clear(); + conventionSet.ForeignKeyPrincipalEndChangedConventions.Clear(); + conventionSet.ForeignKeyPropertiesChangedConventions.Clear(); + conventionSet.ForeignKeyRemovedConventions.Clear(); + conventionSet.ForeignKeyRequirednessChangedConventions.Clear(); + conventionSet.ForeignKeyUniquenessChangedConventions.Clear(); + conventionSet.SkipNavigationForeignKeyChangedConventions.Clear(); + + conventionSet.ModelFinalizingConventions.Add(new ClickHouseDefaultEngineConvention()); + return conventionSet; } + + private static void RemoveForeignKeyIndexConvention(IList conventions) + { + for (var i = conventions.Count - 1; i >= 0; i--) + { + if (conventions[i] is ForeignKeyIndexConvention) + conventions.RemoveAt(i); + } + } } diff --git a/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs new file mode 100644 index 0000000..4a12c29 --- /dev/null +++ b/src/EFCore.ClickHouse/Metadata/Conventions/ClickHouseDefaultEngineConvention.cs @@ -0,0 +1,60 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Metadata.Conventions; + +namespace ClickHouse.EntityFrameworkCore.Metadata.Conventions; + +/// +/// Sets MergeTree as the default engine for entity types that don't have an explicit engine configured. +/// Uses the EF primary key columns as ORDER BY when no explicit ORDER BY is set. +/// +public class ClickHouseDefaultEngineConvention : IModelFinalizingConvention +{ + public void ProcessModelFinalizing(IConventionModelBuilder modelBuilder, IConventionContext context) + { + foreach (var entityType in modelBuilder.Metadata.GetEntityTypes()) + { + // Skip owned types and those without a table mapping + if (entityType.IsOwned() || entityType.GetTableName() is null) + continue; + + var mutableEntityType = (IMutableEntityType)entityType; + + // Set default engine to MergeTree if none configured + if (entityType.GetEngine() is null) + { + mutableEntityType.SetEngine(ClickHouseAnnotationNames.MergeTree); + } + + // Set ORDER BY from primary key if none configured and engine is MergeTree-family + var engine = entityType.GetEngine(); + if (entityType.GetOrderBy() is null && IsMergeTreeFamily(engine)) + { + var primaryKey = entityType.FindPrimaryKey(); + if (primaryKey is not null) + { + var columns = primaryKey.Properties + .Select(p => p.GetColumnName() ?? p.Name) + .ToArray(); + mutableEntityType.SetOrderBy(columns); + } + else + { + mutableEntityType.SetOrderBy(["tuple()"]); + } + } + } + } + + private static bool IsMergeTreeFamily(string? engine) + => engine is ClickHouseAnnotationNames.MergeTree + or ClickHouseAnnotationNames.ReplacingMergeTree + or ClickHouseAnnotationNames.SummingMergeTree + or ClickHouseAnnotationNames.AggregatingMergeTree + or ClickHouseAnnotationNames.CollapsingMergeTree + or ClickHouseAnnotationNames.VersionedCollapsingMergeTree + or ClickHouseAnnotationNames.GraphiteMergeTree; +} diff --git a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs index 8a9eb97..b72723d 100644 --- a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs +++ b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationNames.cs @@ -3,8 +3,47 @@ namespace ClickHouse.EntityFrameworkCore.Metadata.Internal; public static class ClickHouseAnnotationNames { public const string Prefix = "ClickHouse:"; + + // Table-level engine configuration public const string Engine = Prefix + "Engine"; public const string OrderBy = Prefix + "OrderBy"; public const string PartitionBy = Prefix + "PartitionBy"; public const string PrimaryKey = Prefix + "PrimaryKey"; + public const string SampleBy = Prefix + "SampleBy"; + public const string Ttl = Prefix + "Ttl"; + + // Engine-specific parameters + public const string ReplacingMergeTreeVersion = Prefix + "ReplacingMergeTree:Version"; + public const string ReplacingMergeTreeIsDeleted = Prefix + "ReplacingMergeTree:IsDeleted"; + public const string SummingMergeTreeColumns = Prefix + "SummingMergeTree:Columns"; + public const string CollapsingMergeTreeSign = Prefix + "CollapsingMergeTree:Sign"; + public const string VersionedCollapsingMergeTreeSign = Prefix + "VersionedCollapsingMergeTree:Sign"; + public const string VersionedCollapsingMergeTreeVersion = Prefix + "VersionedCollapsingMergeTree:Version"; + public const string GraphiteMergeTreeConfigSection = Prefix + "GraphiteMergeTree:ConfigSection"; + + // Settings (prefix-based key-value storage) + public const string SettingPrefix = Prefix + "Setting:"; + + // Column-level annotations + public const string ColumnCodec = Prefix + "ColumnCodec"; + public const string ColumnTtl = Prefix + "ColumnTtl"; + public const string ColumnComment = Prefix + "ColumnComment"; + + // Data-skipping index annotations + public const string SkippingIndexType = Prefix + "SkippingIndex:Type"; + public const string SkippingIndexGranularity = Prefix + "SkippingIndex:Granularity"; + public const string SkippingIndexParams = Prefix + "SkippingIndex:Params"; + + // Engine name constants + public const string MergeTree = "MergeTree"; + public const string ReplacingMergeTree = "ReplacingMergeTree"; + public const string SummingMergeTree = "SummingMergeTree"; + public const string AggregatingMergeTree = "AggregatingMergeTree"; + public const string CollapsingMergeTree = "CollapsingMergeTree"; + public const string VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"; + public const string GraphiteMergeTree = "GraphiteMergeTree"; + public const string TinyLog = "TinyLog"; + public const string StripeLog = "StripeLog"; + public const string Log = "Log"; + public const string Memory = "Memory"; } diff --git a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs index a86a7ca..1468f44 100644 --- a/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs +++ b/src/EFCore.ClickHouse/Metadata/Internal/ClickHouseAnnotationProvider.cs @@ -1,3 +1,4 @@ +using ClickHouse.EntityFrameworkCore.Extensions; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -9,4 +10,54 @@ public ClickHouseAnnotationProvider(RelationalAnnotationProviderDependencies dep : base(dependencies) { } + + public override IEnumerable For(ITable table, bool designTime) + { + if (!designTime) + yield break; + + var mapping = table.EntityTypeMappings.FirstOrDefault(); + if (mapping is null) + yield break; + + var entityType = (IEntityType)mapping.TypeBase; + + foreach (var annotation in entityType.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } + + public override IEnumerable For(ITableIndex index, bool designTime) + { + if (!designTime) + yield break; + + var modelIndex = index.MappedIndexes.FirstOrDefault(); + if (modelIndex is null) + yield break; + + foreach (var annotation in modelIndex.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } + + public override IEnumerable For(IColumn column, bool designTime) + { + if (!designTime) + yield break; + + var mapping = column.PropertyMappings.FirstOrDefault(); + if (mapping is null) + yield break; + + foreach (var annotation in mapping.Property.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } } diff --git a/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs new file mode 100644 index 0000000..304e19a --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/ClickHouseMigrationsSqlGenerator.cs @@ -0,0 +1,653 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using ClickHouse.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations; + +public class ClickHouseMigrationsSqlGenerator : MigrationsSqlGenerator +{ + public ClickHouseMigrationsSqlGenerator(MigrationsSqlGeneratorDependencies dependencies) + : base(dependencies) + { + } + + // ClickHouse does not support transactions — suppress on all statements. + protected override void EndStatement(MigrationCommandListBuilder builder, bool suppressTransaction = true) + => base.EndStatement(builder, suppressTransaction: true); + + // Custom operation dispatch + + protected override void Generate(MigrationOperation operation, IModel? model, MigrationCommandListBuilder builder) + { + switch (operation) + { + case ClickHouseCreateDatabaseOperation createDb: + Generate(createDb, builder); + return; + case ClickHouseDropDatabaseOperation dropDb: + Generate(dropDb, builder); + return; + default: + base.Generate(operation, model, builder); + return; + } + } + + protected virtual void Generate(ClickHouseCreateDatabaseOperation operation, MigrationCommandListBuilder builder) + { + builder + .Append("CREATE DATABASE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + EndStatement(builder, suppressTransaction: true); + } + + protected virtual void Generate(ClickHouseDropDatabaseOperation operation, MigrationCommandListBuilder builder) + { + builder + .Append("DROP DATABASE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + EndStatement(builder, suppressTransaction: true); + } + + // CREATE TABLE with ENGINE clause + + protected override void Generate( + CreateTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + base.Generate(operation, model, builder, terminate: false); + + GenerateEngineClause(operation, builder); + + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatement(builder); + } + + // Column definition: ClickHouse nullable wrapping, codec, TTL, comment + + protected override void ColumnDefinition( + string? schema, + string table, + string name, + ColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + if (!string.IsNullOrEmpty(operation.ComputedColumnSql)) + { + ComputedColumnDefinition(schema, table, name, operation, model, builder); + return; + } + + var columnType = operation.ColumnType ?? GetColumnType(schema, table, name, operation, model)!; + + // Wrap nullable scalar types in Nullable(T). + // Skip: arrays (CLR T[] or List → Array(T)), Map, Json, Tuple, Variant — + // ClickHouse does not support Nullable(Array(...)) etc. + if (operation.IsNullable && !IsNonNullableContainerType(operation.ClrType, columnType)) + { + columnType = $"Nullable({columnType})"; + } + + builder + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(name)) + .Append(" ") + .Append(columnType); + + // DEFAULT + var defaultValue = operation.DefaultValueSql; + if (string.IsNullOrWhiteSpace(defaultValue) && operation.DefaultValue is not null) + { + var typeMapping = (!string.IsNullOrEmpty(operation.ColumnType) + ? Dependencies.TypeMappingSource.FindMapping(operation.DefaultValue.GetType(), operation.ColumnType) + : null) ?? Dependencies.TypeMappingSource.FindMapping(operation.DefaultValue.GetType())!; + defaultValue = typeMapping.GenerateSqlLiteral(operation.DefaultValue); + } + + if (!string.IsNullOrWhiteSpace(defaultValue)) + builder.Append(" DEFAULT ").Append(defaultValue); + + // ClickHouse column definition order: DEFAULT → COMMENT → CODEC → TTL + + // COMMENT + var comment = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnComment); + if (comment?.Value is string commentStr && !string.IsNullOrWhiteSpace(commentStr)) + { + var escaped = commentStr.Replace("'", "\\'"); + builder.Append($" COMMENT '{escaped}'"); + } + + // CODEC + var codec = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnCodec); + if (codec?.Value is string codecStr && !string.IsNullOrWhiteSpace(codecStr)) + builder.Append($" CODEC({codecStr})"); + + // TTL + var columnTtl = operation.FindAnnotation(ClickHouseAnnotationNames.ColumnTtl); + if (columnTtl?.Value is string ttlStr && !string.IsNullOrWhiteSpace(ttlStr)) + builder.Append($" TTL {ttlStr}"); + } + + protected override void ComputedColumnDefinition( + string? schema, + string table, + string name, + ColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + var keyword = operation.IsStored == true ? " MATERIALIZED " : " ALIAS "; + var columnType = operation.ColumnType ?? GetColumnType(schema, table, name, operation, model)!; + + if (operation.IsNullable && !IsNonNullableContainerType(operation.ClrType, columnType)) + { + columnType = $"Nullable({columnType})"; + } + + builder + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(name)) + .Append(" ") + .Append(columnType) + .Append(keyword) + .Append(operation.ComputedColumnSql!); + } + + // Suppress primary key, foreign key, unique constraints — ClickHouse doesn't support them as SQL constraints + + protected override void CreateTableConstraints( + CreateTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + CreateTableCheckConstraints(operation, model, builder); + } + + // ALTER TABLE operations + + protected override void Generate( + AddColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD COLUMN "); + + ColumnDefinition(operation, model, builder); + EndStatement(builder); + } + + protected override void Generate( + DropColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" DROP COLUMN ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + + EndStatement(builder); + } + + protected override void Generate( + AlterColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" MODIFY COLUMN "); + + ColumnDefinition(operation.Schema, operation.Table, operation.Name, operation, model, builder); + EndStatement(builder); + + // Emit REMOVE statements for column annotations that were present on the old column but not the new one. + // ClickHouse requires explicit REMOVE CODEC / REMOVE TTL / REMOVE COMMENT — a bare MODIFY COLUMN + // does not clear these attributes. + EmitColumnAnnotationRemovals(operation, builder); + } + + private void EmitColumnAnnotationRemovals(AlterColumnOperation operation, MigrationCommandListBuilder builder) + { + ReadOnlySpan removableAnnotations = + [ + ClickHouseAnnotationNames.ColumnCodec, + ClickHouseAnnotationNames.ColumnTtl, + ClickHouseAnnotationNames.ColumnComment, + ]; + + foreach (var annotationName in removableAnnotations) + { + var oldValue = (string?)operation.OldColumn.FindAnnotation(annotationName)?.Value; + var newValue = (string?)operation.FindAnnotation(annotationName)?.Value; + + if (string.IsNullOrWhiteSpace(oldValue) || !string.IsNullOrWhiteSpace(newValue)) + continue; + + var keyword = annotationName switch + { + ClickHouseAnnotationNames.ColumnCodec => "REMOVE CODEC", + ClickHouseAnnotationNames.ColumnTtl => "REMOVE TTL", + ClickHouseAnnotationNames.ColumnComment => "REMOVE COMMENT", + _ => null + }; + + if (keyword is null) + continue; + + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" MODIFY COLUMN ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" ") + .Append(keyword); + EndStatement(builder); + } + } + + protected override void Generate( + RenameColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" RENAME COLUMN ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" TO ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.NewName)); + + EndStatement(builder); + } + + protected override void Generate( + RenameTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + builder + .Append("RENAME TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name, operation.Schema)) + .Append(" TO ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.NewName!, operation.NewSchema)); + + EndStatement(builder); + } + + // ALTER TABLE — reject ClickHouse metadata changes (engine, ORDER BY, etc. are immutable) + + protected override void Generate( + AlterTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder) + { + // Collect all ClickHouse annotations from old and new + var oldAnnotations = operation.OldTable.GetAnnotations() + .Where(a => a.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + .ToDictionary(a => a.Name, a => a.Value); + var newAnnotations = operation.GetAnnotations() + .Where(a => a.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + .ToDictionary(a => a.Name, a => a.Value); + + // Find any annotation that was added, removed, or changed + var allKeys = oldAnnotations.Keys.Union(newAnnotations.Keys); + foreach (var key in allKeys) + { + oldAnnotations.TryGetValue(key, out var oldVal); + newAnnotations.TryGetValue(key, out var newVal); + + if (!AnnotationValuesEqual(oldVal, newVal)) + { + var shortName = key[ClickHouseAnnotationNames.Prefix.Length..]; + throw new NotSupportedException( + $"ClickHouse does not support changing table metadata '{shortName}' via ALTER TABLE. " + + "Recreate the table instead."); + } + } + + // Delegate to base for non-ClickHouse annotation changes (e.g., comments) + base.Generate(operation, model, builder); + } + + private static bool AnnotationValuesEqual(object? a, object? b) + { + if (a is null && b is null) return true; + if (a is null || b is null) return false; + if (a is string[] arrA && b is string[] arrB) return arrA.SequenceEqual(arrB); + return a.Equals(b); + } + + // Data-skipping indices + + protected override void Generate( + CreateIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + if (operation.IsUnique) + throw new NotSupportedException("ClickHouse does not support unique indexes."); + + var indexType = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexType)?.Value; + if (indexType is null) + { + // Standard index — ClickHouse doesn't support CREATE INDEX syntax + // Skip silently rather than error, since EF may generate these for PK-like indices + return; + } + + var indexParams = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexParams)?.Value; + var granularity = (int?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexGranularity)?.Value ?? 1; + + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" (") + .Append(string.Join(", ", operation.Columns.Select(c => + Dependencies.SqlGenerationHelper.DelimitIdentifier(c)))) + .Append(") TYPE ") + .Append(indexType); + + if (!string.IsNullOrWhiteSpace(indexParams)) + builder.Append($"({indexParams})"); + + builder.Append($" GRANULARITY {granularity}"); + EndStatement(builder); + } + + protected override void Generate( + DropIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + // Only emit DROP INDEX for skipping indexes (same symmetry as CreateIndexOperation). + // Standard EF indexes are not created in ClickHouse, so dropping them is a no-op. + var indexType = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexType)?.Value; + if (indexType is null) + return; + + builder + .Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table!, operation.Schema)) + .Append(" DROP INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)); + + EndStatement(builder); + } + + // Unsupported operations + + protected override void Generate(AddForeignKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + => throw new NotSupportedException("ClickHouse does not support foreign key constraints."); + + protected override void Generate(DropForeignKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + => throw new NotSupportedException("ClickHouse does not support foreign key constraints."); + + protected override void Generate(AddUniqueConstraintOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support unique constraints."); + + protected override void Generate(DropUniqueConstraintOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support unique constraints."); + + protected override void Generate(AddPrimaryKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + { + // No-op: ClickHouse primary key is structural (ORDER BY), not a constraint + } + + protected override void Generate(DropPrimaryKeyOperation operation, IModel? model, MigrationCommandListBuilder builder, bool terminate = true) + { + // No-op: ClickHouse primary key is structural (ORDER BY), not a constraint + } + + protected override void Generate(CreateSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(AlterSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(DropSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(RenameSequenceOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support sequences."); + + protected override void Generate(EnsureSchemaOperation operation, IModel? model, MigrationCommandListBuilder builder) + => throw new NotSupportedException("ClickHouse does not support schemas. Use databases instead."); + + // ENGINE clause generation + + private void GenerateEngineClause(CreateTableOperation operation, MigrationCommandListBuilder builder) + { + var engine = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.Engine)?.Value + ?? ClickHouseAnnotationNames.MergeTree; + + builder.AppendLine(); + + // ENGINE = EngineName or ENGINE = EngineName(args) + // Simple engines (Log, TinyLog, StripeLog, Memory) use bare names without parentheses. + if (IsSimpleEngine(engine)) + { + builder.Append($"ENGINE = {engine}"); + } + else + { + builder.Append($"ENGINE = {engine}("); + GenerateEngineArgs(operation, engine, builder); + builder.Append(")"); + } + + // ORDER BY + var orderBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.OrderBy)?.Value; + if (orderBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("ORDER BY ("); + builder.Append(string.Join(", ", orderBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + else if (IsMergeTreeFamily(engine)) + { + builder.AppendLine(); + builder.Append("ORDER BY tuple()"); + } + + // PARTITION BY + var partitionBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.PartitionBy)?.Value; + if (partitionBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("PARTITION BY "); + if (partitionBy.Length == 1) + builder.Append(QuoteColumnOrExpression(partitionBy[0])); + else + { + builder.Append("("); + builder.Append(string.Join(", ", partitionBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + } + + // PRIMARY KEY + var primaryKey = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.PrimaryKey)?.Value; + if (primaryKey is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("PRIMARY KEY ("); + builder.Append(string.Join(", ", primaryKey.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + + // SAMPLE BY + var sampleBy = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.SampleBy)?.Value; + if (sampleBy is { Length: > 0 }) + { + builder.AppendLine(); + builder.Append("SAMPLE BY "); + if (sampleBy.Length == 1) + builder.Append(QuoteColumnOrExpression(sampleBy[0])); + else + { + builder.Append("("); + builder.Append(string.Join(", ", sampleBy.Select(QuoteColumnOrExpression))); + builder.Append(")"); + } + } + + // TTL + var ttl = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.Ttl)?.Value; + if (!string.IsNullOrWhiteSpace(ttl)) + { + builder.AppendLine(); + builder.Append($"TTL {ttl}"); + } + + // SETTINGS + GenerateSettingsClause(operation, builder); + } + + private void GenerateEngineArgs(CreateTableOperation operation, string engine, MigrationCommandListBuilder builder) + { + switch (engine) + { + case ClickHouseAnnotationNames.ReplacingMergeTree: + var version = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion)?.Value; + var isDeleted = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeIsDeleted)?.Value; + var args = new List(); + if (version is not null) + args.Add(QuoteColumnOrExpression(version)); + if (isDeleted is not null) + args.Add(QuoteColumnOrExpression(isDeleted)); + builder.Append(string.Join(", ", args)); + break; + + case ClickHouseAnnotationNames.SummingMergeTree: + var columns = (string[]?)operation.FindAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns)?.Value; + if (columns is { Length: > 0 }) + builder.Append(string.Join(", ", columns.Select(QuoteColumnOrExpression))); + break; + + case ClickHouseAnnotationNames.CollapsingMergeTree: + var sign = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign)?.Value; + if (sign is not null) + builder.Append(QuoteColumnOrExpression(sign)); + break; + + case ClickHouseAnnotationNames.VersionedCollapsingMergeTree: + var vcSign = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign)?.Value; + var vcVersion = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion)?.Value; + var vcArgs = new List(); + if (vcSign is not null) vcArgs.Add(QuoteColumnOrExpression(vcSign)); + if (vcVersion is not null) vcArgs.Add(QuoteColumnOrExpression(vcVersion)); + builder.Append(string.Join(", ", vcArgs)); + break; + + case ClickHouseAnnotationNames.GraphiteMergeTree: + var config = (string?)operation.FindAnnotation(ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection)?.Value; + if (config is not null) + builder.Append($"'{config}'"); + break; + } + } + + private void GenerateSettingsClause(CreateTableOperation operation, MigrationCommandListBuilder builder) + { + var settings = new Dictionary(); + foreach (var annotation in operation.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.SettingPrefix, StringComparison.Ordinal) + && annotation.Value is string value) + { + var key = annotation.Name[ClickHouseAnnotationNames.SettingPrefix.Length..]; + settings[key] = value; + } + } + + if (settings.Count == 0) + return; + + builder.AppendLine(); + builder.Append("SETTINGS "); + builder.Append(string.Join(", ", settings.Select(kv => $"{kv.Key} = {kv.Value}"))); + } + + private string QuoteColumnOrExpression(string columnOrExpr) + { + // Simple identifier (letters, digits, underscore) → backtick quote. + // Anything else (operators, parentheses, spaces) → SQL expression, emit verbatim. + if (IsSimpleIdentifier(columnOrExpr)) + return Dependencies.SqlGenerationHelper.DelimitIdentifier(columnOrExpr); + + return columnOrExpr; + } + + private static bool IsSimpleIdentifier(string s) + { + if (s.Length == 0) + return false; + + if (s[0] != '_' && !char.IsLetter(s[0])) + return false; + + for (var i = 1; i < s.Length; i++) + { + if (s[i] != '_' && !char.IsLetterOrDigit(s[i])) + return false; + } + + return true; + } + + /// + /// Returns true for CLR/store types that ClickHouse does not allow inside Nullable(). + /// Array, Map, Tuple, Variant, Dynamic, Json — and already-Nullable columns. + /// + private static bool IsNonNullableContainerType(Type? clrType, string columnType) + { + // CLR array (T[]) + if (clrType?.IsArray == true) + return true; + + // List → maps to Array(T) at the store level + if (clrType is { IsGenericType: true } && clrType.GetGenericTypeDefinition() == typeof(List<>)) + return true; + + // Store-type check covers Array, Map, Tuple, Variant, Dynamic, Json, and already-wrapped Nullable + return columnType.StartsWith("Nullable(", StringComparison.OrdinalIgnoreCase) + || columnType.StartsWith("Array(", StringComparison.OrdinalIgnoreCase) + || columnType.StartsWith("Map(", StringComparison.OrdinalIgnoreCase) + || columnType.StartsWith("Tuple(", StringComparison.OrdinalIgnoreCase) + || columnType.StartsWith("Variant(", StringComparison.OrdinalIgnoreCase) + || columnType.StartsWith("Json", StringComparison.OrdinalIgnoreCase) + || columnType.Equals("Dynamic", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsMergeTreeFamily(string engine) + => engine is ClickHouseAnnotationNames.MergeTree + or ClickHouseAnnotationNames.ReplacingMergeTree + or ClickHouseAnnotationNames.SummingMergeTree + or ClickHouseAnnotationNames.AggregatingMergeTree + or ClickHouseAnnotationNames.CollapsingMergeTree + or ClickHouseAnnotationNames.VersionedCollapsingMergeTree + or ClickHouseAnnotationNames.GraphiteMergeTree; + + private static bool IsSimpleEngine(string engine) + => engine is ClickHouseAnnotationNames.TinyLog + or ClickHouseAnnotationNames.StripeLog + or ClickHouseAnnotationNames.Log + or ClickHouseAnnotationNames.Memory; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs new file mode 100644 index 0000000..5bb9e79 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseHistoryRepository.cs @@ -0,0 +1,60 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +public class ClickHouseHistoryRepository : HistoryRepository +{ + public ClickHouseHistoryRepository(HistoryRepositoryDependencies dependencies) + : base(dependencies) + { + } + + protected override bool InterpretExistsResult(object? value) + => value is not null and not DBNull && Convert.ToBoolean(value); + + public override string GetCreateIfNotExistsScript() + { + var script = GetCreateScript(); + return script.Insert( + script.IndexOf("CREATE TABLE", StringComparison.Ordinal) + 12, + " IF NOT EXISTS"); + } + + public override string GetBeginIfNotExistsScript(string migrationId) + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override string GetBeginIfExistsScript(string migrationId) + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override string GetEndIfScript() + => throw new NotSupportedException( + "ClickHouse does not support conditional SQL blocks. " + + "Idempotent migration scripts (--idempotent) are not supported by this provider."); + + public override IMigrationsDatabaseLock AcquireDatabaseLock() + => new ClickHouseMigrationDatabaseLock(this); + + public override Task AcquireDatabaseLockAsync(CancellationToken cancellationToken = default) + => Task.FromResult(new ClickHouseMigrationDatabaseLock(this)); + + protected override void ConfigureTable(EntityTypeBuilder history) + { + history.Property(h => h.MigrationId).HasMaxLength(150); + history.Property(h => h.ProductVersion).HasMaxLength(32).IsRequired(); + history.ToTable(TableName, table => table + .HasMergeTreeEngine() + .WithOrderBy("MigrationId")); + } + + public override LockReleaseBehavior LockReleaseBehavior => LockReleaseBehavior.Connection; + + protected override string ExistsSql + => $"EXISTS {SqlGenerationHelper.DelimitIdentifier(TableName, TableSchema)}{SqlGenerationHelper.StatementTerminator}"; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs new file mode 100644 index 0000000..d42c457 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationDatabaseLock.cs @@ -0,0 +1,20 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +/// +/// No-op database lock for ClickHouse (no transaction/lock support). +/// +public class ClickHouseMigrationDatabaseLock : IMigrationsDatabaseLock +{ + public ClickHouseMigrationDatabaseLock(IHistoryRepository historyRepository) + { + HistoryRepository = historyRepository; + } + + public IHistoryRepository HistoryRepository { get; } + + public void Dispose() { } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs new file mode 100644 index 0000000..924bd94 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Internal/ClickHouseMigrationsAnnotationProvider.cs @@ -0,0 +1,42 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Internal; + +public class ClickHouseMigrationsAnnotationProvider : MigrationsAnnotationProvider +{ + public ClickHouseMigrationsAnnotationProvider(MigrationsAnnotationProviderDependencies dependencies) + : base(dependencies) + { + } + + /// + /// Yields ClickHouse annotations for a table index being removed. + /// Without this, DropIndexOperation would lack the SkippingIndexType annotation + /// and the SQL generator would silently skip the DROP INDEX statement. + /// + public override IEnumerable ForRemove(ITableIndex index) + { + foreach (var annotation in index.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } + + /// + /// Yields ClickHouse column annotations (codec, TTL, comment) for a column being removed or altered. + /// Ensures AlterColumnOperation.OldColumn carries the previous annotations + /// so the SQL generator can emit REMOVE statements when they are dropped. + /// + public override IEnumerable ForRemove(IColumn column) + { + foreach (var annotation in column.GetAnnotations()) + { + if (annotation.Name.StartsWith(ClickHouseAnnotationNames.Prefix, StringComparison.Ordinal)) + yield return annotation; + } + } +} diff --git a/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs new file mode 100644 index 0000000..62003e1 --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseCreateDatabaseOperation.cs @@ -0,0 +1,8 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Operations; + +public class ClickHouseCreateDatabaseOperation : MigrationOperation +{ + public required string Name { get; set; } +} diff --git a/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs new file mode 100644 index 0000000..136ee7b --- /dev/null +++ b/src/EFCore.ClickHouse/Migrations/Operations/ClickHouseDropDatabaseOperation.cs @@ -0,0 +1,8 @@ +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace ClickHouse.EntityFrameworkCore.Migrations.Operations; + +public class ClickHouseDropDatabaseOperation : MigrationOperation +{ + public required string Name { get; set; } +} diff --git a/src/EFCore.ClickHouse/README.nuget.md b/src/EFCore.ClickHouse/README.nuget.md index 10e6b5f..b716845 100644 --- a/src/EFCore.ClickHouse/README.nuget.md +++ b/src/EFCore.ClickHouse/README.nuget.md @@ -21,6 +21,19 @@ public class AnalyticsContext : DbContext protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.UseClickHouse("Host=localhost;Port=9000;Database=analytics"); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("page_views", t => t + .HasMergeTreeEngine() + .WithOrderBy("Date", "Path") + .WithPartitionBy("toYYYYMM(Date)")); + e.Property(x => x.UserAgent).HasCodec("ZSTD"); + }); + } } public class PageView @@ -34,14 +47,26 @@ public class PageView ## Supported Types -`String`, `Bool`, `Int8`/`Int16`/`Int32`/`Int64`, `UInt8`/`UInt16`/`UInt32`/`UInt64`, `Float32`/`Float64`, `Decimal(P, S)`, `Date`/`Date32`, `DateTime`, `DateTime64(P, 'TZ')`, `FixedString(N)`, `UUID` +`String`, `Bool`, `Int8`–`Int64`, `UInt8`–`UInt64`, `Float32`/`Float64`, `Decimal(P,S)` (32/64/128/256), `Date`/`Date32`, `DateTime`, `DateTime64`, `FixedString(N)`, `UUID`, `BFloat16`, `Enum8`/`Enum16`, `IPv4`/`IPv6`, `Int128`/`Int256`/`UInt128`/`UInt256`, `Array(T)`, `Map(K,V)`, `Tuple(T1,...)`, `Time`/`Time64`, `Variant(T1,...,TN)`, `Dynamic`, `Json`, geographic types (Point, Ring, Polygon, MultiPolygon, Geometry). ## Supported LINQ Operations -`Where`, `OrderBy`, `Take`, `Skip`, `Select`, `First`, `Single`, `Any`, `Count`, `Sum`, `Min`, `Max`, `Average`, `Distinct`, `GroupBy` +`Where`, `OrderBy`, `Take`, `Skip`, `Select`, `First`, `Single`, `Any`, `Count`, `LongCount`, `Sum`, `Min`, `Max`, `Average`, `Distinct`, `GroupBy` (with DISTINCT and predicate overloads). String methods: `Contains`, `StartsWith`, `EndsWith`, `IndexOf`, `Replace`, `Substring`, `Trim`, `ToLower`, `ToUpper`, `Length`, and string concatenation. +60+ Math/MathF translations: Abs, Floor, Ceiling, Round, Truncate, Pow, Sqrt, Exp, Log, trig functions, etc. + +## Table Engine Configuration + +All MergeTree-family engines (MergeTree, ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree, VersionedCollapsingMergeTree, GraphiteMergeTree) and simple engines (Log, TinyLog, StripeLog, Memory) are supported via fluent API with ORDER BY, PARTITION BY, PRIMARY KEY, SAMPLE BY, TTL, and SETTINGS. + +Column-level features: CODEC, TTL, COMMENT, DEFAULT values. Data-skipping indexes with configurable type and granularity. + +## Migrations + +Full `dotnet ef migrations` support: CREATE TABLE with ENGINE clauses, ALTER TABLE (ADD/DROP/MODIFY/RENAME COLUMN), RENAME TABLE, CREATE/DROP DATABASE, data-skipping index management. + ## Documentation For full documentation, see the [GitHub repository](https://github.com/ClickHouse/ClickHouse.EntityFrameworkCore). diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ListToArrayConverter.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ListToArrayConverter.cs index bdc532e..680fd3b 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ListToArrayConverter.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ListToArrayConverter.cs @@ -11,8 +11,8 @@ public class ListToArrayConverter : ValueConverter, T[]> { public ListToArrayConverter() : base( - list => list.ToArray(), - array => new List(array)) + list => list == null ? Array.Empty() : list.ToArray(), + array => array == null ? new List() : new List(array)) { } } diff --git a/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj b/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj new file mode 100644 index 0000000..f56283f --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/EFCore.ClickHouse.DesignSmoke.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + Exe + enable + enable + + + + + + + + diff --git a/test/EFCore.ClickHouse.DesignSmoke/Program.cs b/test/EFCore.ClickHouse.DesignSmoke/Program.cs new file mode 100644 index 0000000..b537287 --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/Program.cs @@ -0,0 +1,2 @@ +// Minimal entry point required for dotnet-ef to discover the design-time factory. +Console.WriteLine("This project is used for dotnet-ef CLI smoke testing only."); diff --git a/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs new file mode 100644 index 0000000..e1a4b98 --- /dev/null +++ b/test/EFCore.ClickHouse.DesignSmoke/SmokeDbContext.cs @@ -0,0 +1,67 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +namespace EFCore.ClickHouse.DesignSmoke; + +public class SmokeDbContext : DbContext +{ + public SmokeDbContext(DbContextOptions options) : base(options) { } + + public DbSet SensorReadings => Set(); + public DbSet AuditLogs => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(b => + { + b.HasKey(e => e.Id); + b.Property(e => e.Temperature).HasCodec("Delta, ZSTD"); + b.Property(e => e.Timestamp) + .HasColumnComment("Reading timestamp"); + b.HasIndex(e => e.Timestamp) + .HasSkippingIndexType("minmax") + .HasGranularity(4); + b.ToTable("sensor_readings", t => t + .HasReplacingMergeTreeEngine("Version") + .WithOrderBy("Id", "Timestamp") + .WithPartitionBy("toYYYYMM(Timestamp)") + .WithPrimaryKey("Id") + .WithTtl("Timestamp + INTERVAL 1 YEAR") + .WithSetting("index_granularity", "4096")); + }); + + modelBuilder.Entity(b => + { + b.HasKey(e => e.Id); + b.ToTable("audit_logs", t => t.HasMemoryEngine()); + }); + } +} + +public class SensorReading +{ + public long Id { get; set; } + public string SensorId { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public short Temperature { get; set; } + public ulong Version { get; set; } +} + +public class AuditLog +{ + public long Id { get; set; } + public string Message { get; set; } = string.Empty; +} + +public class SmokeDbContextFactory : IDesignTimeDbContextFactory +{ + public SmokeDbContext CreateDbContext(string[] args) + { + var connectionString = Environment.GetEnvironmentVariable("CLICKHOUSE_CONNECTION_STRING") + ?? "Host=localhost;Database=smoke_test"; + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseClickHouse(connectionString); + return new SmokeDbContext(optionsBuilder.Options); + } +} diff --git a/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs b/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs new file mode 100644 index 0000000..f40aed2 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/DotnetEfCliTests.cs @@ -0,0 +1,197 @@ +using System.Diagnostics; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +/// +/// End-to-end tests that shell out to the real dotnet-ef CLI tool +/// and verify the resulting database state against a real ClickHouse instance. +/// These tests skip automatically if dotnet-ef is not installed. +/// +public class DotnetEfCliTests : IAsyncLifetime +{ + private string _connectionString = default!; + private string _smokeProjectDir = default!; + private string? _migrationsDir; + private bool _dotnetEfAvailable; + + public async Task InitializeAsync() + { + _dotnetEfAvailable = await IsDotnetEfInstalled(); + if (!_dotnetEfAvailable) + return; + + _connectionString = await SharedContainer.GetConnectionStringAsync(); + + var testDir = Path.GetDirectoryName(typeof(DotnetEfCliTests).Assembly.Location)!; + var repoRoot = Path.GetFullPath(Path.Combine(testDir, "..", "..", "..", "..", "..")); + _smokeProjectDir = Path.Combine(repoRoot, "test", "EFCore.ClickHouse.DesignSmoke"); + + if (!Directory.Exists(_smokeProjectDir)) + { + _dotnetEfAvailable = false; + return; + } + + _migrationsDir = Path.Combine(_smokeProjectDir, "Migrations"); + if (Directory.Exists(_migrationsDir)) + Directory.Delete(_migrationsDir, recursive: true); + + // Restore the DesignSmoke project (it's not in the solution file) + await RunProcess("dotnet", ["restore", _smokeProjectDir]); + } + + public Task DisposeAsync() + { + if (_migrationsDir is not null && Directory.Exists(_migrationsDir)) + Directory.Delete(_migrationsDir, recursive: true); + return Task.CompletedTask; + } + + [Fact] + public async Task Database_update_creates_correct_schema() + { + if (!_dotnetEfAvailable) + return; // dotnet-ef not installed — skip gracefully + + // Add migration and apply to real ClickHouse + await RunDotnetEfSuccessfully("migrations", "add", "InitialCreate"); + await RunDotnetEfSuccessfully("database", "update"); + + // Verify everything via ClickHouse system tables + using var connection = new global::ClickHouse.Driver.ADO.ClickHouseConnection(_connectionString); + await connection.OpenAsync(); + + // History table tracks the migration + var historyCount = await QueryScalar(connection, + "SELECT count() FROM `__EFMigrationsHistory`"); + Assert.Equal(1UL, historyCount); + + // sensor_readings created with ReplacingMergeTree + var sensorEngine = await QueryScalar(connection, + "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'sensor_readings'"); + Assert.Equal("ReplacingMergeTree", sensorEngine); + + // audit_logs created with Memory + var auditEngine = await QueryScalar(connection, + "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'audit_logs'"); + Assert.Equal("Memory", auditEngine); + + // ORDER BY / sorting key + var sortingKey = await QueryScalar(connection, + "SELECT sorting_key FROM system.tables WHERE database = currentDatabase() AND name = 'sensor_readings'"); + Assert.Contains("Id", sortingKey); + Assert.Contains("Timestamp", sortingKey); + + // PARTITION BY + var partitionKey = await QueryScalar(connection, + "SELECT partition_key FROM system.tables WHERE database = currentDatabase() AND name = 'sensor_readings'"); + Assert.Contains("toYYYYMM(Timestamp)", partitionKey); + + // PRIMARY KEY + var primaryKey = await QueryScalar(connection, + "SELECT primary_key FROM system.tables WHERE database = currentDatabase() AND name = 'sensor_readings'"); + Assert.Contains("Id", primaryKey); + + // Data skipping index exists + var indexCount = await QueryScalar(connection, + "SELECT count() FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'sensor_readings'"); + Assert.True(indexCount > 0, "Expected at least one data skipping index"); + } + + [Fact] + public async Task Idempotent_script_is_rejected() + { + if (!_dotnetEfAvailable) + return; // dotnet-ef not installed — skip gracefully + + await RunDotnetEfSuccessfully("migrations", "add", "InitialCreate"); + + var result = await RunDotnetEf("migrations", "script", "--idempotent"); + Assert.True(result.ExitCode != 0, "Expected --idempotent to fail"); + var output = result.StdOut + result.StdErr; + Assert.Contains("does not support conditional SQL blocks", output); + } + + private async Task RunDotnetEfSuccessfully(params string[] args) + { + var result = await RunDotnetEf(args); + Assert.True(result.ExitCode == 0, + $"dotnet-ef {string.Join(' ', args)} failed (exit {result.ExitCode}):\n{result.StdOut}\n{result.StdErr}"); + } + + private async Task RunDotnetEf(params string[] args) + { + var allArgs = new List(args) + { + "--project", _smokeProjectDir, + "--startup-project", _smokeProjectDir + }; + + var psi = new ProcessStartInfo + { + FileName = "dotnet-ef", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + WorkingDirectory = _smokeProjectDir + }; + psi.Environment["CLICKHOUSE_CONNECTION_STRING"] = _connectionString; + + foreach (var arg in allArgs) + psi.ArgumentList.Add(arg); + + using var process = Process.Start(psi)!; + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + return new DotnetEfResult(process.ExitCode, stdout, stderr); + } + + private static async Task IsDotnetEfInstalled() + { + try + { + var psi = new ProcessStartInfo("dotnet-ef", "--version") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + using var process = Process.Start(psi)!; + await process.WaitForExitAsync(); + return process.ExitCode == 0; + } + catch + { + return false; + } + } + + private static async Task RunProcess(string fileName, string[] args) + { + var psi = new ProcessStartInfo(fileName) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + foreach (var arg in args) + psi.ArgumentList.Add(arg); + + using var process = Process.Start(psi)!; + await process.WaitForExitAsync(); + } + + private static async Task QueryScalar( + global::ClickHouse.Driver.ADO.ClickHouseConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + return (T)Convert.ChangeType(result!, typeof(T)); + } + + private record DotnetEfResult(int ExitCode, string StdOut, string StdErr); +} diff --git a/test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj b/test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj index 88fad5c..e570710 100644 --- a/test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj +++ b/test/EFCore.ClickHouse.Tests/EFCore.ClickHouse.Tests.csproj @@ -10,6 +10,7 @@ + all diff --git a/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs b/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs new file mode 100644 index 0000000..e724361 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/EngineConfigurationTests.cs @@ -0,0 +1,1123 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class EngineConfigurationTests +{ + [Fact] + public void HasMergeTreeEngine_sets_engine_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + } + + [Fact] + public void HasReplacingMergeTreeEngine_sets_version_and_isDeleted() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasReplacingMergeTreeEngine("Version", "IsDeleted")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.ReplacingMergeTree, entityType.GetEngine()); + Assert.Equal("Version", entityType.GetReplacingMergeTreeVersion()); + Assert.Equal("IsDeleted", entityType.GetReplacingMergeTreeIsDeleted()); + } + + [Fact] + public void HasCollapsingMergeTreeEngine_sets_sign() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasCollapsingMergeTreeEngine("Sign")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.CollapsingMergeTree, entityType.GetEngine()); + Assert.Equal("Sign", entityType.GetCollapsingMergeTreeSign()); + } + + [Fact] + public void HasVersionedCollapsingMergeTreeEngine_sets_sign_and_version() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasVersionedCollapsingMergeTreeEngine("Sign", "Ver")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.VersionedCollapsingMergeTree, entityType.GetEngine()); + Assert.Equal("Sign", entityType.GetVersionedCollapsingMergeTreeSign()); + Assert.Equal("Ver", entityType.GetVersionedCollapsingMergeTreeVersion()); + } + + [Fact] + public void HasSummingMergeTreeEngine_sets_columns() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasSummingMergeTreeEngine("Amount", "Count")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.SummingMergeTree, entityType.GetEngine()); + Assert.Equal(["Amount", "Count"], entityType.GetSummingMergeTreeColumns()); + } + + [Fact] + public void HasGraphiteMergeTreeEngine_sets_config_section() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasGraphiteMergeTreeEngine("graphite_rollup")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.GraphiteMergeTree, entityType.GetEngine()); + Assert.Equal("graphite_rollup", entityType.GetGraphiteMergeTreeConfigSection()); + } + + [Fact] + public void WithOrderBy_stores_column_array() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id", "Name")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["Id", "Name"], entityType.GetOrderBy()); + } + + [Fact] + public void WithPartitionBy_stores_expression() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(CreatedAt)")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["toYYYYMM(CreatedAt)"], entityType.GetPartitionBy()); + } + + [Fact] + public void WithSetting_stores_prefix_based_annotations() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithSetting("index_granularity", "4096") + .WithSetting("storage_policy", "'hot_cold'")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + var settings = entityType.GetSettings(); + Assert.Equal(2, settings.Count); + Assert.Equal("4096", settings["index_granularity"]); + Assert.Equal("'hot_cold'", settings["storage_policy"]); + } + + [Fact] + public void WithTtl_stores_ttl_expression() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithTtl("CreatedAt + INTERVAL 30 DAY")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal("CreatedAt + INTERVAL 30 DAY", entityType.GetTtl()); + } + + [Fact] + public void HasMemoryEngine_sets_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.Memory, entityType.GetEngine()); + } + + [Fact] + public void SimpleEngine_WithOrderBy_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine().WithOrderBy("Id")); + }); + }); + }); + } + + [Fact] + public void Index_HasSkippingIndexType_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Name) + .HasSkippingIndexType("minmax") + .HasGranularity(4); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var index = model.FindEntityType(typeof(TestEntity))!.GetIndexes().First(); + Assert.Equal("minmax", index.GetSkippingIndexType()); + Assert.Equal(4, index.GetGranularity()); + } + + [Fact] + public void Property_HasCodec_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasCodec("Delta, ZSTD"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Id")!; + Assert.Equal("Delta, ZSTD", property.GetCodec()); + } + + [Fact] + public void Property_HasColumnTtl_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnTtl("CreatedAt + INTERVAL 1 DAY"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("CreatedAt + INTERVAL 1 DAY", property.GetColumnTtl()); + } + + [Fact] + public void Property_HasColumnComment_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnComment("User's display name"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("User's display name", property.GetColumnComment()); + } + + [Fact] + public void Default_convention_sets_MergeTree_with_PK_as_OrderBy() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test"); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + Assert.Equal(["Id"], entityType.GetOrderBy()); + } + + [Fact] + public void Default_convention_sets_tuple_OrderBy_when_no_PK() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasNoKey(); + e.ToTable("test"); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + Assert.Equal(["tuple()"], entityType.GetOrderBy()); + } + + [Fact] + public void Default_convention_does_not_override_explicit_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasStripeLogEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.StripeLog, entityType.GetEngine()); + Assert.Null(entityType.GetOrderBy()); + } + + // --- Coverage gap tests: engine builders, fluent methods, entry points --- + + [Fact] + public void HasAggregatingMergeTreeEngine_sets_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasAggregatingMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.AggregatingMergeTree, entityType.GetEngine()); + Assert.Equal(["Id"], entityType.GetOrderBy()); + } + + [Fact] + public void HasTinyLogEngine_sets_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasTinyLogEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.TinyLog, entityType.GetEngine()); + } + + [Fact] + public void HasLogEngine_sets_engine() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasLogEngine()); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.Log, entityType.GetEngine()); + } + + [Fact] + public void WithPrimaryKey_stores_columns() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id", "Name") + .WithPrimaryKey("Id")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["Id"], entityType.GetClickHousePrimaryKey()); + } + + [Fact] + public void WithSampleBy_stores_columns() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithSampleBy("Id")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["Id"], entityType.GetSampleBy()); + } + + [Fact] + public void MergeTreeEngine_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(Name)") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.MergeTree, entityType.GetEngine()); + Assert.Equal(["Id"], entityType.GetOrderBy()); + Assert.Equal(["toYYYYMM(Name)"], entityType.GetPartitionBy()); + Assert.Equal(["Id"], entityType.GetClickHousePrimaryKey()); + Assert.Equal(["Id"], entityType.GetSampleBy()); + Assert.Equal("Name + INTERVAL 1 DAY", entityType.GetTtl()); + Assert.Equal("4096", entityType.GetSettings()["index_granularity"]); + } + + [Fact] + public void CollapsingMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasCollapsingMergeTreeEngine("Sign") + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(Name)") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "8192")); + }); + }); + + var entityType = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.CollapsingMergeTree, entityType.GetEngine()); + Assert.Equal("Sign", entityType.GetCollapsingMergeTreeSign()); + Assert.Equal(["Id"], entityType.GetOrderBy()); + Assert.Equal("8192", entityType.GetSettings()["index_granularity"]); + } + + [Fact] + public void ReplacingMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasReplacingMergeTreeEngine("Ver", "IsDeleted") + .WithOrderBy("Id") + .WithPartitionBy("Name") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.ReplacingMergeTree, et.GetEngine()); + Assert.Equal("Ver", et.GetReplacingMergeTreeVersion()); + Assert.Equal(["Id"], et.GetOrderBy()); + Assert.Equal(["Id"], et.GetClickHousePrimaryKey()); + Assert.Equal(["Id"], et.GetSampleBy()); + } + + [Fact] + public void SummingMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasSummingMergeTreeEngine("Name") + .WithOrderBy("Id") + .WithPartitionBy("Name") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.SummingMergeTree, et.GetEngine()); + Assert.Equal(["Id"], et.GetOrderBy()); + Assert.Equal("4096", et.GetSettings()["index_granularity"]); + } + + [Fact] + public void VersionedCollapsingMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasVersionedCollapsingMergeTreeEngine("Sign", "Ver") + .WithOrderBy("Id") + .WithPartitionBy("Name") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.VersionedCollapsingMergeTree, et.GetEngine()); + Assert.Equal("Sign", et.GetVersionedCollapsingMergeTreeSign()); + Assert.Equal(["Id"], et.GetOrderBy()); + } + + [Fact] + public void GraphiteMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasGraphiteMergeTreeEngine("graphite_rollup") + .WithOrderBy("Id") + .WithPartitionBy("Name") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.GraphiteMergeTree, et.GetEngine()); + Assert.Equal("graphite_rollup", et.GetGraphiteMergeTreeConfigSection()); + Assert.Equal(["Id"], et.GetOrderBy()); + } + + [Fact] + public void AggregatingMergeTree_full_fluent_chain() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasAggregatingMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("Name") + .WithPrimaryKey("Id") + .WithSampleBy("Id") + .WithTtl("Name + INTERVAL 1 DAY") + .WithSetting("index_granularity", "4096")); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.AggregatingMergeTree, et.GetEngine()); + Assert.Equal(["Id"], et.GetOrderBy()); + Assert.Equal(["Name"], et.GetPartitionBy()); + Assert.Equal(["Id"], et.GetClickHousePrimaryKey()); + Assert.Equal(["Id"], et.GetSampleBy()); + Assert.Equal("Name + INTERVAL 1 DAY", et.GetTtl()); + Assert.Equal("4096", et.GetSettings()["index_granularity"]); + } + + [Fact] + public void Property_generic_HasCodec_works() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasCodec("LZ4"); + e.ToTable("test"); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Id")!; + Assert.Equal("LZ4", property.GetCodec()); + } + + [Fact] + public void Property_generic_HasColumnTtl_works() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnTtl("ts + INTERVAL 1 DAY"); + e.ToTable("test"); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("ts + INTERVAL 1 DAY", property.GetColumnTtl()); + } + + [Fact] + public void Property_generic_HasColumnComment_works() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasColumnComment("test comment"); + e.ToTable("test"); + }); + }); + + var property = model.FindEntityType(typeof(TestEntity))!.FindProperty("Name")!; + Assert.Equal("test comment", property.GetColumnComment()); + } + + [Fact] + public void Index_HasSkippingIndexParams_stores_annotation() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Name) + .HasSkippingIndexType("set") + .HasGranularity(2) + .HasSkippingIndexParams("100"); + e.ToTable("test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var index = model.FindEntityType(typeof(TestEntity))!.GetIndexes().First(); + Assert.Equal("set", index.GetSkippingIndexType()); + Assert.Equal(2, index.GetGranularity()); + Assert.Equal("100", index.GetSkippingIndexParams()); + } + + [Fact] + public void SimpleEngine_WithPartitionBy_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasTinyLogEngine().WithPartitionBy("Id")); + }); + }); + }); + } + + [Fact] + public void SimpleEngine_WithPrimaryKey_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasLogEngine().WithPrimaryKey("Id")); + }); + }); + }); + } + + [Fact] + public void SimpleEngine_WithSampleBy_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasStripeLogEngine().WithSampleBy("Id")); + }); + }); + }); + } + + [Fact] + public void SimpleEngine_WithTtl_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine().WithTtl("Id + INTERVAL 1 DAY")); + }); + }); + }); + } + + [Fact] + public void SimpleEngine_WithSetting_throws() + { + Assert.Throws(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasMemoryEngine().WithSetting("k", "v")); + }); + }); + }); + } + + // ── Lambda-based overloads ──────────────────────────────────────────── + + [Fact] + public void Lambda_ReplacingMergeTree_sets_version_and_isDeleted() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasReplacingMergeTreeEngine( + x => x.Version, x => x.IsDeleted)); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(ClickHouseAnnotationNames.ReplacingMergeTree, et.GetEngine()); + Assert.Equal("Version", et.GetReplacingMergeTreeVersion()); + Assert.Equal("IsDeleted", et.GetReplacingMergeTreeIsDeleted()); + } + + [Fact] + public void Lambda_ReplacingMergeTree_version_only() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasReplacingMergeTreeEngine( + version: x => x.Version)); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal("Version", et.GetReplacingMergeTreeVersion()); + Assert.Null(et.GetReplacingMergeTreeIsDeleted()); + } + + [Fact] + public void Lambda_CollapsingMergeTree_sets_sign() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasCollapsingMergeTreeEngine( + x => x.Sign)); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal("Sign", et.GetCollapsingMergeTreeSign()); + } + + [Fact] + public void Lambda_VersionedCollapsingMergeTree_sets_sign_and_version() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasVersionedCollapsingMergeTreeEngine( + x => x.Sign, x => x.Version)); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal("Sign", et.GetVersionedCollapsingMergeTreeSign()); + Assert.Equal("Version", et.GetVersionedCollapsingMergeTreeVersion()); + } + + [Fact] + public void Lambda_SummingMergeTree_sets_columns() + { + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t.HasSummingMergeTreeEngine( + x => x.Amount, x => x.Count)); + }); + }); + + var et = model.FindEntityType(typeof(TestEntity))!; + Assert.Equal(["Amount", "Count"], et.GetSummingMergeTreeColumns()); + } + + // ── Validator: column-not-found for engine parameters ──────────────── + + [Fact] + public void VersionedCollapsingMergeTree_warns_on_missing_sign_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasVersionedCollapsingMergeTreeEngine("NonExistentSign", "NonExistentVersion") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentSign", ex.Message); + } + + [Fact] + public void VersionedCollapsingMergeTree_warns_on_missing_version_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasVersionedCollapsingMergeTreeEngine("Sign", "NonExistentVersion") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentVersion", ex.Message); + } + + [Fact] + public void SummingMergeTree_warns_on_missing_sum_columns() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasSummingMergeTreeEngine("NonExistentColumn") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentColumn", ex.Message); + } + + [Fact] + public void CollapsingMergeTree_warns_on_missing_sign_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasCollapsingMergeTreeEngine("NonExistentSign") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentSign", ex.Message); + } + + [Fact] + public void ReplacingMergeTree_warns_on_missing_version_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasReplacingMergeTreeEngine("NonExistentVersion") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentVersion", ex.Message); + } + + [Fact] + public void ReplacingMergeTree_warns_on_missing_isDeleted_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasReplacingMergeTreeEngine("Version", "NonExistentIsDeleted") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("NonExistentIsDeleted", ex.Message); + } + + // ── Validator: wrong CLR type for engine parameters ────────────────── + + [Fact] + public void CollapsingMergeTree_rejects_non_Int8_sign_column() + { + // Sign is int → maps to Int32, but ClickHouse requires Int8 + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasCollapsingMergeTreeEngine("Sign") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("Sign", ex.Message); + Assert.Contains("Int8", ex.Message); + } + + [Fact] + public void VersionedCollapsingMergeTree_rejects_non_Int8_sign_column() + { + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasVersionedCollapsingMergeTreeEngine("Sign", "Version") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("Sign", ex.Message); + Assert.Contains("Int8", ex.Message); + } + + [Fact] + public void ReplacingMergeTree_rejects_non_UInt8_isDeleted_column() + { + // IsDeleted is int → maps to Int32, but ClickHouse requires UInt8 + var ex = Assert.ThrowsAny(() => + { + BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasReplacingMergeTreeEngine("Version", "IsDeleted") + .WithOrderBy("Id")); + }); + }); + }); + Assert.Contains("IsDeleted", ex.Message); + Assert.Contains("UInt8", ex.Message); + } + + [Fact] + public void CollapsingMergeTree_accepts_sbyte_sign_column() + { + // sbyte maps to Int8 — should pass + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasCollapsingMergeTreeEngine("Sign") + .WithOrderBy("Id")); + }); + }); + Assert.Equal("Sign", model.FindEntityType(typeof(TestEntity))!.GetCollapsingMergeTreeSign()); + } + + [Fact] + public void ReplacingMergeTree_accepts_byte_isDeleted_column() + { + // byte maps to UInt8 — should pass + var model = BuildModel(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("test", t => t + .HasReplacingMergeTreeEngine("Version", "IsDeleted") + .WithOrderBy("Id")); + }); + }); + Assert.Equal("IsDeleted", model.FindEntityType(typeof(TestEntity))!.GetReplacingMergeTreeIsDeleted()); + } + + private static Microsoft.EntityFrameworkCore.Metadata.IModel BuildModel(Action configure) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test") + .EnableServiceProviderCaching(false); + using var context = new TestDbContext(optionsBuilder.Options, configure); + return context.Model; + } + + private class TestDbContext : DbContext + { + private readonly Action _configure; + + public TestDbContext(DbContextOptions options, Action configure) + : base(options) + { + _configure = configure; + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) => _configure(modelBuilder); + } + + private class TestEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public sbyte Sign { get; set; } + public ulong Version { get; set; } + public byte IsDeleted { get; set; } + public ulong Ver { get; set; } + public decimal Amount { get; set; } + public long Count { get; set; } + } + + private class VersionedEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public sbyte Sign { get; set; } + public ulong Version { get; set; } + } + + // Sign is int (should be sbyte/Int8) + private class BadSignEntity + { + public long Id { get; set; } + public int Sign { get; set; } + public ulong Version { get; set; } + } + + // IsDeleted is int (should be byte/UInt8) + private class BadIsDeletedEntity + { + public long Id { get; set; } + public ulong Version { get; set; } + public int IsDeleted { get; set; } + } +} diff --git a/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs b/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs new file mode 100644 index 0000000..5bc9f6b --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/EnsureCreatedTests.cs @@ -0,0 +1,241 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class EnsureCreatedTests : IAsyncLifetime +{ + private string _connectionString = default!; + + public async Task InitializeAsync() + { + _connectionString = await SharedContainer.GetConnectionStringAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task EnsureCreated_MergeTree_creates_table() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("mt_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + // Verify table exists and has correct engine via system.tables + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'mt_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_ReplacingMergeTree_creates_table() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("rmt_test", t => t + .HasReplacingMergeTreeEngine("Version") + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'rmt_test'"); + Assert.Equal("ReplacingMergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_default_engine_uses_MergeTree() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("default_engine_test"); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'default_engine_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_with_partitionBy_and_settings() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("partition_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("toYYYYMM(Timestamp)") + .WithSetting("index_granularity", "4096")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'partition_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureCreated_insert_and_query_roundtrip() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("roundtrip_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + context.Set().Add(new SimpleEntity { Id = 1, Name = "test" }); + await context.SaveChangesAsync(); + + await using var readContext = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("roundtrip_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + var entity = await readContext.Set().FirstAsync(e => e.Id == 1); + Assert.Equal("test", entity.Name); + } + + [Fact] + public async Task EnsureCreated_with_codec_column() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasCodec("Delta, ZSTD"); + e.ToTable("codec_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureDeletedAsync(); + await context.Database.EnsureCreatedAsync(); + + // Table creation succeeds means codec was accepted + var engine = await QueryScalar(context, "SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'codec_test'"); + Assert.Equal("MergeTree", engine); + } + + [Fact] + public async Task EnsureDeleted_drops_database() + { + await using var context = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("del_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id")); + }); + }); + + await context.Database.EnsureCreatedAsync(); + Assert.True(await context.Database.EnsureCreatedAsync() is false); // already exists + + await context.Database.EnsureDeletedAsync(); + // After delete, creating again should return true + Assert.True(await context.Database.EnsureCreatedAsync()); + } + + private TestContext CreateContext(Action configure) + { + var options = new DbContextOptionsBuilder() + .UseClickHouse(_connectionString) + .EnableServiceProviderCaching(false) + .Options; + return new TestContext(options, configure); + } + + private static async Task QueryScalar(DbContext context, string sql) + { + var conn = context.Database.GetDbConnection(); + await conn.OpenAsync(); + try + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + return result?.ToString(); + } + finally + { + await conn.CloseAsync(); + } + } + + private class TestContext : DbContext + { + private readonly Action _configure; + + public TestContext(DbContextOptions options, Action configure) + : base(options) + { + _configure = configure; + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) => _configure(modelBuilder); + } + + public class SimpleEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + public class VersionedEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public ulong Version { get; set; } + } + + public class TimestampedEntity + { + public long Id { get; set; } + public DateTime Timestamp { get; set; } + } +} diff --git a/test/EFCore.ClickHouse.Tests/MigrationIntegrationTests.cs b/test/EFCore.ClickHouse.Tests/MigrationIntegrationTests.cs new file mode 100644 index 0000000..a713f19 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/MigrationIntegrationTests.cs @@ -0,0 +1,747 @@ +using ClickHouse.EntityFrameworkCore.Extensions; +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +/// +/// Integration tests that verify migration SQL against a real ClickHouse instance. +/// Each test creates tables, applies migration operations, and checks the resulting +/// database state via system tables — not just the emitted SQL text. +/// +public class MigrationIntegrationTests : IAsyncLifetime +{ + private string _connectionString = default!; + private string _databaseName = default!; + + public async Task InitializeAsync() + { + _connectionString = await SharedContainer.GetConnectionStringAsync(); + _databaseName = System.Text.RegularExpressions.Regex.Match( + _connectionString, @"Database=([^;]+)").Groups[1].Value; + } + + public Task DisposeAsync() => Task.CompletedTask; + + // ── Finding 3: Expression quoting ─────────────────────────────────────── + + [Fact] + public async Task Expression_orderBy_creates_valid_table() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("expr_ob_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id", "Id % 8")); + }); + }); + + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var sortingKey = await QueryScalar(ctx, + $"SELECT sorting_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'expr_ob_test'"); + Assert.Contains("Id % 8", sortingKey!); + } + + [Fact] + public async Task Expression_partitionBy_creates_valid_table() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("expr_pb_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("Id % 4")); + }); + }); + + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var partitionKey = await QueryScalar(ctx, + $"SELECT partition_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'expr_pb_test'"); + Assert.Contains("Id % 4", partitionKey!); + } + + [Fact] + public async Task Multi_column_partitionBy_creates_valid_table() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("multi_part_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("Id") + .WithPartitionBy("Region", "toYYYYMM(CreatedAt)")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var partitionKey = await QueryScalar(ctx, + $"SELECT partition_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'multi_part_test'"); + Assert.Contains("Region", partitionKey!); + Assert.Contains("toYYYYMM(CreatedAt)", partitionKey); + } + + [Fact] + public async Task SampleBy_with_expression_creates_valid_table() + { + // SAMPLE BY requires unsigned integer — use cityHash64() which returns UInt64 + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("sample_expr_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("cityHash64(Id)", "Id") + .WithSampleBy("cityHash64(Id)")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var samplingKey = await QueryScalar(ctx, + $"SELECT sampling_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'sample_expr_test'"); + Assert.Contains("cityHash64(Id)", samplingKey!); + } + + [Fact] + public async Task PartitionBy_sampleBy_and_primaryKey_together() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("combo_clause_test", t => t + .HasMergeTreeEngine() + .WithOrderBy("cityHash64(Id)", "Region", "Id") + .WithPartitionBy("Region", "toYYYYMM(CreatedAt)") + .WithPrimaryKey("cityHash64(Id)", "Region") + .WithSampleBy("cityHash64(Id)")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var partitionKey = await QueryScalar(ctx, + $"SELECT partition_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'combo_clause_test'"); + Assert.Contains("Region", partitionKey!); + Assert.Contains("toYYYYMM(CreatedAt)", partitionKey); + + var primaryKey = await QueryScalar(ctx, + $"SELECT primary_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'combo_clause_test'"); + Assert.Contains("cityHash64(Id)", primaryKey!); + Assert.Contains("Region", primaryKey); + + var samplingKey = await QueryScalar(ctx, + $"SELECT sampling_key FROM system.tables WHERE database = '{_databaseName}' AND name = 'combo_clause_test'"); + Assert.Contains("cityHash64(Id)", samplingKey!); + } + + // ── Finding 2: Column annotation removal ──────────────────────────────── + + [Fact] + public async Task Removing_column_codec_clears_codec_in_database() + { + // Create table with CODEC on a column + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Value).HasCodec("ZSTD"); + e.ToTable("codec_rm_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Verify codec is present + var codecBefore = await QueryScalar(ctx, + $"SELECT compression_codec FROM system.columns WHERE database = '{_databaseName}' AND table = 'codec_rm_test' AND name = 'Value'"); + Assert.Contains("ZSTD", codecBefore!); + + // Generate and execute AlterColumn that removes codec + var op = new AlterColumnOperation + { + Table = "codec_rm_test", Name = "Value", ColumnType = "String", ClrType = typeof(string) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "ZSTD"); + await ApplyMigrationAsync(op); + + // Verify codec is gone + var codecAfter = await QueryScalar(ctx, + $"SELECT compression_codec FROM system.columns WHERE database = '{_databaseName}' AND table = 'codec_rm_test' AND name = 'Value'"); + Assert.DoesNotContain("ZSTD", codecAfter ?? ""); + } + + [Fact] + public async Task Removing_column_comment_clears_comment_in_database() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Value).HasColumnComment("test comment"); + e.ToTable("comment_rm_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var commentBefore = await QueryScalar(ctx, + $"SELECT comment FROM system.columns WHERE database = '{_databaseName}' AND table = 'comment_rm_test' AND name = 'Value'"); + Assert.Equal("test comment", commentBefore); + + var op = new AlterColumnOperation + { + Table = "comment_rm_test", Name = "Value", ColumnType = "String", ClrType = typeof(string) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "test comment"); + await ApplyMigrationAsync(op); + + var commentAfter = await QueryScalar(ctx, + $"SELECT comment FROM system.columns WHERE database = '{_databaseName}' AND table = 'comment_rm_test' AND name = 'Value'"); + Assert.True(string.IsNullOrEmpty(commentAfter), $"Expected empty comment, got: '{commentAfter}'"); + } + + [Fact] + public async Task Removing_column_ttl_clears_ttl_in_database() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Timestamp).HasColumnTtl("Timestamp + INTERVAL 1 DAY"); + e.ToTable("ttl_rm_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Verify TTL is present in CREATE TABLE output + var createBefore = await QueryScalar(ctx, + $"SELECT create_table_query FROM system.tables WHERE database = '{_databaseName}' AND name = 'ttl_rm_test'"); + Assert.Contains("TTL", createBefore!); + + var op = new AlterColumnOperation + { + Table = "ttl_rm_test", Name = "Timestamp", ColumnType = "DateTime", ClrType = typeof(DateTime) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "Timestamp + INTERVAL 1 DAY"); + await ApplyMigrationAsync(op); + + var createAfter = await QueryScalar(ctx, + $"SELECT create_table_query FROM system.tables WHERE database = '{_databaseName}' AND name = 'ttl_rm_test'"); + Assert.DoesNotContain("TTL", createAfter ?? ""); + } + + // ── Finding 1: Skipping index drop ────────────────────────────────────── + + [Fact] + public async Task Drop_skipping_index_removes_index_from_database() + { + // Create table with skipping index via EnsureCreated + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Value) + .HasDatabaseName("idx_value") + .HasSkippingIndexType("set") + .HasGranularity(4) + .HasSkippingIndexParams("100"); + e.ToTable("idx_drop_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Verify index exists + var countBefore = await QueryScalar(ctx, + $"SELECT count() FROM system.data_skipping_indices WHERE database = '{_databaseName}' AND table = 'idx_drop_test' AND name = 'idx_value'"); + Assert.Equal("1", countBefore); + + // Drop the index via migration operation (annotation simulates what ForRemove provides) + var dropOp = new DropIndexOperation { Table = "idx_drop_test", Name = "idx_value" }; + dropOp.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexType, "set"); + await ApplyMigrationAsync(dropOp); + + // Verify index is gone + var countAfter = await QueryScalar(ctx, + $"SELECT count() FROM system.data_skipping_indices WHERE database = '{_databaseName}' AND table = 'idx_drop_test' AND name = 'idx_value'"); + Assert.Equal("0", countAfter); + } + + [Fact] + public async Task Model_differ_carries_skipping_index_annotations_to_DropIndexOperation() + { + // Model v1: table WITH skipping index + await using var ctxWithIndex = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Value) + .HasDatabaseName("idx_val") + .HasSkippingIndexType("set") + .HasGranularity(4); + e.ToTable("differ_idx_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + // Model v2: same table WITHOUT index + await using var ctxNoIndex = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("differ_idx_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var v1Model = ctxWithIndex.GetService().Model.GetRelationalModel(); + var v2Model = ctxNoIndex.GetService().Model.GetRelationalModel(); + var differ = ctxNoIndex.GetService(); + + // Diff: v1 → v2 should produce a DropIndexOperation + var operations = differ.GetDifferences(v1Model, v2Model); + var dropOp = Assert.Single(operations.OfType()); + + // The SkippingIndexType annotation must be present (our ForRemove fix) + var typeAnnotation = dropOp.FindAnnotation(ClickHouseAnnotationNames.SkippingIndexType); + Assert.NotNull(typeAnnotation); + Assert.Equal("set", typeAnnotation.Value); + } + + // ── End-to-end: model differ → SQL gen → execute → verify ─────────────── + + [Fact] + public async Task Model_differ_drop_index_end_to_end() + { + // Step 1: Create table with skipping index + await using var ctxV1 = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.HasIndex(x => x.Value) + .HasDatabaseName("idx_e2e") + .HasSkippingIndexType("minmax") + .HasGranularity(3); + e.ToTable("e2e_idx_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctxV1.Database.EnsureDeletedAsync(); + await ctxV1.Database.EnsureCreatedAsync(); + + // Verify index exists + var countBefore = await QueryScalar(ctxV1, + $"SELECT count() FROM system.data_skipping_indices WHERE database = '{_databaseName}' AND table = 'e2e_idx_test' AND name = 'idx_e2e'"); + Assert.Equal("1", countBefore); + + // Step 2: Model v2 without the index + await using var ctxV2 = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("e2e_idx_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + // Step 3: Diff → Generate → Execute + var v1Model = ctxV1.GetService().Model.GetRelationalModel(); + var v2Model = ctxV2.GetService().Model.GetRelationalModel(); + var differ = ctxV2.GetService(); + var operations = differ.GetDifferences(v1Model, v2Model); + + var generator = ctxV2.GetService(); + var commands = generator.Generate(operations); + + using var conn = new global::ClickHouse.Driver.ADO.ClickHouseConnection(_connectionString); + await conn.OpenAsync(); + foreach (var command in commands) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = command.CommandText; + await cmd.ExecuteNonQueryAsync(); + } + + // Step 4: Verify index is gone + var countAfter = await QueryScalar(ctxV1, + $"SELECT count() FROM system.data_skipping_indices WHERE database = '{_databaseName}' AND table = 'e2e_idx_test' AND name = 'idx_e2e'"); + Assert.Equal("0", countAfter); + } + + // ── Nullable container types ──────────────────────────────────────────── + + [Fact] + public async Task Nullable_List_column_creates_Array_not_Nullable_Array() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Tags).HasColumnType("Array(String)"); + e.ToTable("list_nullable_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Verify column type is Array(String), not Nullable(Array(String)) + var colType = await QueryScalar(ctx, + $"SELECT type FROM system.columns WHERE database = '{_databaseName}' AND table = 'list_nullable_test' AND name = 'Tags'"); + Assert.StartsWith("Array(", colType!); + Assert.DoesNotContain("Nullable(Array", colType); + } + + [Fact] + public async Task Nullable_Map_column_creates_Map_not_Nullable_Map() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Meta).HasColumnType("Map(String, String)"); + e.ToTable("map_nullable_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var colType = await QueryScalar(ctx, + $"SELECT type FROM system.columns WHERE database = '{_databaseName}' AND table = 'map_nullable_test' AND name = 'Meta'"); + Assert.StartsWith("Map(", colType!); + Assert.DoesNotContain("Nullable(Map", colType); + } + + [Fact] + public async Task Null_List_inserts_as_empty_array_and_reads_back() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Tags).HasColumnType("Array(String)"); + e.ToTable("list_null_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Insert with null list + ctx.Set().Add(new ListEntity { Id = 1, Tags = null }); + // Insert with populated list + ctx.Set().Add(new ListEntity { Id = 2, Tags = ["a", "b"] }); + await ctx.SaveChangesAsync(); + + await using var readCtx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Tags).HasColumnType("Array(String)"); + e.ToTable("list_null_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + + var e1 = await readCtx.Set().FirstAsync(e => e.Id == 1); + Assert.NotNull(e1.Tags); + Assert.Empty(e1.Tags); + + var e2 = await readCtx.Set().FirstAsync(e => e.Id == 2); + Assert.Equal(["a", "b"], e2.Tags); + } + + // ── ReplacingMergeTree isDeleted ───────────────────────────────────────── + + [Fact] + public async Task ReplacingMergeTree_with_isDeleted_creates_valid_table() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("rmt_deleted_test", t => t + .HasReplacingMergeTreeEngine("Version", "IsDeleted") + .WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + var engine = await QueryScalar(ctx, + $"SELECT engine FROM system.tables WHERE database = '{_databaseName}' AND name = 'rmt_deleted_test'"); + Assert.Equal("ReplacingMergeTree", engine); + + // Verify the create_table_query includes both version and isDeleted args + var createSql = await QueryScalar(ctx, + $"SELECT create_table_query FROM system.tables WHERE database = '{_databaseName}' AND name = 'rmt_deleted_test'"); + Assert.Contains("ReplacingMergeTree(", createSql!); + Assert.Contains("Version", createSql); + Assert.Contains("IsDeleted", createSql); + } + + [Fact] + public async Task ReplacingMergeTree_with_isDeleted_insert_and_query() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("rmt_deleted_rtrip", t => t + .HasReplacingMergeTreeEngine("Version", "IsDeleted") + .WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + ctx.Set().Add(new VersionedDeleteEntity + { + Id = 1, Name = "test", Version = 1, IsDeleted = 0 + }); + await ctx.SaveChangesAsync(); + + await using var readCtx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.ToTable("rmt_deleted_rtrip", t => t + .HasReplacingMergeTreeEngine("Version", "IsDeleted") + .WithOrderBy("Id")); + }); + }); + + var entity = await readCtx.Set().FirstAsync(e => e.Id == 1); + Assert.Equal("test", entity.Name); + Assert.Equal((byte)0, entity.IsDeleted); + } + + // ── Column with all features: default + codec + comment + TTL ────────── + + [Fact] + public async Task Column_with_default_codec_comment_and_ttl() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name) + .HasDefaultValue("unnamed") + .HasCodec("ZSTD") + .HasColumnComment("The display name"); + e.Property(x => x.CreatedAt) + .HasDefaultValueSql("now()") + .HasColumnTtl("CreatedAt + INTERVAL 30 DAY") + .HasColumnComment("Row creation time") + .HasCodec("Delta, ZSTD"); + e.ToTable("full_feat_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Verify Name column: default, codec, comment + var nameDefault = await QueryScalar(ctx, + $"SELECT default_expression FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'Name'"); + Assert.Contains("unnamed", nameDefault!); + + var nameCodec = await QueryScalar(ctx, + $"SELECT compression_codec FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'Name'"); + Assert.Contains("ZSTD", nameCodec!); + + var nameComment = await QueryScalar(ctx, + $"SELECT comment FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'Name'"); + Assert.Equal("The display name", nameComment); + + // Verify CreatedAt column: default (now()), codec, comment, TTL + var createdDefault = await QueryScalar(ctx, + $"SELECT default_expression FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'CreatedAt'"); + Assert.Contains("now()", createdDefault!); + + var createdCodec = await QueryScalar(ctx, + $"SELECT compression_codec FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'CreatedAt'"); + Assert.Contains("Delta", createdCodec!); + Assert.Contains("ZSTD", createdCodec); + + var createdComment = await QueryScalar(ctx, + $"SELECT comment FROM system.columns WHERE database = '{_databaseName}' AND table = 'full_feat_test' AND name = 'CreatedAt'"); + Assert.Equal("Row creation time", createdComment); + + // TTL appears in the CREATE TABLE statement + var createSql = await QueryScalar(ctx, + $"SELECT create_table_query FROM system.tables WHERE database = '{_databaseName}' AND name = 'full_feat_test'"); + Assert.Contains("TTL", createSql!); + Assert.Contains("CreatedAt", createSql); + } + + [Fact] + public async Task Column_with_default_value_applied_by_clickhouse_on_insert() + { + await using var ctx = CreateContext(b => + { + b.Entity(e => + { + e.HasKey(x => x.Id); + e.Property(x => x.Name).HasDefaultValue("unnamed"); + e.Property(x => x.CreatedAt).HasDefaultValueSql("now()"); + e.ToTable("default_val_test", t => t.HasMergeTreeEngine().WithOrderBy("Id")); + }); + }); + await ctx.Database.EnsureDeletedAsync(); + await ctx.Database.EnsureCreatedAsync(); + + // Insert via raw SQL (omitting Name and CreatedAt so ClickHouse applies defaults) + using var conn = new global::ClickHouse.Driver.ADO.ClickHouseConnection(_connectionString); + await conn.OpenAsync(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "INSERT INTO default_val_test (Id) VALUES (1)"; + await cmd.ExecuteNonQueryAsync(); + + // Verify defaults were applied + var name = await QueryScalar(ctx, "SELECT Name FROM default_val_test WHERE Id = 1"); + Assert.Equal("unnamed", name); + + var createdAt = await QueryScalar(ctx, "SELECT CreatedAt FROM default_val_test WHERE Id = 1"); + Assert.NotNull(createdAt); + Assert.NotEqual("1970-01-01 00:00:00", createdAt); // not epoch — got now() + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + private TestContext CreateContext(Action configure) + { + var options = new DbContextOptionsBuilder() + .UseClickHouse(_connectionString) + .EnableServiceProviderCaching(false) + .Options; + return new TestContext(options, configure); + } + + private async Task ApplyMigrationAsync(params MigrationOperation[] operations) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse(_connectionString) + .EnableServiceProviderCaching(false); + await using var context = new DbContext(optionsBuilder.Options); + var generator = context.GetService(); + var commands = generator.Generate(operations); + + using var conn = new global::ClickHouse.Driver.ADO.ClickHouseConnection(_connectionString); + await conn.OpenAsync(); + foreach (var command in commands) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = command.CommandText; + await cmd.ExecuteNonQueryAsync(); + } + } + + private static async Task QueryScalar(DbContext context, string sql) + { + var conn = context.Database.GetDbConnection(); + await conn.OpenAsync(); + try + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + return result?.ToString(); + } + finally + { + await conn.CloseAsync(); + } + } + + // ── Entity types ──────────────────────────────────────────────────────── + + private class TestContext : DbContext + { + private readonly Action _configure; + public TestContext(DbContextOptions options, Action configure) : base(options) => _configure = configure; + protected override void OnModelCreating(ModelBuilder modelBuilder) => _configure(modelBuilder); + } + + public class IdEntity + { + public long Id { get; set; } + } + + public class IdValueEntity + { + public long Id { get; set; } + public string Value { get; set; } = string.Empty; + } + + public class TimestampEntity + { + public long Id { get; set; } + public DateTime Timestamp { get; set; } + } + + public class ListEntity + { + public long Id { get; set; } + public List? Tags { get; set; } + } + + public class MapEntity + { + public long Id { get; set; } + public Dictionary? Meta { get; set; } + } + + public class VersionedDeleteEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public ulong Version { get; set; } + public byte IsDeleted { get; set; } + } + + public class EventEntity + { + public long Id { get; set; } + public long UserId { get; set; } + public string Region { get; set; } = string.Empty; + public DateTime CreatedAt { get; set; } + } + + public class FullFeaturedEntity + { + public long Id { get; set; } + public string Name { get; set; } = string.Empty; + public DateTime CreatedAt { get; set; } + } +} diff --git a/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs new file mode 100644 index 0000000..7ecfa53 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/MigrationSqlGeneratorTests.cs @@ -0,0 +1,932 @@ +using ClickHouse.EntityFrameworkCore.Metadata.Internal; +using ClickHouse.EntityFrameworkCore.Migrations.Operations; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +public class MigrationSqlGeneratorTests +{ + [Fact] + public void CreateTable_MergeTree_with_OrderBy() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "Name" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }); + }); + + Assert.Contains("ENGINE = MergeTree()", sql); + Assert.Contains("ORDER BY (`Id`, `Name`)", sql); + } + + [Fact] + public void CreateTable_ReplacingMergeTree_with_version() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.ReplacingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Version"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Version", ColumnType = "UInt64", ClrType = typeof(ulong) }); + }); + + Assert.Contains("ENGINE = ReplacingMergeTree(`Version`)", sql); + } + + [Fact] + public void CreateTable_CollapsingMergeTree_with_sign() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.CollapsingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.CollapsingMergeTreeSign, "Sign"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation { Name = "Sign", ColumnType = "Int8", ClrType = typeof(sbyte) }); + }); + + Assert.Contains("ENGINE = CollapsingMergeTree(`Sign`)", sql); + } + + [Fact] + public void CreateTable_StripeLog_no_OrderBy() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.StripeLog); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = StripeLog", sql); + Assert.DoesNotContain("StripeLog()", sql); + Assert.DoesNotContain("ORDER BY", sql); + } + + [Fact] + public void CreateTable_Memory_no_parentheses() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.Memory); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = Memory", sql); + Assert.DoesNotContain("Memory()", sql); + Assert.DoesNotContain("ORDER BY", sql); + } + + [Fact] + public void CreateTable_nullable_column_wraps_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Value", ColumnType = "String", ClrType = typeof(string), IsNullable = true + }); + }); + + Assert.Contains("`Value` Nullable(String)", sql); + Assert.DoesNotContain("NOT NULL", sql); + } + + [Fact] + public void CreateTable_column_with_codec() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Temp", ColumnType = "Int16", ClrType = typeof(short) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "Delta, ZSTD"); + op.Columns.Add(col); + }); + + Assert.Contains("`Temp` Int16 CODEC(Delta, ZSTD)", sql); + } + + [Fact] + public void CreateTable_column_with_ttl() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Created", ColumnType = "DateTime", ClrType = typeof(DateTime) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "Created + INTERVAL 1 MONTH"); + op.Columns.Add(col); + }); + + Assert.Contains("TTL Created + INTERVAL 1 MONTH", sql); + } + + [Fact] + public void CreateTable_column_with_comment() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "User name"); + op.Columns.Add(col); + }); + + Assert.Contains("COMMENT 'User name'", sql); + } + + [Fact] + public void CreateTable_with_partitionBy_expression() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.PartitionBy, new[] { "toYYYYMM(CreatedAt)" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("PARTITION BY toYYYYMM(CreatedAt)", sql); + } + + [Fact] + public void CreateTable_with_settings() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "4096"); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("SETTINGS index_granularity = 4096", sql); + } + + [Fact] + public void CreateTable_with_table_TTL() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.Ttl, "Created + INTERVAL 30 DAY"); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("TTL Created + INTERVAL 30 DAY", sql); + } + + [Fact] + public void CreateTable_MergeTree_no_explicit_OrderBy_falls_back_to_tuple() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY tuple()", sql); + } + + [Fact] + public void AddForeignKey_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new AddForeignKeyOperation + { + Table = "t", + Name = "FK_Test", + Columns = ["Id"], + PrincipalTable = "other", + PrincipalColumns = ["Id"] + }); + }); + } + + [Fact] + public void CreateSequence_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new CreateSequenceOperation { Name = "seq" }); + }); + } + + [Fact] + public void EnsureSchema_throws_NotSupportedException() + { + Assert.Throws(() => + { + Generate(new EnsureSchemaOperation { Name = "dbo" }); + }); + } + + [Fact] + public void RenameTable_generates_RENAME_TABLE() + { + var sql = Generate(new RenameTableOperation { Name = "old_table", NewName = "new_table" }); + Assert.Contains("RENAME TABLE `old_table` TO `new_table`", sql); + } + + [Fact] + public void RenameColumn_generates_ALTER_TABLE_RENAME_COLUMN() + { + var sql = Generate(new RenameColumnOperation { Table = "t", Name = "old_col", NewName = "new_col" }); + Assert.Contains("ALTER TABLE `t` RENAME COLUMN `old_col` TO `new_col`", sql); + } + + [Fact] + public void DropIndex_skipping_generates_ALTER_TABLE_DROP_INDEX() + { + var op = new DropIndexOperation { Table = "t", Name = "idx_name" }; + op.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexType, "minmax"); + var sql = Generate(op); + Assert.Contains("ALTER TABLE `t` DROP INDEX `idx_name`", sql); + } + + // Finding 3: standard index create/drop symmetry + + [Fact] + public void CreateIndex_standard_is_noop() + { + var op = new CreateIndexOperation + { + Name = "IX_Test", Table = "t", Columns = ["Col1"] + }; + var sql = Generate(op); + Assert.DoesNotContain("INDEX", sql); + } + + [Fact] + public void DropIndex_standard_is_noop() + { + var op = new DropIndexOperation { Table = "t", Name = "IX_Test" }; + // No skipping index annotation — should be no-op, symmetric with create + var sql = Generate(op); + Assert.DoesNotContain("INDEX", sql); + } + + // Finding 1: AlterTableOperation rejects ClickHouse metadata changes + + [Fact] + public void AlterTable_engine_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_orderBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "Name" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_partitionBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.PartitionBy, new[] { "toYYYYMM(ts)" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_ttl_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Ttl, "ts + INTERVAL 30 DAY"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Ttl, "ts + INTERVAL 7 DAY"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_primaryKey_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.PrimaryKey, new[] { "Id", "Name" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.PrimaryKey, new[] { "Id" }); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_sampleBy_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.SampleBy, new[] { "Id" }); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "MergeTree"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_settings_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "4096"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.SettingPrefix + "index_granularity", "8192"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_engine_specific_arg_change_throws() + { + var op = new AlterTableOperation { Name = "t" }; + op.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Ver2"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.Engine, "ReplacingMergeTree"); + op.OldTable.AddAnnotation(ClickHouseAnnotationNames.ReplacingMergeTreeVersion, "Ver1"); + + Assert.Throws(() => Generate(op)); + } + + [Fact] + public void AlterTable_no_clickhouse_changes_delegates_to_base() + { + // Non-ClickHouse metadata change (e.g., comment) should not throw + var op = new AlterTableOperation { Name = "t", Comment = "new comment" }; + op.OldTable.Comment = "old comment"; + var sql = Generate(op); + // Should not throw — base handles standard annotation changes + Assert.NotNull(sql); + } + + // Finding 2: idempotent scripts throw + + [Fact] + public void GetBeginIfNotExistsScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetBeginIfNotExistsScript("20260101000000_Init")); + } + + [Fact] + public void GetBeginIfExistsScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetBeginIfExistsScript("20260101000000_Init")); + } + + [Fact] + public void GetEndIfScript_throws_NotSupportedException() + { + var repo = CreateHistoryRepository(); + Assert.Throws(() => repo.GetEndIfScript()); + } + + [Fact] + public void GetCreateIfNotExistsScript_contains_IF_NOT_EXISTS() + { + var repo = CreateHistoryRepository(); + var script = repo.GetCreateIfNotExistsScript(); + Assert.Contains("IF NOT EXISTS", script); + Assert.Contains("CREATE TABLE", script); + } + + // Corner cases from review section D + + [Fact] + public void Column_comment_with_single_quote_is_escaped() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Name", ColumnType = "String", ClrType = typeof(string) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "it's a name"); + op.Columns.Add(col); + }); + + Assert.Contains(@"COMMENT 'it\'s a name'", sql); + } + + [Fact] + public void Column_with_codec_ttl_and_comment_together() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + var col = new AddColumnOperation { Name = "Temp", ColumnType = "Int16", ClrType = typeof(short) }; + col.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "Delta, ZSTD"); + col.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "ts + INTERVAL 1 DAY"); + col.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "temperature"); + op.Columns.Add(col); + }); + + // Verify order per ClickHouse docs: COMMENT → CODEC → TTL + var tempLine = sql.Split('\n').First(l => l.Contains("`Temp`")); + var commentIdx = tempLine.IndexOf("COMMENT ", StringComparison.Ordinal); + var codecIdx = tempLine.IndexOf("CODEC(", StringComparison.Ordinal); + var ttlIdx = tempLine.IndexOf("TTL ", StringComparison.Ordinal); + Assert.True(commentIdx < codecIdx, "COMMENT should come before CODEC"); + Assert.True(codecIdx < ttlIdx, "CODEC should come before TTL"); + } + + [Fact] + public void Nullable_array_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Tags", ColumnType = "Array(String)", ClrType = typeof(string[]), IsNullable = true + }); + }); + + Assert.Contains("`Tags` Array(String)", sql); + Assert.DoesNotContain("Nullable(Array", sql); + } + + [Fact] + public void Nullable_List_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Tags", ColumnType = "Array(String)", ClrType = typeof(List), IsNullable = true + }); + }); + + Assert.Contains("`Tags` Array(String)", sql); + Assert.DoesNotContain("Nullable(Array", sql); + } + + [Fact] + public void Nullable_Map_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Meta", ColumnType = "Map(String, String)", ClrType = typeof(Dictionary), IsNullable = true + }); + }); + + Assert.Contains("`Meta` Map(String, String)", sql); + Assert.DoesNotContain("Nullable(Map", sql); + } + + [Fact] + public void Nullable_Tuple_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Point", ColumnType = "Tuple(Float64, Float64)", ClrType = typeof(Tuple), IsNullable = true + }); + }); + + Assert.Contains("`Point` Tuple(Float64, Float64)", sql); + Assert.DoesNotContain("Nullable(Tuple", sql); + } + + [Fact] + public void Nullable_Dynamic_column_is_not_wrapped_in_Nullable() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "Data", ColumnType = "Dynamic", ClrType = typeof(object), IsNullable = true + }); + }); + + Assert.Contains("`Data` Dynamic", sql); + Assert.DoesNotContain("Nullable(Dynamic", sql); + } + + [Fact] + public void OrderBy_mixed_expressions_and_columns() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "toYYYYMM(ts)" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY (`Id`, toYYYYMM(ts))", sql); + } + + [Fact] + public void VersionedCollapsingMergeTree_both_args() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.VersionedCollapsingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeSign, "Sign"); + op.AddAnnotation(ClickHouseAnnotationNames.VersionedCollapsingMergeTreeVersion, "Ver"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = VersionedCollapsingMergeTree(`Sign`, `Ver`)", sql); + } + + [Fact] + public void SummingMergeTree_multiple_columns() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.SummingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.SummingMergeTreeColumns, new[] { "Amount", "Count" }); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = SummingMergeTree(`Amount`, `Count`)", sql); + } + + [Fact] + public void CreateTable_AggregatingMergeTree() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.AggregatingMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = AggregatingMergeTree()", sql); + Assert.Contains("ORDER BY (`Id`)", sql); + } + + [Fact] + public void CreateTable_GraphiteMergeTree() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.GraphiteMergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.GraphiteMergeTreeConfigSection, "graphite_rollup"); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = GraphiteMergeTree('graphite_rollup')", sql); + } + + [Fact] + public void CreateTable_with_sampleBy() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.SampleBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("SAMPLE BY `Id`", sql); + } + + [Fact] + public void CreateTable_with_primaryKey() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id", "Name" }); + op.AddAnnotation(ClickHouseAnnotationNames.PrimaryKey, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("PRIMARY KEY (`Id`)", sql); + } + + [Fact] + public void CreateTable_computed_column_materialized() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "NameLen", + ColumnType = "UInt32", + ClrType = typeof(uint), + ComputedColumnSql = "length(Name)", + IsStored = true + }); + }); + + Assert.Contains("`NameLen` UInt32 MATERIALIZED length(Name)", sql); + } + + [Fact] + public void CreateTable_computed_column_alias() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + op.Columns.Add(new AddColumnOperation + { + Name = "NameLen", + ColumnType = "UInt32", + ClrType = typeof(uint), + ComputedColumnSql = "length(Name)", + IsStored = false + }); + }); + + Assert.Contains("`NameLen` UInt32 ALIAS length(Name)", sql); + } + + [Fact] + public void CreateIndex_with_skippingIndexParams() + { + var op = new CreateIndexOperation + { + Name = "idx_name", Table = "t", Columns = ["Name"] + }; + op.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexType, "set"); + op.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexGranularity, 2); + op.AddAnnotation(ClickHouseAnnotationNames.SkippingIndexParams, "100"); + + var sql = Generate(op); + Assert.Contains("TYPE set(100) GRANULARITY 2", sql); + } + + [Fact] + public void CreateTable_TinyLog_no_parentheses() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.TinyLog); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = TinyLog", sql); + Assert.DoesNotContain("TinyLog()", sql); + } + + [Fact] + public void CreateTable_Log_no_parentheses() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.Log); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ENGINE = Log", sql); + Assert.DoesNotContain("Log()", sql); + } + + [Fact] + public void AddColumn_generates_ALTER_TABLE() + { + var op = new AddColumnOperation + { + Table = "t", Name = "NewCol", ColumnType = "String", ClrType = typeof(string) + }; + var sql = Generate(op); + Assert.Contains("ALTER TABLE `t` ADD COLUMN `NewCol` String", sql); + } + + [Fact] + public void AlterColumn_generates_MODIFY_COLUMN() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "Int64", ClrType = typeof(long) + }; + var sql = Generate(op); + Assert.Contains("ALTER TABLE `t` MODIFY COLUMN `Col` Int64", sql); + } + + [Fact] + public void DropColumn_generates_ALTER_TABLE() + { + var op = new DropColumnOperation { Table = "t", Name = "OldCol" }; + var sql = Generate(op); + Assert.Contains("ALTER TABLE `t` DROP COLUMN `OldCol`", sql); + } + + [Fact] + public void CreateDatabase_generates_CREATE_DATABASE() + { + var sql = Generate(new ClickHouseCreateDatabaseOperation { Name = "my_db" }); + Assert.Contains("CREATE DATABASE `my_db`", sql); + } + + [Fact] + public void DropDatabase_generates_DROP_DATABASE() + { + var sql = Generate(new ClickHouseDropDatabaseOperation { Name = "my_db" }); + Assert.Contains("DROP DATABASE `my_db`", sql); + } + + // Finding 2: AlterColumn must emit REMOVE statements when annotations are removed + + [Fact] + public void AlterColumn_removing_codec_emits_REMOVE_CODEC() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "String", ClrType = typeof(string) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "ZSTD"); + + var sql = Generate(op); + Assert.Contains("MODIFY COLUMN `Col` String", sql); + Assert.Contains("MODIFY COLUMN `Col` REMOVE CODEC", sql); + } + + [Fact] + public void AlterColumn_removing_ttl_emits_REMOVE_TTL() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "DateTime", ClrType = typeof(DateTime) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "Col + INTERVAL 1 DAY"); + + var sql = Generate(op); + Assert.Contains("MODIFY COLUMN `Col` REMOVE TTL", sql); + } + + [Fact] + public void AlterColumn_removing_comment_emits_REMOVE_COMMENT() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "String", ClrType = typeof(string) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "old comment"); + + var sql = Generate(op); + Assert.Contains("MODIFY COLUMN `Col` REMOVE COMMENT", sql); + } + + [Fact] + public void AlterColumn_removing_all_annotations_emits_all_removes() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "String", ClrType = typeof(string) + }; + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "LZ4"); + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnTtl, "Col + INTERVAL 7 DAY"); + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnComment, "old"); + + var sql = Generate(op); + Assert.Contains("REMOVE CODEC", sql); + Assert.Contains("REMOVE TTL", sql); + Assert.Contains("REMOVE COMMENT", sql); + } + + [Fact] + public void AlterColumn_changing_codec_does_not_emit_remove() + { + var op = new AlterColumnOperation + { + Table = "t", Name = "Col", ColumnType = "String", ClrType = typeof(string) + }; + op.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "ZSTD(3)"); + op.OldColumn.AddAnnotation(ClickHouseAnnotationNames.ColumnCodec, "LZ4"); + + var sql = Generate(op); + Assert.Contains("CODEC(ZSTD(3))", sql); + Assert.DoesNotContain("REMOVE CODEC", sql); + } + + // Finding 3: Expression quoting must handle non-function expressions + + [Fact] + public void OrderBy_arithmetic_expression_not_quoted() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id % 8" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY (Id % 8)", sql); + Assert.DoesNotContain("`Id % 8`", sql); + } + + [Fact] + public void OrderBy_addition_expression_not_quoted() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "a + b" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY (a + b)", sql); + Assert.DoesNotContain("`a + b`", sql); + } + + [Fact] + public void PartitionBy_cast_expression_not_quoted() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "Id" }); + op.AddAnnotation(ClickHouseAnnotationNames.PartitionBy, new[] { "Id % 8" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("PARTITION BY Id % 8", sql); + Assert.DoesNotContain("`Id % 8`", sql); + } + + [Fact] + public void OrderBy_simple_identifier_is_quoted() + { + var sql = GenerateCreateTable(op => + { + op.AddAnnotation(ClickHouseAnnotationNames.Engine, ClickHouseAnnotationNames.MergeTree); + op.AddAnnotation(ClickHouseAnnotationNames.OrderBy, new[] { "MyColumn" }); + op.Columns.Add(new AddColumnOperation { Name = "Id", ColumnType = "Int64", ClrType = typeof(long) }); + }); + + Assert.Contains("ORDER BY (`MyColumn`)", sql); + } + + private string GenerateCreateTable(Action configure) + { + var operation = new CreateTableOperation { Name = "test_table" }; + configure(operation); + return Generate(operation); + } + + private string Generate(params MigrationOperation[] operations) + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test"); + + using var context = new DbContext(optionsBuilder.Options); + var generator = context.GetService(); + var commands = generator.Generate(operations); + return string.Join("\n", commands.Select(c => c.CommandText)); + } + + private static IHistoryRepository CreateHistoryRepository() + { + var optionsBuilder = new DbContextOptionsBuilder() + .UseClickHouse("Host=localhost;Database=test"); + + using var context = new DbContext(optionsBuilder.Options); + return context.GetService(); + } +}