From 0042b79c651b6b3e76207af0db8280d6a12b473e Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Wed, 18 Mar 2026 12:04:59 +0100 Subject: [PATCH] add support for json --- README.md | 54 ++- .../Internal/ClickHouseTypeMappingSource.cs | 23 +- .../Mapping/ClickHouseJsonTypeMapping.cs | 90 ++++ .../JsonTypeMappingTests.cs | 384 ++++++++++++++++++ 4 files changed, 548 insertions(+), 3 deletions(-) create mode 100644 src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseJsonTypeMapping.cs create mode 100644 test/EFCore.ClickHouse.Tests/JsonTypeMappingTests.cs diff --git a/README.md b/README.md index 38fc909..523039a 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ public class PageView | **Tuples** | `Tuple(T1, ...)` | `Tuple<...>` or `ValueTuple<...>` | | **Variant** | `Variant(T1, T2, ...)` | `object` | | **Dynamic** | `Dynamic` | `object` | +| **JSON** | `Json` | `JsonNode` or `string` | | **Wrappers** | `Nullable(T)`, `LowCardinality(T)` | Unwrapped automatically | ## Current Status @@ -135,12 +136,63 @@ long rowsInserted = await ctx.BulkInsertAsync(events); This calls `InsertBinaryAsync` directly, bypassing EF Core's change tracker entirely. Entities are **not** tracked after insert. +### JSON Columns + +The provider supports ClickHouse's `Json` column type, mapping to `System.Text.Json.Nodes.JsonNode` or `string`. + +```csharp +using System.Text.Json.Nodes; + +public class Event +{ + public long Id { get; set; } + public JsonNode? Payload { get; set; } +} + +// In OnModelCreating: +entity.Property(e => e.Payload).HasColumnType("Json"); +``` + +Reading and writing JSON works through both `SaveChanges` and `BulkInsertAsync`: + +```csharp +ctx.Events.Add(new Event +{ + Id = 1, + Payload = JsonNode.Parse("""{"action": "click", "x": 100, "y": 200}""") +}); +await ctx.SaveChangesAsync(); + +var ev = await ctx.Events.Where(e => e.Id == 1).SingleAsync(); +string action = ev.Payload!["action"]!.GetValue(); // "click" +``` + +If you prefer working with raw JSON strings, map the property as `string` with a `Json` column type — the provider applies a `ValueConverter` automatically: + +```csharp +public class Event +{ + public long Id { get; set; } + public string? Payload { get; set; } // raw JSON string +} + +entity.Property(e => e.Payload).HasColumnType("Json"); +``` + +**Limitations:** + +- **No JSON path translation** — `entity.Payload["name"]` in LINQ does not translate to ClickHouse's `data.name` SQL syntax. Filter on non-JSON columns or load entities and inspect JSON in memory. +- **No owned entity mapping** — `.ToJson()` / `StructuralJsonTypeMapping` is not supported. JSON columns are opaque `JsonNode` or `string` values. +- **`JsonElement` / `JsonDocument` not supported** — only `JsonNode` and `string` CLR types are mapped. +- **NULL semantics** — ClickHouse's JSON type returns `{}` (empty object) for NULL values rather than SQL NULL. A row inserted with `Data = null` will read back as an empty `JsonNode`, not `null`. +- **Integer precision** — ClickHouse JSON stores all integers as `Int64` unless the path is typed otherwise. When reading via `JsonNode`, use `GetValue()` rather than `GetValue()`. + ### Not Yet Implemented - UPDATE / DELETE (ClickHouse mutations are async, not OLTP-compatible) - Migrations - JOINs, subqueries, set operations -- Nested type, JSON, Geo types +- Geo types ## Building diff --git a/src/EFCore.ClickHouse/Storage/Internal/ClickHouseTypeMappingSource.cs b/src/EFCore.ClickHouse/Storage/Internal/ClickHouseTypeMappingSource.cs index ca1b9be..838fa5a 100644 --- a/src/EFCore.ClickHouse/Storage/Internal/ClickHouseTypeMappingSource.cs +++ b/src/EFCore.ClickHouse/Storage/Internal/ClickHouseTypeMappingSource.cs @@ -2,6 +2,7 @@ using System.Data; using System.Net; using System.Numerics; +using System.Text.Json.Nodes; using System.Text.RegularExpressions; using ClickHouse.Driver.Numerics; using ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping; @@ -35,6 +36,7 @@ public class ClickHouseTypeMappingSource : RelationalTypeMappingSource private static readonly RelationalTypeMapping UInt128Mapping = new ClickHouseBigIntegerTypeMapping("UInt128"); private static readonly RelationalTypeMapping UInt256Mapping = new ClickHouseBigIntegerTypeMapping("UInt256"); private static readonly RelationalTypeMapping TimeMapping = new ClickHouseTimeSpanTypeMapping(); + private static readonly RelationalTypeMapping JsonMapping = new ClickHouseJsonTypeMapping(); private static readonly Dictionary ClrTypeMappings = new() { @@ -58,6 +60,7 @@ public class ClickHouseTypeMappingSource : RelationalTypeMappingSource { typeof(BigInteger), Int128Mapping }, { typeof(TimeSpan), TimeMapping }, { typeof(ClickHouseDecimal), new ClickHouseBigDecimalTypeMapping() }, + { typeof(JsonNode), JsonMapping }, }; private static readonly Dictionary StoreTypeMappings = @@ -98,6 +101,8 @@ public class ClickHouseTypeMappingSource : RelationalTypeMappingSource ["IPv4"] = IPv4Mapping, ["IPv6"] = IPv6Mapping, + + ["Json"] = JsonMapping, }; // Matches a single-quoted string like 'UTC' or 'Asia/Tokyo' @@ -203,11 +208,12 @@ public ClickHouseTypeMappingSource( return baseName; } - // Array(...), Map(...), Tuple(...), Variant(...) — return base name, inner parsing in FindMapping + // Array(...), Map(...), Tuple(...), Variant(...), Json(...) — return base name, inner parsing in FindMapping if (string.Equals(baseName, "Array", StringComparison.OrdinalIgnoreCase) || string.Equals(baseName, "Map", StringComparison.OrdinalIgnoreCase) || string.Equals(baseName, "Tuple", StringComparison.OrdinalIgnoreCase) - || string.Equals(baseName, "Variant", StringComparison.OrdinalIgnoreCase)) + || string.Equals(baseName, "Variant", StringComparison.OrdinalIgnoreCase) + || string.Equals(baseName, "Json", StringComparison.OrdinalIgnoreCase)) { return baseName; } @@ -228,6 +234,7 @@ public ClickHouseTypeMappingSource( ?? FindTupleMapping(mappingInfo) ?? FindVariantMapping(mappingInfo) ?? FindDynamicMapping(mappingInfo) + ?? FindJsonMapping(mappingInfo) ?? FindEnumMapping(mappingInfo) ?? FindExistingMapping(mappingInfo) ?? FindDecimalMapping(mappingInfo); @@ -445,6 +452,18 @@ public ClickHouseTypeMappingSource( return new ClickHouseDynamicTypeMapping(this); } + private static RelationalTypeMapping? FindJsonMapping(in RelationalTypeMappingInfo mappingInfo) + { + if (!string.Equals(mappingInfo.StoreTypeNameBase, "Json", StringComparison.OrdinalIgnoreCase)) + return null; + + // string CLR type + Json store type → ValueConverter-backed mapping + if (mappingInfo.ClrType == typeof(string)) + return new ClickHouseJsonTypeMapping(typeof(string)); + + return JsonMapping; + } + private static bool IsReferenceTuple(Type? type) => type is not null && type.IsGenericType && type.FullName?.StartsWith("System.Tuple`") == true; diff --git a/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseJsonTypeMapping.cs b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseJsonTypeMapping.cs new file mode 100644 index 0000000..f210f97 --- /dev/null +++ b/src/EFCore.ClickHouse/Storage/Internal/Mapping/ClickHouseJsonTypeMapping.cs @@ -0,0 +1,90 @@ +using System.Data.Common; +using System.Linq.Expressions; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Storage; + +namespace ClickHouse.EntityFrameworkCore.Storage.Internal.Mapping; + +public class ClickHouseJsonTypeMapping : RelationalTypeMapping +{ + private static readonly MethodInfo GetValueMethod = + typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetValue), [typeof(int)])!; + + private static readonly MethodInfo ToJsonStringMethod = + typeof(JsonNode).GetMethod(nameof(JsonNode.ToJsonString), [typeof(JsonSerializerOptions)])!; + + public ClickHouseJsonTypeMapping() + : this(typeof(JsonNode)) + { + } + + public ClickHouseJsonTypeMapping(Type clrType) + : base( + new RelationalTypeMappingParameters( + new CoreTypeMappingParameters( + clrType, + comparer: clrType == typeof(string) + ? new ValueComparer( + (a, b) => a == b, + o => o == null ? 0 : o.GetHashCode(), + source => source) + : CreateJsonNodeComparer()), + "Json", + dbType: System.Data.DbType.Object)) + { + } + + protected ClickHouseJsonTypeMapping(RelationalTypeMappingParameters parameters) + : base(parameters) + { + } + + protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters) + => new ClickHouseJsonTypeMapping(parameters); + + public override MethodInfo GetDataReaderMethod() + => GetValueMethod; + + public override Expression CustomizeDataReaderExpression(Expression expression) + { + // Driver returns JsonObject (subclass of JsonNode) from GetValue(). + var asJsonNode = Expression.Convert(expression, typeof(JsonNode)); + + if (ClrType == typeof(string)) + { + // Call JsonNode.ToJsonString(null) to produce a raw JSON string. + // No ValueConverter needed — the driver accepts string directly for writes. + return Expression.Call( + asJsonNode, + ToJsonStringMethod, + Expression.Constant(null, typeof(JsonSerializerOptions))); + } + + return asJsonNode; + } + + protected override string GenerateNonNullSqlLiteral(object value) + { + var json = value switch + { + JsonNode node => node.ToJsonString(), + string s => s, + _ => throw new InvalidOperationException( + $"Cannot generate SQL literal for JSON column from CLR type '{value.GetType().Name}'.") + }; + + return $"'{EscapeSqlLiteral(json)}'"; + } + + private static string EscapeSqlLiteral(string literal) + => literal.Replace("\\", "\\\\").Replace("'", "\\'"); + + private static ValueComparer CreateJsonNodeComparer() + => new( + (a, b) => (a == null && b == null) || (a != null && b != null && JsonNode.DeepEquals(a, b)), + o => o == null ? 0 : o.ToJsonString().GetHashCode(), + source => source == null ? null : JsonNode.Parse(source.ToJsonString())); +} diff --git a/test/EFCore.ClickHouse.Tests/JsonTypeMappingTests.cs b/test/EFCore.ClickHouse.Tests/JsonTypeMappingTests.cs new file mode 100644 index 0000000..f6e0c55 --- /dev/null +++ b/test/EFCore.ClickHouse.Tests/JsonTypeMappingTests.cs @@ -0,0 +1,384 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace EFCore.ClickHouse.Tests; + +#region Entities + +public class JsonNodeEntity +{ + public long Id { get; set; } + public JsonNode? Data { get; set; } +} + +public class JsonStringEntity +{ + public long Id { get; set; } + public string? Data { get; set; } +} + +#endregion + +#region DbContexts + +public class JsonNodeDbContext : DbContext +{ + public DbSet Entities => Set(); + private readonly string _connectionString; + public JsonNodeDbContext(string cs) => _connectionString = cs; + protected override void OnConfiguring(DbContextOptionsBuilder o) => o.UseClickHouse(_connectionString); + protected override void OnModelCreating(ModelBuilder m) + { + m.Entity(e => + { + e.ToTable("json_node_test"); + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasColumnName("id"); + e.Property(x => x.Data).HasColumnName("data").HasColumnType("Json"); + }); + } +} + +public class JsonStringDbContext : DbContext +{ + public DbSet Entities => Set(); + private readonly string _connectionString; + public JsonStringDbContext(string cs) => _connectionString = cs; + protected override void OnConfiguring(DbContextOptionsBuilder o) => o.UseClickHouse(_connectionString); + protected override void OnModelCreating(ModelBuilder m) + { + m.Entity(e => + { + e.ToTable("json_node_test"); + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasColumnName("id"); + e.Property(x => x.Data).HasColumnName("data").HasColumnType("Json"); + }); + } +} + +#endregion + +#region Fixture + +public class JsonTypesFixture : IAsyncLifetime +{ + public string ConnectionString { get; private set; } = string.Empty; + + public async Task InitializeAsync() + { + ConnectionString = await SharedContainer.GetConnectionStringAsync(); + + using var connection = new global::ClickHouse.Driver.ADO.ClickHouseConnection(ConnectionString); + await connection.OpenAsync(); + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "SET allow_experimental_json_type = 1"; + await cmd.ExecuteNonQueryAsync(); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = """ + CREATE TABLE json_node_test ( + id Int64, + data Json + ) ENGINE = MergeTree() ORDER BY id + SETTINGS allow_experimental_json_type = 1 + """; + await cmd.ExecuteNonQueryAsync(); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = """ + INSERT INTO json_node_test VALUES + (1, '{"name": "Alice", "age": 30}'), + (2, '{"name": "Bob", "tags": ["a", "b"]}') + """; + await cmd.ExecuteNonQueryAsync(); + } + + // NULL must be inserted separately to avoid coercion + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "INSERT INTO json_node_test VALUES (3, NULL)"; + await cmd.ExecuteNonQueryAsync(); + } + + // Table for insert round-trip tests + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = """ + CREATE TABLE json_insert_test ( + id Int64, + data Json + ) ENGINE = MergeTree() ORDER BY id + SETTINGS allow_experimental_json_type = 1 + """; + await cmd.ExecuteNonQueryAsync(); + } + } + + public Task DisposeAsync() => Task.CompletedTask; +} + +#endregion + +#region Collection Fixture + +[CollectionDefinition("JsonTypes")] +public class JsonTypesCollection : ICollectionFixture; + +#endregion + +#region Integration Tests + +[Collection("JsonTypes")] +public class JsonNodeReadTests +{ + private readonly JsonTypesFixture _fixture; + public JsonNodeReadTests(JsonTypesFixture fixture) => _fixture = fixture; + + [Fact] + public async Task ReadAll_JsonNode_RoundTrip() + { + await using var ctx = new JsonNodeDbContext(_fixture.ConnectionString); + var rows = await ctx.Entities.OrderBy(e => e.Id).AsNoTracking().ToListAsync(); + + Assert.Equal(3, rows.Count); + + // Row 1: structured JSON + Assert.NotNull(rows[0].Data); + Assert.Equal("Alice", rows[0].Data!["name"]?.GetValue()); + // ClickHouse JSON stores integers as Int64 + Assert.Equal(30L, rows[0].Data!["age"]?.GetValue()); + + // Row 2: JSON with array + Assert.NotNull(rows[1].Data); + Assert.Equal("Bob", rows[1].Data!["name"]?.GetValue()); + + // Row 3: NULL → ClickHouse JSON returns empty object '{}' for NULL + Assert.NotNull(rows[2].Data); + } + + [Fact] + public async Task Where_JsonTable_FilterById() + { + await using var ctx = new JsonNodeDbContext(_fixture.ConnectionString); + var result = await ctx.Entities + .Where(e => e.Id == 1) + .AsNoTracking().SingleAsync(); + + Assert.NotNull(result.Data); + Assert.Equal("Alice", result.Data!["name"]?.GetValue()); + } +} + +[Collection("JsonTypes")] +public class JsonStringReadTests +{ + private readonly JsonTypesFixture _fixture; + public JsonStringReadTests(JsonTypesFixture fixture) => _fixture = fixture; + + [Fact] + public async Task ReadAll_JsonString_RoundTrip() + { + await using var ctx = new JsonStringDbContext(_fixture.ConnectionString); + var rows = await ctx.Entities.OrderBy(e => e.Id).AsNoTracking().ToListAsync(); + + Assert.Equal(3, rows.Count); + + // Row 1: verify it's valid JSON containing expected data + Assert.NotNull(rows[0].Data); + var parsed = JsonNode.Parse(rows[0].Data!); + Assert.Equal("Alice", parsed!["name"]?.GetValue()); + + // Row 3: ClickHouse JSON returns empty object '{}' for NULL + Assert.NotNull(rows[2].Data); + } +} + +[Collection("JsonTypes")] +public class JsonInsertTests +{ + private readonly JsonTypesFixture _fixture; + public JsonInsertTests(JsonTypesFixture fixture) => _fixture = fixture; + + [Fact] + public async Task Insert_JsonNode_RoundTrip() + { + // Use a dedicated context pointing at insert table + var cs = _fixture.ConnectionString; + await using var insertCtx = new JsonInsertDbContext(cs); + + var entity = new JsonInsertEntity + { + Id = 100, + Data = JsonNode.Parse("""{"key": "value", "num": 42}""") + }; + + insertCtx.Entities.Add(entity); + await insertCtx.SaveChangesAsync(); + + // Read back + await using var readCtx = new JsonInsertDbContext(cs); + var result = await readCtx.Entities + .Where(e => e.Id == 100) + .AsNoTracking().SingleAsync(); + + Assert.NotNull(result.Data); + Assert.Equal("value", result.Data!["key"]?.GetValue()); + Assert.Equal(42L, result.Data!["num"]?.GetValue()); + } + + [Fact] + public async Task Insert_NullJson_RoundTrip() + { + var cs = _fixture.ConnectionString; + await using var insertCtx = new JsonInsertDbContext(cs); + + var entity = new JsonInsertEntity { Id = 101, Data = null }; + insertCtx.Entities.Add(entity); + await insertCtx.SaveChangesAsync(); + + await using var readCtx = new JsonInsertDbContext(cs); + var result = await readCtx.Entities + .Where(e => e.Id == 101) + .AsNoTracking().SingleAsync(); + + // ClickHouse JSON type returns empty object '{}' for NULL, not SQL NULL + Assert.NotNull(result.Data); + } +} + +public class JsonInsertEntity +{ + public long Id { get; set; } + public JsonNode? Data { get; set; } +} + +public class JsonInsertDbContext : DbContext +{ + public DbSet Entities => Set(); + private readonly string _connectionString; + public JsonInsertDbContext(string cs) => _connectionString = cs; + protected override void OnConfiguring(DbContextOptionsBuilder o) => o.UseClickHouse(_connectionString); + protected override void OnModelCreating(ModelBuilder m) + { + m.Entity(e => + { + e.ToTable("json_insert_test"); + e.HasKey(x => x.Id); + e.Property(x => x.Id).HasColumnName("id"); + e.Property(x => x.Data).HasColumnName("data").HasColumnType("Json"); + }); + } +} + +#endregion + +#region Unit Tests + +public class JsonTypeMappingUnitTests +{ + [Fact] + public void SqlLiteral_SimpleJson() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(JsonNode))!; + var node = JsonNode.Parse("""{"name":"Alice"}""")!; + var literal = mapping.GenerateSqlLiteral(node); + Assert.Equal("""'{"name":"Alice"}'""", literal); + } + + [Fact] + public void SqlLiteral_JsonWithBackslash() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(JsonNode))!; + // JSON with a value containing a literal backslash + var node = JsonNode.Parse("""{"path":"C:\\Users"}""")!; + var literal = mapping.GenerateSqlLiteral(node); + // ToJsonString() produces: {"path":"C\\Users"} + // Our escaping doubles backslashes for ClickHouse + Assert.Contains("\\\\", literal); + Assert.StartsWith("'", literal); + Assert.EndsWith("'", literal); + } + + [Fact] + public void SqlLiteral_NullJson_ReturnsNULL() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(JsonNode))!; + var literal = mapping.GenerateSqlLiteral(null); + Assert.Equal("NULL", literal); + } + + [Fact] + public void FindMapping_JsonNode_FromClrType() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(JsonNode)); + Assert.NotNull(mapping); + Assert.Equal("Json", mapping.StoreType); + Assert.Equal(typeof(JsonNode), mapping.ClrType); + } + + [Fact] + public void FindMapping_String_WithJsonStoreType_Resolves() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(string), "Json"); + Assert.NotNull(mapping); + Assert.Equal("Json", mapping.StoreType); + Assert.Equal(typeof(string), mapping.ClrType); + // No ValueConverter — driver accepts string directly for writes, + // and CustomizeDataReaderExpression handles the read-side conversion. + Assert.Null(mapping.Converter); + } + + [Fact] + public void FindMapping_Object_WithJsonStoreType_Resolves() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(object), "Json"); + Assert.NotNull(mapping); + Assert.Equal("Json", mapping.StoreType); + } + + [Fact] + public void FindMapping_JsonWithTypeHints_Resolves() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(object), "Json(name String, age Int32)"); + Assert.NotNull(mapping); + Assert.Equal("Json", mapping.StoreType); + } + + [Fact] + public void FindMapping_String_WithoutJsonStoreType_ResolvesToString() + { + var source = GetTypeMappingSource(); + var mapping = source.FindMapping(typeof(string)); + Assert.NotNull(mapping); + Assert.Equal("String", mapping.StoreType); + Assert.Null(mapping.Converter); + } + + private static Microsoft.EntityFrameworkCore.Storage.IRelationalTypeMappingSource GetTypeMappingSource() + { + var builder = new DbContextOptionsBuilder(); + builder.UseClickHouse("Host=localhost;Protocol=http"); + using var ctx = new DbContext(builder.Options); + return ((IInfrastructure)ctx).Instance + .GetRequiredService(); + } +} + +#endregion