From 24370c43f191e96714d3bf1b71794b8253f19f41 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Sun, 19 May 2024 18:47:55 +0200 Subject: [PATCH 01/18] Refactored Postgres checkpointing repository to take dedicated PostgresConnectionProvider to enable transaction injection This prepares for the further use with EntityFramework --- ...esSubscriptionCheckpointRepositoryTests.cs | 11 ++---- ...ostgresSubscriptionCheckpointRepository.cs | 35 ++++++++++++++++--- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs index 1f60ae88..accf373a 100644 --- a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs +++ b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs @@ -1,8 +1,8 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; -using Npgsql; using Xunit; namespace Core.EventStoreDB.Tests.Subscriptions.Checkpoints; + using static ISubscriptionCheckpointRepository; public class PostgresSubscriptionCheckpointRepositoryTests(PostgresContainerFixture fixture) @@ -10,13 +10,8 @@ public class PostgresSubscriptionCheckpointRepositoryTests(PostgresContainerFixt { private readonly string subscriptionId = Guid.NewGuid().ToString("N"); - private readonly Func> connectionFactory = - _ => - { - var connection = fixture.DataSource.CreateConnection(); - connection.Open(); - return ValueTask.FromResult(connection); - }; + private readonly PostgresConnectionProvider connectionFactory = + PostgresConnectionProvider.From(fixture.DataSource); [Fact] public async Task Store_InitialInsert_Success() diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs index 615ea1e9..a4ce336d 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs @@ -4,17 +4,44 @@ namespace Core.EventStoreDB.Subscriptions.Checkpoints; using static ISubscriptionCheckpointRepository; +public class PostgresConnectionProvider(Func> connectionFactory) +{ + public ValueTask Get(CancellationToken ct) => connectionFactory(ct); + + public void Set(NpgsqlConnection connection) => + connectionFactory = _ => ValueTask.FromResult(connection); + + public void Set(NpgsqlTransaction transaction) => + connectionFactory = _ => ValueTask.FromResult(transaction.Connection!); + + public void Set(NpgsqlDataSource dataSource) => + connectionFactory = async ct => + { + var connection = dataSource.CreateConnection(); + await connection.OpenAsync(ct).ConfigureAwait(false); + return connection; + }; + + public static PostgresConnectionProvider From(NpgsqlDataSource npgsqlDataSource) => + new(async ct => + { + var connection = npgsqlDataSource.CreateConnection(); + await connection.OpenAsync(ct).ConfigureAwait(false); + return connection; + }); +} + public class PostgresSubscriptionCheckpointRepository( // I'm not using data source here, as I'd like to enable option // to update projection in the same transaction in the same transaction as checkpointing // to help handling idempotency - Func> connectionFactory, + PostgresConnectionProvider connectionProvider, PostgresSubscriptionCheckpointSetup checkpointSetup ): ISubscriptionCheckpointRepository { public async ValueTask Load(string subscriptionId, CancellationToken ct) { - var connection = await connectionFactory(ct).ConfigureAwait(false); + var connection = await connectionProvider.Get(ct).ConfigureAwait(false); await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand(SelectCheckpointSql, connection); @@ -36,7 +63,7 @@ public async ValueTask Store( CancellationToken ct ) { - var connection = await connectionFactory(ct).ConfigureAwait(false); + var connection = await connectionProvider.Get(ct).ConfigureAwait(false); await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand("SELECT store_subscription_checkpoint($1, $2, $3)", connection); @@ -59,7 +86,7 @@ CancellationToken ct public async ValueTask Reset(string subscriptionId, CancellationToken ct) { - var connection = await connectionFactory(ct).ConfigureAwait(false); + var connection = await connectionProvider.Get(ct).ConfigureAwait(false); await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand(ResetCheckpointSql, connection); From 2a593e295fffacb9554f85545367ce2467a7f9c7 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Sun, 19 May 2024 19:11:39 +0200 Subject: [PATCH 02/18] Introduced IEventBatchHandler to allow injecting custom batch handling, for instance to use it for projections --- .../Batch/EventsBatchCheckpointer.cs | 3 -- .../Batch/EventsBatchProcessor.cs | 41 +-------------- .../EventStoreDBSubscriptionToAll.cs | 2 - Core/Events/EventBus.cs | 5 +- Core/Events/EventBusBatchHandler.cs | 52 +++++++++++++++++++ Core/Events/IEventBatchHandler.cs | 6 +++ 6 files changed, 63 insertions(+), 46 deletions(-) create mode 100644 Core/Events/EventBusBatchHandler.cs create mode 100644 Core/Events/IEventBatchHandler.cs diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs index 330581ec..bcc14270 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs @@ -1,11 +1,8 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; -using Core.Extensions; using EventStore.Client; namespace Core.EventStoreDB.Subscriptions.Batch; -using static EventStoreClient; - public class EventsBatchCheckpointer( ISubscriptionCheckpointRepository checkpointRepository, EventsBatchProcessor eventsBatchProcessor diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index c697286c..c3b2f58d 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -10,8 +10,7 @@ namespace Core.EventStoreDB.Subscriptions.Batch; public class EventsBatchProcessor( EventTypeMapper eventTypeMapper, - IEventBus eventBus, - IActivityScope activityScope, + IEventBatchHandler batchHandler, ILogger logger ) { @@ -24,11 +23,7 @@ CancellationToken ct var events = TryDeserializeEvents(resolvedEvents, options.IgnoreDeserializationErrors); ulong? lastPosition = null; - foreach (var @event in events) - { - await HandleEvent(@event, ct).ConfigureAwait(false); - lastPosition = @event.Metadata.LogPosition; - } + await batchHandler.Handle(events, ct).ConfigureAwait(false); return lastPosition; } @@ -69,38 +64,6 @@ bool ignoreDeserializationErrors return result.ToArray(); } - private async Task HandleEvent( - IEventEnvelope eventEnvelope, - CancellationToken token - ) - { - try - { - await activityScope.Run($"{nameof(EventStoreDBSubscriptionToAll)}/{nameof(HandleEvent)}", - async (_, ct) => - { - // publish event to internal event bus - await eventBus.Publish(eventEnvelope, ct).ConfigureAwait(false); - }, - new StartActivityOptions - { - Tags = { { TelemetryTags.EventHandling.Event, eventEnvelope.Data.GetType() } }, - Parent = eventEnvelope.Metadata.PropagationContext?.ActivityContext, - Kind = ActivityKind.Consumer - }, - token - ).ConfigureAwait(false); - } - catch (Exception e) - { - logger.LogError("Error consuming message: {ExceptionMessage}{ExceptionStackTrace}", e.Message, - e.StackTrace); - // if you're fine with dropping some events instead of stopping subscription - // then you can add some logic if error should be ignored - throw; - } - } - private bool IsEventWithEmptyData(ResolvedEvent resolvedEvent) { if (resolvedEvent.Event.Data.Length != 0) return false; diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 5133b987..98c24b5f 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -9,8 +9,6 @@ namespace Core.EventStoreDB.Subscriptions; -using static ISubscriptionCheckpointRepository; - public class EventStoreDBSubscriptionToAllOptions { public required string SubscriptionId { get; init; } diff --git a/Core/Events/EventBus.cs b/Core/Events/EventBus.cs index 1699981e..5e69395c 100644 --- a/Core/Events/EventBus.cs +++ b/Core/Events/EventBus.cs @@ -15,8 +15,8 @@ public interface IEventBus public class EventBus( IServiceProvider serviceProvider, IActivityScope activityScope, - AsyncPolicy retryPolicy) - : IEventBus + AsyncPolicy retryPolicy +): IEventBus { private static readonly ConcurrentDictionary PublishMethods = new(); @@ -86,6 +86,7 @@ public static IServiceCollection AddEventBus(this IServiceCollection services, A sp.GetRequiredService(), asyncPolicy ?? Policy.NoOpAsync() )); + services.AddScoped(); services .TryAddSingleton(sp => sp.GetRequiredService()); diff --git a/Core/Events/EventBusBatchHandler.cs b/Core/Events/EventBusBatchHandler.cs new file mode 100644 index 00000000..56d7b9c8 --- /dev/null +++ b/Core/Events/EventBusBatchHandler.cs @@ -0,0 +1,52 @@ +using System.Diagnostics; +using Core.OpenTelemetry; +using Microsoft.Extensions.Logging; + +namespace Core.Events; + +public class EventBusBatchHandler( + IEventBus eventBus, + IActivityScope activityScope, + ILogger logger +): IEventBatchHandler +{ + public async Task Handle(IEventEnvelope[] events, CancellationToken ct) + { + foreach (var @event in events) + { + await HandleEvent(@event, ct).ConfigureAwait(false); + } + } + + private async Task HandleEvent( + IEventEnvelope eventEnvelope, + CancellationToken token + ) + { + try + { + await activityScope.Run($"{nameof(EventBusBatchHandler)}/{nameof(HandleEvent)}", + async (_, ct) => + { + // publish event to internal event bus + await eventBus.Publish(eventEnvelope, ct).ConfigureAwait(false); + }, + new StartActivityOptions + { + Tags = { { TelemetryTags.EventHandling.Event, eventEnvelope.Data.GetType() } }, + Parent = eventEnvelope.Metadata.PropagationContext?.ActivityContext, + Kind = ActivityKind.Consumer + }, + token + ).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogError("Error consuming message: {ExceptionMessage}{ExceptionStackTrace}", e.Message, + e.StackTrace); + // if you're fine with dropping some events instead of stopping subscription + // then you can add some logic if error should be ignored + throw; + } + } +} diff --git a/Core/Events/IEventBatchHandler.cs b/Core/Events/IEventBatchHandler.cs new file mode 100644 index 00000000..9f9061d7 --- /dev/null +++ b/Core/Events/IEventBatchHandler.cs @@ -0,0 +1,6 @@ +namespace Core.Events; + +public interface IEventBatchHandler +{ + Task Handle(IEventEnvelope[] events, CancellationToken ct); +} From 9482330ebb7b2a16423e8fbed8cc4903bde3f1e4 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Sun, 19 May 2024 20:13:50 +0200 Subject: [PATCH 03/18] Added Core.EntityFramework project with EntityFrameworkProjection Implementation is made accordingly to MartenElasticSearchProjection --- .../Core.EntityFramework.csproj | 20 ++++ .../EntityFrameworkProjection.cs | 93 +++++++++++++++++++ .../Batch/EventsBatchProcessor.cs | 3 +- Core/Events/EventBusBatchHandler.cs | 4 +- Core/Events/IEventBatchHandler.cs | 2 +- EventSourcing.NetCore.sln | 7 ++ 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 Core.EntityFramework/Core.EntityFramework.csproj create mode 100644 Core.EntityFramework/EntityFrameworkProjection.cs diff --git a/Core.EntityFramework/Core.EntityFramework.csproj b/Core.EntityFramework/Core.EntityFramework.csproj new file mode 100644 index 00000000..54326c70 --- /dev/null +++ b/Core.EntityFramework/Core.EntityFramework.csproj @@ -0,0 +1,20 @@ + + + + net8.0 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + diff --git a/Core.EntityFramework/EntityFrameworkProjection.cs b/Core.EntityFramework/EntityFrameworkProjection.cs new file mode 100644 index 00000000..dbc9d4f6 --- /dev/null +++ b/Core.EntityFramework/EntityFrameworkProjection.cs @@ -0,0 +1,93 @@ +using Core.Events; +using Core.Reflection; +using Microsoft.EntityFrameworkCore; +using Polly; + +namespace Core.EntityFramework; + +public class EntityFrameworkProjection(TDbContext dbContext, IAsyncPolicy? retryPolicy = null) + : IEventBatchHandler + where TDbContext : DbContext +{ + protected readonly TDbContext DBContext = dbContext; + protected IAsyncPolicy RetryPolicy { get; } = retryPolicy ?? Policy.NoOpAsync(); + private readonly HashSet handledEventTypes = []; + + protected void Projects() => + handledEventTypes.Add(typeof(TEvent)); + + public async Task Handle(IEventEnvelope[] eventInEnvelopes, CancellationToken ct) + { + var events = eventInEnvelopes + .Where(@event => handledEventTypes.Contains(@event.Data.GetType())) + .ToArray(); + + await ApplyAsync(events, ct); + } + + protected virtual Task ApplyAsync(IEventEnvelope[] events, CancellationToken ct) => + ApplyAsync(events.Select(@event => @event.Data).ToArray(), ct); + + protected virtual Task ApplyAsync(object[] events, CancellationToken ct) => + Task.CompletedTask; +} + +public abstract class EntityFrameworkProjection( + TDbContext dbContext, + IAsyncPolicy retryPolicy +): EntityFrameworkProjection(dbContext, retryPolicy) + where TDocument : class + where TDbContext : DbContext +{ + private record ProjectEvent( + Func GetId, + Func Apply + ); + + private readonly Dictionary projectors = new(); + private Func getDocumentId = default!; + + protected void Projects( + Func getId, + Func apply + ) + { + projectors.Add( + typeof(TEvent), + new ProjectEvent( + @event => getId((TEvent)@event), + (document, @event) => apply(document, (TEvent)@event) + ) + ); + Projects(); + } + + protected void DocumentId(Func documentId) => + getDocumentId = documentId; + + protected override Task ApplyAsync(object[] events, CancellationToken token) => + RetryPolicy.ExecuteAsync(async ct => + { + var ids = events.Select(GetDocumentId).ToList(); + + var entities = await DBContext.Set() + .Where(x => ids.Contains(getDocumentId(x))) + .ToListAsync(cancellationToken: ct); + + var existingDocuments = entities.ToDictionary(ks => getDocumentId(ks), vs => vs); + + for(var i = 0; i < events.Length; i++) + { + Apply(existingDocuments.GetValueOrDefault(ids[i], GetDefault(events[i])), events[i]); + } + }, token); + + protected virtual TDocument GetDefault(object @event) => + ObjectFactory.GetDefaultOrUninitialized(); + + private TDocument Apply(TDocument document, object @event) => + projectors[@event.GetType()].Apply(document, @event); + + private object GetDocumentId(object @event) => + projectors[@event.GetType()].GetId(@event); +} diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index c3b2f58d..77bb5d09 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -1,8 +1,6 @@ -using System.Diagnostics; using Core.Events; using Core.EventStoreDB.Events; using Core.EventStoreDB.Subscriptions.Checkpoints; -using Core.OpenTelemetry; using EventStore.Client; using Microsoft.Extensions.Logging; @@ -23,6 +21,7 @@ CancellationToken ct var events = TryDeserializeEvents(resolvedEvents, options.IgnoreDeserializationErrors); ulong? lastPosition = null; + // TODO: How would you implement Dead-Letter Queue here? await batchHandler.Handle(events, ct).ConfigureAwait(false); return lastPosition; diff --git a/Core/Events/EventBusBatchHandler.cs b/Core/Events/EventBusBatchHandler.cs index 56d7b9c8..15a28155 100644 --- a/Core/Events/EventBusBatchHandler.cs +++ b/Core/Events/EventBusBatchHandler.cs @@ -10,9 +10,9 @@ public class EventBusBatchHandler( ILogger logger ): IEventBatchHandler { - public async Task Handle(IEventEnvelope[] events, CancellationToken ct) + public async Task Handle(IEventEnvelope[] eventInEnvelopes, CancellationToken ct) { - foreach (var @event in events) + foreach (var @event in eventInEnvelopes) { await HandleEvent(@event, ct).ConfigureAwait(false); } diff --git a/Core/Events/IEventBatchHandler.cs b/Core/Events/IEventBatchHandler.cs index 9f9061d7..87a69824 100644 --- a/Core/Events/IEventBatchHandler.cs +++ b/Core/Events/IEventBatchHandler.cs @@ -2,5 +2,5 @@ namespace Core.Events; public interface IEventBatchHandler { - Task Handle(IEventEnvelope[] events, CancellationToken ct); + Task Handle(IEventEnvelope[] eventInEnvelopes, CancellationToken ct); } diff --git a/EventSourcing.NetCore.sln b/EventSourcing.NetCore.sln index 7488366d..a1bdf88d 100644 --- a/EventSourcing.NetCore.sln +++ b/EventSourcing.NetCore.sln @@ -486,6 +486,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "ECommerce.FeedConsumer", "S EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ECommerce.Equinox", "ECommerce.Equinox", "{006643C6-E0B6-48E6-ABC6-9BE3DCB293D8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core.EntityFramework", "Core.EntityFramework\Core.EntityFramework.csproj", "{840B5027-91B5-42F2-A431-17DBD05F0BDC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1096,6 +1098,10 @@ Global {C3D0553F-5786-4417-97FD-B65440620274}.Debug|Any CPU.Build.0 = Debug|Any CPU {16B7A55B-E2E6-4CE2-896A-EF0F02259482}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {16B7A55B-E2E6-4CE2-896A-EF0F02259482}.Debug|Any CPU.Build.0 = Debug|Any CPU + {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1299,6 +1305,7 @@ Global {1738AF23-FD36-4457-B9F9-2593207CDEB5} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} {C6383AC1-2D24-4E7A-810F-642920C857EA} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} {E1B97A7B-97C3-4C14-9BE6-ACE0AF45CE61} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} + {840B5027-91B5-42F2-A431-17DBD05F0BDC} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A5F55604-2FF3-43B7-B657-4F18E6E95D3B} From 1789c301e68b5540190a253e3e2e2de9d6d1061c Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Mon, 20 May 2024 20:37:52 +0200 Subject: [PATCH 04/18] Added tests for EntityFrameworkProjection and fixed the implementation. --- .../Core.EntityFramework.Tests.csproj | 48 +++++++ .../EntityFrameworkProjectionTests.cs | 85 +++++++++++++ .../EfCorePostgresContainerFixture.cs | 31 +++++ .../20240520164705_InitialCreate.Designer.cs | 47 +++++++ .../20240520164705_InitialCreate.cs | 35 +++++ .../Migrations/TestDbContextModelSnapshot.cs | 44 +++++++ Core.EntityFramework.Tests/TestDbContext.cs | 53 ++++++++ Core.EntityFramework.Tests/appsettings.json | 13 ++ .../EntityFrameworkProjection.cs | 120 +++++++++++++----- ...esSubscriptionCheckpointRepositoryTests.cs | 1 + Core.Testing/Core.Testing.csproj | 3 + .../Fixtures}/PostgresContainerFixture.cs | 9 +- EventSourcing.NetCore.sln | 7 + .../Projections/EntityFrameworkProjection.cs | 1 + 14 files changed, 462 insertions(+), 35 deletions(-) create mode 100644 Core.EntityFramework.Tests/Core.EntityFramework.Tests.csproj create mode 100644 Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs create mode 100644 Core.EntityFramework.Tests/Fixtures/EfCorePostgresContainerFixture.cs create mode 100644 Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.Designer.cs create mode 100644 Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.cs create mode 100644 Core.EntityFramework.Tests/Migrations/TestDbContextModelSnapshot.cs create mode 100644 Core.EntityFramework.Tests/TestDbContext.cs create mode 100644 Core.EntityFramework.Tests/appsettings.json rename {Core.EventStoreDB.Tests/Subscriptions/Checkpoints => Core.Testing/Fixtures}/PostgresContainerFixture.cs (70%) diff --git a/Core.EntityFramework.Tests/Core.EntityFramework.Tests.csproj b/Core.EntityFramework.Tests/Core.EntityFramework.Tests.csproj new file mode 100644 index 00000000..ea663a69 --- /dev/null +++ b/Core.EntityFramework.Tests/Core.EntityFramework.Tests.csproj @@ -0,0 +1,48 @@ + + + + net8.0 + + + + + PreserveNewest + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + true + PreserveNewest + PreserveNewest + + + + + diff --git a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs new file mode 100644 index 00000000..f5979702 --- /dev/null +++ b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs @@ -0,0 +1,85 @@ +using Core.EntityFramework.Tests.Fixtures; +using Core.Events; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace Core.EntityFramework.Tests; + +public record Opened(Guid ShoppingCartId, Guid ClientId); + +public record ProductItemAdded(Guid ShoppingCartId, int Count); + +public record Cancelled(Guid ShoppingCartId); + +public class ShoppingCartProjection: EntityFrameworkProjection +{ + public ShoppingCartProjection() + { + ViewId(c => c.Id); + + Creates(Create); + Projects(e => e.ShoppingCartId, Apply); + Deletes(e => e.ShoppingCartId); + } + + private ShoppingCart Create(Opened opened) => + new() { Id = opened.ShoppingCartId, ClientId = opened.ClientId, ProductCount = 0 }; + + + private ShoppingCart Apply(ShoppingCart cart, ProductItemAdded added) + { + cart.ProductCount += added.Count; + return cart; + } +} + +public class EntityFrameworkProjectionTests(EfCorePostgresContainerFixture fixture) + : IClassFixture> +{ + private readonly TestDbContext context = fixture.DbContext; + + [Fact] + public async Task Applies_Works_Separately() + { + var projection = new ShoppingCartProjection { DbContext = context }; + + var cartId = Guid.NewGuid(); + var clientId = Guid.NewGuid(); + + await projection.Handle([EventEnvelope.From(new Opened(cartId, clientId))], CancellationToken.None); + await context.SaveChangesAsync(); + + var savedEntity = await context.ShoppingCarts.Where(e => e.Id == cartId).FirstOrDefaultAsync(); + savedEntity.Should().NotBeNull(); + savedEntity.Should().BeEquivalentTo(new ShoppingCart { Id = cartId, ClientId = clientId, ProductCount = 0 }); + + await projection.Handle([EventEnvelope.From(new ProductItemAdded(cartId, 2))], CancellationToken.None); + await context.SaveChangesAsync(); + + savedEntity = await context.ShoppingCarts.Where(e => e.Id == cartId).FirstOrDefaultAsync(); + savedEntity.Should().NotBeNull(); + savedEntity.Should().BeEquivalentTo(new ShoppingCart { Id = cartId, ClientId = clientId, ProductCount = 2 }); + + + await projection.Handle([EventEnvelope.From(new Cancelled(cartId))], CancellationToken.None); + await context.SaveChangesAsync(); + + savedEntity = await context.ShoppingCarts.Where(e => e.Id == cartId).FirstOrDefaultAsync(); + savedEntity.Should().BeNull(); + } + + + [Fact] + public async Task SmokeTest() + { + var entity = new ShoppingCart { Id = Guid.NewGuid(), ProductCount = 2, ClientId = Guid.NewGuid() }; + context.ShoppingCarts.Add(entity); + await context.SaveChangesAsync(); + + var savedEntity = await context.ShoppingCarts.FindAsync(entity.Id); + Assert.NotNull(savedEntity); + savedEntity.Should().NotBeNull(); + savedEntity.Should().BeEquivalentTo(entity); + } +} diff --git a/Core.EntityFramework.Tests/Fixtures/EfCorePostgresContainerFixture.cs b/Core.EntityFramework.Tests/Fixtures/EfCorePostgresContainerFixture.cs new file mode 100644 index 00000000..2e160635 --- /dev/null +++ b/Core.EntityFramework.Tests/Fixtures/EfCorePostgresContainerFixture.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore; +using Core.Testing.Fixtures; +using Xunit; + +namespace Core.EntityFramework.Tests.Fixtures; + +public class EfCorePostgresContainerFixture: IAsyncLifetime where TContext : DbContext +{ + private readonly PostgresContainerFixture postgresContainerFixture = new(); + + public TContext DbContext { get; private set; } = default!; + + public async Task InitializeAsync() + { + await postgresContainerFixture.InitializeAsync().ConfigureAwait(false); + + var optionsBuilder = new DbContextOptionsBuilder() + .UseNpgsql(postgresContainerFixture.DataSource); + + DbContext = (TContext)Activator.CreateInstance(typeof(TContext), optionsBuilder.Options)!; + + await DbContext.Database.MigrateAsync().ConfigureAwait(false); + } + + public async Task DisposeAsync() + { + await DbContext.DisposeAsync().ConfigureAwait(false); + await postgresContainerFixture.DisposeAsync().ConfigureAwait(false); + } +} + diff --git a/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.Designer.cs b/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.Designer.cs new file mode 100644 index 00000000..14bd4aa2 --- /dev/null +++ b/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.Designer.cs @@ -0,0 +1,47 @@ +// +using System; +using Core.EntityFramework.Tests; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Core.EntityFramework.Tests.Migrations +{ + [DbContext(typeof(TestDbContext))] + [Migration("20240520164705_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Core.EntityFramework.Tests.ShoppingCart", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("ProductCount") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("ShoppingCarts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.cs b/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.cs new file mode 100644 index 00000000..e1b34ea5 --- /dev/null +++ b/Core.EntityFramework.Tests/Migrations/20240520164705_InitialCreate.cs @@ -0,0 +1,35 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Core.EntityFramework.Tests.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "ShoppingCarts", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + ClientId = table.Column(type: "uuid", nullable: false), + ProductCount = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ShoppingCarts", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "ShoppingCarts"); + } + } +} diff --git a/Core.EntityFramework.Tests/Migrations/TestDbContextModelSnapshot.cs b/Core.EntityFramework.Tests/Migrations/TestDbContextModelSnapshot.cs new file mode 100644 index 00000000..0ddd32a5 --- /dev/null +++ b/Core.EntityFramework.Tests/Migrations/TestDbContextModelSnapshot.cs @@ -0,0 +1,44 @@ +// +using System; +using Core.EntityFramework.Tests; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Core.EntityFramework.Tests.Migrations +{ + [DbContext(typeof(TestDbContext))] + partial class TestDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Core.EntityFramework.Tests.ShoppingCart", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ClientId") + .HasColumnType("uuid"); + + b.Property("ProductCount") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("ShoppingCarts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Core.EntityFramework.Tests/TestDbContext.cs b/Core.EntityFramework.Tests/TestDbContext.cs new file mode 100644 index 00000000..ecb88011 --- /dev/null +++ b/Core.EntityFramework.Tests/TestDbContext.cs @@ -0,0 +1,53 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Configuration; + +namespace Core.EntityFramework.Tests; + +public class ShoppingCart +{ + public Guid Id { get; set; } + public Guid ClientId { get; set; } + public int ProductCount { get; set; } +} + +public class TestDbContext(DbContextOptions options): DbContext(options) +{ + public DbSet ShoppingCarts { get; set; } = default!; + + protected override void OnModelCreating(ModelBuilder modelBuilder) => + modelBuilder.Entity(); +} + + + +public class TestDbContextFactory: IDesignTimeDbContextFactory +{ + public TestDbContext CreateDbContext(params string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + + if (optionsBuilder.IsConfigured) + return new TestDbContext(optionsBuilder.Options); + + //Called by parameterless ctor Usually Migrations + var environmentName = Environment.GetEnvironmentVariable("EnvironmentName") ?? "Development"; + + var connectionString = + new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("appsettings.json") + .AddJsonFile($"appsettings.{environmentName}.json", optional: true, reloadOnChange: false) + .AddEnvironmentVariables() + .Build() + .GetConnectionString("TestDb"); + + optionsBuilder.UseNpgsql(connectionString); + + return new TestDbContext(optionsBuilder.Options); + } + + public static TestDbContext Create() + => new TestDbContextFactory().CreateDbContext(); +} + diff --git a/Core.EntityFramework.Tests/appsettings.json b/Core.EntityFramework.Tests/appsettings.json new file mode 100644 index 00000000..9c032c36 --- /dev/null +++ b/Core.EntityFramework.Tests/appsettings.json @@ -0,0 +1,13 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "ConnectionStrings": { + "TestDb": "PORT = 5432; HOST = 127.0.0.1; TIMEOUT = 15; POOLING = True; MINPOOLSIZE = 1; MAXPOOLSIZE = 100; COMMANDTIMEOUT = 20; DATABASE = 'postgres'; PASSWORD = 'Password12!'; USER ID = 'postgres'" + }, + "AllowedHosts": "*" +} diff --git a/Core.EntityFramework/EntityFrameworkProjection.cs b/Core.EntityFramework/EntityFrameworkProjection.cs index dbc9d4f6..3b3409db 100644 --- a/Core.EntityFramework/EntityFrameworkProjection.cs +++ b/Core.EntityFramework/EntityFrameworkProjection.cs @@ -1,16 +1,17 @@ -using Core.Events; +using System.Linq.Expressions; +using Core.Events; using Core.Reflection; using Microsoft.EntityFrameworkCore; using Polly; namespace Core.EntityFramework; -public class EntityFrameworkProjection(TDbContext dbContext, IAsyncPolicy? retryPolicy = null) - : IEventBatchHandler +public class EntityFrameworkProjection: IEventBatchHandler where TDbContext : DbContext { - protected readonly TDbContext DBContext = dbContext; - protected IAsyncPolicy RetryPolicy { get; } = retryPolicy ?? Policy.NoOpAsync(); + public TDbContext DbContext { protected get; set; } = default!; + public IAsyncPolicy RetryPolicy { protected get; set; } = Policy.NoOpAsync(); + private readonly HashSet handledEventTypes = []; protected void Projects() => @@ -32,62 +33,121 @@ protected virtual Task ApplyAsync(object[] events, CancellationToken ct) => Task.CompletedTask; } -public abstract class EntityFrameworkProjection( - TDbContext dbContext, - IAsyncPolicy retryPolicy -): EntityFrameworkProjection(dbContext, retryPolicy) - where TDocument : class +public class EntityFrameworkProjection: EntityFrameworkProjection + where TView : class + where TId: struct where TDbContext : DbContext { private record ProjectEvent( - Func GetId, - Func Apply + Func GetId, + Func Apply ); private readonly Dictionary projectors = new(); - private Func getDocumentId = default!; + private Expression> viewIdExpression = default!; + private Func viewId = default!; + + public void ViewId(Expression> id) + { + viewIdExpression = id; + viewId = id.Compile(); + } + + public void Creates( + Func apply + ) + { + projectors.Add( + typeof(TEvent), + new ProjectEvent( + _ => null, + (_, @event) => apply((TEvent)@event) + ) + ); + Projects(); + } - protected void Projects( - Func getId, - Func apply + + public void Deletes( + Func getId ) { projectors.Add( typeof(TEvent), new ProjectEvent( @event => getId((TEvent)@event), - (document, @event) => apply(document, (TEvent)@event) + (_, _) => null ) ); Projects(); } - protected void DocumentId(Func documentId) => - getDocumentId = documentId; + public void Projects( + Func getId, + Func apply + ) + { + projectors.Add( + typeof(TEvent), + new ProjectEvent( + @event => getId((TEvent)@event), + (document, @event) => apply(document, (TEvent)@event) + ) + ); + Projects(); + } protected override Task ApplyAsync(object[] events, CancellationToken token) => RetryPolicy.ExecuteAsync(async ct => { - var ids = events.Select(GetDocumentId).ToList(); + var dbSet = DbContext.Set(); - var entities = await DBContext.Set() - .Where(x => ids.Contains(getDocumentId(x))) - .ToListAsync(cancellationToken: ct); + var ids = events.Select(GetViewId).ToList(); - var existingDocuments = entities.ToDictionary(ks => getDocumentId(ks), vs => vs); + var idPredicate = GetContainsExpression(ids.Where(e => e.HasValue).Cast().ToList()); - for(var i = 0; i < events.Length; i++) + var existingViews = await dbSet + .Where(idPredicate) + .ToDictionaryAsync(viewId, ct); + + for (var i = 0; i < events.Length; i++) { - Apply(existingDocuments.GetValueOrDefault(ids[i], GetDefault(events[i])), events[i]); + var id = ids[i]; + + var current = id.HasValue && existingViews.TryGetValue(id.Value, out var existing) ? existing : null; + + var result = Apply(current ?? GetDefault(events[i]), events[i]); + + if (result == null) + { + if (current != null) + dbSet.Remove(current); + } + else if (current == null) + dbSet.Add(result); + else + dbSet.Update(result); } }, token); - protected virtual TDocument GetDefault(object @event) => - ObjectFactory.GetDefaultOrUninitialized(); + protected virtual TView GetDefault(object @event) => + ObjectFactory.GetDefaultOrUninitialized(); - private TDocument Apply(TDocument document, object @event) => + private TView? Apply(TView document, object @event) => projectors[@event.GetType()].Apply(document, @event); - private object GetDocumentId(object @event) => + private TId? GetViewId(object @event) => projectors[@event.GetType()].GetId(@event); + + private Expression> GetContainsExpression(List ids) + { + var parameter = viewIdExpression.Parameters.Single(); + var body = Expression.Call( + Expression.Constant(ids), + ((Func)ids.Contains).Method, + viewIdExpression.Body + ); + + return Expression.Lambda>(body, parameter); + } } diff --git a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs index accf373a..0acee9b7 100644 --- a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs +++ b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs @@ -1,4 +1,5 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; +using Core.Testing.Fixtures; using Xunit; namespace Core.EventStoreDB.Tests.Subscriptions.Checkpoints; diff --git a/Core.Testing/Core.Testing.csproj b/Core.Testing/Core.Testing.csproj index 4a90b029..a1a8b100 100644 --- a/Core.Testing/Core.Testing.csproj +++ b/Core.Testing/Core.Testing.csproj @@ -21,6 +21,9 @@ + + + diff --git a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresContainerFixture.cs b/Core.Testing/Fixtures/PostgresContainerFixture.cs similarity index 70% rename from Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresContainerFixture.cs rename to Core.Testing/Fixtures/PostgresContainerFixture.cs index 71dbbe6b..3729ceab 100644 --- a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresContainerFixture.cs +++ b/Core.Testing/Fixtures/PostgresContainerFixture.cs @@ -2,7 +2,7 @@ using Testcontainers.PostgreSql; using Xunit; -namespace Core.EventStoreDB.Tests.Subscriptions.Checkpoints; +namespace Core.Testing.Fixtures; public class PostgresContainerFixture: IAsyncLifetime { @@ -14,14 +14,13 @@ public class PostgresContainerFixture: IAsyncLifetime public async Task InitializeAsync() { - await container.StartAsync(); + await container.StartAsync().ConfigureAwait(false); DataSource = new NpgsqlDataSourceBuilder(container.GetConnectionString()).Build(); } public async Task DisposeAsync() { - await DataSource.DisposeAsync(); - await container.StopAsync(); + await DataSource.DisposeAsync().ConfigureAwait(false); + await container.StopAsync().ConfigureAwait(false); } - } diff --git a/EventSourcing.NetCore.sln b/EventSourcing.NetCore.sln index a1bdf88d..f8d9e4b3 100644 --- a/EventSourcing.NetCore.sln +++ b/EventSourcing.NetCore.sln @@ -488,6 +488,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ECommerce.Equinox", "EComme EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core.EntityFramework", "Core.EntityFramework\Core.EntityFramework.csproj", "{840B5027-91B5-42F2-A431-17DBD05F0BDC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core.EntityFramework.Tests", "Core.EntityFramework.Tests\Core.EntityFramework.Tests.csproj", "{040621EF-1D78-443B-8871-2ABC070496BA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1102,6 +1104,10 @@ Global {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Debug|Any CPU.Build.0 = Debug|Any CPU {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Release|Any CPU.ActiveCfg = Release|Any CPU {840B5027-91B5-42F2-A431-17DBD05F0BDC}.Release|Any CPU.Build.0 = Release|Any CPU + {040621EF-1D78-443B-8871-2ABC070496BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {040621EF-1D78-443B-8871-2ABC070496BA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {040621EF-1D78-443B-8871-2ABC070496BA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {040621EF-1D78-443B-8871-2ABC070496BA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1306,6 +1312,7 @@ Global {C6383AC1-2D24-4E7A-810F-642920C857EA} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} {E1B97A7B-97C3-4C14-9BE6-ACE0AF45CE61} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} {840B5027-91B5-42F2-A431-17DBD05F0BDC} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} + {040621EF-1D78-443B-8871-2ABC070496BA} = {0570E45A-2EB6-4C4C-84E4-2C80E1FECEB5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A5F55604-2FF3-43B7-B657-4F18E6E95D3B} diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs b/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs index b424ecb9..10615a2d 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs @@ -27,6 +27,7 @@ public class EntityFrameworkProjectionBuilder(IServiceCollect where TView : class where TDbContext : DbContext { + public EntityFrameworkProjectionBuilder AddOn(Func, TView> handler) where TEvent : notnull { From f3bdd3459d4be071c0ea2b9037843e0f1b967fc8 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Mon, 20 May 2024 21:25:47 +0200 Subject: [PATCH 05/18] Fixed handling for Add, Updates and Removes in the same batch --- .../EntityFrameworkProjectionTests.cs | 40 ++++++++++++ Core.EntityFramework/DbContextExtensions.cs | 17 +++++ .../EntityFrameworkProjection.cs | 63 +++++++++++-------- 3 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 Core.EntityFramework/DbContextExtensions.cs diff --git a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs index f5979702..f1cd30b3 100644 --- a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs +++ b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs @@ -69,6 +69,46 @@ public async Task Applies_Works_Separately() savedEntity.Should().BeNull(); } + [Fact] + public async Task Applies_Works_In_Batch() + { + var projection = new ShoppingCartProjection { DbContext = context }; + + var cartId = Guid.NewGuid(); + var clientId = Guid.NewGuid(); + + await projection.Handle([ + EventEnvelope.From(new Opened(cartId, clientId)), + EventEnvelope.From(new ProductItemAdded(cartId, 2)), + EventEnvelope.From(new ProductItemAdded(cartId, 3)), + EventEnvelope.From(new ProductItemAdded(cartId, 5)) + ], CancellationToken.None); + await context.SaveChangesAsync(); + + var savedEntity = await context.ShoppingCarts.Where(e => e.Id == cartId).FirstOrDefaultAsync(); + savedEntity.Should().NotBeNull(); + savedEntity.Should().BeEquivalentTo(new ShoppingCart { Id = cartId, ClientId = clientId, ProductCount = 10 }); + } + + + + [Fact] + public async Task Applies_Works_In_Batch_With_AddAndDeleteOnTheSameRecord() + { + var projection = new ShoppingCartProjection { DbContext = context }; + + var cartId = Guid.NewGuid(); + var clientId = Guid.NewGuid(); + + await projection.Handle([ + EventEnvelope.From(new Opened(cartId, clientId)), + EventEnvelope.From(new Cancelled(cartId)) + ], CancellationToken.None); + await context.SaveChangesAsync(); + + var savedEntity = await context.ShoppingCarts.Where(e => e.Id == cartId).FirstOrDefaultAsync(); + savedEntity.Should().BeNull(); + } [Fact] public async Task SmokeTest() diff --git a/Core.EntityFramework/DbContextExtensions.cs b/Core.EntityFramework/DbContextExtensions.cs new file mode 100644 index 00000000..b53947d7 --- /dev/null +++ b/Core.EntityFramework/DbContextExtensions.cs @@ -0,0 +1,17 @@ +using Microsoft.EntityFrameworkCore; + +namespace Core.EntityFramework; + +public static class DbContextExtensions +{ + public static void AddOrUpdate(this DbContext dbContext, TEntity entity) where TEntity : class + { + var dbSet = dbContext.Set(); + var entry = dbContext.Entry(entity); + + if (entry.State == EntityState.Detached) + { + dbSet.Add(entity); + } + } +} diff --git a/Core.EntityFramework/EntityFrameworkProjection.cs b/Core.EntityFramework/EntityFrameworkProjection.cs index 3b3409db..cab2c49f 100644 --- a/Core.EntityFramework/EntityFrameworkProjection.cs +++ b/Core.EntityFramework/EntityFrameworkProjection.cs @@ -35,7 +35,7 @@ protected virtual Task ApplyAsync(object[] events, CancellationToken ct) => public class EntityFrameworkProjection: EntityFrameworkProjection where TView : class - where TId: struct + where TId : struct where TDbContext : DbContext { private record ProjectEvent( @@ -97,47 +97,58 @@ Func apply Projects(); } + protected TView? Apply(TView document, object @event) => + projectors[@event.GetType()].Apply(document, @event); + + protected TId? GetViewId(object @event) => + projectors[@event.GetType()].GetId(@event); + protected override Task ApplyAsync(object[] events, CancellationToken token) => RetryPolicy.ExecuteAsync(async ct => { var dbSet = DbContext.Set(); - var ids = events.Select(GetViewId).ToList(); - - var idPredicate = GetContainsExpression(ids.Where(e => e.HasValue).Cast().ToList()); + var eventWithViewIds = events.Select(e => (Event: e, ViewId: GetViewId(e))).ToList(); + var ids = eventWithViewIds.Where(e => e.ViewId.HasValue).Select(e => e.ViewId!.Value).ToList(); - var existingViews = await dbSet - .Where(idPredicate) - .ToDictionaryAsync(viewId, ct); + var existingViews = await GetExistingViews(dbSet, ids, ct); - for (var i = 0; i < events.Length; i++) + foreach (var (@event, id) in eventWithViewIds) { - var id = ids[i]; + ProcessEvent(@event, id, existingViews, dbSet); + } + }, token); - var current = id.HasValue && existingViews.TryGetValue(id.Value, out var existing) ? existing : null; - var result = Apply(current ?? GetDefault(events[i]), events[i]); + private void ProcessEvent(object @event, TId? id, Dictionary existingViews, DbSet dbSet) + { + var current = id.HasValue && existingViews.TryGetValue(id.Value, out var existing) ? existing : null; - if (result == null) - { - if (current != null) - dbSet.Remove(current); - } - else if (current == null) - dbSet.Add(result); - else - dbSet.Update(result); - } - }, token); + var result = Apply(current ?? GetDefault(@event), @event); + + if (result == null) + { + if (current != null) + dbSet.Remove(current); + + return; + } + + DbContext.AddOrUpdate(result); + + if (current == null) + existingViews.Add(viewId(result), result); + } protected virtual TView GetDefault(object @event) => ObjectFactory.GetDefaultOrUninitialized(); - private TView? Apply(TView document, object @event) => - projectors[@event.GetType()].Apply(document, @event); + private Expression> BuildContainsExpression(List ids) => + GetContainsExpression(ids); - private TId? GetViewId(object @event) => - projectors[@event.GetType()].GetId(@event); + private async Task> + GetExistingViews(DbSet dbSet, List ids, CancellationToken ct) => + await dbSet.Where(BuildContainsExpression(ids)).ToDictionaryAsync(viewId, ct); private Expression> GetContainsExpression(List ids) { From 5c2ac46b8639907310696541f20d19caeae18430 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Mon, 20 May 2024 22:52:59 +0200 Subject: [PATCH 06/18] Connected the generic EntityFrameworkProjection with the Simple implementation --- .../EntityFrameworkProjectionTests.cs | 1 + .../Core.EntityFramework.csproj | 1 + .../{ => Extensions}/DbContextExtensions.cs | 2 +- .../EntityFrameworkProjection.cs | 99 ++++++++++------ .../EntityFrameworkProjectionBuilder.cs | 108 ++++++++++++++++++ .../Queries/QueryHandler.cs | 4 +- .../Checkpoints/EFCheckpointTransaction.cs | 16 +++ Core.EventStoreDB/Config.cs | 2 +- .../Batch/EventsBatchCheckpointer.cs | 33 +++--- .../Batch/EventsBatchProcessor.cs | 7 +- ...ostgresSubscriptionCheckpointRepository.cs | 40 +++++++ .../EventStoreDBSubscriptionToAll.cs | 23 +++- .../ECommerce.Core/ECommerce.Core.csproj | 1 + .../Projections/EntityFrameworkProjection.cs | 75 +----------- .../ECommerce/ShoppingCarts/Configuration.cs | 15 ++- .../GettingCartById/ShoppingCartDetails.cs | 8 +- .../GettingCarts/ShoppingCartShortInfo.cs | 8 +- 17 files changed, 294 insertions(+), 149 deletions(-) rename Core.EntityFramework/{ => Extensions}/DbContextExtensions.cs (90%) rename Core.EntityFramework/{ => Projections}/EntityFrameworkProjection.cs (58%) create mode 100644 Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs rename {Sample/EventStoreDB/Simple/ECommerce.Core => Core.EntityFramework}/Queries/QueryHandler.cs (97%) create mode 100644 Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs diff --git a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs index f1cd30b3..431350d7 100644 --- a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs +++ b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs @@ -1,3 +1,4 @@ +using Core.EntityFramework.Projections; using Core.EntityFramework.Tests.Fixtures; using Core.Events; using FluentAssertions; diff --git a/Core.EntityFramework/Core.EntityFramework.csproj b/Core.EntityFramework/Core.EntityFramework.csproj index 54326c70..2216791e 100644 --- a/Core.EntityFramework/Core.EntityFramework.csproj +++ b/Core.EntityFramework/Core.EntityFramework.csproj @@ -14,6 +14,7 @@ + diff --git a/Core.EntityFramework/DbContextExtensions.cs b/Core.EntityFramework/Extensions/DbContextExtensions.cs similarity index 90% rename from Core.EntityFramework/DbContextExtensions.cs rename to Core.EntityFramework/Extensions/DbContextExtensions.cs index b53947d7..6e66ff18 100644 --- a/Core.EntityFramework/DbContextExtensions.cs +++ b/Core.EntityFramework/Extensions/DbContextExtensions.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; -namespace Core.EntityFramework; +namespace Core.EntityFramework.Extensions; public static class DbContextExtensions { diff --git a/Core.EntityFramework/EntityFrameworkProjection.cs b/Core.EntityFramework/Projections/EntityFrameworkProjection.cs similarity index 58% rename from Core.EntityFramework/EntityFrameworkProjection.cs rename to Core.EntityFramework/Projections/EntityFrameworkProjection.cs index cab2c49f..92e40dba 100644 --- a/Core.EntityFramework/EntityFrameworkProjection.cs +++ b/Core.EntityFramework/Projections/EntityFrameworkProjection.cs @@ -1,10 +1,11 @@ using System.Linq.Expressions; +using Core.EntityFramework.Extensions; using Core.Events; using Core.Reflection; using Microsoft.EntityFrameworkCore; using Polly; -namespace Core.EntityFramework; +namespace Core.EntityFramework.Projections; public class EntityFrameworkProjection: IEventBatchHandler where TDbContext : DbContext @@ -38,9 +39,11 @@ public class EntityFrameworkProjection: EntityFrameworkP where TId : struct where TDbContext : DbContext { + public Expression>? Include { protected get; set; } + private record ProjectEvent( - Func GetId, - Func Apply + Func GetId, + Func Apply ); private readonly Dictionary projectors = new(); @@ -54,56 +57,77 @@ public void ViewId(Expression> id) } public void Creates( - Func apply - ) - { - projectors.Add( - typeof(TEvent), + Func, TView> apply + ) where TEvent : notnull => + Projects( new ProjectEvent( _ => null, - (_, @event) => apply((TEvent)@event) + (_, envelope) => apply((EventEnvelope)envelope) ) ); - Projects(); - } + public void Creates( + Func apply + ) where TEvent : notnull => + Creates(envelope => apply(envelope.Data)); public void Deletes( Func getId - ) - { - projectors.Add( - typeof(TEvent), + ) => + Projects( new ProjectEvent( - @event => getId((TEvent)@event), + envelope => getId((TEvent)envelope.Data), (_, _) => null ) ); - Projects(); - } public void Projects( Func getId, - Func apply - ) - { - projectors.Add( - typeof(TEvent), + Func, TView> apply + ) where TEvent : notnull => + Projects( new ProjectEvent( - @event => getId((TEvent)@event), - (document, @event) => apply(document, (TEvent)@event) + envelope => getId((TEvent)envelope.Data), + (document, envelope) => apply(document, (EventEnvelope)envelope) ) ); + + public void Projects( + Func getId, + Action> apply + ) where TEvent : notnull => + Projects(getId, (view, envelope) => + { + apply(view, envelope); + return view; + }); + + public void Projects( + Func getId, + Func apply + ) where TEvent : notnull => + Projects(getId, (view, envelope) => apply(view, envelope.Data)); + + public void Projects( + Func getId, + Action apply + ) where TEvent : notnull => + Projects(getId, (view, envelope) => apply(view, envelope.Data)); + + + private void Projects(ProjectEvent projectEvent) + { + projectors.Add(typeof(TEvent), projectEvent); Projects(); } - protected TView? Apply(TView document, object @event) => - projectors[@event.GetType()].Apply(document, @event); + private TView? Apply(TView document, IEventEnvelope @event) => + projectors[@event.Data.GetType()].Apply(document, @event); - protected TId? GetViewId(object @event) => - projectors[@event.GetType()].GetId(@event); + private TId? GetViewId(IEventEnvelope @event) => + projectors[@event.Data.GetType()].GetId(@event); - protected override Task ApplyAsync(object[] events, CancellationToken token) => + protected override Task ApplyAsync(IEventEnvelope[] events, CancellationToken token) => RetryPolicy.ExecuteAsync(async ct => { var dbSet = DbContext.Set(); @@ -113,18 +137,19 @@ protected override Task ApplyAsync(object[] events, CancellationToken token) => var existingViews = await GetExistingViews(dbSet, ids, ct); - foreach (var (@event, id) in eventWithViewIds) + foreach (var (eventEnvelope, id) in eventWithViewIds) { - ProcessEvent(@event, id, existingViews, dbSet); + ProcessEvent(eventEnvelope, id, existingViews, dbSet); } }, token); - private void ProcessEvent(object @event, TId? id, Dictionary existingViews, DbSet dbSet) + private void ProcessEvent(IEventEnvelope eventEnvelope, TId? id, Dictionary existingViews, + DbSet dbSet) { var current = id.HasValue && existingViews.TryGetValue(id.Value, out var existing) ? existing : null; - var result = Apply(current ?? GetDefault(@event), @event); + var result = Apply(current ?? GetDefault(), eventEnvelope); if (result == null) { @@ -140,7 +165,7 @@ private void ProcessEvent(object @event, TId? id, Dictionary existin existingViews.Add(viewId(result), result); } - protected virtual TView GetDefault(object @event) => + protected virtual TView GetDefault() => ObjectFactory.GetDefaultOrUninitialized(); private Expression> BuildContainsExpression(List ids) => @@ -148,7 +173,9 @@ private Expression> BuildContainsExpression(List ids) => private async Task> GetExistingViews(DbSet dbSet, List ids, CancellationToken ct) => - await dbSet.Where(BuildContainsExpression(ids)).ToDictionaryAsync(viewId, ct); + Include != null + ? await dbSet.Include(Include).Where(BuildContainsExpression(ids)).ToDictionaryAsync(viewId, ct) + : await dbSet.Where(BuildContainsExpression(ids)).ToDictionaryAsync(viewId, ct); private Expression> GetContainsExpression(List ids) { diff --git a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs new file mode 100644 index 00000000..064a5216 --- /dev/null +++ b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs @@ -0,0 +1,108 @@ +using System.Linq.Expressions; +using Core.EntityFramework.Queries; +using Core.EntityFramework.Subscriptions.Checkpoints; +using Core.Events; +using Core.EventStoreDB; +using Core.EventStoreDB.Subscriptions; +using Core.EventStoreDB.Subscriptions.Batch; +using Core.EventStoreDB.Subscriptions.Checkpoints; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace Core.EntityFramework.Projections; + +public static class EntityFrameworkProjection +{ + public static IServiceCollection For( + this IServiceCollection services, + Action> setup + ) + where TView : class + where TDbContext : DbContext + where TId : struct + { + var builder = new EntityFrameworkProjectionBuilder(services); + setup(builder); + + services.AddEventStoreDBSubscriptionToAll( + new EventStoreDBSubscriptionToAllOptions { SubscriptionId = typeof(TView).FullName!, BatchSize = 20 }, + false + ); + + services.AddScoped(sp => + { + var dataSource = sp.GetRequiredService(); + var dbContext = sp.GetRequiredService(); + + var checkpointTransaction = new EFCheckpointTransaction(dbContext); + + return new TransactionalPostgresSubscriptionCheckpointRepository( + new PostgresSubscriptionCheckpointRepository( + new PostgresConnectionProvider( + async ct => + { + await dbContext.Database.BeginTransactionAsync(ct); + + return (NpgsqlConnection)dbContext.Database.GetDbConnection(); + }), + new PostgresSubscriptionCheckpointSetup(dataSource) + ), + checkpointTransaction + ); + }); + + return services; + } +} + +public class EntityFrameworkProjectionBuilder(IServiceCollection services) + where TView : class + where TId : struct + where TDbContext : DbContext +{ + public EntityFrameworkProjection Projection = new(); + + public EntityFrameworkProjectionBuilder AddOn( + Func, TView> handler) + where TEvent : notnull + { + Projection.Creates(handler); + return this; + } + + public EntityFrameworkProjectionBuilder UpdateOn( + Func getViewId, + Action> handler + ) where TEvent : notnull + { + Projection.Projects(getViewId, handler); + return this; + } + + public EntityFrameworkProjectionBuilder Include( + Expression> include) + { + Projection.Include = include; + + return this; + } + + public EntityFrameworkProjectionBuilder QueryWith( + Func, TQuery, CancellationToken, Task> handler + ) + { + services.AddEntityFrameworkQueryHandler(handler); + + return this; + } + + public EntityFrameworkProjectionBuilder QueryWith( + Func, TQuery, CancellationToken, Task>> handler + ) + { + services.AddEntityFrameworkQueryHandler(handler); + + return this; + } +} diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/Queries/QueryHandler.cs b/Core.EntityFramework/Queries/QueryHandler.cs similarity index 97% rename from Sample/EventStoreDB/Simple/ECommerce.Core/Queries/QueryHandler.cs rename to Core.EntityFramework/Queries/QueryHandler.cs index ff74b119..56be50ee 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/Queries/QueryHandler.cs +++ b/Core.EntityFramework/Queries/QueryHandler.cs @@ -1,7 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -namespace ECommerce.Core.Queries; +namespace Core.EntityFramework.Queries; public static class QueryHandler { @@ -46,4 +46,4 @@ public static IServiceCollection AddQueryHandler( Func>> setup ) => services.AddTransient(setup); -} \ No newline at end of file +} diff --git a/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs b/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs new file mode 100644 index 00000000..129a40a7 --- /dev/null +++ b/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs @@ -0,0 +1,16 @@ +using Core.EventStoreDB.Subscriptions.Checkpoints; +using Microsoft.EntityFrameworkCore; + +namespace Core.EntityFramework.Subscriptions.Checkpoints; + +public class EFCheckpointTransaction(DbContext dbContext): ICheckpointTransaction +{ + public Task Commit(CancellationToken cancellationToken = default) => + dbContext.Database.CommitTransactionAsync(cancellationToken); + + public Task Rollback(CancellationToken cancellationToken = default) + { + dbContext.ChangeTracker.Clear(); + return dbContext.Database.RollbackTransactionAsync(cancellationToken); + } +} diff --git a/Core.EventStoreDB/Config.cs b/Core.EventStoreDB/Config.cs index 7bf99008..f99280be 100644 --- a/Core.EventStoreDB/Config.cs +++ b/Core.EventStoreDB/Config.cs @@ -61,7 +61,7 @@ public static IServiceCollection AddEventStoreDBSubscriptionToAll( bool checkpointToEventStoreDB = true) { services.AddScoped(); - services.AddScoped(); + services.AddScoped(); if (checkpointToEventStoreDB) { diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs index bcc14270..8c024a1d 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs @@ -2,33 +2,40 @@ using EventStore.Client; namespace Core.EventStoreDB.Subscriptions.Batch; +using static ISubscriptionCheckpointRepository; + +public interface IEventsBatchCheckpointer +{ + Task Process( + ResolvedEvent[] events, + Checkpoint lastCheckpoint, + EventStoreDBSubscriptionToAllOptions subscriptionOptions, + CancellationToken ct + ); +} public class EventsBatchCheckpointer( ISubscriptionCheckpointRepository checkpointRepository, EventsBatchProcessor eventsBatchProcessor -) +): IEventsBatchCheckpointer { - public async Task Process( + public async Task Process( ResolvedEvent[] events, Checkpoint lastCheckpoint, EventStoreDBSubscriptionToAllOptions subscriptionOptions, CancellationToken ct ) { - var processedPosition = await eventsBatchProcessor.HandleEventsBatch(events, subscriptionOptions, ct) - .ConfigureAwait(false); + var lastPosition = events.LastOrDefault().OriginalPosition?.CommitPosition; - if (!processedPosition.HasValue) - return lastCheckpoint; + if (!lastPosition.HasValue) + return new StoreResult.Ignored(); - var result = await checkpointRepository - .Store(subscriptionOptions.SubscriptionId, processedPosition.Value, lastCheckpoint, ct) + await eventsBatchProcessor.HandleEventsBatch(events, subscriptionOptions, ct) .ConfigureAwait(false); - if (result is not ISubscriptionCheckpointRepository.StoreResult.Success success) - throw new InvalidOperationException( - $"Mismatch while updating Store checkpoint! Ensure that you don't have multiple instances running Subscription with id: {subscriptionOptions.SubscriptionId}"); - - return success.Checkpoint; + return await checkpointRepository + .Store(subscriptionOptions.SubscriptionId, lastPosition.Value, lastCheckpoint, ct) + .ConfigureAwait(false); } } diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index 77bb5d09..0e8785ab 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -12,19 +12,16 @@ public class EventsBatchProcessor( ILogger logger ) { - public async Task HandleEventsBatch( + public Task HandleEventsBatch( ResolvedEvent[] resolvedEvents, EventStoreDBSubscriptionToAllOptions options, CancellationToken ct ) { var events = TryDeserializeEvents(resolvedEvents, options.IgnoreDeserializationErrors); - ulong? lastPosition = null; // TODO: How would you implement Dead-Letter Queue here? - await batchHandler.Handle(events, ct).ConfigureAwait(false); - - return lastPosition; + return batchHandler.Handle(events, ct); } private IEventEnvelope[] TryDeserializeEvents( diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs index a4ce336d..997f2e58 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs @@ -1,3 +1,4 @@ +using System.Data.Common; using Npgsql; namespace Core.EventStoreDB.Subscriptions.Checkpoints; @@ -120,6 +121,45 @@ ON CONFLICT ("id") WHERE "position" = $3 """; } +public interface ICheckpointTransaction +{ + Task Commit(CancellationToken cancellationToken = default); + + Task Rollback(CancellationToken cancellationToken = default); +} + +public class TransactionalPostgresSubscriptionCheckpointRepository( + PostgresSubscriptionCheckpointRepository inner, + ICheckpointTransaction checkpointTransaction +): ISubscriptionCheckpointRepository +{ + public ValueTask Load(string subscriptionId, CancellationToken ct) => + inner.Load(subscriptionId, ct); + + public async ValueTask Store( + string subscriptionId, + ulong position, + Checkpoint previousCheckpoint, + CancellationToken ct + ) + { + var result = await inner.Store(subscriptionId, position, previousCheckpoint, ct).ConfigureAwait(false); + + if (result is not StoreResult.Success) + { + await checkpointTransaction.Rollback(ct).ConfigureAwait(false); + return result; + } + + await checkpointTransaction.Commit(ct).ConfigureAwait(false); + + return result; + } + + public ValueTask Reset(string subscriptionId, CancellationToken ct) => + inner.Reset(subscriptionId, ct); +} + public class PostgresSubscriptionCheckpointSetup(NpgsqlDataSource dataSource) { private bool wasCreated; diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 98c24b5f..18e11ad7 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -9,6 +9,8 @@ namespace Core.EventStoreDB.Subscriptions; +using static ISubscriptionCheckpointRepository; + public class EventStoreDBSubscriptionToAllOptions { public required string SubscriptionId { get; init; } @@ -56,9 +58,22 @@ public async Task SubscribeToAll(EventStoreDBSubscriptionToAllOptions subscripti logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); - await foreach (var events in subscription.BatchAsync(subscriptionOptions.BatchSize, ct).ConfigureAwait(false)) + await foreach (var events in subscription.BatchAsync(subscriptionOptions.BatchSize, ct) + .ConfigureAwait(false)) { - checkpoint = await ProcessBatch(events, checkpoint, subscriptionOptions, ct).ConfigureAwait(false); + var result = await ProcessBatch(events, checkpoint, subscriptionOptions, ct).ConfigureAwait(false); + + switch (result) + { + case StoreResult.Success success: + checkpoint = success.Checkpoint; + break; + case StoreResult.Ignored ignored: + break; + default: + throw new InvalidOperationException( + "Checkpoint mismatch, ensure that you have a single subscription running!"); + } } } catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || @@ -87,14 +102,14 @@ private ValueTask LoadCheckpoint(CancellationToken ct) return scope.ServiceProvider.GetRequiredService().Load(SubscriptionId, ct); } - private Task ProcessBatch( + private Task ProcessBatch( ResolvedEvent[] events, Checkpoint lastCheckpoint, EventStoreDBSubscriptionToAllOptions subscriptionOptions, CancellationToken ct) { using var scope = serviceProvider.CreateScope(); - return scope.ServiceProvider.GetRequiredService() + return scope.ServiceProvider.GetRequiredService() .Process(events, lastCheckpoint, subscriptionOptions, ct); } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj b/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj index 2b9cdd78..806f4fd9 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj @@ -22,6 +22,7 @@ + diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs b/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs index 10615a2d..657c2fcd 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/Projections/EntityFrameworkProjection.cs @@ -1,6 +1,7 @@ -using Core.Events; +using System.Linq.Expressions; +using Core.EntityFramework; +using Core.Events; using Core.Projections; -using ECommerce.Core.Queries; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.Extensions.DependencyInjection; @@ -9,71 +10,6 @@ namespace ECommerce.Core.Projections; -public static class EntityFrameworkProjection -{ - public static IServiceCollection For( - this IServiceCollection services, - Action> setup - ) - where TView : class - where TDbContext : DbContext - { - setup(new EntityFrameworkProjectionBuilder(services)); - return services; - } -} - -public class EntityFrameworkProjectionBuilder(IServiceCollection services) - where TView : class - where TDbContext : DbContext -{ - - public EntityFrameworkProjectionBuilder AddOn(Func, TView> handler) - where TEvent : notnull - { - services.AddSingleton(handler); - services.AddTransient>, AddProjection>(); - - return this; - } - - public EntityFrameworkProjectionBuilder UpdateOn( - Func getViewId, - Action, TView> handler, - Func, CancellationToken, Task>? prepare = null - ) where TEvent : notnull - { - services.AddSingleton(getViewId); - services.AddSingleton(handler); - services.AddTransient>, UpdateProjection>(); - - if (prepare != null) - { - services.AddSingleton(prepare); - } - - return this; - } - - public EntityFrameworkProjectionBuilder QueryWith( - Func, TQuery, CancellationToken, Task> handler - ) - { - services.AddEntityFrameworkQueryHandler(handler); - - return this; - } - - public EntityFrameworkProjectionBuilder QueryWith( - Func, TQuery, CancellationToken, Task>> handler - ) - { - services.AddEntityFrameworkQueryHandler(handler); - - return this; - } -} - public class AddProjection( TDbContext dbContext, Func, TView> create, @@ -104,8 +40,7 @@ public class UpdateProjection( TDbContext dbContext, ILogger> logger, Func getViewId, - Action, TView> update, - Func, CancellationToken, Task>? prepare = null + Action, TView> update ): IEventHandler> where TView : class where TDbContext : DbContext @@ -134,8 +69,6 @@ public async Task Handle(EventEnvelope eventEnvelope, CancellationToken break; } - prepare?.Invoke(dbContext.Entry(view), ct); - update(eventEnvelope, view); await dbContext.SaveChangesAsync(ct); diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs index 268a9332..c8139185 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs @@ -1,5 +1,5 @@ -using ECommerce.Core.Commands; -using ECommerce.Core.Projections; +using Core.EntityFramework.Projections; +using ECommerce.Core.Commands; using ECommerce.Pricing.ProductPricing; using ECommerce.ShoppingCarts.AddingProductItem; using ECommerce.ShoppingCarts.Canceling; @@ -49,18 +49,16 @@ public static IServiceCollection AddShoppingCartsModule(this IServiceCollection command => ShoppingCart.MapToStreamId(command.ShoppingCartId) ) ) - .For( + .For( builder => builder .AddOn(ShoppingCartDetailsProjection.Handle) .UpdateOn( e => e.ShoppingCartId, - ShoppingCartDetailsProjection.Handle, - (entry, ct) => entry.Collection(x => x.ProductItems).LoadAsync(ct) + ShoppingCartDetailsProjection.Handle ) .UpdateOn( e => e.ShoppingCartId, - ShoppingCartDetailsProjection.Handle, - (entry, ct) => entry.Collection(x => x.ProductItems).LoadAsync(ct) + ShoppingCartDetailsProjection.Handle ) .UpdateOn( e => e.ShoppingCartId, @@ -70,9 +68,10 @@ public static IServiceCollection AddShoppingCartsModule(this IServiceCollection e => e.ShoppingCartId, ShoppingCartDetailsProjection.Handle ) + .Include(x => x.ProductItems) .QueryWith(GetCartById.Handle) ) - .For( + .For( builder => builder .AddOn(ShoppingCartShortInfoProjection.Handle) .UpdateOn( diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs index c400c31a..095f2398 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs @@ -35,7 +35,7 @@ public static ShoppingCartDetails Handle(EventEnvelope event }; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartDetails view) + public static void Handle(ShoppingCartDetails view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -62,7 +62,7 @@ public static void Handle(EventEnvelope eventEnv view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartDetails view) + public static void Handle(ShoppingCartDetails view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -84,7 +84,7 @@ public static void Handle(EventEnvelope even view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartDetails view) + public static void Handle(ShoppingCartDetails view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -94,7 +94,7 @@ public static void Handle(EventEnvelope eventEnvelope, Sh view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartDetails view) + public static void Handle(ShoppingCartDetails view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCarts/ShoppingCartShortInfo.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCarts/ShoppingCartShortInfo.cs index c888907a..9521dbab 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCarts/ShoppingCartShortInfo.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCarts/ShoppingCartShortInfo.cs @@ -31,7 +31,7 @@ public static ShoppingCartShortInfo Handle(EventEnvelope eve }; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartShortInfo view) + public static void Handle(ShoppingCartShortInfo view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -44,7 +44,7 @@ public static void Handle(EventEnvelope eventEnv view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartShortInfo view) + public static void Handle(ShoppingCartShortInfo view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -57,7 +57,7 @@ public static void Handle(EventEnvelope even view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartShortInfo view) + public static void Handle(ShoppingCartShortInfo view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; @@ -67,7 +67,7 @@ public static void Handle(EventEnvelope eventEnvelope, Sh view.LastProcessedPosition = eventEnvelope.Metadata.LogPosition; } - public static void Handle(EventEnvelope eventEnvelope, ShoppingCartShortInfo view) + public static void Handle(ShoppingCartShortInfo view, EventEnvelope eventEnvelope) { if (view.LastProcessedPosition >= eventEnvelope.Metadata.LogPosition) return; From 8cb8cb5a9d1f01be6a84e909fcc4132a2d64fdc3 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 21 May 2024 07:19:46 +0200 Subject: [PATCH 07/18] Refactored Checkpointing, moved setup to hosted service --- .../EntityFrameworkProjectionBuilder.cs | 52 ++++---- .../Checkpoints/EFCheckpointTransaction.cs | 1 + ...esSubscriptionCheckpointRepositoryTests.cs | 13 +- .../Checkpoints/Postgres/Configuration.cs | 26 ++++ ...ostgresSubscriptionCheckpointRepository.cs | 118 +++--------------- .../PostgresSubscriptionCheckpointSetup.cs | 100 +++++++++++++++ .../Simple/ECommerce.Api/Program.cs | 1 - .../Simple/ECommerce/Configuration.cs | 7 +- .../Simple/ECommerce/ECommerce.csproj | 1 + 9 files changed, 188 insertions(+), 131 deletions(-) create mode 100644 Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs rename Core.EventStoreDB/Subscriptions/Checkpoints/{ => Postgres}/PostgresSubscriptionCheckpointRepository.cs (58%) create mode 100644 Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs diff --git a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs index 064a5216..41bf7d47 100644 --- a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs +++ b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs @@ -1,12 +1,14 @@ +using System.Data; using System.Linq.Expressions; using Core.EntityFramework.Queries; using Core.EntityFramework.Subscriptions.Checkpoints; using Core.Events; using Core.EventStoreDB; using Core.EventStoreDB.Subscriptions; -using Core.EventStoreDB.Subscriptions.Batch; using Core.EventStoreDB.Subscriptions.Checkpoints; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; using Npgsql; @@ -14,6 +16,33 @@ namespace Core.EntityFramework.Projections; public static class EntityFrameworkProjection { + public static IServiceCollection AddEntityFrameworkProject( + this IServiceCollection services + ) where TDbContext : DbContext => + services.AddPostgresCheckpointing() + .AddSingleton() + .AddScoped(sp => + { + var dbContext = sp.GetRequiredService(); + return (NpgsqlTransaction)dbContext.Database.BeginTransaction().GetDbTransaction(); + }) + .AddScoped(sp => + PostgresConnectionProvider.From(sp.GetRequiredService()) + ) + .AddScoped(sp => + { + var dbContext = sp.GetRequiredService(); + + var checkpointTransaction = new EFCheckpointTransaction(dbContext); + + var connectionProvider = sp.GetRequiredService(); + + return new TransactionalPostgresSubscriptionCheckpointRepository( + new PostgresSubscriptionCheckpointRepository(connectionProvider), + checkpointTransaction + ); + }); + public static IServiceCollection For( this IServiceCollection services, Action> setup @@ -30,27 +59,6 @@ Action> setup false ); - services.AddScoped(sp => - { - var dataSource = sp.GetRequiredService(); - var dbContext = sp.GetRequiredService(); - - var checkpointTransaction = new EFCheckpointTransaction(dbContext); - - return new TransactionalPostgresSubscriptionCheckpointRepository( - new PostgresSubscriptionCheckpointRepository( - new PostgresConnectionProvider( - async ct => - { - await dbContext.Database.BeginTransactionAsync(ct); - - return (NpgsqlConnection)dbContext.Database.GetDbConnection(); - }), - new PostgresSubscriptionCheckpointSetup(dataSource) - ), - checkpointTransaction - ); - }); return services; } diff --git a/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs b/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs index 129a40a7..e8744dfb 100644 --- a/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs +++ b/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs @@ -1,4 +1,5 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Microsoft.EntityFrameworkCore; namespace Core.EntityFramework.Subscriptions.Checkpoints; diff --git a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs index 0acee9b7..7e13468f 100644 --- a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs +++ b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs @@ -1,4 +1,5 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Core.Testing.Fixtures; using Xunit; @@ -18,7 +19,8 @@ public class PostgresSubscriptionCheckpointRepositoryTests(PostgresContainerFixt public async Task Store_InitialInsert_Success() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory, checkpointTableCreator); + await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); + var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); var result = await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); @@ -29,7 +31,8 @@ public async Task Store_InitialInsert_Success() public async Task Store_UpdatePosition_Success() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory, checkpointTableCreator); + await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); + var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); var result = await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); @@ -41,7 +44,8 @@ public async Task Store_UpdatePosition_Success() public async Task Store_IdempotentCheck_ReturnsZero() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory, checkpointTableCreator); + await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); + var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); @@ -54,7 +58,8 @@ public async Task Store_IdempotentCheck_ReturnsZero() public async Task Store_InvalidUpdate_Failure() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory, checkpointTableCreator); + await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); + var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs new file mode 100644 index 00000000..6bf4d4e7 --- /dev/null +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs @@ -0,0 +1,26 @@ +using Core.BackgroundWorkers; +using Core.OpenTelemetry; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; + +public static class PostgresCheckpointingConfiguration +{ + public static IServiceCollection AddPostgresCheckpointing(this IServiceCollection services) => + services.AddScoped() + .AddScoped() + .AddHostedService(serviceProvider => + { + var logger = serviceProvider.GetRequiredService>(); + var checkpointSetup = serviceProvider.GetRequiredService(); + + TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); + + return new BackgroundWorker( + logger, + async ct => await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false) + ); + } + ); +} diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs similarity index 58% rename from Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs rename to Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs index 997f2e58..45b96372 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs @@ -1,7 +1,7 @@ -using System.Data.Common; +using System.Data; using Npgsql; -namespace Core.EventStoreDB.Subscriptions.Checkpoints; +namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using static ISubscriptionCheckpointRepository; @@ -30,20 +30,30 @@ public static PostgresConnectionProvider From(NpgsqlDataSource npgsqlDataSource) await connection.OpenAsync(ct).ConfigureAwait(false); return connection; }); + + public static PostgresConnectionProvider From(NpgsqlTransaction transaction) => + new(async ct => + { + if (transaction.Connection == null) + throw new InvalidOperationException("Transaction connection is not opened!"); + + if (transaction.Connection.State == ConnectionState.Closed) + await transaction.Connection.OpenAsync(ct).ConfigureAwait(false); + + return transaction.Connection; + }); } public class PostgresSubscriptionCheckpointRepository( // I'm not using data source here, as I'd like to enable option // to update projection in the same transaction in the same transaction as checkpointing // to help handling idempotency - PostgresConnectionProvider connectionProvider, - PostgresSubscriptionCheckpointSetup checkpointSetup + PostgresConnectionProvider connectionProvider ): ISubscriptionCheckpointRepository { public async ValueTask Load(string subscriptionId, CancellationToken ct) { var connection = await connectionProvider.Get(ct).ConfigureAwait(false); - await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand(SelectCheckpointSql, connection); command.Parameters.AddWithValue(subscriptionId); @@ -65,7 +75,6 @@ CancellationToken ct ) { var connection = await connectionProvider.Get(ct).ConfigureAwait(false); - await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand("SELECT store_subscription_checkpoint($1, $2, $3)", connection); command.Parameters.AddWithValue(subscriptionId); @@ -88,7 +97,6 @@ CancellationToken ct public async ValueTask Reset(string subscriptionId, CancellationToken ct) { var connection = await connectionProvider.Get(ct).ConfigureAwait(false); - await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false); await using var command = new NpgsqlCommand(ResetCheckpointSql, connection); command.Parameters.AddWithValue(subscriptionId); @@ -159,99 +167,3 @@ CancellationToken ct public ValueTask Reset(string subscriptionId, CancellationToken ct) => inner.Reset(subscriptionId, ct); } - -public class PostgresSubscriptionCheckpointSetup(NpgsqlDataSource dataSource) -{ - private bool wasCreated; - private readonly SemaphoreSlim tableLock = new(1, 1); - - public async ValueTask EnsureCheckpointsTableExist(CancellationToken ct) - { - if (wasCreated) - return; - - await tableLock.WaitAsync(ct).ConfigureAwait(false); - - if (wasCreated) - return; - - try - { - await using var cmd = dataSource.CreateCommand(SetupSql); - await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); - wasCreated = false; - } - finally - { - tableLock.Release(); - } - } - - private const string SetupSql = $"{CreateCheckpointsTableSql}\n{CreateStoreCheckpointsProcedureSql}"; - - private const string CreateCheckpointsTableSql = - """ - CREATE TABLE IF NOT EXISTS "subscription_checkpoints" ( - "id" VARCHAR(100) PRIMARY KEY, - "position" BIGINT NULL, - "revision" BIGINT - ); - """; - - private const string CreateStoreCheckpointsProcedureSql = - """ - CREATE OR REPLACE FUNCTION store_subscription_checkpoint( - p_id VARCHAR(100), - p_position BIGINT, - check_position BIGINT DEFAULT NULL - ) RETURNS INT AS $$ - DECLARE - current_position BIGINT; - BEGIN - -- Handle the case when check_position is provided - IF check_position IS NOT NULL THEN - -- Try to update if the position matches check_position - UPDATE "subscription_checkpoints" - SET "position" = p_position - WHERE "id" = p_id AND "position" = check_position; - - IF FOUND THEN - RETURN 1; -- Successfully updated - END IF; - - -- Retrieve the current position - SELECT "position" INTO current_position - FROM "subscription_checkpoints" - WHERE "id" = p_id; - - -- Return appropriate codes based on current position - IF current_position = p_position THEN - RETURN 0; -- Idempotent check: position already set - ELSIF current_position > check_position THEN - RETURN 2; -- Failure: current position is greater - ELSE - RETURN 2; -- Default failure case for mismatched positions - END IF; - END IF; - - -- Handle the case when check_position is NULL: Insert if not exists - BEGIN - INSERT INTO "subscription_checkpoints"("id", "position") - VALUES (p_id, p_position); - RETURN 1; -- Successfully inserted - EXCEPTION WHEN unique_violation THEN - -- If insertion failed, it means the row already exists - SELECT "position" INTO current_position - FROM "subscription_checkpoints" - WHERE "id" = p_id; - - IF current_position = p_position THEN - RETURN 0; -- Idempotent check: position already set - ELSE - RETURN 2; -- Insertion failed, row already exists with different position - END IF; - END; - END; - $$ LANGUAGE plpgsql; - """; -} diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs new file mode 100644 index 00000000..97d6460d --- /dev/null +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs @@ -0,0 +1,100 @@ +using Npgsql; + +namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; + +public class PostgresSubscriptionCheckpointSetup(NpgsqlDataSource dataSource) +{ + private bool wasCreated; + private readonly SemaphoreSlim tableLock = new(1, 1); + + public async ValueTask EnsureCheckpointsTableExist(CancellationToken ct) + { + if (wasCreated) + return; + + await tableLock.WaitAsync(ct).ConfigureAwait(false); + + if (wasCreated) + return; + + try + { + await using var cmd = dataSource.CreateCommand(SetupSql); + + await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + wasCreated = false; + } + finally + { + tableLock.Release(); + } + } + + private const string SetupSql = $"{CreateCheckpointsTableSql}\n{CreateStoreCheckpointsProcedureSql}"; + + private const string CreateCheckpointsTableSql = + """ + CREATE TABLE IF NOT EXISTS "subscription_checkpoints" ( + "id" VARCHAR(100) PRIMARY KEY, + "position" BIGINT NULL, + "revision" BIGINT + ); + """; + + private const string CreateStoreCheckpointsProcedureSql = + """ + CREATE OR REPLACE FUNCTION store_subscription_checkpoint( + p_id VARCHAR(100), + p_position BIGINT, + check_position BIGINT DEFAULT NULL + ) RETURNS INT AS $$ + DECLARE + current_position BIGINT; + BEGIN + -- Handle the case when check_position is provided + IF check_position IS NOT NULL THEN + -- Try to update if the position matches check_position + UPDATE "subscription_checkpoints" + SET "position" = p_position + WHERE "id" = p_id AND "position" = check_position; + + IF FOUND THEN + RETURN 1; -- Successfully updated + END IF; + + -- Retrieve the current position + SELECT "position" INTO current_position + FROM "subscription_checkpoints" + WHERE "id" = p_id; + + -- Return appropriate codes based on current position + IF current_position = p_position THEN + RETURN 0; -- Idempotent check: position already set + ELSIF current_position > check_position THEN + RETURN 2; -- Failure: current position is greater + ELSE + RETURN 2; -- Default failure case for mismatched positions + END IF; + END IF; + + -- Handle the case when check_position is NULL: Insert if not exists + BEGIN + INSERT INTO "subscription_checkpoints"("id", "position") + VALUES (p_id, p_position); + RETURN 1; -- Successfully inserted + EXCEPTION WHEN unique_violation THEN + -- If insertion failed, it means the row already exists + SELECT "position" INTO current_position + FROM "subscription_checkpoints" + WHERE "id" = p_id; + + IF current_position = p_position THEN + RETURN 0; -- Idempotent check: position already set + ELSE + RETURN 2; -- Insertion failed, row already exists with different position + END IF; + END; + END; + $$ LANGUAGE plpgsql; + """; +} diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api/Program.cs b/Sample/EventStoreDB/Simple/ECommerce.Api/Program.cs index 22f1170d..c66bd307 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api/Program.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api/Program.cs @@ -27,7 +27,6 @@ WrongExpectedVersionException => exception.MapToProblemDetails(StatusCodes.Status412PreconditionFailed), _ => null }) - .AddEventStoreDBSubscriptionToAll(new EventStoreDBSubscriptionToAllOptions { SubscriptionId = "SimpleESDB"}) .AddECommerceModule(builder.Configuration) .AddOptimisticConcurrencyMiddleware() .AddOpenTelemetry("Carts", OpenTelemetryOptions.Build(options => diff --git a/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs index c7b935fa..21d58fc6 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs @@ -1,4 +1,7 @@ -using ECommerce.Pricing; +using Core.Configuration; +using Core.EntityFramework.Projections; +using Core.EventStoreDB.Subscriptions.Checkpoints; +using ECommerce.Pricing; using ECommerce.ShoppingCarts; using ECommerce.Storage; using Microsoft.EntityFrameworkCore; @@ -11,6 +14,8 @@ public static class Configuration { public static IServiceCollection AddECommerceModule(this IServiceCollection services, IConfiguration config) => services + .AddNpgsqlDataSource(config.GetRequiredConnectionString("ECommerceDB")) + .AddEntityFrameworkProject() .AddShoppingCartsModule() .AddPricingModule() .AddDbContext( diff --git a/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj b/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj index b1185723..106b7e61 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj +++ b/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj @@ -5,6 +5,7 @@ + From 6499ef43855dcee68e8bcb3b573d7bd5e8dbe6e8 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 21 May 2024 08:08:20 +0200 Subject: [PATCH 08/18] f --- Core.EventStoreDB/Config.cs | 44 ++++++----- .../Checkpoints/Postgres/Configuration.cs | 8 +- ...EventStoreDBSubscriptioToAllCoordinator.cs | 12 +++ .../EventStoreDBSubscriptionToAll.cs | 8 +- Core/BackgroundWorkers/BackgroundWorker.cs | 8 ++ Core/Config.cs | 2 + Core/Extensions/DIExtensions.cs | 73 +++++++++++++++++++ .../Simple/ECommerce.Core/Configuration.cs | 2 + 8 files changed, 131 insertions(+), 26 deletions(-) create mode 100644 Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs create mode 100644 Core/Extensions/DIExtensions.cs diff --git a/Core.EventStoreDB/Config.cs b/Core.EventStoreDB/Config.cs index f99280be..4f834f15 100644 --- a/Core.EventStoreDB/Config.cs +++ b/Core.EventStoreDB/Config.cs @@ -43,8 +43,7 @@ public static IServiceCollection AddEventStoreDB( { services .AddSingleton(EventTypeMapper.Instance) - .AddSingleton(new EventStoreClient(EventStoreClientSettings.Create(eventStoreDBConfig.ConnectionString))) - .AddTransient(); + .AddSingleton(new EventStoreClient(EventStoreClientSettings.Create(eventStoreDBConfig.ConnectionString))); if (options?.UseInternalCheckpointing != false) { @@ -52,7 +51,22 @@ public static IServiceCollection AddEventStoreDB( .AddTransient(); } - return services; + return services.AddHostedService(serviceProvider => + { + var logger = + serviceProvider.GetRequiredService>(); + + var coordinator = serviceProvider.GetRequiredService(); + + TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); + + return new BackgroundWorker( + coordinator, + logger, + (c, ct) => c.SubscribeToAll(ct) + ); + } + ); } public static IServiceCollection AddEventStoreDBSubscriptionToAll( @@ -62,6 +76,7 @@ public static IServiceCollection AddEventStoreDBSubscriptionToAll( { services.AddScoped(); services.AddScoped(); + services.AddSingleton(); if (checkpointToEventStoreDB) { @@ -69,21 +84,14 @@ public static IServiceCollection AddEventStoreDBSubscriptionToAll( .AddSingleton(); } - return services.AddHostedService(serviceProvider => - { - var logger = - serviceProvider.GetRequiredService>(); - - var eventStoreDBSubscriptionToAll = - serviceProvider.GetRequiredService(); - - TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); - - return new BackgroundWorker( - logger, - ct => eventStoreDBSubscriptionToAll.SubscribeToAll(subscriptionOptions, ct) - ); - } + return services.AddKeyedSingleton( + $"ESDB_subscription-{subscriptionOptions.SubscriptionId}", + (sp, _) => new EventStoreDBSubscriptionToAll( + subscriptionOptions, + sp.GetRequiredService(), + sp, + sp.GetRequiredService>() + ) ); } } diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs index 6bf4d4e7..e83d0dd2 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs @@ -12,14 +12,16 @@ public static IServiceCollection AddPostgresCheckpointing(this IServiceCollectio .AddScoped() .AddHostedService(serviceProvider => { - var logger = serviceProvider.GetRequiredService>(); + var logger = serviceProvider.GetRequiredService>>(); var checkpointSetup = serviceProvider.GetRequiredService(); TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); - return new BackgroundWorker( + return new BackgroundWorker( + checkpointSetup, logger, - async ct => await checkpointSetup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false) + async (setup, ct) => await setup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false) ); } ); diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs new file mode 100644 index 00000000..95e1c0d8 --- /dev/null +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs @@ -0,0 +1,12 @@ +namespace Core.EventStoreDB.Subscriptions; + +public class EventStoreDBSubscriptioToAllCoordinator(IDictionary subscriptions) +{ + public async Task SubscribeToAll(CancellationToken ct) + { + // see: https://github.com/dotnet/runtime/issues/36063 + await Task.Yield(); + + await Task.WhenAll(subscriptions.Values.Select(s => s.SubscribeToAll(ct))).ConfigureAwait(false); + } +} diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 18e11ad7..b5878f96 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -27,21 +27,19 @@ public class EventStoreDBSubscriptionToAllOptions } public class EventStoreDBSubscriptionToAll( + EventStoreDBSubscriptionToAllOptions subscriptionOptions, EventStoreClient eventStoreClient, IServiceProvider serviceProvider, ILogger logger ) { - private EventStoreDBSubscriptionToAllOptions subscriptionOptions = default!; private string SubscriptionId => subscriptionOptions.SubscriptionId; - public async Task SubscribeToAll(EventStoreDBSubscriptionToAllOptions subscriptionOptions, CancellationToken ct) + public async Task SubscribeToAll(CancellationToken ct) { // see: https://github.com/dotnet/runtime/issues/36063 await Task.Yield(); - this.subscriptionOptions = subscriptionOptions; - logger.LogInformation("Subscription to all '{SubscriptionId}'", subscriptionOptions.SubscriptionId); try @@ -92,7 +90,7 @@ public async Task SubscribeToAll(EventStoreDBSubscriptionToAllOptions subscripti // Randomness added to reduce the chance of multiple subscriptions trying to reconnect at the same time Thread.Sleep(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)); - await SubscribeToAll(this.subscriptionOptions, ct).ConfigureAwait(false); + await SubscribeToAll(ct).ConfigureAwait(false); } } diff --git a/Core/BackgroundWorkers/BackgroundWorker.cs b/Core/BackgroundWorkers/BackgroundWorker.cs index e0959b11..8e63c2c5 100644 --- a/Core/BackgroundWorkers/BackgroundWorker.cs +++ b/Core/BackgroundWorkers/BackgroundWorker.cs @@ -3,6 +3,14 @@ namespace Core.BackgroundWorkers; +public class BackgroundWorker( + TService service, + ILogger logger, + Func perform) + : BackgroundWorker(logger, ct => perform(service, ct)) +{ +} + public class BackgroundWorker( ILogger logger, Func perform) diff --git a/Core/Config.cs b/Core/Config.cs index 29b4c8b4..e0a6780e 100644 --- a/Core/Config.cs +++ b/Core/Config.cs @@ -1,5 +1,6 @@ using Core.Commands; using Core.Events; +using Core.Extensions; using Core.Ids; using Core.OpenTelemetry; using Core.Queries; @@ -14,6 +15,7 @@ public static class Config public static IServiceCollection AddCoreServices(this IServiceCollection services) { services + .AllowResolvingKeyedServicesAsDictionary() .AddSingleton(TimeProvider.System) .AddSingleton(ActivityScope.Instance) .AddEventBus() diff --git a/Core/Extensions/DIExtensions.cs b/Core/Extensions/DIExtensions.cs new file mode 100644 index 00000000..118eaee2 --- /dev/null +++ b/Core/Extensions/DIExtensions.cs @@ -0,0 +1,73 @@ +using System.Collections.ObjectModel; +using Microsoft.Extensions.DependencyInjection; + +namespace Core.Extensions; + +public static class DIExtensions +{ + // Taken from: https://stackoverflow.com/a/77559901 + public static IServiceCollection AllowResolvingKeyedServicesAsDictionary( + this IServiceCollection sc) + { + // KeyedServiceCache caches all the keys of a given type for a + // specific service type. By making it a singleton we only have + // determine the keys once, which makes resolving the dict very fast. + sc.AddSingleton(typeof(KeyedServiceCache<,>)); + + // KeyedServiceCache depends on the IServiceCollection to get + // the list of keys. That's why we register that here as well, as it + // is not registered by default in MS.DI. + sc.AddSingleton(sc); + + // Last we make the registration for the dictionary itself, which maps + // to our custom type below. This registration must be transient, as + // the containing services could have any lifetime and this registration + // should by itself not cause Captive Dependencies. + sc.AddTransient(typeof(IDictionary<,>), typeof(KeyedServiceDictionary<,>)); + + // For completeness, let's also allow IReadOnlyDictionary to be resolved. + sc.AddTransient( + typeof(IReadOnlyDictionary<,>), typeof(KeyedServiceDictionary<,>)); + + return sc; + } + + // We inherit from ReadOnlyDictionary, to disallow consumers from changing + // the wrapped dependencies while reusing all its functionality. This way + // we don't have to implement IDictionary ourselves; too much work. + private sealed class KeyedServiceDictionary( + KeyedServiceCache keys, IServiceProvider provider) + : ReadOnlyDictionary(Create(keys, provider)) + where TKey : notnull + where TService : notnull + { + private static Dictionary Create( + KeyedServiceCache keys, IServiceProvider provider) + { + var dict = new Dictionary(capacity: keys.Keys.Length); + + foreach (TKey key in keys.Keys) + { + dict[key] = provider.GetRequiredKeyedService(key); + } + + return dict; + } + } + + private sealed class KeyedServiceCache(IServiceCollection sc) + where TKey : notnull + where TService : notnull + { + // Once this class is resolved, all registrations are guaranteed to be + // made, so we can, at that point, safely iterate the collection to get + // the keys for the service type. + public TKey[] Keys { get; } = ( + from service in sc + where service.ServiceKey != null + where service.ServiceKey!.GetType() == typeof(TKey) + where service.ServiceType == typeof(TService) + select (TKey)service.ServiceKey!) + .ToArray(); + } +} diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs index be6e7bc1..dd8bc6cc 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs @@ -1,5 +1,6 @@ using Core.Events; using Core.EventStoreDB; +using Core.Extensions; using Core.OpenTelemetry; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -13,6 +14,7 @@ public static IServiceCollection AddCoreServices( IConfiguration configuration ) => services + .AllowResolvingKeyedServicesAsDictionary() .AddSingleton() .AddEventBus() .AddEventStoreDB(configuration); From 4a242c501b2bd2c97c8d5abc032d54a3a9ee6c15 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 21 May 2024 19:22:59 +0200 Subject: [PATCH 09/18] Added EventStoreDB subscription coordinator with handling multiple subscriptions --- .../Core.EntityFramework.csproj | 2 +- .../EntityFrameworkProjectionBuilder.cs | 55 +++---- .../Checkpoints/EFCheckpointTransaction.cs | 17 -- ...ctionalDbContextEventsBatchCheckpointer.cs | 51 ++++++ .../EventStoreDBAsyncCommandBusTests.cs | 7 +- ...esSubscriptionCheckpointRepositoryTests.cs | 24 +-- Core.EventStoreDB/Config.cs | 52 +++--- .../Batch/EventsBatchCheckpointer.cs | 9 +- .../Batch/EventsBatchProcessor.cs | 18 ++- .../ISubscriptionCheckpointRepository.cs | 12 ++ .../Checkpoints/Postgres/Configuration.cs | 19 +-- ...ostgresSubscriptionCheckpointRepository.cs | 148 +++++++----------- .../PostgresSubscriptionCheckpointSetup.cs | 17 +- ...EventStoreDBSubscriptioToAllCoordinator.cs | 105 ++++++++++++- .../EventStoreDBSubscriptionToAll.cs | 88 ++++++----- Core/Events/EventBus.cs | 3 +- Core/Extensions/AsyncEnumerableExtensions.cs | 37 +++-- .../01-CRUD/ECommerce/ECommerce.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../ECommerce.Domain/ECommerce.Domain.csproj | 2 +- .../ECommerce/ECommerce.csproj | 2 +- .../Carts/Carts.Api/Carts.Api.csproj | 2 +- .../Orders/Orders.Api/Orders.Api.csproj | 2 +- .../Payments/Payments.Api/Payments.Api.csproj | 2 +- .../Shipments.Api/Shipments.Api.csproj | 2 +- .../Shipments/Shipments/Shipments.csproj | 2 +- .../ECommerce/Carts/Carts.Api/Program.cs | 3 +- .../ECommerce/Carts/Carts/Config.cs | 1 + .../Simple/ECommerce.Core/Configuration.cs | 4 +- .../ECommerce.Core/ECommerce.Core.csproj | 2 +- .../Simple/ECommerce/Configuration.cs | 34 ++-- .../Simple/ECommerce/ECommerce.csproj | 2 +- .../ECommerce/ShoppingCarts/Configuration.cs | 2 + Sample/Tickets/Tickets.Api/Tickets.Api.csproj | 2 +- .../Warehouse.Api/Warehouse.Api.csproj | 2 +- Sample/Warehouse/Warehouse/Warehouse.csproj | 2 +- 48 files changed, 454 insertions(+), 306 deletions(-) delete mode 100644 Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs create mode 100644 Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs diff --git a/Core.EntityFramework/Core.EntityFramework.csproj b/Core.EntityFramework/Core.EntityFramework.csproj index 2216791e..7d5384b6 100644 --- a/Core.EntityFramework/Core.EntityFramework.csproj +++ b/Core.EntityFramework/Core.EntityFramework.csproj @@ -10,7 +10,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs index 41bf7d47..64158db1 100644 --- a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs +++ b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs @@ -1,51 +1,27 @@ -using System.Data; using System.Linq.Expressions; using Core.EntityFramework.Queries; using Core.EntityFramework.Subscriptions.Checkpoints; using Core.Events; using Core.EventStoreDB; using Core.EventStoreDB.Subscriptions; -using Core.EventStoreDB.Subscriptions.Checkpoints; -using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; +using Core.EventStoreDB.Subscriptions.Batch; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; -using Npgsql; +using Polly; namespace Core.EntityFramework.Projections; public static class EntityFrameworkProjection { - public static IServiceCollection AddEntityFrameworkProject( + public static IServiceCollection AddEntityFrameworkProjections( this IServiceCollection services ) where TDbContext : DbContext => - services.AddPostgresCheckpointing() - .AddSingleton() - .AddScoped(sp => - { - var dbContext = sp.GetRequiredService(); - return (NpgsqlTransaction)dbContext.Database.BeginTransaction().GetDbTransaction(); - }) - .AddScoped(sp => - PostgresConnectionProvider.From(sp.GetRequiredService()) - ) - .AddScoped(sp => - { - var dbContext = sp.GetRequiredService(); - - var checkpointTransaction = new EFCheckpointTransaction(dbContext); - - var connectionProvider = sp.GetRequiredService(); - - return new TransactionalPostgresSubscriptionCheckpointRepository( - new PostgresSubscriptionCheckpointRepository(connectionProvider), - checkpointTransaction - ); - }); + services.AddScoped>(); public static IServiceCollection For( this IServiceCollection services, - Action> setup + Action> setup, + int batchSize = 20 ) where TView : class where TDbContext : DbContext @@ -55,10 +31,17 @@ Action> setup setup(builder); services.AddEventStoreDBSubscriptionToAll( - new EventStoreDBSubscriptionToAllOptions { SubscriptionId = typeof(TView).FullName!, BatchSize = 20 }, - false - ); + new EventStoreDBSubscriptionToAllOptions { SubscriptionId = typeof(TView).FullName!, BatchSize = batchSize }, + sp => + { + var dbContext = sp.GetRequiredService(); + var projection = builder.Projection; + projection.DbContext = dbContext; + + return [projection]; + } + ); return services; } @@ -71,6 +54,12 @@ public class EntityFrameworkProjectionBuilder(IServiceCo { public EntityFrameworkProjection Projection = new(); + public EntityFrameworkProjectionBuilder ViewId(Expression> id) + { + Projection.ViewId(id); + return this; + } + public EntityFrameworkProjectionBuilder AddOn( Func, TView> handler) where TEvent : notnull diff --git a/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs b/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs deleted file mode 100644 index e8744dfb..00000000 --- a/Core.EntityFramework/Subscriptions/Checkpoints/EFCheckpointTransaction.cs +++ /dev/null @@ -1,17 +0,0 @@ -using Core.EventStoreDB.Subscriptions.Checkpoints; -using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; -using Microsoft.EntityFrameworkCore; - -namespace Core.EntityFramework.Subscriptions.Checkpoints; - -public class EFCheckpointTransaction(DbContext dbContext): ICheckpointTransaction -{ - public Task Commit(CancellationToken cancellationToken = default) => - dbContext.Database.CommitTransactionAsync(cancellationToken); - - public Task Rollback(CancellationToken cancellationToken = default) - { - dbContext.ChangeTracker.Clear(); - return dbContext.Database.RollbackTransactionAsync(cancellationToken); - } -} diff --git a/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs b/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs new file mode 100644 index 00000000..9ff445a8 --- /dev/null +++ b/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs @@ -0,0 +1,51 @@ +using Core.Events; +using Core.EventStoreDB.Subscriptions; +using Core.EventStoreDB.Subscriptions.Batch; +using Core.EventStoreDB.Subscriptions.Checkpoints; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; +using EventStore.Client; +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace Core.EntityFramework.Subscriptions.Checkpoints; + +using static ISubscriptionCheckpointRepository; + +public class TransactionalDbContextEventsBatchCheckpointer( + TDbContext dbContext, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + EventsBatchProcessor batchProcessor +): IEventsBatchCheckpointer + where TDbContext : DbContext +{ + public async Task Process( + ResolvedEvent[] events, + Checkpoint lastCheckpoint, + BatchProcessingOptions options, + CancellationToken ct + ) + { + await dbContext.Database.UseTransactionAsync(transaction, ct); + var inner = new EventsBatchCheckpointer( + new PostgresSubscriptionCheckpointRepository(connection, transaction), + batchProcessor + ); + + var result = await inner.Process(events, lastCheckpoint, options, ct) + .ConfigureAwait(false); + + await dbContext.SaveChangesAsync(ct); + + if (result is not StoreResult.Success) + { + dbContext.ChangeTracker.Clear(); + await transaction.RollbackAsync(ct); + return result; + } + + await transaction.CommitAsync(ct).ConfigureAwait(false); + + return result; + } +} diff --git a/Core.EventStoreDB.Tests/Commands/EventStoreDBAsyncCommandBusTests.cs b/Core.EventStoreDB.Tests/Commands/EventStoreDBAsyncCommandBusTests.cs index 8e012058..8468d9ac 100644 --- a/Core.EventStoreDB.Tests/Commands/EventStoreDBAsyncCommandBusTests.cs +++ b/Core.EventStoreDB.Tests/Commands/EventStoreDBAsyncCommandBusTests.cs @@ -44,11 +44,8 @@ public EventStoreDBAsyncCommandBusTests() ) ) .AddCoreServices() - .AddEventStoreDB(new EventStoreDBConfig - { - ConnectionString = "esdb://localhost:2113?tls=false" - }) - .AddEventStoreDBSubscriptionToAll(new EventStoreDBSubscriptionToAllOptions(){SubscriptionId = "AsyncCommandBusTest"}) + .AddEventStoreDB(new EventStoreDBConfig { ConnectionString = "esdb://localhost:2113?tls=false" }) + .AddEventStoreDBSubscriptionToAll("AsyncCommandBusTest") .AddCommandHandler( _ => new AddUserCommandHandler(userIds) ) diff --git a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs index 7e13468f..2a551dc4 100644 --- a/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs +++ b/Core.EventStoreDB.Tests/Subscriptions/Checkpoints/PostgresSubscriptionCheckpointRepositoryTests.cs @@ -1,6 +1,7 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Core.Testing.Fixtures; +using Npgsql; using Xunit; namespace Core.EventStoreDB.Tests.Subscriptions.Checkpoints; @@ -12,15 +13,13 @@ public class PostgresSubscriptionCheckpointRepositoryTests(PostgresContainerFixt { private readonly string subscriptionId = Guid.NewGuid().ToString("N"); - private readonly PostgresConnectionProvider connectionFactory = - PostgresConnectionProvider.From(fixture.DataSource); - [Fact] public async Task Store_InitialInsert_Success() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); + await checkpointTableCreator.EnsureStoreExists(CancellationToken.None); + await using var connection = fixture.DataSource.CreateConnection(); + var repository = new PostgresSubscriptionCheckpointRepository(connection); var result = await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); @@ -31,8 +30,9 @@ public async Task Store_InitialInsert_Success() public async Task Store_UpdatePosition_Success() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); + await checkpointTableCreator.EnsureStoreExists(CancellationToken.None); + await using var connection = fixture.DataSource.CreateConnection(); + var repository = new PostgresSubscriptionCheckpointRepository(connection); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); var result = await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); @@ -44,8 +44,9 @@ public async Task Store_UpdatePosition_Success() public async Task Store_IdempotentCheck_ReturnsZero() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); + await checkpointTableCreator.EnsureStoreExists(CancellationToken.None); + await using var connection = fixture.DataSource.CreateConnection(); + var repository = new PostgresSubscriptionCheckpointRepository(connection); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); @@ -58,8 +59,9 @@ public async Task Store_IdempotentCheck_ReturnsZero() public async Task Store_InvalidUpdate_Failure() { var checkpointTableCreator = new PostgresSubscriptionCheckpointSetup(fixture.DataSource); - await checkpointTableCreator.EnsureCheckpointsTableExist(CancellationToken.None); - var repository = new PostgresSubscriptionCheckpointRepository(connectionFactory); + await checkpointTableCreator.EnsureStoreExists(CancellationToken.None); + await using var connection = fixture.DataSource.CreateConnection(); + var repository = new PostgresSubscriptionCheckpointRepository(connection); await repository.Store(subscriptionId, 1, Checkpoint.None, CancellationToken.None); await repository.Store(subscriptionId, 2, Checkpoint.From(1), CancellationToken.None); diff --git a/Core.EventStoreDB/Config.cs b/Core.EventStoreDB/Config.cs index 4f834f15..71f43ea0 100644 --- a/Core.EventStoreDB/Config.cs +++ b/Core.EventStoreDB/Config.cs @@ -43,7 +43,10 @@ public static IServiceCollection AddEventStoreDB( { services .AddSingleton(EventTypeMapper.Instance) - .AddSingleton(new EventStoreClient(EventStoreClientSettings.Create(eventStoreDBConfig.ConnectionString))); + .AddSingleton(new EventStoreClient(EventStoreClientSettings.Create(eventStoreDBConfig.ConnectionString))) + .AddScoped() + .AddScoped() + .AddSingleton(); if (options?.UseInternalCheckpointing != false) { @@ -69,29 +72,42 @@ public static IServiceCollection AddEventStoreDB( ); } + + public static IServiceCollection AddEventStoreDBSubscriptionToAll( + this IServiceCollection services, + string subscriptionId + ) where THandler : IEventBatchHandler => + services.AddEventStoreDBSubscriptionToAll( + new EventStoreDBSubscriptionToAllOptions { SubscriptionId = subscriptionId }, + sp => [sp.GetRequiredService()] + ); + + + public static IServiceCollection AddEventStoreDBSubscriptionToAll( + this IServiceCollection services, + EventStoreDBSubscriptionToAllOptions subscriptionOptions + ) where THandler : IEventBatchHandler => + services.AddEventStoreDBSubscriptionToAll(subscriptionOptions, sp => [sp.GetRequiredService()]); + public static IServiceCollection AddEventStoreDBSubscriptionToAll( this IServiceCollection services, EventStoreDBSubscriptionToAllOptions subscriptionOptions, - bool checkpointToEventStoreDB = true) + Func handlers + ) { - services.AddScoped(); - services.AddScoped(); services.AddSingleton(); - if (checkpointToEventStoreDB) - { - services - .AddSingleton(); - } - return services.AddKeyedSingleton( - $"ESDB_subscription-{subscriptionOptions.SubscriptionId}", - (sp, _) => new EventStoreDBSubscriptionToAll( - subscriptionOptions, - sp.GetRequiredService(), - sp, - sp.GetRequiredService>() - ) - ); + subscriptionOptions.SubscriptionId, + (sp, _) => + { + var subscription = new EventStoreDBSubscriptionToAll( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>() + ) { Options = subscriptionOptions, GetHandlers = handlers }; + + return subscription; + }); } } diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs index 8c024a1d..e6245b1e 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs @@ -1,3 +1,4 @@ +using Core.Events; using Core.EventStoreDB.Subscriptions.Checkpoints; using EventStore.Client; @@ -9,7 +10,7 @@ public interface IEventsBatchCheckpointer Task Process( ResolvedEvent[] events, Checkpoint lastCheckpoint, - EventStoreDBSubscriptionToAllOptions subscriptionOptions, + BatchProcessingOptions batchProcessingOptions, CancellationToken ct ); } @@ -22,7 +23,7 @@ EventsBatchProcessor eventsBatchProcessor public async Task Process( ResolvedEvent[] events, Checkpoint lastCheckpoint, - EventStoreDBSubscriptionToAllOptions subscriptionOptions, + BatchProcessingOptions options, CancellationToken ct ) { @@ -31,11 +32,11 @@ CancellationToken ct if (!lastPosition.HasValue) return new StoreResult.Ignored(); - await eventsBatchProcessor.HandleEventsBatch(events, subscriptionOptions, ct) + await eventsBatchProcessor.HandleEventsBatch(events, options, ct) .ConfigureAwait(false); return await checkpointRepository - .Store(subscriptionOptions.SubscriptionId, lastPosition.Value, lastCheckpoint, ct) + .Store(options.SubscriptionId, lastPosition.Value, lastCheckpoint, ct) .ConfigureAwait(false); } } diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index 0e8785ab..3c6b375f 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -6,22 +6,30 @@ namespace Core.EventStoreDB.Subscriptions.Batch; +public record BatchProcessingOptions( + string SubscriptionId, + bool IgnoreDeserializationErrors, + IEventBatchHandler[] BatchHandlers +); + public class EventsBatchProcessor( EventTypeMapper eventTypeMapper, - IEventBatchHandler batchHandler, ILogger logger ) { - public Task HandleEventsBatch( + public async Task HandleEventsBatch( ResolvedEvent[] resolvedEvents, - EventStoreDBSubscriptionToAllOptions options, + BatchProcessingOptions options, CancellationToken ct ) { var events = TryDeserializeEvents(resolvedEvents, options.IgnoreDeserializationErrors); - // TODO: How would you implement Dead-Letter Queue here? - return batchHandler.Handle(events, ct); + foreach (var batchHandler in options.BatchHandlers) + { + // TODO: How would you implement Dead-Letter Queue here? + await batchHandler.Handle(events, ct).ConfigureAwait(false); + } } private IEventEnvelope[] TryDeserializeEvents( diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/ISubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/ISubscriptionCheckpointRepository.cs index 6c7ec446..314ab58f 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/ISubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/ISubscriptionCheckpointRepository.cs @@ -15,3 +15,15 @@ public record Ignored: StoreResult; public record Mismatch: StoreResult; } } + + +public interface ISubscriptionStoreSetup +{ + ValueTask EnsureStoreExists(CancellationToken ct); +} + +public class NulloSubscriptionStoreSetup: ISubscriptionStoreSetup +{ + public ValueTask EnsureStoreExists(CancellationToken ct) => + ValueTask.CompletedTask; +} diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs index e83d0dd2..2526eb9e 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/Configuration.cs @@ -2,27 +2,14 @@ using Core.OpenTelemetry; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Npgsql; namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; public static class PostgresCheckpointingConfiguration { public static IServiceCollection AddPostgresCheckpointing(this IServiceCollection services) => - services.AddScoped() + services .AddScoped() - .AddHostedService(serviceProvider => - { - var logger = serviceProvider.GetRequiredService>>(); - var checkpointSetup = serviceProvider.GetRequiredService(); - - TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); - - return new BackgroundWorker( - checkpointSetup, - logger, - async (setup, ct) => await setup.EnsureCheckpointsTableExist(ct).ConfigureAwait(false) - ); - } - ); + .AddSingleton(); } diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs index 45b96372..e38ead06 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs @@ -1,70 +1,78 @@ using System.Data; using Npgsql; +using Polly; namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using static ISubscriptionCheckpointRepository; -public class PostgresConnectionProvider(Func> connectionFactory) -{ - public ValueTask Get(CancellationToken ct) => connectionFactory(ct); - - public void Set(NpgsqlConnection connection) => - connectionFactory = _ => ValueTask.FromResult(connection); - - public void Set(NpgsqlTransaction transaction) => - connectionFactory = _ => ValueTask.FromResult(transaction.Connection!); - - public void Set(NpgsqlDataSource dataSource) => - connectionFactory = async ct => - { - var connection = dataSource.CreateConnection(); - await connection.OpenAsync(ct).ConfigureAwait(false); - return connection; - }; - - public static PostgresConnectionProvider From(NpgsqlDataSource npgsqlDataSource) => - new(async ct => - { - var connection = npgsqlDataSource.CreateConnection(); - await connection.OpenAsync(ct).ConfigureAwait(false); - return connection; - }); - - public static PostgresConnectionProvider From(NpgsqlTransaction transaction) => - new(async ct => - { - if (transaction.Connection == null) - throw new InvalidOperationException("Transaction connection is not opened!"); - - if (transaction.Connection.State == ConnectionState.Closed) - await transaction.Connection.OpenAsync(ct).ConfigureAwait(false); - - return transaction.Connection; - }); -} +// public class PostgresConnectionProvider(Func> connectionFactory) +// { +// public ValueTask Get(CancellationToken ct) => connectionFactory(ct); +// +// public void Set(NpgsqlConnection connection) => +// connectionFactory = _ => ValueTask.FromResult(connection); +// +// public void Set(NpgsqlTransaction transaction) => +// connectionFactory = _ => ValueTask.FromResult(transaction.Connection!); +// +// public void Set(NpgsqlDataSource dataSource) => +// connectionFactory = async ct => +// { +// var connection = dataSource.CreateConnection(); +// await connection.OpenAsync(ct).ConfigureAwait(false); +// return connection; +// }; +// +// public static PostgresConnectionProvider From(NpgsqlDataSource npgsqlDataSource) => +// new(async ct => +// { +// var connection = npgsqlDataSource.CreateConnection(); +// await connection.OpenAsync(ct).ConfigureAwait(false); +// return connection; +// }); +// +// public static PostgresConnectionProvider From(NpgsqlTransaction transaction) => +// new(async ct => +// { +// if (transaction.Connection == null) +// throw new InvalidOperationException("Transaction connection is not opened!"); +// +// if (transaction.Connection.State == ConnectionState.Closed) +// await transaction.Connection.OpenAsync(ct).ConfigureAwait(false); +// +// return transaction.Connection; +// }); +// } public class PostgresSubscriptionCheckpointRepository( // I'm not using data source here, as I'd like to enable option // to update projection in the same transaction in the same transaction as checkpointing // to help handling idempotency - PostgresConnectionProvider connectionProvider + NpgsqlConnection connection, + NpgsqlTransaction? transaction = null ): ISubscriptionCheckpointRepository { public async ValueTask Load(string subscriptionId, CancellationToken ct) { - var connection = await connectionProvider.Get(ct).ConfigureAwait(false); + if (connection.State != ConnectionState.Open) + await connection.OpenAsync(ct).ConfigureAwait(false); - await using var command = new NpgsqlCommand(SelectCheckpointSql, connection); + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = SelectCheckpointSql; command.Parameters.AddWithValue(subscriptionId); await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false); - return (await reader.ReadAsync(ct).ConfigureAwait(false)) - ? Checkpoint.From( - await reader.IsDBNullAsync(0, ct).ConfigureAwait(false) ? (ulong)reader.GetInt64(0) : null - ) - : Checkpoint.None; + if(!await reader.ReadAsync(ct).ConfigureAwait(false)) + return Checkpoint.None; + + var value = reader.GetValue(0); + + var checkpoint = Checkpoint.From(Convert.ToUInt64(value)); + + return checkpoint; } public async ValueTask Store( @@ -74,9 +82,12 @@ public async ValueTask Store( CancellationToken ct ) { - var connection = await connectionProvider.Get(ct).ConfigureAwait(false); + if (connection.State == ConnectionState.Closed) + await connection.OpenAsync(ct).ConfigureAwait(false); - await using var command = new NpgsqlCommand("SELECT store_subscription_checkpoint($1, $2, $3)", connection); + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "SELECT store_subscription_checkpoint($1, $2, $3)"; command.Parameters.AddWithValue(subscriptionId); command.Parameters.AddWithValue((long)position); command.Parameters.AddWithValue(previousCheckpoint != Checkpoint.None @@ -96,8 +107,6 @@ CancellationToken ct public async ValueTask Reset(string subscriptionId, CancellationToken ct) { - var connection = await connectionProvider.Get(ct).ConfigureAwait(false); - await using var command = new NpgsqlCommand(ResetCheckpointSql, connection); command.Parameters.AddWithValue(subscriptionId); @@ -128,42 +137,3 @@ ON CONFLICT ("id") WHERE "position" = $3 WHERE "id" = $1; """; } - -public interface ICheckpointTransaction -{ - Task Commit(CancellationToken cancellationToken = default); - - Task Rollback(CancellationToken cancellationToken = default); -} - -public class TransactionalPostgresSubscriptionCheckpointRepository( - PostgresSubscriptionCheckpointRepository inner, - ICheckpointTransaction checkpointTransaction -): ISubscriptionCheckpointRepository -{ - public ValueTask Load(string subscriptionId, CancellationToken ct) => - inner.Load(subscriptionId, ct); - - public async ValueTask Store( - string subscriptionId, - ulong position, - Checkpoint previousCheckpoint, - CancellationToken ct - ) - { - var result = await inner.Store(subscriptionId, position, previousCheckpoint, ct).ConfigureAwait(false); - - if (result is not StoreResult.Success) - { - await checkpointTransaction.Rollback(ct).ConfigureAwait(false); - return result; - } - - await checkpointTransaction.Commit(ct).ConfigureAwait(false); - - return result; - } - - public ValueTask Reset(string subscriptionId, CancellationToken ct) => - inner.Reset(subscriptionId, ct); -} diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs index 97d6460d..7a85fb8a 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointSetup.cs @@ -2,18 +2,12 @@ namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; -public class PostgresSubscriptionCheckpointSetup(NpgsqlDataSource dataSource) +public class PostgresSubscriptionCheckpointSetup(NpgsqlDataSource dataSource): ISubscriptionStoreSetup { private bool wasCreated; - private readonly SemaphoreSlim tableLock = new(1, 1); - public async ValueTask EnsureCheckpointsTableExist(CancellationToken ct) + public async ValueTask EnsureStoreExists(CancellationToken ct) { - if (wasCreated) - return; - - await tableLock.WaitAsync(ct).ConfigureAwait(false); - if (wasCreated) return; @@ -24,9 +18,9 @@ public async ValueTask EnsureCheckpointsTableExist(CancellationToken ct) await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); wasCreated = false; } - finally + catch { - tableLock.Release(); + Console.WriteLine(); } } @@ -36,8 +30,7 @@ public async ValueTask EnsureCheckpointsTableExist(CancellationToken ct) """ CREATE TABLE IF NOT EXISTS "subscription_checkpoints" ( "id" VARCHAR(100) PRIMARY KEY, - "position" BIGINT NULL, - "revision" BIGINT + "position" BIGINT NULL ); """; diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs index 95e1c0d8..9a70e90d 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs @@ -1,12 +1,113 @@ +using System.Threading.Channels; +using Core.EventStoreDB.Subscriptions.Batch; +using Core.EventStoreDB.Subscriptions.Checkpoints; +using EventStore.Client; +using Microsoft.Extensions.DependencyInjection; +using Polly; + namespace Core.EventStoreDB.Subscriptions; -public class EventStoreDBSubscriptioToAllCoordinator(IDictionary subscriptions) +public class SubscriptionInfo +{ + public required EventStoreDBSubscriptionToAll Subscription { get; set; } + public required Checkpoint LastCheckpoint { get; set; } +}; + +public class EventStoreDBSubscriptioToAllCoordinator { + private readonly IDictionary subscriptions; + private readonly IServiceScopeFactory serviceScopeFactory; + + private readonly Channel events = Channel.CreateBounded( + new BoundedChannelOptions(1) + { + SingleWriter = false, + SingleReader = true, + AllowSynchronousContinuations = false, + FullMode = BoundedChannelFullMode.Wait + } + ); + + public EventStoreDBSubscriptioToAllCoordinator(IDictionary subscriptions, + IServiceScopeFactory serviceScopeFactory) + { + this.subscriptions = + subscriptions.ToDictionary(ks => ks.Key, + vs => new SubscriptionInfo { Subscription = vs.Value, LastCheckpoint = Checkpoint.None } + ); + this.serviceScopeFactory = serviceScopeFactory; + } + + public ChannelReader Reader => events.Reader; + public ChannelWriter Writer => events.Writer; + public async Task SubscribeToAll(CancellationToken ct) { // see: https://github.com/dotnet/runtime/issues/36063 await Task.Yield(); - await Task.WhenAll(subscriptions.Values.Select(s => s.SubscribeToAll(ct))).ConfigureAwait(false); + var tasks = subscriptions.Select(s => Task.Run(async () => + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var token = cts.Token; + + var checkpoint = await LoadCheckpoint(s.Key, token).ConfigureAwait(false); + subscriptions[s.Key].LastCheckpoint = checkpoint; + + await s.Value.Subscription.SubscribeToAll(checkpoint, Writer, token).ConfigureAwait(false); + }, ct)).ToList(); + var process = ProcessMessages(ct); + + await Task.WhenAll([process, ..tasks]).ConfigureAwait(false); } + + public async Task ProcessMessages(CancellationToken ct) + { + while (!Reader.Completion.IsCompleted && await Reader.WaitToReadAsync(ct).ConfigureAwait(false)) + { + if (!Reader.TryPeek(out var batch)) continue; + + try + { + using var scope = serviceScopeFactory.CreateScope(); + var checkpointer = scope.ServiceProvider.GetRequiredService(); + + var subscriptionInfo = subscriptions[batch.SubscriptionId]; + + var result = await checkpointer.Process( + batch.Events, + subscriptionInfo.LastCheckpoint, + new BatchProcessingOptions( + batch.SubscriptionId, + subscriptionInfo.Subscription.Options.IgnoreDeserializationErrors, + subscriptionInfo.Subscription.GetHandlers(scope.ServiceProvider) + ), + ct + ) + .ConfigureAwait(false); + + + if (result is ISubscriptionCheckpointRepository.StoreResult.Success success) + { + subscriptionInfo.LastCheckpoint = success.Checkpoint; + Reader.TryRead(out _); + } + } + catch (Exception exc) + { + Console.WriteLine(exc); + } + } + } + + private Task LoadCheckpoint(string subscriptionId, CancellationToken token) => + Policy.Handle().RetryAsync(3) + .ExecuteAsync(async ct => + { + using var scope = serviceScopeFactory.CreateScope(); + return await scope.ServiceProvider.GetRequiredService() + .Load(subscriptionId, ct).ConfigureAwait(false); + }, token); } + +public record EventBatch(string SubscriptionId, ResolvedEvent[] Events); diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index b5878f96..0c98c832 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -1,3 +1,5 @@ +using System.Threading.Channels; +using Core.Events; using Core.EventStoreDB.Subscriptions.Batch; using Core.EventStoreDB.Subscriptions.Checkpoints; using Core.Extensions; @@ -5,6 +7,7 @@ using Grpc.Core; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Polly; using EventTypeFilter = EventStore.Client.EventTypeFilter; namespace Core.EventStoreDB.Subscriptions; @@ -27,51 +30,61 @@ public class EventStoreDBSubscriptionToAllOptions } public class EventStoreDBSubscriptionToAll( - EventStoreDBSubscriptionToAllOptions subscriptionOptions, EventStoreClient eventStoreClient, - IServiceProvider serviceProvider, + ISubscriptionStoreSetup storeSetup, ILogger logger ) { - private string SubscriptionId => subscriptionOptions.SubscriptionId; + public enum ProcessingStatus + { + NotStarted, + Starting, + Started, + Paused, + Errored, + Stopped + } + + public EventStoreDBSubscriptionToAllOptions Options { get; set; } = default!; + + public Func GetHandlers { get; set; } = default!; + + public ProcessingStatus Status = ProcessingStatus.NotStarted; - public async Task SubscribeToAll(CancellationToken ct) + private string SubscriptionId => Options.SubscriptionId; + + public async Task SubscribeToAll(Checkpoint checkpoint, ChannelWriter cw, CancellationToken ct) { + Status = ProcessingStatus.Starting; // see: https://github.com/dotnet/runtime/issues/36063 await Task.Yield(); - logger.LogInformation("Subscription to all '{SubscriptionId}'", subscriptionOptions.SubscriptionId); + logger.LogInformation("Subscription to all '{SubscriptionId}'", Options.SubscriptionId); try { - var checkpoint = await LoadCheckpoint(ct).ConfigureAwait(false); + await storeSetup.EnsureStoreExists(ct).ConfigureAwait(false); var subscription = eventStoreClient.SubscribeToAll( checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, - subscriptionOptions.ResolveLinkTos, - subscriptionOptions.FilterOptions, - subscriptionOptions.Credentials, + Options.ResolveLinkTos, + Options.FilterOptions, + Options.Credentials, ct ); + Status = ProcessingStatus.Started; + logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); - await foreach (var events in subscription.BatchAsync(subscriptionOptions.BatchSize, ct) - .ConfigureAwait(false)) + await foreach (var @event in subscription) + // TODO: Add proper batching here! + // .BatchAsync(subscriptionOptions.BatchSize, TimeSpan.FromMilliseconds(100), ct) + // .ConfigureAwait(false)) { - var result = await ProcessBatch(events, checkpoint, subscriptionOptions, ct).ConfigureAwait(false); - - switch (result) - { - case StoreResult.Success success: - checkpoint = success.Checkpoint; - break; - case StoreResult.Ignored ignored: - break; - default: - throw new InvalidOperationException( - "Checkpoint mismatch, ensure that you have a single subscription running!"); - } + ResolvedEvent[] events = [@event]; + await cw.WriteAsync(new EventBatch(Options.SubscriptionId, events), ct) + .ConfigureAwait(false); } } catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || @@ -82,32 +95,23 @@ public async Task SubscribeToAll(CancellationToken ct) SubscriptionId ); } + catch (OperationCanceledException) + { + logger.LogWarning( + "Subscription to all '{SubscriptionId}' dropped by client", + SubscriptionId + ); + } catch (Exception ex) { + Status = ProcessingStatus.Errored; logger.LogWarning("Subscription was dropped: {Exception}", ex); // Sleep between reconnections to not flood the database or not kill the CPU with infinite loop // Randomness added to reduce the chance of multiple subscriptions trying to reconnect at the same time Thread.Sleep(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)); - await SubscribeToAll(ct).ConfigureAwait(false); + await SubscribeToAll(checkpoint, cw, ct).ConfigureAwait(false); } } - - private ValueTask LoadCheckpoint(CancellationToken ct) - { - using var scope = serviceProvider.CreateScope(); - return scope.ServiceProvider.GetRequiredService().Load(SubscriptionId, ct); - } - - private Task ProcessBatch( - ResolvedEvent[] events, - Checkpoint lastCheckpoint, - EventStoreDBSubscriptionToAllOptions subscriptionOptions, - CancellationToken ct) - { - using var scope = serviceProvider.CreateScope(); - return scope.ServiceProvider.GetRequiredService() - .Process(events, lastCheckpoint, subscriptionOptions, ct); - } } diff --git a/Core/Events/EventBus.cs b/Core/Events/EventBus.cs index 5e69395c..a649edf7 100644 --- a/Core/Events/EventBus.cs +++ b/Core/Events/EventBus.cs @@ -86,7 +86,8 @@ public static IServiceCollection AddEventBus(this IServiceCollection services, A sp.GetRequiredService(), asyncPolicy ?? Policy.NoOpAsync() )); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(sp => sp.GetRequiredService()); services .TryAddSingleton(sp => sp.GetRequiredService()); diff --git a/Core/Extensions/AsyncEnumerableExtensions.cs b/Core/Extensions/AsyncEnumerableExtensions.cs index 568b7070..d52fbc74 100644 --- a/Core/Extensions/AsyncEnumerableExtensions.cs +++ b/Core/Extensions/AsyncEnumerableExtensions.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Runtime.CompilerServices; namespace Core.Extensions; @@ -7,26 +8,36 @@ public static class AsyncEnumerableExtensions public static async IAsyncEnumerable BatchAsync( this IAsyncEnumerable source, int batchSize, - [EnumeratorCancellation]CancellationToken cancellationToken = default) + TimeSpan maxBatchTime, + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var batch = new List(batchSize); + var batch = new List(); + var stopwatch = new Stopwatch(); - await foreach (var item in source.WithCancellation(cancellationToken)) + try { - if (cancellationToken.IsCancellationRequested) - yield break; + await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + batch.Add(item); + if (batch.Count == 1) + stopwatch.Start(); // Start the stopwatch when the first item is added to the batch - batch.Add(item); + if (batch.Count >= batchSize || stopwatch.Elapsed >= maxBatchTime) + { + yield return batch.ToArray(); // Yield the current batch + batch.Clear(); // Clear the batch + stopwatch.Restart(); // Restart the stopwatch + } + } - if (batch.Count >= batchSize) + if (batch.Count > 0) { - yield return batch.ToArray(); - - batch.Clear(); + yield return batch.ToArray(); // Yield any remaining items in the batch } } - - if (batch.Count > 0) - yield return batch.ToArray(); + finally + { + stopwatch.Stop(); // Stop the stopwatch + } } } diff --git a/Sample/CRUDToCQRS/01-CRUD/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/01-CRUD/ECommerce/ECommerce.csproj index ff44dba6..b4dd2351 100644 --- a/Sample/CRUDToCQRS/01-CRUD/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/01-CRUD/ECommerce/ECommerce.csproj @@ -12,7 +12,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/02-CRUDWithCQRS/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/02-CRUDWithCQRS/ECommerce/ECommerce.csproj index ff44dba6..b4dd2351 100644 --- a/Sample/CRUDToCQRS/02-CRUDWithCQRS/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/02-CRUDWithCQRS/ECommerce/ECommerce.csproj @@ -12,7 +12,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce.Domain/ECommerce.Domain.csproj index b8c78b14..06d14eeb 100644 --- a/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce.Domain/ECommerce.Domain.csproj @@ -7,7 +7,7 @@ - + diff --git a/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce/ECommerce.csproj index 6f68e9cf..362e3a95 100644 --- a/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/03-DomainGrouping/ECommerce/ECommerce.csproj @@ -12,7 +12,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce.Domain/ECommerce.Domain.csproj index b8c78b14..06d14eeb 100644 --- a/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce.Domain/ECommerce.Domain.csproj @@ -7,7 +7,7 @@ - + diff --git a/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce/ECommerce.csproj index 6f68e9cf..362e3a95 100644 --- a/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/04-SlimmedDomain/ECommerce/ECommerce.csproj @@ -12,7 +12,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce.Domain/ECommerce.Domain.csproj index 59cec750..e58e7866 100644 --- a/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce.Domain/ECommerce.Domain.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce/ECommerce.csproj index 6833ac25..bd62bd2e 100644 --- a/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/05-ExplicitDomain/ECommerce/ECommerce.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce.Domain/ECommerce.Domain.csproj index 59cec750..e58e7866 100644 --- a/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce.Domain/ECommerce.Domain.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce/ECommerce.csproj index 6833ac25..bd62bd2e 100644 --- a/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/06-SlicedDomain/ECommerce/ECommerce.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce.Domain/ECommerce.Domain.csproj index 59cec750..e58e7866 100644 --- a/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce.Domain/ECommerce.Domain.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce/ECommerce.csproj index 6833ac25..bd62bd2e 100644 --- a/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/07-FlattenedLayers/ECommerce/ECommerce.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce.Domain/ECommerce.Domain.csproj index 667ddd48..61cacff2 100644 --- a/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce.Domain/ECommerce.Domain.csproj @@ -10,7 +10,7 @@ - + diff --git a/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce/ECommerce.csproj index 6833ac25..bd62bd2e 100644 --- a/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/08-SlicedEndpoints/ECommerce/ECommerce.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/CRUDToCQRS/09-MultipleModules/ECommerce.Domain/ECommerce.Domain.csproj b/Sample/CRUDToCQRS/09-MultipleModules/ECommerce.Domain/ECommerce.Domain.csproj index 667ddd48..61cacff2 100644 --- a/Sample/CRUDToCQRS/09-MultipleModules/ECommerce.Domain/ECommerce.Domain.csproj +++ b/Sample/CRUDToCQRS/09-MultipleModules/ECommerce.Domain/ECommerce.Domain.csproj @@ -10,7 +10,7 @@ - + diff --git a/Sample/CRUDToCQRS/09-MultipleModules/ECommerce/ECommerce.csproj b/Sample/CRUDToCQRS/09-MultipleModules/ECommerce/ECommerce.csproj index 6833ac25..bd62bd2e 100644 --- a/Sample/CRUDToCQRS/09-MultipleModules/ECommerce/ECommerce.csproj +++ b/Sample/CRUDToCQRS/09-MultipleModules/ECommerce/ECommerce.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/ECommerce/Carts/Carts.Api/Carts.Api.csproj b/Sample/ECommerce/Carts/Carts.Api/Carts.Api.csproj index 9e769cb3..f12a4b46 100644 --- a/Sample/ECommerce/Carts/Carts.Api/Carts.Api.csproj +++ b/Sample/ECommerce/Carts/Carts.Api/Carts.Api.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/ECommerce/Orders/Orders.Api/Orders.Api.csproj b/Sample/ECommerce/Orders/Orders.Api/Orders.Api.csproj index faeb52e8..9de0afe5 100644 --- a/Sample/ECommerce/Orders/Orders.Api/Orders.Api.csproj +++ b/Sample/ECommerce/Orders/Orders.Api/Orders.Api.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/ECommerce/Payments/Payments.Api/Payments.Api.csproj b/Sample/ECommerce/Payments/Payments.Api/Payments.Api.csproj index 08101a12..b1696fde 100644 --- a/Sample/ECommerce/Payments/Payments.Api/Payments.Api.csproj +++ b/Sample/ECommerce/Payments/Payments.Api/Payments.Api.csproj @@ -6,7 +6,7 @@ - + diff --git a/Sample/ECommerce/Shipments/Shipments.Api/Shipments.Api.csproj b/Sample/ECommerce/Shipments/Shipments.Api/Shipments.Api.csproj index be947918..76674e2b 100644 --- a/Sample/ECommerce/Shipments/Shipments.Api/Shipments.Api.csproj +++ b/Sample/ECommerce/Shipments/Shipments.Api/Shipments.Api.csproj @@ -5,7 +5,7 @@ - + diff --git a/Sample/ECommerce/Shipments/Shipments/Shipments.csproj b/Sample/ECommerce/Shipments/Shipments/Shipments.csproj index 42c02e2e..8edda464 100644 --- a/Sample/ECommerce/Shipments/Shipments/Shipments.csproj +++ b/Sample/ECommerce/Shipments/Shipments/Shipments.csproj @@ -23,7 +23,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Program.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Program.cs index 1a5abc11..c37a7531 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Program.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Program.cs @@ -1,5 +1,6 @@ using Carts; using Core; +using Core.Events; using Core.EventStoreDB; using Core.EventStoreDB.Subscriptions; using Core.Exceptions; @@ -27,7 +28,7 @@ WrongExpectedVersionException => exception.MapToProblemDetails(StatusCodes.Status412PreconditionFailed), _ => null }) - .AddEventStoreDBSubscriptionToAll(new EventStoreDBSubscriptionToAllOptions { SubscriptionId = "DDDESDB" }) + .AddEventStoreDBSubscriptionToAll("DDDESDB") .AddCartsModule(builder.Configuration) .AddOptimisticConcurrencyMiddleware() .AddOpenTelemetry("Carts", OpenTelemetryOptions.Build(options => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs index c5238924..6bd4d627 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs @@ -1,5 +1,6 @@ using Carts.ShoppingCarts; using Core.EventStoreDB; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Core.Marten; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs index dd8bc6cc..0cef0e51 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/Configuration.cs @@ -1,5 +1,6 @@ using Core.Events; using Core.EventStoreDB; +using Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using Core.Extensions; using Core.OpenTelemetry; using Microsoft.Extensions.Configuration; @@ -17,5 +18,6 @@ IConfiguration configuration .AllowResolvingKeyedServicesAsDictionary() .AddSingleton() .AddEventBus() - .AddEventStoreDB(configuration); + .AddEventStoreDB(configuration) + .AddPostgresCheckpointing(); } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj b/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj index 806f4fd9..0e92397f 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj +++ b/Sample/EventStoreDB/Simple/ECommerce.Core/ECommerce.Core.csproj @@ -18,7 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs index 21d58fc6..022e411f 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/Configuration.cs @@ -7,28 +7,44 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Npgsql; namespace ECommerce; public static class Configuration { - public static IServiceCollection AddECommerceModule(this IServiceCollection services, IConfiguration config) => - services - .AddNpgsqlDataSource(config.GetRequiredConnectionString("ECommerceDB")) - .AddEntityFrameworkProject() + public static IServiceCollection AddECommerceModule(this IServiceCollection services, IConfiguration config) + { + var schemaName = // Environment.GetEnvironmentVariable("SchemaName") ?? + "simple_esdb_ecommerce"; + var connectionString = $"{config.GetConnectionString("ECommerceDB")};searchpath = {schemaName.ToLower()}"; + + return services + .AddNpgsqlDataSource(connectionString) + .AddScoped(_ => + { + var connection = new NpgsqlConnection(connectionString); + connection.Open(); + return connection; + }) + .AddScoped(sp => + { + var connection = sp.GetRequiredService(); + return connection.BeginTransaction(); + }) + .AddEntityFrameworkProjections() .AddShoppingCartsModule() .AddPricingModule() .AddDbContext( - options => + (sp, options) => { - var connectionString = config.GetConnectionString("ECommerceDB"); - var schemaName = Environment.GetEnvironmentVariable("SchemaName") ?? "simple_esdb_ecommerce"; + var connection = sp.GetRequiredService(); - options.UseNpgsql( - $"{connectionString};searchpath = {schemaName.ToLower()}", + options.UseNpgsql(connection, x => x.MigrationsHistoryTable("__EFMigrationsHistory", schemaName.ToLower())); }) .AddSingleton>(Guid.NewGuid); + } public static void UseECommerceModule(this IServiceProvider serviceProvider) { diff --git a/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj b/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj index 106b7e61..7eed3ff4 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj +++ b/Sample/EventStoreDB/Simple/ECommerce/ECommerce.csproj @@ -19,7 +19,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs index c8139185..ba732a8a 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs @@ -51,6 +51,7 @@ public static IServiceCollection AddShoppingCartsModule(this IServiceCollection ) .For( builder => builder + .ViewId(v => v.Id) .AddOn(ShoppingCartDetailsProjection.Handle) .UpdateOn( e => e.ShoppingCartId, @@ -73,6 +74,7 @@ public static IServiceCollection AddShoppingCartsModule(this IServiceCollection ) .For( builder => builder + .ViewId(v => v.Id) .AddOn(ShoppingCartShortInfoProjection.Handle) .UpdateOn( e => e.ShoppingCartId, diff --git a/Sample/Tickets/Tickets.Api/Tickets.Api.csproj b/Sample/Tickets/Tickets.Api/Tickets.Api.csproj index 66817851..b58cf449 100644 --- a/Sample/Tickets/Tickets.Api/Tickets.Api.csproj +++ b/Sample/Tickets/Tickets.Api/Tickets.Api.csproj @@ -5,7 +5,7 @@ - + diff --git a/Sample/Warehouse.MinimalAPI/Warehouse.Api/Warehouse.Api.csproj b/Sample/Warehouse.MinimalAPI/Warehouse.Api/Warehouse.Api.csproj index bfcf1b07..b4fab470 100644 --- a/Sample/Warehouse.MinimalAPI/Warehouse.Api/Warehouse.Api.csproj +++ b/Sample/Warehouse.MinimalAPI/Warehouse.Api/Warehouse.Api.csproj @@ -15,7 +15,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Sample/Warehouse/Warehouse/Warehouse.csproj b/Sample/Warehouse/Warehouse/Warehouse.csproj index 7c062785..2b02b1a7 100644 --- a/Sample/Warehouse/Warehouse/Warehouse.csproj +++ b/Sample/Warehouse/Warehouse/Warehouse.csproj @@ -23,7 +23,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 2f51b6ebc4f28665d408355fb1ff733b8380565f Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 21 May 2024 19:33:45 +0200 Subject: [PATCH 10/18] Improved batching --- ...EventStoreDBSubscriptioToAllCoordinator.cs | 51 ++++++++++--------- .../EventStoreDBSubscriptionToAll.cs | 11 ++-- Core/Extensions/AsyncEnumerableExtensions.cs | 38 ++++++++++---- 3 files changed, 62 insertions(+), 38 deletions(-) diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs index 9a70e90d..d1425980 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs @@ -69,29 +69,7 @@ public async Task ProcessMessages(CancellationToken ct) try { - using var scope = serviceScopeFactory.CreateScope(); - var checkpointer = scope.ServiceProvider.GetRequiredService(); - - var subscriptionInfo = subscriptions[batch.SubscriptionId]; - - var result = await checkpointer.Process( - batch.Events, - subscriptionInfo.LastCheckpoint, - new BatchProcessingOptions( - batch.SubscriptionId, - subscriptionInfo.Subscription.Options.IgnoreDeserializationErrors, - subscriptionInfo.Subscription.GetHandlers(scope.ServiceProvider) - ), - ct - ) - .ConfigureAwait(false); - - - if (result is ISubscriptionCheckpointRepository.StoreResult.Success success) - { - subscriptionInfo.LastCheckpoint = success.Checkpoint; - Reader.TryRead(out _); - } + await ProcessBatch(ct, batch).ConfigureAwait(false); } catch (Exception exc) { @@ -100,6 +78,33 @@ public async Task ProcessMessages(CancellationToken ct) } } + private async Task ProcessBatch(CancellationToken ct, EventBatch batch) + { + using var scope = serviceScopeFactory.CreateScope(); + var checkpointer = scope.ServiceProvider.GetRequiredService(); + + var subscriptionInfo = subscriptions[batch.SubscriptionId]; + + var result = await checkpointer.Process( + batch.Events, + subscriptionInfo.LastCheckpoint, + new BatchProcessingOptions( + batch.SubscriptionId, + subscriptionInfo.Subscription.Options.IgnoreDeserializationErrors, + subscriptionInfo.Subscription.GetHandlers(scope.ServiceProvider) + ), + ct + ) + .ConfigureAwait(false); + + + if (result is ISubscriptionCheckpointRepository.StoreResult.Success success) + { + subscriptionInfo.LastCheckpoint = success.Checkpoint; + Reader.TryRead(out _); + } + } + private Task LoadCheckpoint(string subscriptionId, CancellationToken token) => Policy.Handle().RetryAsync(3) .ExecuteAsync(async ct => diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 0c98c832..88fdba22 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -77,12 +77,13 @@ public async Task SubscribeToAll(Checkpoint checkpoint, ChannelWriter BatchAsync( @@ -12,32 +20,42 @@ public static async IAsyncEnumerable BatchAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { var batch = new List(); - var stopwatch = new Stopwatch(); + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); try { - await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (var item in source.WithCancellation(cts.Token).ConfigureAwait(false)) { batch.Add(item); - if (batch.Count == 1) - stopwatch.Start(); // Start the stopwatch when the first item is added to the batch + if (batch.Count == 1) // Reset the timer when the first item is added + { + cts.CancelAfter(maxBatchTime); // Set or reset the deadline + } - if (batch.Count >= batchSize || stopwatch.Elapsed >= maxBatchTime) + if (batch.Count >= batchSize) { - yield return batch.ToArray(); // Yield the current batch - batch.Clear(); // Clear the batch - stopwatch.Restart(); // Restart the stopwatch + yield return batch.ToArray(); + batch.Clear(); + cts.CancelAfter(maxBatchTime); // Reset the deadline for the new batch } } if (batch.Count > 0) { - yield return batch.ToArray(); // Yield any remaining items in the batch + yield return batch.ToArray(); // Return any remaining items as a batch + } + } + catch (OperationCanceledException) + { + if (batch.Count > 0) + { + yield return batch.ToArray(); // Yield whatever is in the batch when the timeout occurs } + // Optionally, rethrow or handle the cancellation if needed } finally { - stopwatch.Stop(); // Stop the stopwatch + cts.Dispose(); // Ensure the CancellationTokenSource is disposed to free resources } } } From 3e59018289e262eef7322ba8cd714ead75096e62 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 21 May 2024 22:55:08 +0200 Subject: [PATCH 11/18] Fixed Batching and disabled AsyncDaemon for EVentStoreDB + Marten configuration --- Core.EventStoreDB/Config.cs | 1 - Core.EventStoreDB/Core.EventStoreDB.csproj | 1 + ...EventStoreDBSubscriptioToAllCoordinator.cs | 15 +++-- .../EventStoreDBSubscriptionToAll.cs | 34 +++++----- Core.Marten/MartenConfig.cs | 19 ++++-- Core.Testing/Core.Testing.csproj | 2 +- Core/Core.csproj | 1 + Core/Extensions/AsyncEnumerableExtensions.cs | 67 ++++++------------- .../Confirming/ConfirmShoppingCartTests.cs | 2 +- .../Opening/OpenShoppingCartTests.cs | 2 +- .../AddingProduct/AddProductTests.cs | 2 +- .../Canceling/CancelShoppingCartTests.cs | 2 +- .../Confirming/ConfirmShoppingCartTests.cs | 2 +- .../RemovingProduct/RemoveProductTests.cs | 2 +- .../ECommerce/Carts/Carts/Config.cs | 2 +- .../AddingProduct/AddProductTests.cs | 2 +- .../Canceling/CancelShoppingCartTests.cs | 2 +- .../Opening/OpenShoppingCartTests.cs | 2 +- .../GettingCartById/ShoppingCartDetails.cs | 2 +- 19 files changed, 76 insertions(+), 86 deletions(-) diff --git a/Core.EventStoreDB/Config.cs b/Core.EventStoreDB/Config.cs index 71f43ea0..933df9a4 100644 --- a/Core.EventStoreDB/Config.cs +++ b/Core.EventStoreDB/Config.cs @@ -103,7 +103,6 @@ Func handlers { var subscription = new EventStoreDBSubscriptionToAll( sp.GetRequiredService(), - sp.GetRequiredService(), sp.GetRequiredService>() ) { Options = subscriptionOptions, GetHandlers = handlers }; diff --git a/Core.EventStoreDB/Core.EventStoreDB.csproj b/Core.EventStoreDB/Core.EventStoreDB.csproj index 9b98769d..ff05da80 100644 --- a/Core.EventStoreDB/Core.EventStoreDB.csproj +++ b/Core.EventStoreDB/Core.EventStoreDB.csproj @@ -14,6 +14,7 @@ + diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs index d1425980..2d9f0701 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs @@ -16,6 +16,7 @@ public class SubscriptionInfo public class EventStoreDBSubscriptioToAllCoordinator { private readonly IDictionary subscriptions; + private readonly ISubscriptionStoreSetup storeSetup; private readonly IServiceScopeFactory serviceScopeFactory; private readonly Channel events = Channel.CreateBounded( @@ -28,24 +29,30 @@ public class EventStoreDBSubscriptioToAllCoordinator } ); - public EventStoreDBSubscriptioToAllCoordinator(IDictionary subscriptions, - IServiceScopeFactory serviceScopeFactory) + public EventStoreDBSubscriptioToAllCoordinator( + IDictionary subscriptions, + ISubscriptionStoreSetup storeSetup, + IServiceScopeFactory serviceScopeFactory + ) { this.subscriptions = subscriptions.ToDictionary(ks => ks.Key, vs => new SubscriptionInfo { Subscription = vs.Value, LastCheckpoint = Checkpoint.None } ); + this.storeSetup = storeSetup; this.serviceScopeFactory = serviceScopeFactory; } - public ChannelReader Reader => events.Reader; - public ChannelWriter Writer => events.Writer; + private ChannelReader Reader => events.Reader; + private ChannelWriter Writer => events.Writer; public async Task SubscribeToAll(CancellationToken ct) { // see: https://github.com/dotnet/runtime/issues/36063 await Task.Yield(); + await storeSetup.EnsureStoreExists(ct).ConfigureAwait(false); + var tasks = subscriptions.Select(s => Task.Run(async () => { var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 88fdba22..e0da24dd 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -1,19 +1,16 @@ using System.Threading.Channels; using Core.Events; -using Core.EventStoreDB.Subscriptions.Batch; using Core.EventStoreDB.Subscriptions.Checkpoints; using Core.Extensions; using EventStore.Client; using Grpc.Core; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Open.ChannelExtensions; using Polly; using EventTypeFilter = EventStore.Client.EventTypeFilter; namespace Core.EventStoreDB.Subscriptions; -using static ISubscriptionCheckpointRepository; - public class EventStoreDBSubscriptionToAllOptions { public required string SubscriptionId { get; init; } @@ -27,11 +24,11 @@ public class EventStoreDBSubscriptionToAllOptions public bool IgnoreDeserializationErrors { get; set; } = true; public int BatchSize { get; set; } = 1; + public int BatchDeadline { get; set; } = 50; } public class EventStoreDBSubscriptionToAll( EventStoreClient eventStoreClient, - ISubscriptionStoreSetup storeSetup, ILogger logger ) { @@ -63,8 +60,6 @@ public async Task SubscribeToAll(Checkpoint checkpoint, ChannelWriter( + cw, + events => new EventBatch(Options.SubscriptionId, events.ToArray()), + Options.BatchSize, + Options.BatchDeadline, + ct + ).ConfigureAwait(false); } catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || rpcException.InnerException is ObjectDisposedException) @@ -115,4 +108,11 @@ await cw.WriteAsync(new EventBatch(Options.SubscriptionId, events), ct) await SubscribeToAll(checkpoint, cw, ct).ConfigureAwait(false); } } + // + // private AsyncPolicy retry = Policy.Handle(rpcException => + // rpcException is { StatusCode: StatusCode.Cancelled } || + // rpcException.InnerException is ObjectDisposedException) + // .Or() + // .F + // .WaitAndRetryForeverAsync(_ => TimeSpan.FromMilliseconds(500)); } diff --git a/Core.Marten/MartenConfig.cs b/Core.Marten/MartenConfig.cs index 8b8570a8..733263de 100644 --- a/Core.Marten/MartenConfig.cs +++ b/Core.Marten/MartenConfig.cs @@ -38,19 +38,22 @@ public static MartenServiceCollectionExtensions.MartenConfigurationExpression Ad IConfiguration configuration, Action? configureOptions = null, string configKey = DefaultConfigKey, - bool useExternalBus = false + bool useExternalBus = false, + bool disableAsyncDaemon = false ) => services.AddMarten( configuration.GetRequiredConfig(configKey), configureOptions, - useExternalBus + useExternalBus, + disableAsyncDaemon ); public static MartenServiceCollectionExtensions.MartenConfigurationExpression AddMarten( this IServiceCollection services, MartenConfig martenConfig, Action? configureOptions = null, - bool useExternalBus = false + bool useExternalBus = false, + bool disableAsyncDaemon = false ) { var config = services @@ -67,10 +70,14 @@ public static MartenServiceCollectionExtensions.MartenConfigurationExpression Ad return SetStoreOptions(martenConfig, configureOptions); }) .UseLightweightSessions() - .ApplyAllDatabaseChangesOnStartup() + .ApplyAllDatabaseChangesOnStartup(); + + if (!disableAsyncDaemon) + { //.OptimizeArtifactWorkflow() - .AddAsyncDaemon(martenConfig.DaemonMode) - .AddSubscriptionWithServices(ServiceLifetime.Scoped); + config.AddAsyncDaemon(martenConfig.DaemonMode) + .AddSubscriptionWithServices(ServiceLifetime.Scoped); + } if (useExternalBus) services.AddMartenAsyncCommandBus(); diff --git a/Core.Testing/Core.Testing.csproj b/Core.Testing/Core.Testing.csproj index a1a8b100..839df821 100644 --- a/Core.Testing/Core.Testing.csproj +++ b/Core.Testing/Core.Testing.csproj @@ -11,7 +11,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Core/Core.csproj b/Core/Core.csproj index 691f5ea8..c514fd09 100644 --- a/Core/Core.csproj +++ b/Core/Core.csproj @@ -11,6 +11,7 @@ + diff --git a/Core/Extensions/AsyncEnumerableExtensions.cs b/Core/Extensions/AsyncEnumerableExtensions.cs index f2c81d7d..eeee3cb5 100644 --- a/Core/Extensions/AsyncEnumerableExtensions.cs +++ b/Core/Extensions/AsyncEnumerableExtensions.cs @@ -1,61 +1,36 @@ -using System.Diagnostics; -using System.Runtime.CompilerServices; +using System.Threading.Channels; +using Open.ChannelExtensions; namespace Core.Extensions; public static class AsyncEnumerableExtensions { -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; - -public static class AsyncEnumerableExtensions -{ - public static async IAsyncEnumerable BatchAsync( - this IAsyncEnumerable source, + public static async Task Pipe( + this IAsyncEnumerable enumerable, + ChannelWriter cw, + Func, TResult> transform, int batchSize, - TimeSpan maxBatchTime, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + int timeout, + CancellationToken ct + ) { - var batch = new List(); - var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - - try - { - await foreach (var item in source.WithCancellation(cts.Token).ConfigureAwait(false)) + var channel = Channel.CreateUnbounded( + new UnboundedChannelOptions { - batch.Add(item); - if (batch.Count == 1) // Reset the timer when the first item is added - { - cts.CancelAfter(maxBatchTime); // Set or reset the deadline - } - - if (batch.Count >= batchSize) - { - yield return batch.ToArray(); - batch.Clear(); - cts.CancelAfter(maxBatchTime); // Reset the deadline for the new batch - } + SingleWriter = false, SingleReader = true, AllowSynchronousContinuations = false } + ); - if (batch.Count > 0) - { - yield return batch.ToArray(); // Return any remaining items as a batch - } - } - catch (OperationCanceledException) + channel.Reader.Batch(batchSize).WithTimeout(timeout).PipeAsync(async batch => { - if (batch.Count > 0) - { - yield return batch.ToArray(); // Yield whatever is in the batch when the timeout occurs - } - // Optionally, rethrow or handle the cancellation if needed - } - finally + await cw.WriteAsync(transform(batch), ct).ConfigureAwait(false); + + return batch; + }); + + await foreach (var @event in enumerable.WithCancellation(ct)) { - cts.Dispose(); // Ensure the CancellationTokenSource is disposed to free resources + await channel.Writer.WriteAsync(@event, ct).ConfigureAwait(false); } } } diff --git a/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 142994b3..fa27fd0b 100644 --- a/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -29,7 +29,7 @@ public Task Put_Should_Return_OK_And_Confirm_Shopping_Cart() => .Then(OK) .And() .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) - .Until(RESPONSE_ETAG_IS(3)) + .Until(RESPONSE_ETAG_IS(3), 10) .Then( OK, RESPONSE_BODY((details, ctx) => diff --git a/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index d38c3034..42fe8245 100644 --- a/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -23,7 +23,7 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => response => api.Given() .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, RESPONSE_BODY(details => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index c0daab8f..bee73760 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -30,7 +30,7 @@ await api .Then(OK) .And() .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( RESPONSE_BODY((details, ctx) => { diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index a77acab7..c9c422d8 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -26,7 +26,7 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => .Then(OK) .And() .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, RESPONSE_BODY((details, ctx) => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index c54e0531..7e59fe33 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -43,7 +43,7 @@ await api await api .Given() .When(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, RESPONSE_BODY(details => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index fdd1eb98..01506c9a 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -63,7 +63,7 @@ await api .Then(NO_CONTENT) .And() .When(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) - .Until(RESPONSE_ETAG_IS(2)) + .Until(RESPONSE_ETAG_IS(2), 10) .Then( OK, RESPONSE_BODY(details => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs index 6bd4d627..7d36d6d4 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts/Config.cs @@ -12,7 +12,7 @@ public static class Config public static IServiceCollection AddCartsModule(this IServiceCollection services, IConfiguration config) => services // Document Part used for projections - .AddMarten(config, configKey: "ReadModel_Marten") + .AddMarten(config, configKey: "ReadModel_Marten", disableAsyncDaemon: true) .Services .AddCarts() .AddEventStoreDB(config); diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index 21ef0d8c..9a055d5c 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -30,7 +30,7 @@ public Task Post_Should_AddProductItem_To_ShoppingCart() .Then(OK) .And() .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( RESPONSE_BODY((details, ctx) => { diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index fcf79e16..17997dc0 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -26,7 +26,7 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => .Then(OK) .And() .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) - .Until(RESPONSE_ETAG_IS(1)) + .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, RESPONSE_BODY((details, ctx) => diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index 0aaf6bea..7c50d74b 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -26,7 +26,7 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => response => API.Given() .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}")) - .Until(RESPONSE_ETAG_IS(0)) + .Until(RESPONSE_ETAG_IS(0), 10) .Then( OK, RESPONSE_BODY(details => diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs index 095f2398..a6096681 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/ShoppingCartDetails.cs @@ -7,7 +7,7 @@ public class ShoppingCartDetails public Guid Id { get; set; } public Guid ClientId { get; set; } public ShoppingCartStatus Status { get; set; } - public List ProductItems { get; set; } = default!; + public List ProductItems { get; set; } = new(); public int Version { get; set; } public ulong LastProcessedPosition { get; set; } } From 02e55b589fe5e8e2f2aa71712b8af501672c9a66 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 07:33:13 +0200 Subject: [PATCH 12/18] Added Collection Fixture to have Host setup only once --- .../AddingProduct/AddProductTests.cs | 8 +- .../Canceling/CancelShoppingCartTests.cs | 8 +- .../Confirming/ConfirmShoppingCartTests.cs | 40 ++++----- .../Opening/OpenShoppingCartTests.cs | 9 ++- .../RemovingProduct/RemoveProductTests.cs | 81 ++++++++++--------- .../ShoppingCartsApplicationFactory.cs | 28 +++++++ 6 files changed, 101 insertions(+), 73 deletions(-) diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index 9a055d5c..a11281fc 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -10,7 +10,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.AddingProduct; using static ShoppingCartsApi; -public class AddProductTests(ApiSpecification api): IClassFixture> +public class AddProductTests: ApiTest { [Fact] [Trait("Category", "Acceptance")] @@ -18,7 +18,7 @@ public Task Post_Should_AddProductItem_To_ShoppingCart() { var product = new ProductItemRequest(Guid.NewGuid(), 1); - return api + return API .Given("Opened Shopping Cart", OpenShoppingCart()) .When( "Add new product", @@ -43,4 +43,8 @@ public Task Post_Should_AddProductItem_To_ShoppingCart() }) ); } + + public AddProductTests(ApiFixture fixture) : base(fixture) + { + } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index 17997dc0..cf2278c7 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -8,14 +8,14 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Canceling; -public class CancelShoppingCartTests(ApiSpecification api): IClassFixture> +public class CancelShoppingCartTests: ApiTest { public readonly Guid ClientId = Guid.NewGuid(); [Fact] [Trait("Category", "Acceptance")] public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => - api + API .Given("Opened Shopping Cart", OpenShoppingCart(ClientId)) .When( "Cancel Shopping Cart", @@ -37,4 +37,8 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => details.ClientId.Should().Be(ClientId); details.Version.Should().Be(1); })); + + public CancelShoppingCartTests(ApiFixture fixture) : base(fixture) + { + } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 76d4f85f..28dd791f 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -8,51 +8,41 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Confirming; -public class ConfirmShoppingCartFixture() - : ApiSpecification(new ShoppingCartsApplicationFactory()), IAsyncLifetime -{ - public Guid ShoppingCartId { get; private set; } - - public readonly Guid ClientId = Guid.NewGuid(); - - public async Task InitializeAsync() - { - ShoppingCartId = await Given() - .When(POST, URI("/api/ShoppingCarts"), BODY(new OpenShoppingCartRequest(ClientId))) - .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)) - .GetCreatedId(); - } +using static ShoppingCartsApi; - public Task DisposeAsync() => Task.CompletedTask; -} - -public class ConfirmShoppingCartTests(ConfirmShoppingCartFixture api): IClassFixture +public class ConfirmShoppingCartTests: ApiTest { + private Guid ClientId = Guid.NewGuid(); + [Fact] [Trait("Category", "Acceptance")] public async Task Put_Should_Return_OK_And_Confirm_Shopping_Cart() { - await api - .Given() + await API + .Given("Opened Shopping Cart", OpenShoppingCart(ClientId)) .When( PUT, - URI($"/api/ShoppingCarts/{api.ShoppingCartId}/confirmation"), + URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}/confirmation"), HEADERS(IF_MATCH(0)) ) .Then(OK) - .AndWhen(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) + .AndWhen(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) .Until(RESPONSE_ETAG_IS(1), maxNumberOfRetries: 10) .Then( OK, - RESPONSE_BODY(details => + RESPONSE_BODY((details,ctx) => { - details.Id.Should().Be(api.ShoppingCartId); + details.Id.Should().Be(ctx.OpenedShoppingCartId()); details.Status.Should().Be(ShoppingCartStatus.Confirmed); details.ProductItems.Should().BeEmpty(); - details.ClientId.Should().Be(api.ClientId); + details.ClientId.Should().Be(ClientId); details.Version.Should().Be(1); })); // API.PublishedExternalEventsOfType(); } + + public ConfirmShoppingCartTests(ApiFixture fixture) : base(fixture) + { + } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index 7c50d74b..2a38dc64 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -8,11 +8,8 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Opening; -public class OpenShoppingCartTests(ShoppingCartsApplicationFactory applicationFactory) - : IClassFixture +public class OpenShoppingCartTests: ApiTest { - private readonly ApiSpecification API = ApiSpecification.Setup(applicationFactory); - [Fact] public Task Post_ShouldReturn_CreatedStatus_With_CartId() => API.Scenario( @@ -40,4 +37,8 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => ); public readonly Guid ClientId = Guid.NewGuid(); + + public OpenShoppingCartTests(ApiFixture fixture) : base(fixture) + { + } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index 9fe6932f..037c946b 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -9,8 +9,44 @@ namespace ECommerce.Api.Tests.ShoppingCarts.RemovingProduct; -public class RemoveProductFixture(): ApiSpecification(new ShoppingCartsApplicationFactory()), IAsyncLifetime +public class RemoveProductTests: ApiTest, IAsyncLifetime { + [Fact] + [Trait("Category", "Acceptance")] + public async Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() + { + await API + .Given() + .When( + DELETE, + URI( + $"/api/ShoppingCarts/{ShoppingCartId}/products/{ProductItem.ProductId}?quantity={RemovedCount}&unitPrice={UnitPrice.ToString(CultureInfo.InvariantCulture)}"), + HEADERS(IF_MATCH(1)) + ) + .Then(NO_CONTENT) + .And() + .When(GET, URI($"/api/ShoppingCarts/{ShoppingCartId}")) + .Until(RESPONSE_ETAG_IS(2), maxNumberOfRetries: 10) + .Then( + OK, + RESPONSE_BODY(details => + { + details.Id.Should().Be(ShoppingCartId); + details.Status.Should().Be(ShoppingCartStatus.Pending); + details.ProductItems.Should().HaveCount(1); + var productItem = details.ProductItems.Single(); + productItem.Should().BeEquivalentTo( + new ShoppingCartDetailsProductItem + { + ProductId = ProductItem.ProductId!.Value, + Quantity = ProductItem.Quantity!.Value - RemovedCount, + UnitPrice = UnitPrice + }); + details.ClientId.Should().Be(ClientId); + details.Version.Should().Be(2); + })); + } + public Guid ShoppingCartId { get; private set; } public readonly Guid ClientId = Guid.NewGuid(); @@ -19,9 +55,11 @@ public class RemoveProductFixture(): ApiSpecification(new ShoppingCarts public decimal UnitPrice; + private readonly int RemovedCount = 5; + public async Task InitializeAsync() { - var cartDetails = await Given() + var cartDetails = await API.Given() .When(POST, URI("/api/ShoppingCarts"), BODY(new OpenShoppingCartRequest(ClientId))) .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)) .And() @@ -43,45 +81,8 @@ public async Task InitializeAsync() } public Task DisposeAsync() => Task.CompletedTask; -} -public class RemoveProductTests(RemoveProductFixture api): IClassFixture -{ - [Fact] - [Trait("Category", "Acceptance")] - public async Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() + public RemoveProductTests(ApiFixture fixture) : base(fixture) { - await api - .Given() - .When( - DELETE, - URI( - $"/api/ShoppingCarts/{api.ShoppingCartId}/products/{api.ProductItem.ProductId}?quantity={RemovedCount}&unitPrice={api.UnitPrice.ToString(CultureInfo.InvariantCulture)}"), - HEADERS(IF_MATCH(1)) - ) - .Then(NO_CONTENT) - .And() - .When(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) - .Until(RESPONSE_ETAG_IS(2), maxNumberOfRetries: 10) - .Then( - OK, - RESPONSE_BODY(details => - { - details.Id.Should().Be(api.ShoppingCartId); - details.Status.Should().Be(ShoppingCartStatus.Pending); - details.ProductItems.Should().HaveCount(1); - var productItem = details.ProductItems.Single(); - productItem.Should().BeEquivalentTo( - new ShoppingCartDetailsProductItem - { - ProductId = api.ProductItem.ProductId!.Value, - Quantity = api.ProductItem.Quantity!.Value - RemovedCount, - UnitPrice = api.UnitPrice - }); - details.ClientId.Should().Be(api.ClientId); - details.Version.Should().Be(2); - })); } - - private readonly int RemovedCount = 5; } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs index 89a397bb..25dba1b4 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs @@ -1,4 +1,6 @@ using Core.Testing; +using Ogooreck.API; +using Xunit; namespace ECommerce.Api.Tests; @@ -18,3 +20,29 @@ public class ShoppingCartsApplicationFactory: TestWebApplicationFactory // return host; // } } + +public class ApiFixture: IDisposable +{ + public ApiFixture() + { + + } + + public ApiSpecification API { get; } = + ApiSpecification.Setup(new ShoppingCartsApplicationFactory()); + + public void Dispose() => + API.Dispose(); +} + +[CollectionDefinition("ApiTests")] +public class DatabaseCollection: ICollectionFixture; + +[Collection("ApiTests")] +public abstract class ApiTest +{ + protected readonly ApiSpecification API; + + protected ApiTest(ApiFixture fixture) => + API = fixture.API; +} From 85b7824cb7b7449e4d5f86a1aeb98cd6c5d7e2df Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 12:59:23 +0200 Subject: [PATCH 13/18] Made collection test for the EventStoreDB + Marten samples --- .../Carts/Carts.Api.Tests/ApiTest.cs | 25 ++++++ .../AddingProduct/AddProductTests.cs | 4 +- .../Canceling/CancelShoppingCartTests.cs | 4 +- .../Confirming/ConfirmShoppingCartTests.cs | 43 +++------- .../Opening/OpenShoppingCartTests.cs | 8 +- .../RemovingProduct/RemoveProductTests.cs | 86 +++++++++---------- ...gCartsApplicationFactory.cs => ApiTest.cs} | 12 +-- .../AddingProduct/AddProductTests.cs | 6 +- .../Canceling/CancelShoppingCartTests.cs | 6 +- .../Confirming/ConfirmShoppingCartTests.cs | 6 +- .../Opening/OpenShoppingCartTests.cs | 6 +- .../RemovingProduct/RemoveProductTests.cs | 6 +- 12 files changed, 94 insertions(+), 118 deletions(-) create mode 100644 Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ApiTest.cs rename Sample/EventStoreDB/Simple/ECommerce.Api.Tests/{ShoppingCartsApplicationFactory.cs => ApiTest.cs} (85%) diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ApiTest.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ApiTest.cs new file mode 100644 index 00000000..cba3512f --- /dev/null +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ApiTest.cs @@ -0,0 +1,25 @@ +using Core.Testing; +using Ogooreck.API; +using Xunit; + +namespace Carts.Api.Tests; + +public class ShoppingCartsApplicationFactory: TestWebApplicationFactory; + +public class ApiFixture: IDisposable +{ + public ApiSpecification API { get; } = + ApiSpecification.Setup(new ShoppingCartsApplicationFactory()); + + public void Dispose() => + API.Dispose(); +} + +[CollectionDefinition("ApiTests")] +public class DatabaseCollection: ICollectionFixture; + +[Collection("ApiTests")] +public abstract class ApiTest(ApiFixture fixture) +{ + protected readonly ApiSpecification API = fixture.API; +} diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index bee73760..82c1b81e 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -10,7 +10,7 @@ namespace Carts.Api.Tests.ShoppingCarts.AddingProduct; using static ShoppingCartsApi; -public class AddProductTests(ApiSpecification api): IClassFixture> +public class AddProductTests(ApiFixture fixture): ApiTest(fixture) { [Fact] [Trait("Category", "Acceptance")] @@ -18,7 +18,7 @@ public async Task Post_Should_AddProductItem_To_ShoppingCart() { var product = new ProductItemRequest(Guid.NewGuid(), 1); - await api + await API .Given("Opened Shopping Cart", OpenShoppingCart()) .When( "Add new product", diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index c9c422d8..ce738dab 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -8,14 +8,14 @@ namespace Carts.Api.Tests.ShoppingCarts.Canceling; -public class CancelShoppingCartTests(ApiSpecification api): IClassFixture> +public class CancelShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { public readonly Guid ClientId = Guid.NewGuid(); [Fact] [Trait("Category", "Acceptance")] public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => - api + API .Given("Opened Shopping Cart", OpenShoppingCart(ClientId)) .When( "Cancel Shopping Cart", diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 7e59fe33..0bd94706 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -1,60 +1,43 @@ -using Carts.Api.Requests; using Carts.ShoppingCarts; using Carts.ShoppingCarts.GettingCartById; using FluentAssertions; -using Ogooreck.API; using Xunit; using static Ogooreck.API.ApiSpecification; namespace Carts.Api.Tests.ShoppingCarts.Confirming; -public class ConfirmShoppingCartFixture: ApiSpecification, IAsyncLifetime -{ - public Guid ShoppingCartId { get; private set; } - - public readonly Guid ClientId = Guid.NewGuid(); - - public async Task InitializeAsync() - { - ShoppingCartId = await Given() - .When(POST, URI("/api/ShoppingCarts"), BODY(new OpenShoppingCartRequest(ClientId))) - .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)) - .GetCreatedId(); - } +using static ShoppingCartsApi; - public Task DisposeAsync() => Task.CompletedTask; -} - -public class ConfirmShoppingCartTests(ConfirmShoppingCartFixture api): IClassFixture +public class ConfirmShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { [Fact] [Trait("Category", "Acceptance")] public async Task Put_Should_Return_OK_And_Cancel_Shopping_Cart() { - await api - .Given() + await API + .Given(OpenShoppingCart(ClientId)) .When( PUT, - URI($"/api/ShoppingCarts/{api.ShoppingCartId}/confirmation"), + URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}/confirmation"), HEADERS(IF_MATCH(0)) ) - .Then(OK); - - await api - .Given() - .When(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) + .Then(OK) + .And() + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, - RESPONSE_BODY(details => + RESPONSE_BODY((details, ctx) => { - details.Id.Should().Be(api.ShoppingCartId); + details.Id.Should().Be(ctx.OpenedShoppingCartId()); details.Status.Should().Be(ShoppingCartStatus.Confirmed); details.ProductItems.Should().BeEmpty(); - details.ClientId.Should().Be(api.ClientId); + details.ClientId.Should().Be(ClientId); details.Version.Should().Be(1); })); // API.PublishedExternalEventsOfType(); } + + private readonly Guid ClientId = Guid.NewGuid(); } diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index e94b335b..77063617 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -8,12 +8,12 @@ namespace Carts.Api.Tests.ShoppingCarts.Opening; -public class OpenShoppingCartTests(ApiSpecification api): IClassFixture> +public class OpenShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { [Fact] public Task Post_ShouldReturn_CreatedStatus_With_CartId() => - api.Scenario( - api.Given() + API.Scenario( + API.Given() .When( POST, URI("/api/ShoppingCarts/"), @@ -21,7 +21,7 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => ) .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)), response => - api.Given() + API.Given() .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}")) .Until(RESPONSE_ETAG_IS(0)) .Then( diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index 01506c9a..94ac1772 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -10,65 +10,29 @@ namespace Carts.Api.Tests.ShoppingCarts.RemovingProduct; -public class RemoveProductFixture: ApiSpecification, IAsyncLifetime -{ - public Guid ShoppingCartId { get; private set; } - - public readonly Guid ClientId = Guid.NewGuid(); - - public readonly ProductItemRequest ProductItem = new(Guid.NewGuid(), 10); - - public decimal UnitPrice; - - public async Task InitializeAsync() - { - var cartDetails = await Given() - .When(POST, URI("/api/ShoppingCarts"), BODY(new OpenShoppingCartRequest(ClientId))) - .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)) - .And() - .When( - POST, - URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}/products"), - BODY(new AddProductRequest(ProductItem)), - HEADERS(IF_MATCH(0)) - ) - .Then(OK) - .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}")) - .Until(RESPONSE_ETAG_IS(1)) - .Then(OK) - .GetResponseBody(); - - ShoppingCartId = cartDetails.Id; - UnitPrice = cartDetails.ProductItems.Single().UnitPrice; - } - - public Task DisposeAsync() => Task.CompletedTask; -} - -public class RemoveProductTests(RemoveProductFixture api): IClassFixture +public class RemoveProductTests(ApiFixture fixture): ApiTest(fixture), IAsyncLifetime { [Fact] [Trait("Category", "Acceptance")] public async Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() { - await api + await API .Given() .When( DELETE, URI( - $"/api/ShoppingCarts/{api.ShoppingCartId}/products/{api.ProductItem.ProductId}?quantity={RemovedCount}&unitPrice={api.UnitPrice.ToString(CultureInfo.InvariantCulture)}"), + $"/api/ShoppingCarts/{ShoppingCartId}/products/{ProductItem.ProductId}?quantity={RemovedCount}&unitPrice={UnitPrice.ToString(CultureInfo.InvariantCulture)}"), HEADERS(IF_MATCH(1)) ) .Then(NO_CONTENT) .And() - .When(GET, URI($"/api/ShoppingCarts/{api.ShoppingCartId}")) + .When(GET, URI($"/api/ShoppingCarts/{ShoppingCartId}")) .Until(RESPONSE_ETAG_IS(2), 10) .Then( OK, RESPONSE_BODY(details => { - details.Id.Should().Be(api.ShoppingCartId); + details.Id.Should().Be(ShoppingCartId); details.Status.Should().Be(ShoppingCartStatus.Pending); details.ProductItems.Should().HaveCount(1); var productItem = details.ProductItems.Single(); @@ -76,15 +40,47 @@ await api new PricedProductItem( new ProductItem ( - api.ProductItem.ProductId!.Value, - api.ProductItem.Quantity!.Value - RemovedCount + ProductItem.ProductId!.Value, + ProductItem.Quantity!.Value - RemovedCount ), - api.UnitPrice + UnitPrice )); - details.ClientId.Should().Be(api.ClientId); + details.ClientId.Should().Be(ClientId); details.Version.Should().Be(2); })); } + public Guid ShoppingCartId { get; private set; } + + public readonly Guid ClientId = Guid.NewGuid(); + + public readonly ProductItemRequest ProductItem = new(Guid.NewGuid(), 10); + + public decimal UnitPrice; + private readonly int RemovedCount = 5; + public async Task InitializeAsync() + { + var cartDetails = await API.Given() + .When(POST, URI("/api/ShoppingCarts"), BODY(new OpenShoppingCartRequest(ClientId))) + .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)) + .And() + .When( + POST, + URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}/products"), + BODY(new AddProductRequest(ProductItem)), + HEADERS(IF_MATCH(0)) + ) + .Then(OK) + .And() + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}")) + .Until(RESPONSE_ETAG_IS(1)) + .Then(OK) + .GetResponseBody(); + + ShoppingCartId = cartDetails.Id; + UnitPrice = cartDetails.ProductItems.Single().UnitPrice; + } + + public Task DisposeAsync() => Task.CompletedTask; } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ApiTest.cs similarity index 85% rename from Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs rename to Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ApiTest.cs index 25dba1b4..46dfe76d 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCartsApplicationFactory.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ApiTest.cs @@ -23,11 +23,6 @@ public class ShoppingCartsApplicationFactory: TestWebApplicationFactory public class ApiFixture: IDisposable { - public ApiFixture() - { - - } - public ApiSpecification API { get; } = ApiSpecification.Setup(new ShoppingCartsApplicationFactory()); @@ -39,10 +34,7 @@ public void Dispose() => public class DatabaseCollection: ICollectionFixture; [Collection("ApiTests")] -public abstract class ApiTest +public abstract class ApiTest(ApiFixture fixture) { - protected readonly ApiSpecification API; - - protected ApiTest(ApiFixture fixture) => - API = fixture.API; + protected readonly ApiSpecification API = fixture.API; } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index a11281fc..0658dcab 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -10,7 +10,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.AddingProduct; using static ShoppingCartsApi; -public class AddProductTests: ApiTest +public class AddProductTests(ApiFixture fixture): ApiTest(fixture) { [Fact] [Trait("Category", "Acceptance")] @@ -43,8 +43,4 @@ public Task Post_Should_AddProductItem_To_ShoppingCart() }) ); } - - public AddProductTests(ApiFixture fixture) : base(fixture) - { - } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index cf2278c7..a73b3d80 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -8,7 +8,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Canceling; -public class CancelShoppingCartTests: ApiTest +public class CancelShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { public readonly Guid ClientId = Guid.NewGuid(); @@ -37,8 +37,4 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => details.ClientId.Should().Be(ClientId); details.Version.Should().Be(1); })); - - public CancelShoppingCartTests(ApiFixture fixture) : base(fixture) - { - } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 28dd791f..9a54fdd1 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -10,7 +10,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Confirming; using static ShoppingCartsApi; -public class ConfirmShoppingCartTests: ApiTest +public class ConfirmShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { private Guid ClientId = Guid.NewGuid(); @@ -41,8 +41,4 @@ await API // API.PublishedExternalEventsOfType(); } - - public ConfirmShoppingCartTests(ApiFixture fixture) : base(fixture) - { - } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index 2a38dc64..4b7c93d3 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -8,7 +8,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.Opening; -public class OpenShoppingCartTests: ApiTest +public class OpenShoppingCartTests(ApiFixture fixture): ApiTest(fixture) { [Fact] public Task Post_ShouldReturn_CreatedStatus_With_CartId() => @@ -37,8 +37,4 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => ); public readonly Guid ClientId = Guid.NewGuid(); - - public OpenShoppingCartTests(ApiFixture fixture) : base(fixture) - { - } } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index 037c946b..19733e37 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -9,7 +9,7 @@ namespace ECommerce.Api.Tests.ShoppingCarts.RemovingProduct; -public class RemoveProductTests: ApiTest, IAsyncLifetime +public class RemoveProductTests(ApiFixture fixture): ApiTest(fixture), IAsyncLifetime { [Fact] [Trait("Category", "Acceptance")] @@ -81,8 +81,4 @@ public async Task InitializeAsync() } public Task DisposeAsync() => Task.CompletedTask; - - public RemoveProductTests(ApiFixture fixture) : base(fixture) - { - } } From 77129e4ca9f537076b8a4ad298419dd5018aa0e0 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 14:10:34 +0200 Subject: [PATCH 14/18] Fixed scope handling in the batch processing --- ...ctionalDbContextEventsBatchCheckpointer.cs | 15 ++++---- ...EventStoreDBSubscriptioToAllCoordinator.cs | 34 +++++++++++-------- .../ECommerce/Storage/ECommerceDbContext.cs | 12 ++++++- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs b/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs index 9ff445a8..aa490dff 100644 --- a/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs +++ b/Core.EntityFramework/Subscriptions/Checkpoints/TransactionalDbContextEventsBatchCheckpointer.cs @@ -24,28 +24,27 @@ public async Task Process( Checkpoint lastCheckpoint, BatchProcessingOptions options, CancellationToken ct - ) + ) { await dbContext.Database.UseTransactionAsync(transaction, ct); var inner = new EventsBatchCheckpointer( new PostgresSubscriptionCheckpointRepository(connection, transaction), batchProcessor ); - var result = await inner.Process(events, lastCheckpoint, options, ct) .ConfigureAwait(false); - await dbContext.SaveChangesAsync(ct); - - if (result is not StoreResult.Success) + if (result is StoreResult.Success) + { + await dbContext.SaveChangesAsync(ct); + await transaction.CommitAsync(ct).ConfigureAwait(false); + } + else { - dbContext.ChangeTracker.Clear(); await transaction.RollbackAsync(ct); return result; } - await transaction.CommitAsync(ct).ConfigureAwait(false); - return result; } } diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs index 2d9f0701..b34c4e38 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs @@ -3,9 +3,11 @@ using Core.EventStoreDB.Subscriptions.Checkpoints; using EventStore.Client; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Polly; namespace Core.EventStoreDB.Subscriptions; +using static ISubscriptionCheckpointRepository; public class SubscriptionInfo { @@ -18,6 +20,7 @@ public class EventStoreDBSubscriptioToAllCoordinator private readonly IDictionary subscriptions; private readonly ISubscriptionStoreSetup storeSetup; private readonly IServiceScopeFactory serviceScopeFactory; + private readonly ILogger logger; private readonly Channel events = Channel.CreateBounded( new BoundedChannelOptions(1) @@ -32,7 +35,8 @@ public class EventStoreDBSubscriptioToAllCoordinator public EventStoreDBSubscriptioToAllCoordinator( IDictionary subscriptions, ISubscriptionStoreSetup storeSetup, - IServiceScopeFactory serviceScopeFactory + IServiceScopeFactory serviceScopeFactory, + ILogger logger ) { this.subscriptions = @@ -41,6 +45,7 @@ IServiceScopeFactory serviceScopeFactory ); this.storeSetup = storeSetup; this.serviceScopeFactory = serviceScopeFactory; + this.logger = logger; } private ChannelReader Reader => events.Reader; @@ -76,22 +81,29 @@ public async Task ProcessMessages(CancellationToken ct) try { - await ProcessBatch(ct, batch).ConfigureAwait(false); + + var subscriptionInfo = subscriptions[batch.SubscriptionId]; + + var result = await ProcessBatch(subscriptionInfo, batch, ct).ConfigureAwait(false); + + if (result is StoreResult.Success success) + { + subscriptionInfo.LastCheckpoint = success.Checkpoint; + Reader.TryRead(out _); + } } catch (Exception exc) { - Console.WriteLine(exc); + logger.LogError(exc, "Error processing batch for: {batch}", batch); } } } - private async Task ProcessBatch(CancellationToken ct, EventBatch batch) + private async Task ProcessBatch(SubscriptionInfo subscriptionInfo, EventBatch batch, CancellationToken ct) { using var scope = serviceScopeFactory.CreateScope(); var checkpointer = scope.ServiceProvider.GetRequiredService(); - var subscriptionInfo = subscriptions[batch.SubscriptionId]; - var result = await checkpointer.Process( batch.Events, subscriptionInfo.LastCheckpoint, @@ -101,15 +113,9 @@ private async Task ProcessBatch(CancellationToken ct, EventBatch batch) subscriptionInfo.Subscription.GetHandlers(scope.ServiceProvider) ), ct - ) - .ConfigureAwait(false); + ).ConfigureAwait(false); - - if (result is ISubscriptionCheckpointRepository.StoreResult.Success success) - { - subscriptionInfo.LastCheckpoint = success.Checkpoint; - Reader.TryRead(out _); - } + return result; } private Task LoadCheckpoint(string subscriptionId, CancellationToken token) => diff --git a/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs b/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs index 0d3a8883..4f9b4ca2 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs @@ -4,12 +4,22 @@ namespace ECommerce.Storage; -public class ECommerceDbContext(DbContextOptions options): DbContext(options) +public class ECommerceDbContext(DbContextOptions options): DbContext(options), IDisposable, IAsyncDisposable { protected override void OnModelCreating(ModelBuilder modelBuilder) { modelBuilder.SetupShoppingCartsReadModels(); } + + void IDisposable.Dispose() + { + base.Dispose(); + } + + ValueTask IAsyncDisposable.DisposeAsync() + { + return base.DisposeAsync(); + } } public class ECommerceDBContextFactory: IDesignTimeDbContextFactory From 99ba1ac8ef959bf1eb625c326c470aa1981689a3 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 15:39:11 +0200 Subject: [PATCH 15/18] Added long polling example --- .../EntityFrameworkProjectionBuilder.cs | 9 +++ Core.EntityFramework/Queries/QueryHandler.cs | 14 ++++- Core.WebApi/Headers/ETagExtensions.cs | 24 ++++++++ .../AddingProduct/AddProductTests.cs | 2 +- .../Canceling/CancelShoppingCartTests.cs | 2 +- .../Confirming/ConfirmShoppingCartTests.cs | 2 +- .../Opening/OpenShoppingCartTests.cs | 2 +- .../RemovingProduct/RemoveProductTests.cs | 2 +- .../Controllers/ShoppingCartsController.cs | 20 ++++-- .../GettingCartById/GetCartById.cs | 61 +++++++++++++++---- .../AddingProduct/AddProductTests.cs | 2 +- .../Canceling/CancelShoppingCartTests.cs | 2 +- .../Confirming/ConfirmShoppingCartTests.cs | 2 +- .../Opening/OpenShoppingCartTests.cs | 2 +- .../RemovingProduct/RemoveProductTests.cs | 4 +- .../Controllers/ShoppingCartsController.cs | 11 ++-- .../ECommerce/ShoppingCarts/Configuration.cs | 2 +- .../GettingCartById/GetCartById.cs | 34 ++++++----- 18 files changed, 147 insertions(+), 50 deletions(-) diff --git a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs index 64158db1..97425ab0 100644 --- a/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs +++ b/Core.EntityFramework/Projections/EntityFrameworkProjectionBuilder.cs @@ -94,6 +94,15 @@ Func, TQuery, CancellationToken, Task> handler return this; } + public EntityFrameworkProjectionBuilder QueryWith( + Func, TQuery, CancellationToken, Task> handler + ) + { + services.AddEntityFrameworkQueryHandler(handler); + + return this; + } + public EntityFrameworkProjectionBuilder QueryWith( Func, TQuery, CancellationToken, Task>> handler ) diff --git a/Core.EntityFramework/Queries/QueryHandler.cs b/Core.EntityFramework/Queries/QueryHandler.cs index 56be50ee..ac191133 100644 --- a/Core.EntityFramework/Queries/QueryHandler.cs +++ b/Core.EntityFramework/Queries/QueryHandler.cs @@ -5,17 +5,27 @@ namespace Core.EntityFramework.Queries; public static class QueryHandler { + public static IServiceCollection AddEntityFrameworkQueryHandler( this IServiceCollection services, Func, TQuery, CancellationToken, Task> handler ) - where TDbContext : DbContext where TResult : class + where TDbContext : DbContext + where TResult : class + => services.AddEntityFrameworkQueryHandler(handler); + + public static IServiceCollection AddEntityFrameworkQueryHandler( + this IServiceCollection services, + Func, TQuery, CancellationToken, Task> handler + ) + where TDbContext : DbContext + where TView : class => services.AddQueryHandler(sp => { var queryable = sp.GetRequiredService() - .Set() + .Set() .AsNoTracking() .AsQueryable(); diff --git a/Core.WebApi/Headers/ETagExtensions.cs b/Core.WebApi/Headers/ETagExtensions.cs index 06043864..2dcd304c 100644 --- a/Core.WebApi/Headers/ETagExtensions.cs +++ b/Core.WebApi/Headers/ETagExtensions.cs @@ -1,5 +1,6 @@ using Core.WebApi.Responses; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Net.Http.Headers; namespace Core.WebApi.Headers; @@ -28,4 +29,27 @@ public static string GetSanitizedValue(this EntityTagHeaderValue eTag) // trim first and last quote characters return value.Substring(1, value.Length - 2); } + + public static int ToExpectedVersion(string? eTag) + { + if (eTag is null) + throw new ArgumentNullException(nameof(eTag)); + + var value = EntityTagHeaderValue.Parse(eTag).Tag.Value; + + if (value is null) + throw new ArgumentNullException(nameof(eTag)); + + return int.Parse(value.Substring(1, value.Length - 2)); + } +} + + +[AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property, AllowMultiple = false, Inherited = true)] +public class FromIfMatchHeaderAttribute: FromHeaderAttribute +{ + public FromIfMatchHeaderAttribute() + { + Name = "If-Match"; + } } diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index 82c1b81e..f7da311b 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -29,7 +29,7 @@ await API ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), 10) .Then( RESPONSE_BODY((details, ctx) => diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index ce738dab..c2f0e1da 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -25,7 +25,7 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 0bd94706..116ac374 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -23,7 +23,7 @@ await API ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index 77063617..24503cdb 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -22,7 +22,7 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)), response => API.Given() - .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}")) + .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}"), HEADERS(IF_MATCH(0))) .Until(RESPONSE_ETAG_IS(0)) .Then( OK, diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index 94ac1772..d3b2fe31 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -73,7 +73,7 @@ public async Task InitializeAsync() ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1)) .Then(OK) .GetResponseBody(); diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Controllers/ShoppingCartsController.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Controllers/ShoppingCartsController.cs index 30eb45b3..af48bde1 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Controllers/ShoppingCartsController.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts.Api/Controllers/ShoppingCartsController.cs @@ -18,6 +18,8 @@ namespace Carts.Api.Controllers; +using static ETagExtensions; + [Route("api/[controller]")] public class ShoppingCartsController( ICommandBus commandBus, @@ -62,9 +64,9 @@ [FromBody] AddProductRequest? request [HttpDelete("{id}/products/{productId}")] public async Task RemoveProduct( Guid id, - [FromRoute]Guid? productId, - [FromQuery]int? quantity, - [FromQuery]decimal? unitPrice + [FromRoute] Guid? productId, + [FromQuery] int? quantity, + [FromQuery] decimal? unitPrice ) { var command = ShoppingCarts.RemovingProduct.RemoveProduct.Create( @@ -108,9 +110,14 @@ public async Task CancelCart(Guid id) } [HttpGet("{id}")] - public async Task Get(Guid id) + public async Task Get( + Guid id, + [FromIfMatchHeader] string? eTag + ) { - var result = await queryBus.Query(GetCartById.Create(id)); + var result = await queryBus.Query( + GetCartById.From(id, eTag != null ? ToExpectedVersion(eTag) : null) + ); Response.TrySetETagResponseHeader(result.Version); @@ -121,7 +128,8 @@ public async Task Get(Guid id) public async Task> Get([FromQuery] int pageNumber = 1, [FromQuery] int pageSize = 20) { - var pagedList = await queryBus.Query>(GetCarts.Create(pageNumber, pageSize)); + var pagedList = + await queryBus.Query>(GetCarts.Create(pageNumber, pageSize)); return pagedList.ToResponse(); } diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs index c5b091a1..e1bf45f5 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs @@ -1,29 +1,68 @@ using Core.Exceptions; using Core.Queries; +using Core.Validation; using Marten; +using Polly; namespace Carts.ShoppingCarts.GettingCartById; public record GetCartById( - Guid CartId + Guid CartId, + int? ExpectedVersion ) { - public static GetCartById Create(Guid? cartId) - { - if (cartId == null || cartId == Guid.Empty) - throw new ArgumentOutOfRangeException(nameof(cartId)); - - return new GetCartById(cartId.Value); - } + public static GetCartById From(Guid cartId, int? expectedVersion) => + new(cartId.NotEmpty(), expectedVersion); } internal class HandleGetCartById(IQuerySession querySession): IQueryHandler { - public async Task Handle(GetCartById request, CancellationToken cancellationToken) + public async Task Handle(GetCartById query, CancellationToken token) { - var cart = await querySession.LoadAsync(request.CartId, cancellationToken); + var expectedVersion = query.ExpectedVersion; - return cart ?? throw AggregateNotFoundException.For(request.CartId); + if (!expectedVersion.HasValue) + return await querySession.LoadAsync(query.CartId, token) + ?? throw AggregateNotFoundException.For(query.CartId); + + return await Policy + .HandleResult(cart => + cart == null || cart.Version < expectedVersion + ) + .WaitAndRetryAsync(5, i => TimeSpan.FromMilliseconds(50 * Math.Pow(i, 2))) + .ExecuteAsync( + ct => querySession.Query() + .SingleOrDefaultAsync(x => x.Id == query.CartId && x.Version >= expectedVersion, ct), token) + ?? throw AggregateNotFoundException.For(query.CartId); } } + + +// public record GetCartById( +// Guid ShoppingCartId, +// int? ExpectedVersion +// ) +// { +// public static GetCartById From(Guid cartId, int? expectedVersion) => +// new(cartId.NotEmpty(), expectedVersion); +// +// public static Task Handle( +// IQueryable shoppingCarts, +// GetCartById query, +// CancellationToken token +// ) +// { +// var expectedVersion = query.ExpectedVersion; +// +// if (!expectedVersion.HasValue) +// return shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId, token); +// +// return Policy +// .HandleResult(cart => +// cart == null || cart.Version < expectedVersion +// ) +// .WaitAndRetryAsync(5, i => TimeSpan.FromMilliseconds(50 * Math.Pow(i, 2))) +// .ExecuteAsync(ct => shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId && x.Version >= expectedVersion, ct), token); +// } +// } diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs index 0658dcab..8fb781e0 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/AddingProduct/AddProductTests.cs @@ -29,7 +29,7 @@ public Task Post_Should_AddProductItem_To_ShoppingCart() ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), 10) .Then( RESPONSE_BODY((details, ctx) => diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs index a73b3d80..b75745a7 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Canceling/CancelShoppingCartTests.cs @@ -25,7 +25,7 @@ public Task Delete_Should_Return_OK_And_Cancel_Shopping_Cart() => ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), 10) .Then( OK, diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs index 9a54fdd1..0c46a394 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Confirming/ConfirmShoppingCartTests.cs @@ -26,7 +26,7 @@ await API HEADERS(IF_MATCH(0)) ) .Then(OK) - .AndWhen(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}")) + .AndWhen(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.OpenedShoppingCartId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), maxNumberOfRetries: 10) .Then( OK, diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs index 4b7c93d3..5f5180be 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/Opening/OpenShoppingCartTests.cs @@ -22,7 +22,7 @@ public Task Post_ShouldReturn_CreatedStatus_With_CartId() => .Then(CREATED_WITH_DEFAULT_HEADERS(eTag: 0)), response => API.Given() - .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}")) + .When(GET, URI($"/api/ShoppingCarts/{response.GetCreatedId()}"), HEADERS(IF_MATCH(0))) .Until(RESPONSE_ETAG_IS(0), 10) .Then( OK, diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs index 19733e37..7d3d7032 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api.Tests/ShoppingCarts/RemovingProduct/RemoveProductTests.cs @@ -25,7 +25,7 @@ await API ) .Then(NO_CONTENT) .And() - .When(GET, URI($"/api/ShoppingCarts/{ShoppingCartId}")) + .When(GET, URI($"/api/ShoppingCarts/{ShoppingCartId}"), HEADERS(IF_MATCH(2))) .Until(RESPONSE_ETAG_IS(2), maxNumberOfRetries: 10) .Then( OK, @@ -71,7 +71,7 @@ public async Task InitializeAsync() ) .Then(OK) .And() - .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}")) + .When(GET, URI(ctx => $"/api/ShoppingCarts/{ctx.GetCreatedId()}"), HEADERS(IF_MATCH(1))) .Until(RESPONSE_ETAG_IS(1), maxNumberOfRetries: 10) .Then(OK) .GetResponseBody(); diff --git a/Sample/EventStoreDB/Simple/ECommerce.Api/Controllers/ShoppingCartsController.cs b/Sample/EventStoreDB/Simple/ECommerce.Api/Controllers/ShoppingCartsController.cs index 54ba11b3..68fa1959 100644 --- a/Sample/EventStoreDB/Simple/ECommerce.Api/Controllers/ShoppingCartsController.cs +++ b/Sample/EventStoreDB/Simple/ECommerce.Api/Controllers/ShoppingCartsController.cs @@ -12,6 +12,8 @@ namespace ECommerce.Api.Controllers; +using static ETagExtensions; + [Route("api/[controller]")] public class ShoppingCartsController: Controller { @@ -66,9 +68,9 @@ CancellationToken ct public async Task RemoveProduct( [FromServices] Func handle, Guid id, - [FromRoute]Guid? productId, - [FromQuery]int? quantity, - [FromQuery]decimal? unitPrice, + [FromRoute] Guid? productId, + [FromQuery] int? quantity, + [FromQuery] decimal? unitPrice, CancellationToken ct ) { @@ -120,10 +122,11 @@ CancellationToken ct public async Task Get( [FromServices] Func> query, Guid id, + [FromIfMatchHeader] string? eTag, CancellationToken ct ) { - var result = await query(GetCartById.From(id), ct); + var result = await query(GetCartById.From(id, eTag != null ? ToExpectedVersion(eTag) : null), ct); if (result == null) return NotFound(); diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs index ba732a8a..1cab5917 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/Configuration.cs @@ -70,7 +70,7 @@ public static IServiceCollection AddShoppingCartsModule(this IServiceCollection ShoppingCartDetailsProjection.Handle ) .Include(x => x.ProductItems) - .QueryWith(GetCartById.Handle) + .QueryWith(GetCartById.Handle) ) .For( builder => builder diff --git a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/GetCartById.cs b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/GetCartById.cs index f29ef7c8..3147e1a8 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/GetCartById.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/ShoppingCarts/GettingCartById/GetCartById.cs @@ -1,29 +1,33 @@ -using Core.Exceptions; +using Core.Validation; using Microsoft.EntityFrameworkCore; +using Polly; namespace ECommerce.ShoppingCarts.GettingCartById; public record GetCartById( - Guid ShoppingCartId + Guid ShoppingCartId, + int? ExpectedVersion ) { - public static GetCartById From(Guid cartId) - { - if (cartId == Guid.Empty) - throw new ArgumentOutOfRangeException(nameof(cartId)); - - return new GetCartById(cartId); - } + public static GetCartById From(Guid cartId, int? expectedVersion) => + new(cartId.NotEmpty(), expectedVersion); - public static async Task Handle( + public static Task Handle( IQueryable shoppingCarts, GetCartById query, - CancellationToken ct + CancellationToken token ) { - return await shoppingCarts - .SingleOrDefaultAsync( - x => x.Id == query.ShoppingCartId, ct - ) ?? throw AggregateNotFoundException.For(query.ShoppingCartId); + var expectedVersion = query.ExpectedVersion; + + if (!expectedVersion.HasValue) + return shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId, token); + + return Policy + .HandleResult(cart => + cart == null || cart.Version < expectedVersion + ) + .WaitAndRetryAsync(10, i => TimeSpan.FromMilliseconds(50 * Math.Pow(i, 2))) + .ExecuteAsync(ct => shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId && x.Version >= expectedVersion, ct), token); } } From 4e5b32cc10824325827014c0448989b85d6d12d8 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 17:13:20 +0200 Subject: [PATCH 16/18] Added Polly retry policy to SubscribeToAll --- .../EventStoreDBSubscriptionToAll.cs | 112 +++++++++--------- Core/Extensions/AsyncEnumerableExtensions.cs | 2 +- 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index e0da24dd..9549ebd6 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Open.ChannelExtensions; using Polly; +using Polly.Wrap; using EventTypeFilter = EventStore.Client.EventTypeFilter; namespace Core.EventStoreDB.Subscriptions; @@ -53,66 +54,67 @@ public enum ProcessingStatus public async Task SubscribeToAll(Checkpoint checkpoint, ChannelWriter cw, CancellationToken ct) { Status = ProcessingStatus.Starting; - // see: https://github.com/dotnet/runtime/issues/36063 - await Task.Yield(); logger.LogInformation("Subscription to all '{SubscriptionId}'", Options.SubscriptionId); - try - { - var subscription = eventStoreClient.SubscribeToAll( - checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, - Options.ResolveLinkTos, - Options.FilterOptions, - Options.Credentials, - ct - ); - - Status = ProcessingStatus.Started; - - logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); - - - await subscription.Pipe( - cw, - events => new EventBatch(Options.SubscriptionId, events.ToArray()), - Options.BatchSize, - Options.BatchDeadline, - ct - ).ConfigureAwait(false); - } - catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || - rpcException.InnerException is ObjectDisposedException) - { - logger.LogWarning( - "Subscription to all '{SubscriptionId}' dropped by client", - SubscriptionId - ); - } - catch (OperationCanceledException) - { - logger.LogWarning( - "Subscription to all '{SubscriptionId}' dropped by client", - SubscriptionId - ); - } - catch (Exception ex) - { - Status = ProcessingStatus.Errored; - logger.LogWarning("Subscription was dropped: {Exception}", ex); + await RetryPolicy.ExecuteAsync(token => + OnSubscribe(checkpoint, cw, token), ct + ).ConfigureAwait(false); + } - // Sleep between reconnections to not flood the database or not kill the CPU with infinite loop - // Randomness added to reduce the chance of multiple subscriptions trying to reconnect at the same time - Thread.Sleep(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)); + private async Task OnSubscribe(Checkpoint checkpoint, ChannelWriter cw, CancellationToken ct) + { + var subscription = eventStoreClient.SubscribeToAll( + checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, + Options.ResolveLinkTos, + Options.FilterOptions, + Options.Credentials, + ct + ); + + Status = ProcessingStatus.Started; + + logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); + + await subscription.Pipe( + cw, + events => new EventBatch(Options.SubscriptionId, events.ToArray()), + Options.BatchSize, + Options.BatchDeadline, + ct + ).ConfigureAwait(false); + } - await SubscribeToAll(checkpoint, cw, ct).ConfigureAwait(false); + private AsyncPolicyWrap RetryPolicy + { + get + { + var generalPolicy = Policy.Handle(ex => !IsCancelledByUser(ex)) + .WaitAndRetryForeverAsync( + sleepDurationProvider: _ => + TimeSpan.FromMilliseconds(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)), + onRetry: (exception, _, _) => + logger.LogWarning("Subscription was dropped: {Exception}", exception) + ); + + var fallbackPolicy = Policy.Handle() + .Or(IsCancelledByUser) + .FallbackAsync(_ => + { + logger.LogWarning("Subscription to all '{SubscriptionId}' dropped by client", SubscriptionId); + return Task.CompletedTask; + } + ); + + return Policy.WrapAsync(generalPolicy, fallbackPolicy); } } - // - // private AsyncPolicy retry = Policy.Handle(rpcException => - // rpcException is { StatusCode: StatusCode.Cancelled } || - // rpcException.InnerException is ObjectDisposedException) - // .Or() - // .F - // .WaitAndRetryForeverAsync(_ => TimeSpan.FromMilliseconds(500)); + + private static bool IsCancelledByUser(RpcException rpcException) => + rpcException.StatusCode == StatusCode.Cancelled + || rpcException.InnerException is ObjectDisposedException; + + private static bool IsCancelledByUser(Exception exception) => + exception is OperationCanceledException + || exception is RpcException rpcException && IsCancelledByUser(rpcException); } diff --git a/Core/Extensions/AsyncEnumerableExtensions.cs b/Core/Extensions/AsyncEnumerableExtensions.cs index eeee3cb5..a587ea39 100644 --- a/Core/Extensions/AsyncEnumerableExtensions.cs +++ b/Core/Extensions/AsyncEnumerableExtensions.cs @@ -26,7 +26,7 @@ CancellationToken ct await cw.WriteAsync(transform(batch), ct).ConfigureAwait(false); return batch; - }); + }, cancellationToken: ct); await foreach (var @event in enumerable.WithCancellation(ct)) { From e6e870ae80629a633eaf53e5c34f3e4267ad4bd6 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 18:50:48 +0200 Subject: [PATCH 17/18] It appeared that there's no need (for now) for Channels besides the extensions as the issue was with wrongly disposed scope --- Core.EventStoreDB/Config.cs | 7 +- .../Batch/EventsBatchCheckpointer.cs | 2 +- .../Batch/EventsBatchProcessor.cs | 2 +- ...EventStoreDBSubscriptioToAllCoordinator.cs | 131 ------------------ .../EventStoreDBSubscriptionToAll.cs | 58 ++++++-- ...entStoreDBSubscriptionsToAllCoordinator.cs | 39 ++++++ .../SimpleEventStoreDBSubscriptionToAll.cs | 101 ++++++++++++++ Core/Extensions/AsyncEnumerableExtensions.cs | 38 +++++ 8 files changed, 229 insertions(+), 149 deletions(-) delete mode 100644 Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs create mode 100644 Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionsToAllCoordinator.cs create mode 100644 Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs diff --git a/Core.EventStoreDB/Config.cs b/Core.EventStoreDB/Config.cs index 933df9a4..fafc4101 100644 --- a/Core.EventStoreDB/Config.cs +++ b/Core.EventStoreDB/Config.cs @@ -59,11 +59,11 @@ public static IServiceCollection AddEventStoreDB( var logger = serviceProvider.GetRequiredService>(); - var coordinator = serviceProvider.GetRequiredService(); + var coordinator = serviceProvider.GetRequiredService(); TelemetryPropagator.UseDefaultCompositeTextMapPropagator(); - return new BackgroundWorker( + return new BackgroundWorker( coordinator, logger, (c, ct) => c.SubscribeToAll(ct) @@ -95,7 +95,7 @@ public static IServiceCollection AddEventStoreDBSubscriptionToAll( Func handlers ) { - services.AddSingleton(); + services.AddSingleton(); return services.AddKeyedSingleton( subscriptionOptions.SubscriptionId, @@ -103,6 +103,7 @@ Func handlers { var subscription = new EventStoreDBSubscriptionToAll( sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService>() ) { Options = subscriptionOptions, GetHandlers = handlers }; diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs index e6245b1e..fc5b7fba 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchCheckpointer.cs @@ -32,7 +32,7 @@ CancellationToken ct if (!lastPosition.HasValue) return new StoreResult.Ignored(); - await eventsBatchProcessor.HandleEventsBatch(events, options, ct) + await eventsBatchProcessor.Handle(events, options, ct) .ConfigureAwait(false); return await checkpointRepository diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index 3c6b375f..71ba4a39 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -17,7 +17,7 @@ public class EventsBatchProcessor( ILogger logger ) { - public async Task HandleEventsBatch( + public async Task Handle( ResolvedEvent[] resolvedEvents, BatchProcessingOptions options, CancellationToken ct diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs deleted file mode 100644 index b34c4e38..00000000 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptioToAllCoordinator.cs +++ /dev/null @@ -1,131 +0,0 @@ -using System.Threading.Channels; -using Core.EventStoreDB.Subscriptions.Batch; -using Core.EventStoreDB.Subscriptions.Checkpoints; -using EventStore.Client; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Polly; - -namespace Core.EventStoreDB.Subscriptions; -using static ISubscriptionCheckpointRepository; - -public class SubscriptionInfo -{ - public required EventStoreDBSubscriptionToAll Subscription { get; set; } - public required Checkpoint LastCheckpoint { get; set; } -}; - -public class EventStoreDBSubscriptioToAllCoordinator -{ - private readonly IDictionary subscriptions; - private readonly ISubscriptionStoreSetup storeSetup; - private readonly IServiceScopeFactory serviceScopeFactory; - private readonly ILogger logger; - - private readonly Channel events = Channel.CreateBounded( - new BoundedChannelOptions(1) - { - SingleWriter = false, - SingleReader = true, - AllowSynchronousContinuations = false, - FullMode = BoundedChannelFullMode.Wait - } - ); - - public EventStoreDBSubscriptioToAllCoordinator( - IDictionary subscriptions, - ISubscriptionStoreSetup storeSetup, - IServiceScopeFactory serviceScopeFactory, - ILogger logger - ) - { - this.subscriptions = - subscriptions.ToDictionary(ks => ks.Key, - vs => new SubscriptionInfo { Subscription = vs.Value, LastCheckpoint = Checkpoint.None } - ); - this.storeSetup = storeSetup; - this.serviceScopeFactory = serviceScopeFactory; - this.logger = logger; - } - - private ChannelReader Reader => events.Reader; - private ChannelWriter Writer => events.Writer; - - public async Task SubscribeToAll(CancellationToken ct) - { - // see: https://github.com/dotnet/runtime/issues/36063 - await Task.Yield(); - - await storeSetup.EnsureStoreExists(ct).ConfigureAwait(false); - - var tasks = subscriptions.Select(s => Task.Run(async () => - { - var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); - var token = cts.Token; - - var checkpoint = await LoadCheckpoint(s.Key, token).ConfigureAwait(false); - subscriptions[s.Key].LastCheckpoint = checkpoint; - - await s.Value.Subscription.SubscribeToAll(checkpoint, Writer, token).ConfigureAwait(false); - }, ct)).ToList(); - var process = ProcessMessages(ct); - - await Task.WhenAll([process, ..tasks]).ConfigureAwait(false); - } - - public async Task ProcessMessages(CancellationToken ct) - { - while (!Reader.Completion.IsCompleted && await Reader.WaitToReadAsync(ct).ConfigureAwait(false)) - { - if (!Reader.TryPeek(out var batch)) continue; - - try - { - - var subscriptionInfo = subscriptions[batch.SubscriptionId]; - - var result = await ProcessBatch(subscriptionInfo, batch, ct).ConfigureAwait(false); - - if (result is StoreResult.Success success) - { - subscriptionInfo.LastCheckpoint = success.Checkpoint; - Reader.TryRead(out _); - } - } - catch (Exception exc) - { - logger.LogError(exc, "Error processing batch for: {batch}", batch); - } - } - } - - private async Task ProcessBatch(SubscriptionInfo subscriptionInfo, EventBatch batch, CancellationToken ct) - { - using var scope = serviceScopeFactory.CreateScope(); - var checkpointer = scope.ServiceProvider.GetRequiredService(); - - var result = await checkpointer.Process( - batch.Events, - subscriptionInfo.LastCheckpoint, - new BatchProcessingOptions( - batch.SubscriptionId, - subscriptionInfo.Subscription.Options.IgnoreDeserializationErrors, - subscriptionInfo.Subscription.GetHandlers(scope.ServiceProvider) - ), - ct - ).ConfigureAwait(false); - - return result; - } - - private Task LoadCheckpoint(string subscriptionId, CancellationToken token) => - Policy.Handle().RetryAsync(3) - .ExecuteAsync(async ct => - { - using var scope = serviceScopeFactory.CreateScope(); - return await scope.ServiceProvider.GetRequiredService() - .Load(subscriptionId, ct).ConfigureAwait(false); - }, token); -} - -public record EventBatch(string SubscriptionId, ResolvedEvent[] Events); diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 9549ebd6..9c04a1c7 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -1,16 +1,16 @@ -using System.Threading.Channels; using Core.Events; +using Core.EventStoreDB.Subscriptions.Batch; using Core.EventStoreDB.Subscriptions.Checkpoints; using Core.Extensions; using EventStore.Client; using Grpc.Core; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Open.ChannelExtensions; using Polly; using Polly.Wrap; -using EventTypeFilter = EventStore.Client.EventTypeFilter; namespace Core.EventStoreDB.Subscriptions; +using static ISubscriptionCheckpointRepository; public class EventStoreDBSubscriptionToAllOptions { @@ -30,6 +30,7 @@ public class EventStoreDBSubscriptionToAllOptions public class EventStoreDBSubscriptionToAll( EventStoreClient eventStoreClient, + IServiceScopeFactory serviceScopeFactory, ILogger logger ) { @@ -43,6 +44,8 @@ public enum ProcessingStatus Stopped } + public record EventBatch(string SubscriptionId, ResolvedEvent[] Events); + public EventStoreDBSubscriptionToAllOptions Options { get; set; } = default!; public Func GetHandlers { get; set; } = default!; @@ -51,18 +54,18 @@ public enum ProcessingStatus private string SubscriptionId => Options.SubscriptionId; - public async Task SubscribeToAll(Checkpoint checkpoint, ChannelWriter cw, CancellationToken ct) + public async Task SubscribeToAll(Checkpoint checkpoint, CancellationToken ct) { Status = ProcessingStatus.Starting; logger.LogInformation("Subscription to all '{SubscriptionId}'", Options.SubscriptionId); await RetryPolicy.ExecuteAsync(token => - OnSubscribe(checkpoint, cw, token), ct + OnSubscribe(checkpoint, token), ct ).ConfigureAwait(false); } - private async Task OnSubscribe(Checkpoint checkpoint, ChannelWriter cw, CancellationToken ct) + private async Task OnSubscribe(Checkpoint checkpoint, CancellationToken ct) { var subscription = eventStoreClient.SubscribeToAll( checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, @@ -76,13 +79,41 @@ private async Task OnSubscribe(Checkpoint checkpoint, ChannelWriter logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); - await subscription.Pipe( - cw, - events => new EventBatch(Options.SubscriptionId, events.ToArray()), - Options.BatchSize, - Options.BatchDeadline, - ct - ).ConfigureAwait(false); + await foreach(var events in subscription.Batch(Options.BatchSize, Options.BatchDeadline, ct)) + { + var batch = new EventBatch(Options.SubscriptionId, events.ToArray()); + var result = await ProcessBatch(batch, checkpoint, ct).ConfigureAwait(false); + + if (result is StoreResult.Success success) + { + checkpoint = success.Checkpoint; + } + } + } + + private async Task ProcessBatch(EventBatch batch, Checkpoint lastCheckpoint, CancellationToken ct) + { + try + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + var checkpointer = scope.ServiceProvider.GetRequiredService(); + + return await checkpointer.Process( + batch.Events, + lastCheckpoint, + new BatchProcessingOptions( + batch.SubscriptionId, + Options.IgnoreDeserializationErrors, + GetHandlers(scope.ServiceProvider) + ), + ct + ).ConfigureAwait(false); + } + catch (Exception exc) + { + logger.LogError(exc, "Error while handling batch"); + throw; + } } private AsyncPolicyWrap RetryPolicy @@ -117,4 +148,5 @@ private static bool IsCancelledByUser(RpcException rpcException) => private static bool IsCancelledByUser(Exception exception) => exception is OperationCanceledException || exception is RpcException rpcException && IsCancelledByUser(rpcException); + } diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionsToAllCoordinator.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionsToAllCoordinator.cs new file mode 100644 index 00000000..3cc08c55 --- /dev/null +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionsToAllCoordinator.cs @@ -0,0 +1,39 @@ +using Core.EventStoreDB.Subscriptions.Checkpoints; +using EventStore.Client; +using Microsoft.Extensions.DependencyInjection; +using Polly; + +namespace Core.EventStoreDB.Subscriptions; + +public class EventStoreDBSubscriptionsToAllCoordinator( + IDictionary subscriptions, + ISubscriptionStoreSetup checkpointStoreSetup, + IServiceScopeFactory serviceScopeFactory +) +{ + public async Task SubscribeToAll(CancellationToken ct) + { + await checkpointStoreSetup.EnsureStoreExists(ct).ConfigureAwait(false); + + var tasks = subscriptions.Select(s => Task.Run(async () => + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var token = cts.Token; + + var checkpoint = await LoadCheckpoint(s.Key, token).ConfigureAwait(false); + + await s.Value.SubscribeToAll(checkpoint, ct).ConfigureAwait(false); + }, ct)).ToArray(); + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + private Task LoadCheckpoint(string subscriptionId, CancellationToken token) => + Policy.Handle().RetryAsync(3) + .ExecuteAsync(async ct => + { + using var scope = serviceScopeFactory.CreateScope(); + return await scope.ServiceProvider.GetRequiredService() + .Load(subscriptionId, ct).ConfigureAwait(false); + }, token); +} diff --git a/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs new file mode 100644 index 00000000..ab1600d8 --- /dev/null +++ b/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs @@ -0,0 +1,101 @@ +// using Core.EventStoreDB.Subscriptions.Batch; +// using Core.EventStoreDB.Subscriptions.Checkpoints; +// using Core.Extensions; +// using EventStore.Client; +// using Grpc.Core; +// using Microsoft.Extensions.DependencyInjection; +// using Microsoft.Extensions.Logging; +// using EventTypeFilter = EventStore.Client.EventTypeFilter; +// +// namespace Core.EventStoreDB.Subscriptions; +// +// public class SimpleEventStoreDBSubscriptionToAllOptions +// { +// public required string SubscriptionId { get; init; } +// +// public SubscriptionFilterOptions FilterOptions { get; set; } = +// new(EventTypeFilter.ExcludeSystemEvents()); +// +// public Action? ConfigureOperation { get; set; } +// public UserCredentials? Credentials { get; set; } +// public bool ResolveLinkTos { get; set; } +// public bool IgnoreDeserializationErrors { get; set; } = true; +// +// public int BatchSize { get; set; } = 1; +// } +// +// public class SimpleEventStoreDBSubscriptionToAll( +// EventStoreClient eventStoreClient, +// IServiceProvider serviceProvider, +// ILogger logger +// ) +// { +// private SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions = default!; +// private string SubscriptionId => subscriptionOptions.SubscriptionId; +// +// public async Task SubscribeToAll(SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions, CancellationToken ct) +// { +// // see: https://github.com/dotnet/runtime/issues/36063 +// await Task.Yield(); +// +// this.subscriptionOptions = subscriptionOptions; +// +// logger.LogInformation("Subscription to all '{SubscriptionId}'", subscriptionOptions.SubscriptionId); +// +// try +// { +// var checkpoint = await LoadCheckpoint(ct).ConfigureAwait(false); +// +// var subscription = eventStoreClient.SubscribeToAll( +// checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, +// subscriptionOptions.ResolveLinkTos, +// subscriptionOptions.FilterOptions, +// subscriptionOptions.Credentials, +// ct +// ); +// +// logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); +// +// await foreach (var events in subscription.Batch(subscriptionOptions.BatchSize, ct).ConfigureAwait(false)) +// { +// checkpoint = await ProcessBatch(events, checkpoint, subscriptionOptions, ct).ConfigureAwait(false); +// } +// } +// catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || +// rpcException.InnerException is ObjectDisposedException) +// { +// logger.LogWarning( +// "Subscription to all '{SubscriptionId}' dropped by client", +// SubscriptionId +// ); +// } +// catch (Exception ex) +// { +// logger.LogWarning("Subscription was dropped: {Exception}", ex); +// +// // Sleep between reconnections to not flood the database or not kill the CPU with infinite loop +// // Randomness added to reduce the chance of multiple subscriptions trying to reconnect at the same time +// Thread.Sleep(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)); +// +// await SubscribeToAll(this.subscriptionOptions, ct).ConfigureAwait(false); +// } +// } +// +// private ValueTask LoadCheckpoint(CancellationToken ct) +// { +// using var scope = serviceProvider.CreateScope(); +// return scope.ServiceProvider.GetRequiredService().Load(SubscriptionId, ct); +// } +// +// private Task ProcessBatch( +// ResolvedEvent[] events, +// Checkpoint lastCheckpoint, +// SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions, +// CancellationToken ct) +// { +// using var scope = serviceProvider.CreateScope(); +// return scope.ServiceProvider.GetRequiredService() +// .Process(events, lastCheckpoint, subscriptionOptions, ct); +// } +// } +// diff --git a/Core/Extensions/AsyncEnumerableExtensions.cs b/Core/Extensions/AsyncEnumerableExtensions.cs index a587ea39..82d8d3da 100644 --- a/Core/Extensions/AsyncEnumerableExtensions.cs +++ b/Core/Extensions/AsyncEnumerableExtensions.cs @@ -33,4 +33,42 @@ CancellationToken ct await channel.Writer.WriteAsync(@event, ct).ConfigureAwait(false); } } + + public static IAsyncEnumerable> Batch( + this IAsyncEnumerable enumerable, + int batchSize, + int timeout, + CancellationToken ct + ) + { + var channel = Channel.CreateUnbounded( + new UnboundedChannelOptions + { + SingleWriter = false, SingleReader = true, AllowSynchronousContinuations = false + } + ); + + // Start the producer in the background + _ = ProduceAsync(enumerable, channel, ct).ContinueWith(async t => + { + if (t.IsFaulted) + { + await channel.CompleteAsync(t.Exception).ConfigureAwait(false); + } + }, ct, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Current); + + + return channel.Reader.Batch(batchSize).WithTimeout(timeout).AsAsyncEnumerable(cancellationToken: ct); + } + + + private static async Task ProduceAsync(this IAsyncEnumerable enumerable, Channel channel, CancellationToken ct) + { + await foreach (var @event in enumerable.WithCancellation(ct)) + { + await channel.Writer.WriteAsync(@event, ct).ConfigureAwait(false); + } + + await channel.CompleteAsync().ConfigureAwait(false); + } } From 45583feaf151f2940d42546c51f635063d3c57e4 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 22 May 2024 19:05:15 +0200 Subject: [PATCH 18/18] Cleaned up leftovers after the EventStoreDB refactorings --- .../EntityFrameworkProjectionTests.cs | 2 - Core.EntityFramework.Tests/TestDbContext.cs | 2 - .../Batch/EventsBatchProcessor.cs | 1 - ...ostgresSubscriptionCheckpointRepository.cs | 39 ------- .../EventStoreDBSubscriptionToAll.cs | 1 + .../SimpleEventStoreDBSubscriptionToAll.cs | 101 ------------------ .../GettingCartById/GetCartById.cs | 29 ----- .../ECommerce/Storage/ECommerceDbContext.cs | 10 -- 8 files changed, 1 insertion(+), 184 deletions(-) delete mode 100644 Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs diff --git a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs index 431350d7..3689a440 100644 --- a/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs +++ b/Core.EntityFramework.Tests/EntityFrameworkProjectionTests.cs @@ -91,8 +91,6 @@ await projection.Handle([ savedEntity.Should().BeEquivalentTo(new ShoppingCart { Id = cartId, ClientId = clientId, ProductCount = 10 }); } - - [Fact] public async Task Applies_Works_In_Batch_With_AddAndDeleteOnTheSameRecord() { diff --git a/Core.EntityFramework.Tests/TestDbContext.cs b/Core.EntityFramework.Tests/TestDbContext.cs index ecb88011..62cc3347 100644 --- a/Core.EntityFramework.Tests/TestDbContext.cs +++ b/Core.EntityFramework.Tests/TestDbContext.cs @@ -19,8 +19,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) => modelBuilder.Entity(); } - - public class TestDbContextFactory: IDesignTimeDbContextFactory { public TestDbContext CreateDbContext(params string[] args) diff --git a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs index 71ba4a39..c1d6f828 100644 --- a/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs +++ b/Core.EventStoreDB/Subscriptions/Batch/EventsBatchProcessor.cs @@ -27,7 +27,6 @@ CancellationToken ct foreach (var batchHandler in options.BatchHandlers) { - // TODO: How would you implement Dead-Letter Queue here? await batchHandler.Handle(events, ct).ConfigureAwait(false); } } diff --git a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs index e38ead06..0d8e03f6 100644 --- a/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs +++ b/Core.EventStoreDB/Subscriptions/Checkpoints/Postgres/PostgresSubscriptionCheckpointRepository.cs @@ -6,45 +6,6 @@ namespace Core.EventStoreDB.Subscriptions.Checkpoints.Postgres; using static ISubscriptionCheckpointRepository; -// public class PostgresConnectionProvider(Func> connectionFactory) -// { -// public ValueTask Get(CancellationToken ct) => connectionFactory(ct); -// -// public void Set(NpgsqlConnection connection) => -// connectionFactory = _ => ValueTask.FromResult(connection); -// -// public void Set(NpgsqlTransaction transaction) => -// connectionFactory = _ => ValueTask.FromResult(transaction.Connection!); -// -// public void Set(NpgsqlDataSource dataSource) => -// connectionFactory = async ct => -// { -// var connection = dataSource.CreateConnection(); -// await connection.OpenAsync(ct).ConfigureAwait(false); -// return connection; -// }; -// -// public static PostgresConnectionProvider From(NpgsqlDataSource npgsqlDataSource) => -// new(async ct => -// { -// var connection = npgsqlDataSource.CreateConnection(); -// await connection.OpenAsync(ct).ConfigureAwait(false); -// return connection; -// }); -// -// public static PostgresConnectionProvider From(NpgsqlTransaction transaction) => -// new(async ct => -// { -// if (transaction.Connection == null) -// throw new InvalidOperationException("Transaction connection is not opened!"); -// -// if (transaction.Connection.State == ConnectionState.Closed) -// await transaction.Connection.OpenAsync(ct).ConfigureAwait(false); -// -// return transaction.Connection; -// }); -// } - public class PostgresSubscriptionCheckpointRepository( // I'm not using data source here, as I'd like to enable option // to update projection in the same transaction in the same transaction as checkpointing diff --git a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs index 9c04a1c7..b8ca0de0 100644 --- a/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs +++ b/Core.EventStoreDB/Subscriptions/EventStoreDBSubscriptionToAll.cs @@ -111,6 +111,7 @@ private async Task ProcessBatch(EventBatch batch, Checkpoint lastCh } catch (Exception exc) { + // TODO: How would you implement Dead-Letter Queue here? logger.LogError(exc, "Error while handling batch"); throw; } diff --git a/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs b/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs deleted file mode 100644 index ab1600d8..00000000 --- a/Core.EventStoreDB/Subscriptions/SimpleEventStoreDBSubscriptionToAll.cs +++ /dev/null @@ -1,101 +0,0 @@ -// using Core.EventStoreDB.Subscriptions.Batch; -// using Core.EventStoreDB.Subscriptions.Checkpoints; -// using Core.Extensions; -// using EventStore.Client; -// using Grpc.Core; -// using Microsoft.Extensions.DependencyInjection; -// using Microsoft.Extensions.Logging; -// using EventTypeFilter = EventStore.Client.EventTypeFilter; -// -// namespace Core.EventStoreDB.Subscriptions; -// -// public class SimpleEventStoreDBSubscriptionToAllOptions -// { -// public required string SubscriptionId { get; init; } -// -// public SubscriptionFilterOptions FilterOptions { get; set; } = -// new(EventTypeFilter.ExcludeSystemEvents()); -// -// public Action? ConfigureOperation { get; set; } -// public UserCredentials? Credentials { get; set; } -// public bool ResolveLinkTos { get; set; } -// public bool IgnoreDeserializationErrors { get; set; } = true; -// -// public int BatchSize { get; set; } = 1; -// } -// -// public class SimpleEventStoreDBSubscriptionToAll( -// EventStoreClient eventStoreClient, -// IServiceProvider serviceProvider, -// ILogger logger -// ) -// { -// private SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions = default!; -// private string SubscriptionId => subscriptionOptions.SubscriptionId; -// -// public async Task SubscribeToAll(SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions, CancellationToken ct) -// { -// // see: https://github.com/dotnet/runtime/issues/36063 -// await Task.Yield(); -// -// this.subscriptionOptions = subscriptionOptions; -// -// logger.LogInformation("Subscription to all '{SubscriptionId}'", subscriptionOptions.SubscriptionId); -// -// try -// { -// var checkpoint = await LoadCheckpoint(ct).ConfigureAwait(false); -// -// var subscription = eventStoreClient.SubscribeToAll( -// checkpoint != Checkpoint.None ? FromAll.After(checkpoint) : FromAll.Start, -// subscriptionOptions.ResolveLinkTos, -// subscriptionOptions.FilterOptions, -// subscriptionOptions.Credentials, -// ct -// ); -// -// logger.LogInformation("Subscription to all '{SubscriptionId}' started", SubscriptionId); -// -// await foreach (var events in subscription.Batch(subscriptionOptions.BatchSize, ct).ConfigureAwait(false)) -// { -// checkpoint = await ProcessBatch(events, checkpoint, subscriptionOptions, ct).ConfigureAwait(false); -// } -// } -// catch (RpcException rpcException) when (rpcException is { StatusCode: StatusCode.Cancelled } || -// rpcException.InnerException is ObjectDisposedException) -// { -// logger.LogWarning( -// "Subscription to all '{SubscriptionId}' dropped by client", -// SubscriptionId -// ); -// } -// catch (Exception ex) -// { -// logger.LogWarning("Subscription was dropped: {Exception}", ex); -// -// // Sleep between reconnections to not flood the database or not kill the CPU with infinite loop -// // Randomness added to reduce the chance of multiple subscriptions trying to reconnect at the same time -// Thread.Sleep(1000 + new Random((int)DateTime.UtcNow.Ticks).Next(1000)); -// -// await SubscribeToAll(this.subscriptionOptions, ct).ConfigureAwait(false); -// } -// } -// -// private ValueTask LoadCheckpoint(CancellationToken ct) -// { -// using var scope = serviceProvider.CreateScope(); -// return scope.ServiceProvider.GetRequiredService().Load(SubscriptionId, ct); -// } -// -// private Task ProcessBatch( -// ResolvedEvent[] events, -// Checkpoint lastCheckpoint, -// SimpleEventStoreDBSubscriptionToAllOptions subscriptionOptions, -// CancellationToken ct) -// { -// using var scope = serviceProvider.CreateScope(); -// return scope.ServiceProvider.GetRequiredService() -// .Process(events, lastCheckpoint, subscriptionOptions, ct); -// } -// } -// diff --git a/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs b/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs index e1bf45f5..d3dffeea 100644 --- a/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs +++ b/Sample/EventStoreDB/ECommerce/Carts/Carts/ShoppingCarts/GettingCartById/GetCartById.cs @@ -37,32 +37,3 @@ public async Task Handle(GetCartById query, CancellationTok ?? throw AggregateNotFoundException.For(query.CartId); } } - - -// public record GetCartById( -// Guid ShoppingCartId, -// int? ExpectedVersion -// ) -// { -// public static GetCartById From(Guid cartId, int? expectedVersion) => -// new(cartId.NotEmpty(), expectedVersion); -// -// public static Task Handle( -// IQueryable shoppingCarts, -// GetCartById query, -// CancellationToken token -// ) -// { -// var expectedVersion = query.ExpectedVersion; -// -// if (!expectedVersion.HasValue) -// return shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId, token); -// -// return Policy -// .HandleResult(cart => -// cart == null || cart.Version < expectedVersion -// ) -// .WaitAndRetryAsync(5, i => TimeSpan.FromMilliseconds(50 * Math.Pow(i, 2))) -// .ExecuteAsync(ct => shoppingCarts.SingleOrDefaultAsync(x => x.Id == query.ShoppingCartId && x.Version >= expectedVersion, ct), token); -// } -// } diff --git a/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs b/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs index 4f9b4ca2..0b5ac3f0 100644 --- a/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs +++ b/Sample/EventStoreDB/Simple/ECommerce/Storage/ECommerceDbContext.cs @@ -10,16 +10,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { modelBuilder.SetupShoppingCartsReadModels(); } - - void IDisposable.Dispose() - { - base.Dispose(); - } - - ValueTask IAsyncDisposable.DisposeAsync() - { - return base.DisposeAsync(); - } } public class ECommerceDBContextFactory: IDesignTimeDbContextFactory