From ac34e7aa39c92ebde4f582e8cbddff9e9f576ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 19:42:06 -0500 Subject: [PATCH 01/44] Add MessageDb feed source --- Propulsion.sln | 6 ++ src/Propulsion.MessageDb/MessageDbSource.fs | 101 ++++++++++++++++++ .../Propulsion.MessageDb.fsproj | 27 +++++ src/Propulsion.MessageDb/ReaderCheckpoint.fs | 77 +++++++++++++ src/Propulsion.MessageDb/Types.fs | 6 ++ 5 files changed, 217 insertions(+) create mode 100644 src/Propulsion.MessageDb/MessageDbSource.fs create mode 100644 src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj create mode 100644 src/Propulsion.MessageDb/ReaderCheckpoint.fs create mode 100644 src/Propulsion.MessageDb/Types.fs diff --git a/Propulsion.sln b/Propulsion.sln index ef90beba..db1ee342 100644 --- a/Propulsion.sln +++ b/Propulsion.sln @@ -55,6 +55,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.MemoryStore", "s EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.DynamoStore.Lambda", "src\Propulsion.DynamoStore.Lambda\Propulsion.DynamoStore.Lambda.fsproj", "{7AEA3BB7-E5C4-4653-ABBF-F6C8476E77AF}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.MessageDb", "src\Propulsion.MessageDb\Propulsion.MessageDb.fsproj", "{BCAEFE2C-8D09-4F4E-B27E-62077497C752}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -149,6 +151,10 @@ Global {7AEA3BB7-E5C4-4653-ABBF-F6C8476E77AF}.Debug|Any CPU.Build.0 = Debug|Any CPU {7AEA3BB7-E5C4-4653-ABBF-F6C8476E77AF}.Release|Any CPU.ActiveCfg = Release|Any CPU {7AEA3BB7-E5C4-4653-ABBF-F6C8476E77AF}.Release|Any CPU.Build.0 = Release|Any CPU + {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs new file mode 100644 index 00000000..909572a4 --- /dev/null +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -0,0 +1,101 @@ +namespace Propulsion.MessageDb + +open System +open System.Data.Common +open System.Diagnostics +open FSharp.Control +open FsCodec +open FsCodec.Core +open Npgsql +open NpgsqlTypes +open Propulsion.Feed + +type MessageDbCategoryReader(connectionString) = + let readonly (bytes: byte array) = ReadOnlyMemory.op_Implicit(bytes) + let readRow (reader: DbDataReader) = + let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) + let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) + let streamName = reader.GetString(9) + let event =TimelineEvent.Create( + index = reader.GetInt64(0), + eventType = reader.GetString(1), + data = (reader.GetFieldValue(2) |> readonly), + meta = (reader.GetFieldValue(3) |> readonly), + eventId = reader.GetGuid(4), + ?correlationId = readNullableString 5, + ?causationId = readNullableString 6, + timestamp = timestamp) + + struct(StreamName.parse streamName, event) + member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { + use conn = new NpgsqlConnection(connectionString) + do! conn.OpenAsync(ct) + let command = conn.CreateCommand() + command.CommandText <- "select + global_position, type, data, metadata, id::uuid, + (metadata::jsonb->>'$correlationId')::text, + (metadata::jsonb->>'$causationId')::text, + time, stream_name + from get_category_messages(@Category, @Position, @BatchSize);" + command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore + command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore + command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore + + let! reader = command.ExecuteReaderAsync(ct) + let! hasRow = reader.ReadAsync(ct) + let mutable hasRow = hasRow + + let events = ResizeArray() + while hasRow do + events.Add(readRow reader) + let! nextHasRow = reader.ReadAsync(ct) + hasRow <- nextHasRow + return events.ToArray() } + member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { + use conn = new NpgsqlConnection(connectionString) + do! conn.OpenAsync(ct) + let command = conn.CreateCommand() + command.CommandText <- "select max(global_position) from messages where category(stream_name) = @Category;" + command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore + + let! reader = command.ExecuteReaderAsync(ct) + let! hasRow = reader.ReadAsync(ct) + return if hasRow then reader.GetInt64(0) else 0L + } + +module private Impl = + open Propulsion.Infrastructure // AwaitTaskCorrect + + let readBatch batchSize (store : MessageDbCategoryReader) (category, pos) : Async> = async { + let! ct = Async.CancellationToken + let positionInclusive = Position.toInt64 pos + let! page = store.ReadCategoryMessages(category, positionInclusive, batchSize, ct) |> Async.AwaitTaskCorrect + let checkpoint = match Array.tryLast page with Some struct(_, evt) -> evt.Index + 1L | None -> positionInclusive + return { checkpoint = Position.parse checkpoint; items = page; isTail = false } } + + let readTailPositionForTranche (store : MessageDbCategoryReader) trancheId : Async = async { + let! ct = Async.CancellationToken + let! lastEventPos = store.ReadCategoryLastVersion(trancheId, ct) |> Async.AwaitTaskCorrect + return Position.parse(lastEventPos + 1L) } + +type MessageDbSource + ( log : Serilog.ILogger, statsInterval, + reader: MessageDbCategoryReader, batchSize, tailSleepInterval, + checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, + // Override default start position to be at the tail of the index. Default: Replay all events. + ?startFromTail, + ?sourceId) = + inherit Propulsion.Feed.Core.TailingFeedSource + ( log, statsInterval, defaultArg sourceId FeedSourceId.wellKnownId, tailSleepInterval, checkpoints, + ( if startFromTail <> Some true then None + else Some (Impl.readTailPositionForTranche reader)), + sink, + (fun req -> asyncSeq { + let sw = Stopwatch.StartNew() + let! b = Impl.readBatch batchSize reader req + yield sw.Elapsed, b }), + string + ) + // ( log, statsInterval, defaultArg sourceId FeedSourceId.wellKnownId, tailSleepInterval, + // checkpoints, sink, + // Impl.readPage (hydrateBodies = Some true) batchSize store) diff --git a/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj new file mode 100644 index 00000000..97eeffe4 --- /dev/null +++ b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj @@ -0,0 +1,27 @@ + + + + net6.0 + true + + + + + Infrastructure.fs + + + + + + + + + + + + + + + + + diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs new file mode 100644 index 00000000..608f350d --- /dev/null +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -0,0 +1,77 @@ +module Propulsion.MessageDb.ReaderCheckpoint + +open Npgsql +open NpgsqlTypes +open Propulsion.Feed +open Propulsion.Infrastructure +open System + +let createConnection connString = new NpgsqlConnection(connString) + +let createIfNotExists (conn : NpgsqlConnection, schema: string) = + let cmd = conn.CreateCommand() + cmd.CommandText <- $" + create table if not exists {schema}.propulsion_checkpoints ( + stream_name text not null, + consumer_group text not null, + global_position bigint not null, + primary key (stream_name, consumer_group) + ); + " + cmd.ExecuteNonQueryAsync() |> Async.AwaitTaskCorrect |> Async.Ignore + +let commitPosition (conn : NpgsqlConnection, schema: string) (stream : string) (consumerGroup : string) (position : int64) = async { + let cmd = conn.CreateCommand() + cmd.CommandText <- + $"insert into {schema}.propulsion_checkpoints(stream_name, consumer_group, global_position) + values (@StreamName, @ConsumerGroup, @GlobalPosition) + on conflict (stream_name, consumer_group) + do update set global_position = @GlobalPosition;" + cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, stream) |> ignore + cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore + cmd.Parameters.AddWithValue("GlobalPosition", NpgsqlDbType.Bigint, position) |> ignore + + let! ct = Async.CancellationToken + do! cmd.ExecuteNonQueryAsync(ct) |> Async.AwaitTaskCorrect |> Async.Ignore } + +let tryGetPosition (conn : NpgsqlConnection, schema : string) (stream : string) (consumerGroup : string) = async { + let cmd = conn.CreateCommand() + cmd.CommandText <- + "select global_position from {schema}.propulsion_checkpoints where stream_name = @StreamName and consumer_group = @ConsumerGroup" + + cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, stream) |> ignore + cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore + + use reader = cmd.ExecuteReader() + let! ct = Async.CancellationToken + let! hasRow = reader.ReadAsync(ct) |> Async.AwaitTaskCorrect + return if hasRow then Some (reader.GetInt64 0) else None } + +type Service(connString : string, schema: string, consumerGroupName, defaultCheckpointFrequency) = + + let streamName source tranche = + match SourceId.toString source, TrancheId.toString tranche with + | s, null -> s + | s, tid -> String.Join("_", s, tid) + + member _.CreateSchemaIfNotExists() = async { + use conn = new NpgsqlConnection(connString) + return! createIfNotExists (conn, schema) } + + interface IFeedCheckpointStore with + + member _.Start(source, tranche, ?establishOrigin) = async { + use conn = createConnection connString + let! maybePos = tryGetPosition (conn, schema) (streamName source tranche) consumerGroupName + let! pos = + match maybePos, establishOrigin with + | Some pos, _ -> async { return Position.parse pos } + | None, Some f -> f + | None, None -> async { return Position.initial } + return defaultCheckpointFrequency, pos } + + member _.Commit(source, tranche, pos) = async { + use conn = createConnection connString + return! commitPosition (conn, schema) (streamName source tranche) consumerGroupName (Position.toInt64 pos) } + + diff --git a/src/Propulsion.MessageDb/Types.fs b/src/Propulsion.MessageDb/Types.fs new file mode 100644 index 00000000..6e0fbb7b --- /dev/null +++ b/src/Propulsion.MessageDb/Types.fs @@ -0,0 +1,6 @@ +namespace Propulsion.MessageDb + +open FSharp.UMX + +module internal FeedSourceId = + let wellKnownId : Propulsion.Feed.SourceId = UMX.tag "messageDb" From 5e8e0e166adc358fd528b845a32527ad5a6a9840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 19:43:21 -0500 Subject: [PATCH 02/44] spaces --- src/Propulsion.MessageDb/MessageDbSource.fs | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 909572a4..97c5271b 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -13,20 +13,20 @@ open Propulsion.Feed type MessageDbCategoryReader(connectionString) = let readonly (bytes: byte array) = ReadOnlyMemory.op_Implicit(bytes) let readRow (reader: DbDataReader) = - let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) - let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) - let streamName = reader.GetString(9) - let event =TimelineEvent.Create( - index = reader.GetInt64(0), - eventType = reader.GetString(1), - data = (reader.GetFieldValue(2) |> readonly), - meta = (reader.GetFieldValue(3) |> readonly), - eventId = reader.GetGuid(4), - ?correlationId = readNullableString 5, - ?causationId = readNullableString 6, - timestamp = timestamp) + let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) + let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) + let streamName = reader.GetString(9) + let event =TimelineEvent.Create( + index = reader.GetInt64(0), + eventType = reader.GetString(1), + data = (reader.GetFieldValue(2) |> readonly), + meta = (reader.GetFieldValue(3) |> readonly), + eventId = reader.GetGuid(4), + ?correlationId = readNullableString 5, + ?causationId = readNullableString 6, + timestamp = timestamp) - struct(StreamName.parse streamName, event) + struct(StreamName.parse streamName, event) member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { use conn = new NpgsqlConnection(connectionString) do! conn.OpenAsync(ct) From 47bc5160e94cfc6274604105a9e503f69b4906a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 19:44:43 -0500 Subject: [PATCH 03/44] formatting --- src/Propulsion.MessageDb/MessageDbSource.fs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 97c5271b..7ad2ad4c 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -94,8 +94,4 @@ type MessageDbSource let sw = Stopwatch.StartNew() let! b = Impl.readBatch batchSize reader req yield sw.Elapsed, b }), - string - ) - // ( log, statsInterval, defaultArg sourceId FeedSourceId.wellKnownId, tailSleepInterval, - // checkpoints, sink, - // Impl.readPage (hydrateBodies = Some true) batchSize store) + string) From 496746b1cb2be6bfc1a5cd10eb7dd7b4cee104ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 19:45:14 -0500 Subject: [PATCH 04/44] fix bug --- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 608f350d..b0758e7a 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -37,7 +37,7 @@ let commitPosition (conn : NpgsqlConnection, schema: string) (stream : string) ( let tryGetPosition (conn : NpgsqlConnection, schema : string) (stream : string) (consumerGroup : string) = async { let cmd = conn.CreateCommand() cmd.CommandText <- - "select global_position from {schema}.propulsion_checkpoints where stream_name = @StreamName and consumer_group = @ConsumerGroup" + $"select global_position from {schema}.propulsion_checkpoints where stream_name = @StreamName and consumer_group = @ConsumerGroup" cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, stream) |> ignore cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore From 32c13ad15c0d6503058068dbc5758747bbb61911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 19:57:25 -0500 Subject: [PATCH 05/44] remove conn helper --- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index b0758e7a..4a045e10 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -6,8 +6,6 @@ open Propulsion.Feed open Propulsion.Infrastructure open System -let createConnection connString = new NpgsqlConnection(connString) - let createIfNotExists (conn : NpgsqlConnection, schema: string) = let cmd = conn.CreateCommand() cmd.CommandText <- $" @@ -61,7 +59,7 @@ type Service(connString : string, schema: string, consumerGroupName, defaultChec interface IFeedCheckpointStore with member _.Start(source, tranche, ?establishOrigin) = async { - use conn = createConnection connString + use conn = new NpgsqlConnection(connString) let! maybePos = tryGetPosition (conn, schema) (streamName source tranche) consumerGroupName let! pos = match maybePos, establishOrigin with @@ -71,7 +69,7 @@ type Service(connString : string, schema: string, consumerGroupName, defaultChec return defaultCheckpointFrequency, pos } member _.Commit(source, tranche, pos) = async { - use conn = createConnection connString + use conn = new NpgsqlConnection(connString) return! commitPosition (conn, schema) (streamName source tranche) consumerGroupName (Position.toInt64 pos) } From 06422dd67e8be681b2f61275251597721fd065a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 21:20:49 -0500 Subject: [PATCH 06/44] add to build --- .gitignore | 1 + build.proj | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6486412d..b2eaf6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ packages.config ## Visual Studio 2015+ cache/options directory .vs/ *.fsproj.user +*.DotSettings.user ## JetBrains Rider .idea/ diff --git a/build.proj b/build.proj index df64beac..78cf9c1e 100644 --- a/build.proj +++ b/build.proj @@ -23,6 +23,7 @@ + From 2e6aff37d2d18cd2a2cdafd4625455dd05dc2a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 11 Nov 2022 23:45:44 -0500 Subject: [PATCH 07/44] fix bugs with positions --- Propulsion.sln | 6 ++ src/Propulsion.MessageDb/MessageDbSource.fs | 6 +- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 6 ++ .../Program.fs | 2 + .../Propulsion.MessageDb.Integration.fsproj | 38 +++++++++ .../Propulsion.MessageDb.Integration/Tests.fs | 78 +++++++++++++++++++ 6 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 tests/Propulsion.MessageDb.Integration/Program.fs create mode 100644 tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj create mode 100644 tests/Propulsion.MessageDb.Integration/Tests.fs diff --git a/Propulsion.sln b/Propulsion.sln index db1ee342..91c6dbcc 100644 --- a/Propulsion.sln +++ b/Propulsion.sln @@ -57,6 +57,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.DynamoStore.Lamb EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.MessageDb", "src\Propulsion.MessageDb\Propulsion.MessageDb.fsproj", "{BCAEFE2C-8D09-4F4E-B27E-62077497C752}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Propulsion.MessageDb.Integration", "tests\Propulsion.MessageDb.Integration\Propulsion.MessageDb.Integration.fsproj", "{9738D2C1-EE7C-400F-8B14-31B5B7B66839}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -155,6 +157,10 @@ Global {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Debug|Any CPU.Build.0 = Debug|Any CPU {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Release|Any CPU.ActiveCfg = Release|Any CPU {BCAEFE2C-8D09-4F4E-B27E-62077497C752}.Release|Any CPU.Build.0 = Release|Any CPU + {9738D2C1-EE7C-400F-8B14-31B5B7B66839}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9738D2C1-EE7C-400F-8B14-31B5B7B66839}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9738D2C1-EE7C-400F-8B14-31B5B7B66839}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9738D2C1-EE7C-400F-8B14-31B5B7B66839}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 7ad2ad4c..84937d29 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -15,7 +15,7 @@ type MessageDbCategoryReader(connectionString) = let readRow (reader: DbDataReader) = let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) - let streamName = reader.GetString(9) + let streamName = reader.GetString(8) let event =TimelineEvent.Create( index = reader.GetInt64(0), eventType = reader.GetString(1), @@ -70,13 +70,13 @@ module private Impl = let! ct = Async.CancellationToken let positionInclusive = Position.toInt64 pos let! page = store.ReadCategoryMessages(category, positionInclusive, batchSize, ct) |> Async.AwaitTaskCorrect - let checkpoint = match Array.tryLast page with Some struct(_, evt) -> evt.Index + 1L | None -> positionInclusive + let checkpoint = match Array.tryLast page with Some struct(_, evt) -> evt.Index | None -> positionInclusive return { checkpoint = Position.parse checkpoint; items = page; isTail = false } } let readTailPositionForTranche (store : MessageDbCategoryReader) trancheId : Async = async { let! ct = Async.CancellationToken let! lastEventPos = store.ReadCategoryLastVersion(trancheId, ct) |> Async.AwaitTaskCorrect - return Position.parse(lastEventPos + 1L) } + return Position.parse(lastEventPos) } type MessageDbSource ( log : Serilog.ILogger, statsInterval, diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 4a045e10..608e2f68 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -54,12 +54,16 @@ type Service(connString : string, schema: string, consumerGroupName, defaultChec member _.CreateSchemaIfNotExists() = async { use conn = new NpgsqlConnection(connString) + let! ct = Async.CancellationToken + do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect return! createIfNotExists (conn, schema) } interface IFeedCheckpointStore with member _.Start(source, tranche, ?establishOrigin) = async { use conn = new NpgsqlConnection(connString) + let! ct = Async.CancellationToken + do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect let! maybePos = tryGetPosition (conn, schema) (streamName source tranche) consumerGroupName let! pos = match maybePos, establishOrigin with @@ -70,6 +74,8 @@ type Service(connString : string, schema: string, consumerGroupName, defaultChec member _.Commit(source, tranche, pos) = async { use conn = new NpgsqlConnection(connString) + let! ct = Async.CancellationToken + do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect return! commitPosition (conn, schema) (streamName source tranche) consumerGroupName (Position.toInt64 pos) } diff --git a/tests/Propulsion.MessageDb.Integration/Program.fs b/tests/Propulsion.MessageDb.Integration/Program.fs new file mode 100644 index 00000000..a2b1c636 --- /dev/null +++ b/tests/Propulsion.MessageDb.Integration/Program.fs @@ -0,0 +1,2 @@ +module Program = let [] main _ = 0 + diff --git a/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj new file mode 100644 index 00000000..47bbbe2b --- /dev/null +++ b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj @@ -0,0 +1,38 @@ + + + + net6.0 + + false + false + + + + + Infrastructure.fs + + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs new file mode 100644 index 00000000..4fe53d84 --- /dev/null +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -0,0 +1,78 @@ +module Tests + +open System +open System.Collections.Generic +open System.Threading +open System.Threading.Tasks +open FSharp.Control +open Npgsql +open NpgsqlTypes +open Propulsion.Feed +open Propulsion.Streams +open TypeShape.UnionContract +open Xunit +open Propulsion.MessageDb +open Swensen.Unquote +open Propulsion.Infrastructure + +module Simple = + type Hello = {name: string} + type Event = + | Hello of Hello + interface IUnionContract + let codec = FsCodec.SystemTextJson.Codec.Create() + +let writeMessagesToCategory category = task { + use conn = new NpgsqlConnection("Host=localhost; Port=5433; Username=message_store; Password=;") + do! conn.OpenAsync() + let batch = conn.CreateBatch() + for _ in 1..100 do + let streamName = $"{category}-{Guid.NewGuid():N}" + for _ in 1..20 do + let cmd = NpgsqlBatchCommand() + cmd.CommandText <- "select 1 from write_message(@Id::text, @StreamName, @EventType, @Data, 'null', null)" + cmd.Parameters.AddWithValue("Id", NpgsqlDbType.Uuid, Guid.NewGuid()) |> ignore + cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, streamName) |> ignore + cmd.Parameters.AddWithValue("EventType", NpgsqlDbType.Text, "Hello") |> ignore + cmd.Parameters.AddWithValue("Data", NpgsqlDbType.Jsonb, """{"name": "world"}""") |> ignore + + batch.BatchCommands.Add(cmd) + do! batch.ExecuteNonQueryAsync() :> Task } + +module Array = + let chooseV f (arr: _ array) = [| for item in arr do match f item with ValueSome v -> yield v | ValueNone -> () |] +[] +let ``It processes events for a category`` () = async { + let log = Serilog.Log.Logger + let consumerGroup = $"{Guid.NewGuid():N}" + let category = $"{Guid.NewGuid():N}" + do! writeMessagesToCategory category |> Async.AwaitTaskCorrect + let reader = MessageDbCategoryReader("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") + let checkpoints = ReaderCheckpoint.Service("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) + do! checkpoints.CreateSchemaIfNotExists() + let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) + with member _.HandleExn(log, x) = () + member _.HandleOk x = () } + let stop = ref (fun () -> () ) + let handled = HashSet<_>() + let handle struct(stream, evts: StreamSpan<_>) = async { + lock handled (fun _ -> for evt in evts do handled.Add((stream, evt.EventId)) |> ignore) + test <@ Array.chooseV Simple.codec.TryDecode evts |> Array.forall ((=) (Simple.Hello {name = "world"})) @> + if handled.Count >= 2000 then + stop.contents() + return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) + } + use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) + let source = MessageDbSource(log, TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) + use src = source.Start(source.Pump(fun _ -> async { return [| TrancheId.parse category |] })) + // who says you can't do backwards referencing in F# + stop.contents <- src.Stop + + Task.Delay(TimeSpan.FromSeconds 20).ContinueWith(fun _ -> src.Stop()) |> ignore + + do! src.AwaitShutdown() + // 2000 total events + test <@ handled.Count = 2000 @> + // 20 in each stream + test <@ handled |> Array.ofSeq |> Array.groupBy fst |> Array.map (snd >> Array.length) |> Array.forall ((=) 20) @> +} From 5fd81a6b6d9a4557ea713abb0c61c27dd8cb9e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 08:36:17 -0500 Subject: [PATCH 08/44] fix typo --- build.proj | 2 +- tests/Propulsion.MessageDb.Integration/Tests.fs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.proj b/build.proj index 78cf9c1e..68a15169 100644 --- a/build.proj +++ b/build.proj @@ -23,7 +23,7 @@ - + diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 4fe53d84..0a125180 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -60,8 +60,7 @@ let ``It processes events for a category`` () = async { test <@ Array.chooseV Simple.codec.TryDecode evts |> Array.forall ((=) (Simple.Hello {name = "world"})) @> if handled.Count >= 2000 then stop.contents() - return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) - } + return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) let source = MessageDbSource(log, TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) use src = source.Start(source.Pump(fun _ -> async { return [| TrancheId.parse category |] })) From be656aba7dce81eb31714c553c6bf139d8570461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 08:36:40 -0500 Subject: [PATCH 09/44] space --- tests/Propulsion.MessageDb.Integration/Tests.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 0a125180..0ca1467b 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -53,7 +53,7 @@ let ``It processes events for a category`` () = async { let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) with member _.HandleExn(log, x) = () member _.HandleOk x = () } - let stop = ref (fun () -> () ) + let stop = ref (fun () -> ()) let handled = HashSet<_>() let handle struct(stream, evts: StreamSpan<_>) = async { lock handled (fun _ -> for evt in evts do handled.Add((stream, evt.EventId)) |> ignore) From cac17689899a49c630eba0b265912ae0219165a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 09:34:02 -0500 Subject: [PATCH 10/44] Add readme and changelog --- CHANGELOG.md | 1 + README.md | 4 ++++ src/Propulsion.MessageDb/ReaderCheckpoint.fs | 2 +- tests/Propulsion.MessageDb.Integration/Tests.fs | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf1a2cea..62215254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The `Unreleased` section name is replaced by the expected version of next releas - `Propulsion.EventStoreDb`: Ported `EventStore` to target `Equinox.EventStore` >= `4.0.0` (using the gRPC interface) [#139](https://github.com/jet/propulsion/pull/139) - `Propulsion.CosmosStore3`: Special cased version of `Propulsion.CosmosStore` to target `Equinox.CosmosStore` v `[3.0.7`-`3.99.0]` **Deprecated; Please migrate to `Propulsion.CosmosStore` by updating `Equinox.CosmosStore` dependencies to `4.0.0`** [#139](https://github.com/jet/propulsion/pull/139) - `Propulsion.DynamoStore`: `Equinox.CosmosStore`-equivalent functionality for `Equinox.DynamoStore`. Combines elements of `CosmosStore`, `SqlStreamStore`, `Feed` [#140](https://github.com/jet/propulsion/pull/143) [#140](https://github.com/jet/propulsion/pull/143) [#177](https://github.com/jet/propulsion/pull/177) +- `Propulsion.MessageDb`: `FeedSource` for [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181) - `Propulsion.MemoryStore`: `MemoryStoreSource` to align with other sources for integration testing. Includes *deterministic* `AwaitCompletion` as per `Propulsion.Feed`-based Sources [#165](https://github.com/jet/propulsion/pull/165) - `Propulsion.SqlStreamStore`: Added `startFromTail` [#173](https://github.com/jet/propulsion/pull/173) - `Propulsion.Tool`: `checkpoint` commandline option; enables viewing or overriding checkpoints [#141](https://github.com/jet/propulsion/pull/141) diff --git a/README.md b/README.md index d649f35d..419ac66b 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,10 @@ If you're looking for a good discussion forum on these kinds of topics, look no - `Propulsion.Kafka` [![NuGet](https://img.shields.io/nuget/v/Propulsion.Kafka.svg)](https://www.nuget.org/packages/Propulsion.Kafka/) Provides bindings for producing and consuming both streamwise and in parallel. Includes a standard codec for use with streamwise projection and consumption, `Propulsion.Kafka.Codec.NewtonsoftJson.RenderedSpan`. [Depends](https://www.fuget.org/packages/Propulsion.Kafka) on `FsKafka` v `1.7.0`-`1.9.99`, `Serilog` +- `Propulsion.MessageDb` [![NuGet](https://img.shields.io/nuget/v/Propulsion.MessageDb.svg)](https://www.nuget.org/packages/Propulsion.MessageDb/). Provides bindings to [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181), maintaining checkpoints in a postgres table [Depends](https://www.fuget.org/packages/Propulsion.MessageDb) on `Propulsion.Feed`, `Npgsql` >= `6.0.7` + 1. `MessageDbSource`: reading from a MessageDb category into a `Propulsion.Sink` + 2. `ReaderCheckpoint`: checkpoint storage for `Propulsion.Feed` using `Npgsql` + - `Propulsion.SqlStreamStore` [![NuGet](https://img.shields.io/nuget/v/Propulsion.SqlStreamStore.svg)](https://www.nuget.org/packages/Propulsion.SqlStreamStore/). Provides bindings to [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore), maintaining checkpoints in a SQL table using Dapper. [Depends](https://www.fuget.org/packages/Propulsion.SqlStreamStore) on `Propulsion.Feed`, `SqlStreamStore`, `Dapper` v `2.0`, `Microsoft.Data.SqlClient` v `1.1.3`, `Serilog` 1. `SqlStreamStoreSource`: reading from a SqlStreamStore `$all` stream into a `Propulsion.Sink` diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 608e2f68..a309f201 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -45,7 +45,7 @@ let tryGetPosition (conn : NpgsqlConnection, schema : string) (stream : string) let! hasRow = reader.ReadAsync(ct) |> Async.AwaitTaskCorrect return if hasRow then Some (reader.GetInt64 0) else None } -type Service(connString : string, schema: string, consumerGroupName, defaultCheckpointFrequency) = +type NpgsqlCheckpointStore(connString : string, schema: string, consumerGroupName, defaultCheckpointFrequency) = let streamName source tranche = match SourceId.toString source, TrancheId.toString tranche with diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 0ca1467b..cf566be9 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -48,7 +48,7 @@ let ``It processes events for a category`` () = async { let category = $"{Guid.NewGuid():N}" do! writeMessagesToCategory category |> Async.AwaitTaskCorrect let reader = MessageDbCategoryReader("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") - let checkpoints = ReaderCheckpoint.Service("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) + let checkpoints = ReaderCheckpoint.NpgsqlCheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) do! checkpoints.CreateSchemaIfNotExists() let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) with member _.HandleExn(log, x) = () From 2a3f73f6c0a2a64186b1511b33e533b6bb698a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 09:34:23 -0500 Subject: [PATCH 11/44] clarify --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 419ac66b..365c7e0f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ If you're looking for a good discussion forum on these kinds of topics, look no - `Propulsion.MessageDb` [![NuGet](https://img.shields.io/nuget/v/Propulsion.MessageDb.svg)](https://www.nuget.org/packages/Propulsion.MessageDb/). Provides bindings to [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181), maintaining checkpoints in a postgres table [Depends](https://www.fuget.org/packages/Propulsion.MessageDb) on `Propulsion.Feed`, `Npgsql` >= `6.0.7` 1. `MessageDbSource`: reading from a MessageDb category into a `Propulsion.Sink` - 2. `ReaderCheckpoint`: checkpoint storage for `Propulsion.Feed` using `Npgsql` + 2. `NpgsqlCheckpointStore`: checkpoint storage for `Propulsion.Feed` using `Npgsql` - `Propulsion.SqlStreamStore` [![NuGet](https://img.shields.io/nuget/v/Propulsion.SqlStreamStore.svg)](https://www.nuget.org/packages/Propulsion.SqlStreamStore/). Provides bindings to [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore), maintaining checkpoints in a SQL table using Dapper. [Depends](https://www.fuget.org/packages/Propulsion.SqlStreamStore) on `Propulsion.Feed`, `SqlStreamStore`, `Dapper` v `2.0`, `Microsoft.Data.SqlClient` v `1.1.3`, `Serilog` From 9f69cf9d1a4a05b52bde1f6f95b0328745a4ea3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 09:43:55 -0500 Subject: [PATCH 12/44] add assert on the order of events --- tests/Propulsion.MessageDb.Integration/Tests.fs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index cf566be9..f1c61bf2 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -41,6 +41,7 @@ let writeMessagesToCategory category = task { module Array = let chooseV f (arr: _ array) = [| for item in arr do match f item with ValueSome v -> yield v | ValueNone -> () |] + let isAscending arr = Array.sort arr = arr [] let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger @@ -56,7 +57,7 @@ let ``It processes events for a category`` () = async { let stop = ref (fun () -> ()) let handled = HashSet<_>() let handle struct(stream, evts: StreamSpan<_>) = async { - lock handled (fun _ -> for evt in evts do handled.Add((stream, evt.EventId)) |> ignore) + lock handled (fun _ -> for evt in evts do handled.Add((stream, evt.Index)) |> ignore) test <@ Array.chooseV Simple.codec.TryDecode evts |> Array.forall ((=) (Simple.Hello {name = "world"})) @> if handled.Count >= 2000 then stop.contents() @@ -74,4 +75,6 @@ let ``It processes events for a category`` () = async { test <@ handled.Count = 2000 @> // 20 in each stream test <@ handled |> Array.ofSeq |> Array.groupBy fst |> Array.map (snd >> Array.length) |> Array.forall ((=) 20) @> + // they were handled in order within streams + test <@ handled |> Array.ofSeq |> Array.groupBy fst |> Array.map snd |> Array.forall Array.isAscending @> } From c51bd81f34ee0881480100d8266dbcace873f4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 11:54:11 -0500 Subject: [PATCH 13/44] perf: voptions shouldn't be turned into options --- src/Propulsion/Internal.fs | 11 ++++++++--- tests/Propulsion.MessageDb.Integration/Tests.fs | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Propulsion/Internal.fs b/src/Propulsion/Internal.fs index fe76bcd4..ad6d4ff0 100644 --- a/src/Propulsion/Internal.fs +++ b/src/Propulsion/Internal.fs @@ -123,13 +123,18 @@ module ValueOption = module Seq = - let tryPickV f xs = Seq.tryPick (f >> ValueOption.toOption) xs |> ValueOption.ofOption - let inline chooseV f = Seq.choose (f >> ValueOption.toOption) + let tryPickV f (xs: _ seq) = + use e = xs.GetEnumerator() + let mutable res = ValueNone + while (ValueOption.isNone res && e.MoveNext()) do + res <- f e.Current + res + let inline chooseV f xs = seq { for x in xs do match f x with ValueSome v -> yield v | ValueNone -> () } module Array = let inline any xs = (not << Array.isEmpty) xs - let inline chooseV f = Array.choose (f >> ValueOption.toOption) + let inline chooseV f xs = [| for item in xs do match f item with ValueSome v -> yield v | ValueNone -> () |] module Stats = diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index f1c61bf2..00955de8 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -2,7 +2,6 @@ module Tests open System open System.Collections.Generic -open System.Threading open System.Threading.Tasks open FSharp.Control open Npgsql From dee01162537fd1a6d5fb6c748ff455248ec82ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Sat, 12 Nov 2022 14:38:39 -0500 Subject: [PATCH 14/44] use the internal chooseV --- tests/Propulsion.MessageDb.Integration/Tests.fs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 00955de8..8b661b82 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -13,6 +13,7 @@ open Xunit open Propulsion.MessageDb open Swensen.Unquote open Propulsion.Infrastructure +open Propulsion.Internal module Simple = type Hello = {name: string} @@ -39,8 +40,9 @@ let writeMessagesToCategory category = task { do! batch.ExecuteNonQueryAsync() :> Task } module Array = - let chooseV f (arr: _ array) = [| for item in arr do match f item with ValueSome v -> yield v | ValueNone -> () |] + let isAscending arr = Array.sort arr = arr + [] let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger From 5394519732f903221d653cbedf5770c0afffed47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 22:23:08 -0500 Subject: [PATCH 15/44] fixes from code review --- src/Propulsion.MessageDb/Internal.fs | 13 ++++ src/Propulsion.MessageDb/MessageDbSource.fs | 53 ++++++------- .../Propulsion.MessageDb.fsproj | 2 +- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 77 +++++++++---------- src/Propulsion.MessageDb/Types.fs | 6 -- .../Program.fs | 2 - .../Propulsion.MessageDb.Integration.fsproj | 12 +-- .../Propulsion.MessageDb.Integration/Tests.fs | 4 +- 8 files changed, 82 insertions(+), 87 deletions(-) create mode 100644 src/Propulsion.MessageDb/Internal.fs delete mode 100644 src/Propulsion.MessageDb/Types.fs delete mode 100644 tests/Propulsion.MessageDb.Integration/Program.fs diff --git a/src/Propulsion.MessageDb/Internal.fs b/src/Propulsion.MessageDb/Internal.fs new file mode 100644 index 00000000..8b842704 --- /dev/null +++ b/src/Propulsion.MessageDb/Internal.fs @@ -0,0 +1,13 @@ +namespace Propulsion.MessageDb + +open FSharp.UMX +open Npgsql + +module internal FeedSourceId = + let wellKnownId : Propulsion.Feed.SourceId = UMX.tag "messageDb" + +module internal Npgsql = + let connect connectionString ct = task { + let conn = new NpgsqlConnection(connectionString) + do! conn.OpenAsync(ct) + return conn } diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 84937d29..0d9900f0 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -1,26 +1,28 @@ namespace Propulsion.MessageDb -open System -open System.Data.Common -open System.Diagnostics open FSharp.Control open FsCodec open FsCodec.Core open Npgsql open NpgsqlTypes open Propulsion.Feed +open System +open System.Data.Common +open System.Diagnostics +open Propulsion.Feed.Core + type MessageDbCategoryReader(connectionString) = - let readonly (bytes: byte array) = ReadOnlyMemory.op_Implicit(bytes) + let connect = Npgsql.connect connectionString let readRow (reader: DbDataReader) = let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) let streamName = reader.GetString(8) - let event =TimelineEvent.Create( + let event = TimelineEvent.Create( index = reader.GetInt64(0), eventType = reader.GetString(1), - data = (reader.GetFieldValue(2) |> readonly), - meta = (reader.GetFieldValue(3) |> readonly), + data = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 2)), + meta = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 3)), eventId = reader.GetGuid(4), ?correlationId = readNullableString 5, ?causationId = readNullableString 6, @@ -28,39 +30,35 @@ type MessageDbCategoryReader(connectionString) = struct(StreamName.parse streamName, event) member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { - use conn = new NpgsqlConnection(connectionString) - do! conn.OpenAsync(ct) + use! conn = connect ct let command = conn.CreateCommand() command.CommandText <- "select - global_position, type, data, metadata, id::uuid, + position, type, data, metadata, id::uuid, (metadata::jsonb->>'$correlationId')::text, (metadata::jsonb->>'$causationId')::text, - time, stream_name + time, stream_name, global_position from get_category_messages(@Category, @Position, @BatchSize);" command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore - let! reader = command.ExecuteReaderAsync(ct) - let! hasRow = reader.ReadAsync(ct) - let mutable hasRow = hasRow - + let mutable checkpoint = fromPositionInclusive let events = ResizeArray() - while hasRow do + + use! reader = command.ExecuteReaderAsync(ct) + while reader.Read() do events.Add(readRow reader) - let! nextHasRow = reader.ReadAsync(ct) - hasRow <- nextHasRow - return events.ToArray() } + checkpoint <- reader.GetInt64(9) + + return { checkpoint = Position.parse checkpoint; items = events.ToArray(); isTail = false } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { - use conn = new NpgsqlConnection(connectionString) - do! conn.OpenAsync(ct) + use! conn = connect ct let command = conn.CreateCommand() command.CommandText <- "select max(global_position) from messages where category(stream_name) = @Category;" command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore - let! reader = command.ExecuteReaderAsync(ct) - let! hasRow = reader.ReadAsync(ct) - return if hasRow then reader.GetInt64(0) else 0L + use! reader = command.ExecuteReaderAsync(ct) + return if reader.Read() then reader.GetInt64(0) else 0L } module private Impl = @@ -69,14 +67,13 @@ module private Impl = let readBatch batchSize (store : MessageDbCategoryReader) (category, pos) : Async> = async { let! ct = Async.CancellationToken let positionInclusive = Position.toInt64 pos - let! page = store.ReadCategoryMessages(category, positionInclusive, batchSize, ct) |> Async.AwaitTaskCorrect - let checkpoint = match Array.tryLast page with Some struct(_, evt) -> evt.Index | None -> positionInclusive - return { checkpoint = Position.parse checkpoint; items = page; isTail = false } } + let! x = store.ReadCategoryMessages(category, positionInclusive, batchSize, ct) |> Async.AwaitTaskCorrect + return x } let readTailPositionForTranche (store : MessageDbCategoryReader) trancheId : Async = async { let! ct = Async.CancellationToken let! lastEventPos = store.ReadCategoryLastVersion(trancheId, ct) |> Async.AwaitTaskCorrect - return Position.parse(lastEventPos) } + return Position.parse lastEventPos } type MessageDbSource ( log : Serilog.ILogger, statsInterval, diff --git a/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj index 97eeffe4..07514a13 100644 --- a/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj +++ b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj @@ -9,7 +9,7 @@ Infrastructure.fs - + diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index a309f201..9034f497 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -4,53 +4,54 @@ open Npgsql open NpgsqlTypes open Propulsion.Feed open Propulsion.Infrastructure -open System let createIfNotExists (conn : NpgsqlConnection, schema: string) = let cmd = conn.CreateCommand() cmd.CommandText <- $" - create table if not exists {schema}.propulsion_checkpoints ( - stream_name text not null, + create table if not exists {schema}.propulsion_checkpoint ( + source text not null, + tranche text not null, consumer_group text not null, - global_position bigint not null, - primary key (stream_name, consumer_group) - ); + position bigint not null, + primary key (source, tranche, consumer_group) + ) " cmd.ExecuteNonQueryAsync() |> Async.AwaitTaskCorrect |> Async.Ignore -let commitPosition (conn : NpgsqlConnection, schema: string) (stream : string) (consumerGroup : string) (position : int64) = async { - let cmd = conn.CreateCommand() - cmd.CommandText <- - $"insert into {schema}.propulsion_checkpoints(stream_name, consumer_group, global_position) - values (@StreamName, @ConsumerGroup, @GlobalPosition) - on conflict (stream_name, consumer_group) - do update set global_position = @GlobalPosition;" - cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, stream) |> ignore - cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore - cmd.Parameters.AddWithValue("GlobalPosition", NpgsqlDbType.Bigint, position) |> ignore +let commitPosition (conn : NpgsqlConnection, schema: string) source tranche (consumerGroup : string) (position : int64) + = async { + let cmd = conn.CreateCommand() + cmd.CommandText <- + $"insert into {schema}.propulsion_checkpoint(source, tranche, consumer_group, position) + values (@Source, @Tranche, @ConsumerGroup, @GlobalPosition) + on conflict (source, tranche, consumer_group) + do update set position = @Position;" + cmd.Parameters.AddWithValue("Source", NpgsqlDbType.Text, SourceId.toString source) |> ignore + cmd.Parameters.AddWithValue("Tranche", NpgsqlDbType.Text, TrancheId.toString tranche) |> ignore + cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore + cmd.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, position) |> ignore - let! ct = Async.CancellationToken - do! cmd.ExecuteNonQueryAsync(ct) |> Async.AwaitTaskCorrect |> Async.Ignore } + let! ct = Async.CancellationToken + do! cmd.ExecuteNonQueryAsync(ct) |> Async.AwaitTaskCorrect |> Async.Ignore } -let tryGetPosition (conn : NpgsqlConnection, schema : string) (stream : string) (consumerGroup : string) = async { +let tryGetPosition (conn : NpgsqlConnection, schema : string) source tranche (consumerGroup : string) = async { let cmd = conn.CreateCommand() cmd.CommandText <- - $"select global_position from {schema}.propulsion_checkpoints where stream_name = @StreamName and consumer_group = @ConsumerGroup" + $"select position from {schema}.propulsion_checkpoint + where source = @Source + and tranche = @Tranche + and consumer_group = @ConsumerGroup" - cmd.Parameters.AddWithValue("StreamName", NpgsqlDbType.Text, stream) |> ignore + cmd.Parameters.AddWithValue("Source", NpgsqlDbType.Text, SourceId.toString source) |> ignore + cmd.Parameters.AddWithValue("Tranche", NpgsqlDbType.Text, TrancheId.toString tranche) |> ignore cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore - use reader = cmd.ExecuteReader() let! ct = Async.CancellationToken - let! hasRow = reader.ReadAsync(ct) |> Async.AwaitTaskCorrect - return if hasRow then Some (reader.GetInt64 0) else None } - -type NpgsqlCheckpointStore(connString : string, schema: string, consumerGroupName, defaultCheckpointFrequency) = + use! reader = cmd.ExecuteReaderAsync(ct) |> Async.AwaitTaskCorrect + return if reader.Read() then ValueSome (reader.GetInt64 0) else ValueNone } - let streamName source tranche = - match SourceId.toString source, TrancheId.toString tranche with - | s, null -> s - | s, tid -> String.Join("_", s, tid) +type CheckpointStore(connString : string, schema: string, consumerGroupName, defaultCheckpointFrequency) = + let connect = Npgsql.connect connString member _.CreateSchemaIfNotExists() = async { use conn = new NpgsqlConnection(connString) @@ -61,21 +62,19 @@ type NpgsqlCheckpointStore(connString : string, schema: string, consumerGroupNam interface IFeedCheckpointStore with member _.Start(source, tranche, ?establishOrigin) = async { - use conn = new NpgsqlConnection(connString) let! ct = Async.CancellationToken - do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect - let! maybePos = tryGetPosition (conn, schema) (streamName source tranche) consumerGroupName + use! conn = connect ct |> Async.AwaitTaskCorrect + let! maybePos = tryGetPosition (conn, schema) source tranche consumerGroupName let! pos = match maybePos, establishOrigin with - | Some pos, _ -> async { return Position.parse pos } - | None, Some f -> f - | None, None -> async { return Position.initial } + | ValueSome pos, _ -> async { return Position.parse pos } + | ValueNone, Some f -> f + | ValueNone, None -> async { return Position.initial } return defaultCheckpointFrequency, pos } member _.Commit(source, tranche, pos) = async { - use conn = new NpgsqlConnection(connString) let! ct = Async.CancellationToken - do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect - return! commitPosition (conn, schema) (streamName source tranche) consumerGroupName (Position.toInt64 pos) } + use! conn = connect ct |> Async.AwaitTaskCorrect + return! commitPosition (conn, schema) source tranche consumerGroupName (Position.toInt64 pos) } diff --git a/src/Propulsion.MessageDb/Types.fs b/src/Propulsion.MessageDb/Types.fs deleted file mode 100644 index 6e0fbb7b..00000000 --- a/src/Propulsion.MessageDb/Types.fs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Propulsion.MessageDb - -open FSharp.UMX - -module internal FeedSourceId = - let wellKnownId : Propulsion.Feed.SourceId = UMX.tag "messageDb" diff --git a/tests/Propulsion.MessageDb.Integration/Program.fs b/tests/Propulsion.MessageDb.Integration/Program.fs deleted file mode 100644 index a2b1c636..00000000 --- a/tests/Propulsion.MessageDb.Integration/Program.fs +++ /dev/null @@ -1,2 +0,0 @@ -module Program = let [] main _ = 0 - diff --git a/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj index 47bbbe2b..a5bef440 100644 --- a/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj +++ b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj @@ -2,9 +2,6 @@ net6.0 - - false - false @@ -12,23 +9,20 @@ Infrastructure.fs - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 8b661b82..a96fad1e 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -50,7 +50,7 @@ let ``It processes events for a category`` () = async { let category = $"{Guid.NewGuid():N}" do! writeMessagesToCategory category |> Async.AwaitTaskCorrect let reader = MessageDbCategoryReader("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") - let checkpoints = ReaderCheckpoint.NpgsqlCheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) + let checkpoints = ReaderCheckpoint.CheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) do! checkpoints.CreateSchemaIfNotExists() let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) with member _.HandleExn(log, x) = () @@ -69,7 +69,7 @@ let ``It processes events for a category`` () = async { // who says you can't do backwards referencing in F# stop.contents <- src.Stop - Task.Delay(TimeSpan.FromSeconds 20).ContinueWith(fun _ -> src.Stop()) |> ignore + Task.Delay(TimeSpan.FromSeconds 30).ContinueWith(fun _ -> src.Stop()) |> ignore do! src.AwaitShutdown() // 2000 total events From a001b001d7c695efe19a28d948bdddac324c97a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 22:29:06 -0500 Subject: [PATCH 16/44] fix --- tests/Propulsion.MessageDb.Integration/Tests.fs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index a96fad1e..71802f48 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -39,10 +39,6 @@ let writeMessagesToCategory category = task { batch.BatchCommands.Add(cmd) do! batch.ExecuteNonQueryAsync() :> Task } -module Array = - - let isAscending arr = Array.sort arr = arr - [] let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger @@ -77,5 +73,6 @@ let ``It processes events for a category`` () = async { // 20 in each stream test <@ handled |> Array.ofSeq |> Array.groupBy fst |> Array.map (snd >> Array.length) |> Array.forall ((=) 20) @> // they were handled in order within streams - test <@ handled |> Array.ofSeq |> Array.groupBy fst |> Array.map snd |> Array.forall Array.isAscending @> + let ordering = handled |> Array.ofSeq |> Array.groupBy fst |> Array.map (snd >> Array.map snd) + test <@ ordering |> Array.forall ((=) [| 0L..19L |]) @> } From e2aec05d124954cc8870a4dd9a62c0ee76a9cc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 23:19:30 -0500 Subject: [PATCH 17/44] Update CHANGELOG.md Co-authored-by: Ruben Bartelink --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62215254..86554491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ The `Unreleased` section name is replaced by the expected version of next releas - `Propulsion.EventStoreDb`: Ported `EventStore` to target `Equinox.EventStore` >= `4.0.0` (using the gRPC interface) [#139](https://github.com/jet/propulsion/pull/139) - `Propulsion.CosmosStore3`: Special cased version of `Propulsion.CosmosStore` to target `Equinox.CosmosStore` v `[3.0.7`-`3.99.0]` **Deprecated; Please migrate to `Propulsion.CosmosStore` by updating `Equinox.CosmosStore` dependencies to `4.0.0`** [#139](https://github.com/jet/propulsion/pull/139) - `Propulsion.DynamoStore`: `Equinox.CosmosStore`-equivalent functionality for `Equinox.DynamoStore`. Combines elements of `CosmosStore`, `SqlStreamStore`, `Feed` [#140](https://github.com/jet/propulsion/pull/143) [#140](https://github.com/jet/propulsion/pull/143) [#177](https://github.com/jet/propulsion/pull/177) -- `Propulsion.MessageDb`: `FeedSource` for [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181) +- `Propulsion.MessageDb`: `FeedSource` for [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181) :pray: [@nordfjord](https://github.com/nordfjord) - `Propulsion.MemoryStore`: `MemoryStoreSource` to align with other sources for integration testing. Includes *deterministic* `AwaitCompletion` as per `Propulsion.Feed`-based Sources [#165](https://github.com/jet/propulsion/pull/165) - `Propulsion.SqlStreamStore`: Added `startFromTail` [#173](https://github.com/jet/propulsion/pull/173) - `Propulsion.Tool`: `checkpoint` commandline option; enables viewing or overriding checkpoints [#141](https://github.com/jet/propulsion/pull/141) From c7db625c7d970d1279350fd1a99407d619425cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 23:19:42 -0500 Subject: [PATCH 18/44] Update README.md Co-authored-by: Ruben Bartelink --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 365c7e0f..50a6ca05 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ If you're looking for a good discussion forum on these kinds of topics, look no - `Propulsion.Kafka` [![NuGet](https://img.shields.io/nuget/v/Propulsion.Kafka.svg)](https://www.nuget.org/packages/Propulsion.Kafka/) Provides bindings for producing and consuming both streamwise and in parallel. Includes a standard codec for use with streamwise projection and consumption, `Propulsion.Kafka.Codec.NewtonsoftJson.RenderedSpan`. [Depends](https://www.fuget.org/packages/Propulsion.Kafka) on `FsKafka` v `1.7.0`-`1.9.99`, `Serilog` - `Propulsion.MessageDb` [![NuGet](https://img.shields.io/nuget/v/Propulsion.MessageDb.svg)](https://www.nuget.org/packages/Propulsion.MessageDb/). Provides bindings to [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181), maintaining checkpoints in a postgres table [Depends](https://www.fuget.org/packages/Propulsion.MessageDb) on `Propulsion.Feed`, `Npgsql` >= `6.0.7` - 1. `MessageDbSource`: reading from a MessageDb category into a `Propulsion.Sink` + 1. `MessageDbSource`: reading from one or more MessageDb categories into a `Propulsion.Sink` 2. `NpgsqlCheckpointStore`: checkpoint storage for `Propulsion.Feed` using `Npgsql` - `Propulsion.SqlStreamStore` [![NuGet](https://img.shields.io/nuget/v/Propulsion.SqlStreamStore.svg)](https://www.nuget.org/packages/Propulsion.SqlStreamStore/). Provides bindings to [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore), maintaining checkpoints in a SQL table using Dapper. [Depends](https://www.fuget.org/packages/Propulsion.SqlStreamStore) on `Propulsion.Feed`, `SqlStreamStore`, `Dapper` v `2.0`, `Microsoft.Data.SqlClient` v `1.1.3`, `Serilog` From 578182042d02c2a459cbfb9357732fa70e8d8b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 23:38:05 -0500 Subject: [PATCH 19/44] take categories as an argument --- src/Propulsion.MessageDb/MessageDbSource.fs | 11 ++++++++++- tests/Propulsion.MessageDb.Integration/Tests.fs | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 0d9900f0..48180f00 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -76,7 +76,7 @@ module private Impl = return Position.parse lastEventPos } type MessageDbSource - ( log : Serilog.ILogger, statsInterval, + ( log : Serilog.ILogger, categories, statsInterval, reader: MessageDbCategoryReader, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, // Override default start position to be at the tail of the index. Default: Replay all events. @@ -92,3 +92,12 @@ type MessageDbSource let! b = Impl.readBatch batchSize reader req yield sw.Elapsed, b }), string) + + abstract member ListTranches : unit -> Async + default _.ListTranches() = async { return categories |> Array.map TrancheId.parse } + + abstract member Pump : unit -> Async + default x.Pump() = base.Pump(x.ListTranches) + + abstract member Start : unit -> Propulsion.SourcePipeline + default x.Start() = base.Start(x.Pump()) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 71802f48..98cfb4ee 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -60,8 +60,8 @@ let ``It processes events for a category`` () = async { stop.contents() return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) - let source = MessageDbSource(log, TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) - use src = source.Start(source.Pump(fun _ -> async { return [| TrancheId.parse category |] })) + let source = MessageDbSource(log, [| category |], TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) + use src = source.Start() // who says you can't do backwards referencing in F# stop.contents <- src.Stop From 2fea86a8044ae3fe670e5dc09d6f4b67fafd0f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Mon, 14 Nov 2022 23:47:53 -0500 Subject: [PATCH 20/44] change the test to make sure it reads across categories --- tests/Propulsion.MessageDb.Integration/Tests.fs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 98cfb4ee..29da7441 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -26,7 +26,7 @@ let writeMessagesToCategory category = task { use conn = new NpgsqlConnection("Host=localhost; Port=5433; Username=message_store; Password=;") do! conn.OpenAsync() let batch = conn.CreateBatch() - for _ in 1..100 do + for _ in 1..50 do let streamName = $"{category}-{Guid.NewGuid():N}" for _ in 1..20 do let cmd = NpgsqlBatchCommand() @@ -43,8 +43,10 @@ let writeMessagesToCategory category = task { let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger let consumerGroup = $"{Guid.NewGuid():N}" - let category = $"{Guid.NewGuid():N}" - do! writeMessagesToCategory category |> Async.AwaitTaskCorrect + let category1 = $"{Guid.NewGuid():N}" + let category2 = $"{Guid.NewGuid():N}" + do! writeMessagesToCategory category1 |> Async.AwaitTaskCorrect + do! writeMessagesToCategory category2 |> Async.AwaitTaskCorrect let reader = MessageDbCategoryReader("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") let checkpoints = ReaderCheckpoint.CheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) do! checkpoints.CreateSchemaIfNotExists() @@ -60,7 +62,7 @@ let ``It processes events for a category`` () = async { stop.contents() return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) - let source = MessageDbSource(log, [| category |], TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) + let source = MessageDbSource(log, [| category1; category2 |], TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) use src = source.Start() // who says you can't do backwards referencing in F# stop.contents <- src.Stop From 7fc78dcd62aee29efa0fb95ac48f10f3f04a56e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Tue, 15 Nov 2022 00:00:39 -0500 Subject: [PATCH 21/44] add tail semantics when reaching an empty page --- src/Propulsion.MessageDb/MessageDbSource.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 48180f00..4e99116d 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -50,7 +50,8 @@ type MessageDbCategoryReader(connectionString) = events.Add(readRow reader) checkpoint <- reader.GetInt64(9) - return { checkpoint = Position.parse checkpoint; items = events.ToArray(); isTail = false } } + let events = events.ToArray() + return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { use! conn = connect ct let command = conn.CreateCommand() From d305f1192ffa61f8c38f0800e75b7f8aa5a82508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Wed, 16 Nov 2022 09:19:23 -0500 Subject: [PATCH 22/44] reviews --- src/Propulsion.MessageDb/MessageDbSource.fs | 18 ++++++++++++++++++ src/Propulsion.MessageDb/ReaderCheckpoint.fs | 3 +-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 4e99116d..6318b15d 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -10,6 +10,7 @@ open System open System.Data.Common open System.Diagnostics open Propulsion.Feed.Core +open Propulsion.Internal type MessageDbCategoryReader(connectionString) = @@ -102,3 +103,20 @@ type MessageDbSource abstract member Start : unit -> Propulsion.SourcePipeline default x.Start() = base.Start(x.Pump()) + + + /// Pumps to the Sink until either the specified timeout has been reached, or all items in the Source have been fully consumed + member x.RunUntilCaughtUp(timeout : TimeSpan, statsInterval : IntervalTimer) = task { + let sw = Stopwatch.start () + use pipeline = x.Start() + + try System.Threading.Tasks.Task.Delay(timeout).ContinueWith(fun _ -> pipeline.Stop()) |> ignore + + let initialReaderTimeout = TimeSpan.FromMinutes 1. + do! pipeline.Monitor.AwaitCompletion(initialReaderTimeout, awaitFullyCaughtUp = true, logInterval = TimeSpan.FromSeconds 30) + pipeline.Stop() + + if sw.ElapsedSeconds > 2 then statsInterval.Trigger() + // force a final attempt to flush anything not already checkpointed (normally checkpointing is at 5s intervals) + return! x.Checkpoint() + finally statsInterval.SleepUntilTriggerCleared() } diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 9034f497..8db55bc0 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -54,9 +54,8 @@ type CheckpointStore(connString : string, schema: string, consumerGroupName, def let connect = Npgsql.connect connString member _.CreateSchemaIfNotExists() = async { - use conn = new NpgsqlConnection(connString) let! ct = Async.CancellationToken - do! conn.OpenAsync(ct) |> Async.AwaitTaskCorrect + use! conn = connect ct |> Async.AwaitTaskCorrect return! createIfNotExists (conn, schema) } interface IFeedCheckpointStore with From 30a7991060d879f007b422947865e89db145c9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Wed, 16 Nov 2022 20:43:56 -0500 Subject: [PATCH 23/44] add initpg command --- tools/Propulsion.Tool/Args.fs | 31 ++++++++++++++++++++ tools/Propulsion.Tool/Program.fs | 3 ++ tools/Propulsion.Tool/Propulsion.Tool.fsproj | 1 + 3 files changed, 35 insertions(+) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 1b54404a..abb2cb8c 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -1,6 +1,7 @@ module Propulsion.Tool.Args open Argu +open Npgsql open Serilog open System @@ -29,6 +30,10 @@ module Configuration = let [] BROKER = "PROPULSION_KAFKA_BROKER" let [] TOPIC = "PROPULSION_KAFKA_TOPIC" + module MessageDb = + let [] CONNECTION_STRING = "MDB_CONNECTION_STRING" + let [] SCHEMA = "MDB_SCHEMA" + type Configuration(tryGet : string -> string option) = member val tryGet = tryGet @@ -50,6 +55,9 @@ type Configuration(tryGet : string -> string option) = member x.KafkaBroker = x.get Configuration.Kafka.BROKER member x.KafkaTopic = x.get Configuration.Kafka.TOPIC + member x.MdbConnectionString = x.get Configuration.MessageDb.CONNECTION_STRING + member x.MdbSchema = x.get Configuration.MessageDb.SCHEMA + module Cosmos = open Configuration.Cosmos @@ -253,3 +261,26 @@ module Dynamo = member x.CreateCheckpointStore(group, cache, storeLog) = let context = DynamoStoreContext.create indexReadClient.Value Propulsion.Feed.ReaderCheckpoint.DynamoStore.create storeLog (group, checkpointInterval) (context, cache) + +module MessageDb = + type [] Parameters = + | [] ConnectionString of string + | [] Schema of string + interface IArgParserTemplate with + member a.Usage = a |> function + | ConnectionString _ -> "Connection string for the postgres database housing message-db" + | Schema _ -> "Schema that should contain the checkpoints table" + + type Arguments(c : Configuration, p : ParseResults) = + let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) + let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) + + member x.CreateCheckpointStore() = async { + let log = Log.Logger + let connStringWithoutPassword = NpgsqlConnectionStringBuilder(conn) + connStringWithoutPassword.Password <- null + log.Information("Authenticating with postgres using {connection_string}", connStringWithoutPassword.ToString()) + log.Information("Creating checkpoints table as {table}", $"{schema}.propulsion_checkpoint") + let checkpointStore = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(conn, schema, "nil", TimeSpan.FromSeconds 5.) + do! checkpointStore.CreateSchemaIfNotExists() + log.Information("Table created") } diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index df5ef632..1fc7480d 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -15,6 +15,7 @@ type Parameters = | [] VerboseConsole | [] VerboseStore | [] Init of ParseResults + | [] InitPg of ParseResults | [] Index of ParseResults | [] Checkpoint of ParseResults | [] Project of ParseResults @@ -24,6 +25,7 @@ type Parameters = | VerboseConsole -> "Include low level test and store actions logging in on-screen output to console." | VerboseStore -> "Include low level Store logging" | Init _ -> "Initialize auxiliary store (Supported for `cosmos` Only)." + | InitPg _ -> "Initialize a postgres checkpoint store" | Index _ -> "Validate index (optionally, ingest events from a DynamoDB JSON S3 export to remediate missing events)." | Checkpoint _ -> "Display or override checkpoints in Cosmos or Dynamo" | Project _ -> "Project from store specified as the last argument." @@ -377,6 +379,7 @@ let main argv = let c = Args.Configuration(Environment.GetEnvironmentVariable >> Option.ofObj) try match a.GetSubCommand() with | Init a -> CosmosInit.aux (c, a) |> Async.Ignore |> Async.RunSynchronously + | InitPg a -> MessageDb.Arguments(c, a).CreateCheckpointStore() |> Async.RunSynchronously | Checkpoint a -> Checkpoints.readOrOverride (c, a) |> Async.RunSynchronously | Index a -> Indexer.run (c, a) |> Async.RunSynchronously | Project a -> Project.run (c, a) |> Async.RunSynchronously diff --git a/tools/Propulsion.Tool/Propulsion.Tool.fsproj b/tools/Propulsion.Tool/Propulsion.Tool.fsproj index 13113cfc..eaa73513 100644 --- a/tools/Propulsion.Tool/Propulsion.Tool.fsproj +++ b/tools/Propulsion.Tool/Propulsion.Tool.fsproj @@ -21,6 +21,7 @@ + From 2df580f08f1b8a9c5e65afd911848f442144df03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Wed, 16 Nov 2022 20:46:44 -0500 Subject: [PATCH 24/44] docs --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 50a6ca05..600f9a9e 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ If you're looking for a good discussion forum on these kinds of topics, look no - `Propulsion.MessageDb` [![NuGet](https://img.shields.io/nuget/v/Propulsion.MessageDb.svg)](https://www.nuget.org/packages/Propulsion.MessageDb/). Provides bindings to [MessageDb](http://docs.eventide-project.org/user-guide/message-db/) [#181](https://github.com/jet/propulsion/pull/181), maintaining checkpoints in a postgres table [Depends](https://www.fuget.org/packages/Propulsion.MessageDb) on `Propulsion.Feed`, `Npgsql` >= `6.0.7` 1. `MessageDbSource`: reading from one or more MessageDb categories into a `Propulsion.Sink` - 2. `NpgsqlCheckpointStore`: checkpoint storage for `Propulsion.Feed` using `Npgsql` + 2. `CheckpointStore`: checkpoint storage for `Propulsion.Feed` using `Npgsql` (can be initialized via `propulsion initpg -c connstr -s schema`) - `Propulsion.SqlStreamStore` [![NuGet](https://img.shields.io/nuget/v/Propulsion.SqlStreamStore.svg)](https://www.nuget.org/packages/Propulsion.SqlStreamStore/). Provides bindings to [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore), maintaining checkpoints in a SQL table using Dapper. [Depends](https://www.fuget.org/packages/Propulsion.SqlStreamStore) on `Propulsion.Feed`, `SqlStreamStore`, `Dapper` v `2.0`, `Microsoft.Data.SqlClient` v `1.1.3`, `Serilog` @@ -112,6 +112,7 @@ The ubiquitous `Serilog` dependency is solely on the core module, not any sinks. - CosmosDB/DynamoStore/EventStoreDB/Feed/SqlStreamStore: adjust checkpoints - CosmosDB/DynamoStore/EventStoreDB: walk change feeds/indexes and/or project to Kafka - DynamoStore: validate and/or reindex DynamoStore Index + - MessageDb: Initialize a checkpoints table ## Deprecated components From e4a6d388dcff0529b4ba2d2f98a8bbf1129531f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 18:34:57 -0500 Subject: [PATCH 25/44] Review --- src/Propulsion.MessageDb/MessageDbSource.fs | 42 ++++++++----------- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 5 ++- .../Propulsion.MessageDb.Integration/Tests.fs | 2 +- tools/Propulsion.Tool/Args.fs | 12 +++--- 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 6318b15d..cda8558d 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -17,41 +17,35 @@ type MessageDbCategoryReader(connectionString) = let connect = Npgsql.connect connectionString let readRow (reader: DbDataReader) = let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) - let timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)) let streamName = reader.GetString(8) let event = TimelineEvent.Create( - index = reader.GetInt64(0), - eventType = reader.GetString(1), - data = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 2)), - meta = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 3)), - eventId = reader.GetGuid(4), - ?correlationId = readNullableString 5, - ?causationId = readNullableString 6, - timestamp = timestamp) + index = reader.GetInt64(0), + eventType = reader.GetString(1), + data = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 2)), + meta = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 3)), + eventId = reader.GetGuid(4), + ?correlationId = readNullableString 5, + ?causationId = readNullableString 6, + timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))) struct(StreamName.parse streamName, event) member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { use! conn = connect ct - let command = conn.CreateCommand() - command.CommandText <- "select - position, type, data, metadata, id::uuid, - (metadata::jsonb->>'$correlationId')::text, - (metadata::jsonb->>'$causationId')::text, - time, stream_name, global_position - from get_category_messages(@Category, @Position, @BatchSize);" + let command = conn.CreateCommand( + CommandText = "select position, type, data, metadata, id::uuid, + (metadata::jsonb->>'$correlationId')::text, + (metadata::jsonb->>'$causationId')::text, + time, stream_name, global_position + from get_category_messages(@Category, @Position, @BatchSize);") command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore let mutable checkpoint = fromPositionInclusive - let events = ResizeArray() use! reader = command.ExecuteReaderAsync(ct) - while reader.Read() do - events.Add(readRow reader) - checkpoint <- reader.GetInt64(9) + let events = [| while reader.Read() do yield readRow reader |] - let events = events.ToArray() return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { use! conn = connect ct @@ -60,8 +54,7 @@ type MessageDbCategoryReader(connectionString) = command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore use! reader = command.ExecuteReaderAsync(ct) - return if reader.Read() then reader.GetInt64(0) else 0L - } + return if reader.Read() then reader.GetInt64(0) else 0L } module private Impl = open Propulsion.Infrastructure // AwaitTaskCorrect @@ -78,9 +71,10 @@ module private Impl = return Position.parse lastEventPos } type MessageDbSource - ( log : Serilog.ILogger, categories, statsInterval, + ( log : Serilog.ILogger, statsInterval, reader: MessageDbCategoryReader, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, + categories, // Override default start position to be at the tail of the index. Default: Replay all events. ?startFromTail, ?sourceId) = diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 8db55bc0..3549e081 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -5,10 +5,13 @@ open NpgsqlTypes open Propulsion.Feed open Propulsion.Infrastructure + +let table = "propulsion_checkpoint" + let createIfNotExists (conn : NpgsqlConnection, schema: string) = let cmd = conn.CreateCommand() cmd.CommandText <- $" - create table if not exists {schema}.propulsion_checkpoint ( + create table if not exists {schema}.{table} ( source text not null, tranche text not null, consumer_group text not null, diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 29da7441..adbbfdf5 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -62,7 +62,7 @@ let ``It processes events for a category`` () = async { stop.contents() return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) - let source = MessageDbSource(log, [| category1; category2 |], TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink) + let source = MessageDbSource(log, TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink, [| category1; category2 |]) use src = source.Start() // who says you can't do backwards referencing in F# stop.contents <- src.Stop diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index abb2cb8c..51bad3c6 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -263,13 +263,14 @@ module Dynamo = Propulsion.Feed.ReaderCheckpoint.DynamoStore.create storeLog (group, checkpointInterval) (context, cache) module MessageDb = + open Configuration.MessageDb type [] Parameters = | [] ConnectionString of string | [] Schema of string interface IArgParserTemplate with member a.Usage = a |> function - | ConnectionString _ -> "Connection string for the postgres database housing message-db" - | Schema _ -> "Schema that should contain the checkpoints table" + | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" + | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" type Arguments(c : Configuration, p : ParseResults) = let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) @@ -277,10 +278,9 @@ module MessageDb = member x.CreateCheckpointStore() = async { let log = Log.Logger - let connStringWithoutPassword = NpgsqlConnectionStringBuilder(conn) - connStringWithoutPassword.Password <- null - log.Information("Authenticating with postgres using {connection_string}", connStringWithoutPassword.ToString()) - log.Information("Creating checkpoints table as {table}", $"{schema}.propulsion_checkpoint") + let connStringWithoutPassword = NpgsqlConnectionStringBuilder(conn, Password = null) + log.Information("Authenticating with postgres using {connectionString}", connStringWithoutPassword.ToString()) + log.Information("Creating checkpoints table as {table}", $"{schema}.{Propulsion.MessageDb.ReaderCheckpoint.table}") let checkpointStore = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(conn, schema, "nil", TimeSpan.FromSeconds 5.) do! checkpointStore.CreateSchemaIfNotExists() log.Information("Table created") } From f440284fe6b5ae70c250dd451335ed12b9a79a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 19:36:16 -0500 Subject: [PATCH 26/44] rename reader to client --- src/Propulsion.MessageDb/MessageDbSource.fs | 8 ++++---- .../Propulsion.MessageDb.Integration/Tests.fs | 2 +- tools/Propulsion.Tool/Program.fs | 19 +++++++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index cda8558d..095797d9 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -13,7 +13,7 @@ open Propulsion.Feed.Core open Propulsion.Internal -type MessageDbCategoryReader(connectionString) = +type MessageDbCategoryClient(connectionString) = let connect = Npgsql.connect connectionString let readRow (reader: DbDataReader) = let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) @@ -59,20 +59,20 @@ type MessageDbCategoryReader(connectionString) = module private Impl = open Propulsion.Infrastructure // AwaitTaskCorrect - let readBatch batchSize (store : MessageDbCategoryReader) (category, pos) : Async> = async { + let readBatch batchSize (store : MessageDbCategoryClient) (category, pos) : Async> = async { let! ct = Async.CancellationToken let positionInclusive = Position.toInt64 pos let! x = store.ReadCategoryMessages(category, positionInclusive, batchSize, ct) |> Async.AwaitTaskCorrect return x } - let readTailPositionForTranche (store : MessageDbCategoryReader) trancheId : Async = async { + let readTailPositionForTranche (store : MessageDbCategoryClient) trancheId : Async = async { let! ct = Async.CancellationToken let! lastEventPos = store.ReadCategoryLastVersion(trancheId, ct) |> Async.AwaitTaskCorrect return Position.parse lastEventPos } type MessageDbSource ( log : Serilog.ILogger, statsInterval, - reader: MessageDbCategoryReader, batchSize, tailSleepInterval, + reader: MessageDbCategoryClient, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, categories, // Override default start position to be at the tail of the index. Default: Replay all events. diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index adbbfdf5..8cb37e86 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -47,7 +47,7 @@ let ``It processes events for a category`` () = async { let category2 = $"{Guid.NewGuid():N}" do! writeMessagesToCategory category1 |> Async.AwaitTaskCorrect do! writeMessagesToCategory category2 |> Async.AwaitTaskCorrect - let reader = MessageDbCategoryReader("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") + let reader = MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") let checkpoints = ReaderCheckpoint.CheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) do! checkpoints.CreateSchemaIfNotExists() let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index 1fc7480d..3da996c7 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -111,19 +111,23 @@ and [] KafkaParameters = | [] Broker of string | [] Cosmos of ParseResults | [] Dynamo of ParseResults + | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Topic _ -> "Specify target topic. Default: Use $env:PROPULSION_KAFKA_TOPIC" | Broker _ -> "Specify target broker. Default: Use $env:PROPULSION_KAFKA_BROKER" | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." + | MessageDb _ -> "Specify MessageDb parameters." and [] StatsParameters = | [] Cosmos of ParseResults | [] Dynamo of ParseResults + | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." + | MessageDb _ -> "Specify MessageDb parameters." let [] appName = "propulsion-tool" @@ -282,8 +286,9 @@ module Project = type StatsArguments(c, p : ParseResults) = member val StoreArgs = match p.GetSubCommand() with - | StatsParameters.Cosmos p -> Choice1Of2 (Args.Cosmos.Arguments (c, p)) - | StatsParameters.Dynamo p -> Choice2Of2 (Args.Dynamo.Arguments (c, p)) + | StatsParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) + | StatsParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) + | StatsParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) type Arguments(c, p : ParseResults) = member val IdleDelay = TimeSpan.FromMilliseconds 10. @@ -306,8 +311,9 @@ module Project = let a = Arguments(c, p) let storeArgs, dumpStoreStats = match a.StoreArgs with - | Choice1Of2 sa -> Choice1Of2 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump - | Choice2Of2 sa -> Choice2Of2 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump + | Choice1Of3 sa -> Choice1Of3 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump + | Choice2Of3 sa -> Choice2Of3 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump + | Choice3Of3 sa -> Choice3Of3 sa, Equinox.MessageDb.Core.Log.InternalMetrics.dump let group, startFromTail, maxItems = p.GetResult ConsumerGroupName, p.Contains FromTail, p.TryGetResult MaxItems match maxItems with None -> () | Some bs -> Log.Information("ChangeFeed Max items Count {changeFeedMaxItems}", bs) if startFromTail then Log.Warning("ChangeFeed (If new projector group) Skipping projection of all existing events.") @@ -334,7 +340,7 @@ module Project = let source = let nullFilter _ = true match storeArgs with - | Choice1Of2 sa -> + | Choice1Of3 sa -> let monitored = sa.MonitoredContainer() let leases = sa.ConnectLeases() let parseFeedDoc = Propulsion.CosmosStore.EquinoxSystemTextJsonParser.enumStreamEvents nullFilter @@ -342,7 +348,7 @@ module Project = Propulsion.CosmosStore.CosmosStoreSource.Start ( Log.Logger, monitored, leases, group, observer, startFromTail = startFromTail, ?maxItems = maxItems, ?lagReportFreq = sa.MaybeLogLagInterval) - | Choice2Of2 sa -> + | Choice2Of3 sa -> let (indexStore, indexFilter), maybeHydrate = sa.MonitoringParams() let checkpoints = let cache = Equinox.Cache (appName, sizeMb = 1) @@ -358,6 +364,7 @@ module Project = checkpoints, sink, loadMode, startFromTail = startFromTail, storeLog = Log.forMetrics, ?trancheIds = indexFilter ).Start() + | Choice3Of3 _ -> () let work = [ Async.AwaitKeyboardInterruptAsTaskCanceledException() sink.AwaitWithStopOnCancellation() From f5ff105af386d28332335eabb442bdc053e3c61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 19:36:51 -0500 Subject: [PATCH 27/44] renames --- src/Propulsion.MessageDb/MessageDbSource.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 095797d9..8298de68 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -15,7 +15,7 @@ open Propulsion.Internal type MessageDbCategoryClient(connectionString) = let connect = Npgsql.connect connectionString - let readRow (reader: DbDataReader) = + let parseRow (reader: DbDataReader) = let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) let streamName = reader.GetString(8) let event = TimelineEvent.Create( @@ -44,7 +44,7 @@ type MessageDbCategoryClient(connectionString) = let mutable checkpoint = fromPositionInclusive use! reader = command.ExecuteReaderAsync(ct) - let events = [| while reader.Read() do yield readRow reader |] + let events = [| while reader.Read() do yield parseRow reader |] return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { From 782842013cc9d434c1d76c0709e0a8d7e53e475e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 19:58:19 -0500 Subject: [PATCH 28/44] move to rider --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b2eaf6b1..9612cb6e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,11 +21,11 @@ packages.config ## Visual Studio 2015+ cache/options directory .vs/ *.fsproj.user -*.DotSettings.user ## JetBrains Rider .idea/ *.sln.iml +*.DotSettings.user ## CodeRush .cr/ From 3abd627eae2eca9233a883f10207eb9eecb95486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 20:01:06 -0500 Subject: [PATCH 29/44] Add constructor with connection string --- src/Propulsion.MessageDb/MessageDbSource.fs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 8298de68..2c2213b1 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -88,6 +88,9 @@ type MessageDbSource let! b = Impl.readBatch batchSize reader req yield sw.Elapsed, b }), string) + new (log, statsInterval, connectionString, batchSize, tailSleepInterval, checkpoints, sink, categories, ?startFromTail, ?sourceId) = + MessageDbSource(log, statsInterval, MessageDbCategoryClient(connectionString), + batchSize, tailSleepInterval, checkpoints, sink, categories, ?startFromTail=startFromTail, ?sourceId=sourceId) abstract member ListTranches : unit -> Async default _.ListTranches() = async { return categories |> Array.map TrancheId.parse } From 33494a29c19c8de689e826d71c0a92c4ae82c5a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 20:11:45 -0500 Subject: [PATCH 30/44] Update README.md Co-authored-by: Ruben Bartelink --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 600f9a9e..e5028fb8 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ The ubiquitous `Serilog` dependency is solely on the core module, not any sinks. - CosmosDB/DynamoStore/EventStoreDB/Feed/SqlStreamStore: adjust checkpoints - CosmosDB/DynamoStore/EventStoreDB: walk change feeds/indexes and/or project to Kafka - DynamoStore: validate and/or reindex DynamoStore Index - - MessageDb: Initialize a checkpoints table + - MessageDb: Initialize a checkpoints table in a Postgres Database ## Deprecated components From 8d1d0cc4e47bbd340a167dfa57e8dfbc263896fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 20:13:53 -0500 Subject: [PATCH 31/44] revert silly --- tools/Propulsion.Tool/Program.fs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index 3da996c7..1fc7480d 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -111,23 +111,19 @@ and [] KafkaParameters = | [] Broker of string | [] Cosmos of ParseResults | [] Dynamo of ParseResults - | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Topic _ -> "Specify target topic. Default: Use $env:PROPULSION_KAFKA_TOPIC" | Broker _ -> "Specify target broker. Default: Use $env:PROPULSION_KAFKA_BROKER" | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." - | MessageDb _ -> "Specify MessageDb parameters." and [] StatsParameters = | [] Cosmos of ParseResults | [] Dynamo of ParseResults - | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." - | MessageDb _ -> "Specify MessageDb parameters." let [] appName = "propulsion-tool" @@ -286,9 +282,8 @@ module Project = type StatsArguments(c, p : ParseResults) = member val StoreArgs = match p.GetSubCommand() with - | StatsParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) - | StatsParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) - | StatsParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) + | StatsParameters.Cosmos p -> Choice1Of2 (Args.Cosmos.Arguments (c, p)) + | StatsParameters.Dynamo p -> Choice2Of2 (Args.Dynamo.Arguments (c, p)) type Arguments(c, p : ParseResults) = member val IdleDelay = TimeSpan.FromMilliseconds 10. @@ -311,9 +306,8 @@ module Project = let a = Arguments(c, p) let storeArgs, dumpStoreStats = match a.StoreArgs with - | Choice1Of3 sa -> Choice1Of3 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump - | Choice2Of3 sa -> Choice2Of3 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump - | Choice3Of3 sa -> Choice3Of3 sa, Equinox.MessageDb.Core.Log.InternalMetrics.dump + | Choice1Of2 sa -> Choice1Of2 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump + | Choice2Of2 sa -> Choice2Of2 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump let group, startFromTail, maxItems = p.GetResult ConsumerGroupName, p.Contains FromTail, p.TryGetResult MaxItems match maxItems with None -> () | Some bs -> Log.Information("ChangeFeed Max items Count {changeFeedMaxItems}", bs) if startFromTail then Log.Warning("ChangeFeed (If new projector group) Skipping projection of all existing events.") @@ -340,7 +334,7 @@ module Project = let source = let nullFilter _ = true match storeArgs with - | Choice1Of3 sa -> + | Choice1Of2 sa -> let monitored = sa.MonitoredContainer() let leases = sa.ConnectLeases() let parseFeedDoc = Propulsion.CosmosStore.EquinoxSystemTextJsonParser.enumStreamEvents nullFilter @@ -348,7 +342,7 @@ module Project = Propulsion.CosmosStore.CosmosStoreSource.Start ( Log.Logger, monitored, leases, group, observer, startFromTail = startFromTail, ?maxItems = maxItems, ?lagReportFreq = sa.MaybeLogLagInterval) - | Choice2Of3 sa -> + | Choice2Of2 sa -> let (indexStore, indexFilter), maybeHydrate = sa.MonitoringParams() let checkpoints = let cache = Equinox.Cache (appName, sizeMb = 1) @@ -364,7 +358,6 @@ module Project = checkpoints, sink, loadMode, startFromTail = startFromTail, storeLog = Log.forMetrics, ?trancheIds = indexFilter ).Start() - | Choice3Of3 _ -> () let work = [ Async.AwaitKeyboardInterruptAsTaskCanceledException() sink.AwaitWithStopOnCancellation() From 22f2778e5e858c1f291cba2ab5d40ae0dab91516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 21:29:13 -0500 Subject: [PATCH 32/44] wire up kafka and project command --- src/Propulsion.MessageDb/MessageDbSource.fs | 8 ++--- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 6 ++-- tools/Propulsion.Tool/Args.fs | 13 ++++++-- tools/Propulsion.Tool/Program.fs | 34 ++++++++++++++------ 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 2c2213b1..a0fcc6e4 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -74,7 +74,7 @@ type MessageDbSource ( log : Serilog.ILogger, statsInterval, reader: MessageDbCategoryClient, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, - categories, + trancheIds, // Override default start position to be at the tail of the index. Default: Replay all events. ?startFromTail, ?sourceId) = @@ -88,12 +88,12 @@ type MessageDbSource let! b = Impl.readBatch batchSize reader req yield sw.Elapsed, b }), string) - new (log, statsInterval, connectionString, batchSize, tailSleepInterval, checkpoints, sink, categories, ?startFromTail, ?sourceId) = + new (log, statsInterval, connectionString, batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail, ?sourceId) = MessageDbSource(log, statsInterval, MessageDbCategoryClient(connectionString), - batchSize, tailSleepInterval, checkpoints, sink, categories, ?startFromTail=startFromTail, ?sourceId=sourceId) + batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail=startFromTail, ?sourceId=sourceId) abstract member ListTranches : unit -> Async - default _.ListTranches() = async { return categories |> Array.map TrancheId.parse } + default _.ListTranches() = async { return trancheIds } abstract member Pump : unit -> Async default x.Pump() = base.Pump(x.ListTranches) diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 3549e081..2a5cfe2c 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -25,8 +25,8 @@ let commitPosition (conn : NpgsqlConnection, schema: string) source tranche (con = async { let cmd = conn.CreateCommand() cmd.CommandText <- - $"insert into {schema}.propulsion_checkpoint(source, tranche, consumer_group, position) - values (@Source, @Tranche, @ConsumerGroup, @GlobalPosition) + $"insert into {schema}.{table}(source, tranche, consumer_group, position) + values (@Source, @Tranche, @ConsumerGroup, @Position) on conflict (source, tranche, consumer_group) do update set position = @Position;" cmd.Parameters.AddWithValue("Source", NpgsqlDbType.Text, SourceId.toString source) |> ignore @@ -40,7 +40,7 @@ let commitPosition (conn : NpgsqlConnection, schema: string) source tranche (con let tryGetPosition (conn : NpgsqlConnection, schema : string) source tranche (consumerGroup : string) = async { let cmd = conn.CreateCommand() cmd.CommandText <- - $"select position from {schema}.propulsion_checkpoint + $"select position from {schema}.{table} where source = @Source and tranche = @Tranche and consumer_group = @ConsumerGroup" diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 51bad3c6..af8d2650 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -267,20 +267,27 @@ module MessageDb = type [] Parameters = | [] ConnectionString of string | [] Schema of string + | [] Tranches of Propulsion.Feed.TrancheId interface IArgParserTemplate with member a.Usage = a |> function | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" + | Tranches _ -> "The message-db categories to load" type Arguments(c : Configuration, p : ParseResults) = let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) - member x.CreateCheckpointStore() = async { + member x.CreateClient() = Array.ofList (p.GetResults Tranches), Propulsion.MessageDb.MessageDbCategoryClient(conn) + + member x.CreateCheckpointStore(group) = + Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(conn, schema, group, TimeSpan.FromSeconds 5.) + member x.CreateCheckpointStoreTable() = async { let log = Log.Logger let connStringWithoutPassword = NpgsqlConnectionStringBuilder(conn, Password = null) log.Information("Authenticating with postgres using {connectionString}", connStringWithoutPassword.ToString()) log.Information("Creating checkpoints table as {table}", $"{schema}.{Propulsion.MessageDb.ReaderCheckpoint.table}") - let checkpointStore = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(conn, schema, "nil", TimeSpan.FromSeconds 5.) + let checkpointStore = x.CreateCheckpointStore("nil") do! checkpointStore.CreateSchemaIfNotExists() - log.Information("Table created") } + log.Information("Table created") + } diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index 1fc7480d..b585efeb 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -1,6 +1,7 @@ module Propulsion.Tool.Program open Argu +open Propulsion.Feed open Propulsion.Internal // AwaitKeyboardInterruptAsTaskCanceledException open Propulsion.Tool.Args open Serilog @@ -111,19 +112,23 @@ and [] KafkaParameters = | [] Broker of string | [] Cosmos of ParseResults | [] Dynamo of ParseResults + | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Topic _ -> "Specify target topic. Default: Use $env:PROPULSION_KAFKA_TOPIC" | Broker _ -> "Specify target broker. Default: Use $env:PROPULSION_KAFKA_BROKER" | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." + | MessageDb _ -> "Specify MessageDb parameters." and [] StatsParameters = | [] Cosmos of ParseResults | [] Dynamo of ParseResults + | [] MessageDb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." + | MessageDb _ -> "Specify MessageDb parameters." let [] appName = "propulsion-tool" @@ -275,15 +280,17 @@ module Project = member _.Topic = p.TryGetResult Topic |> Option.defaultWith (fun () -> c.KafkaTopic) member val StoreArgs = match p.GetSubCommand() with - | KafkaParameters.Cosmos p -> Choice1Of2 (Args.Cosmos.Arguments (c, p)) - | KafkaParameters.Dynamo p -> Choice2Of2 (Args.Dynamo.Arguments (c, p)) + | KafkaParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) + | KafkaParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) + | KafkaParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) | x -> missingArg $"unexpected subcommand %A{x}" type StatsArguments(c, p : ParseResults) = member val StoreArgs = match p.GetSubCommand() with - | StatsParameters.Cosmos p -> Choice1Of2 (Args.Cosmos.Arguments (c, p)) - | StatsParameters.Dynamo p -> Choice2Of2 (Args.Dynamo.Arguments (c, p)) + | StatsParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) + | StatsParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) + | StatsParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) type Arguments(c, p : ParseResults) = member val IdleDelay = TimeSpan.FromMilliseconds 10. @@ -306,8 +313,9 @@ module Project = let a = Arguments(c, p) let storeArgs, dumpStoreStats = match a.StoreArgs with - | Choice1Of2 sa -> Choice1Of2 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump - | Choice2Of2 sa -> Choice2Of2 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump + | Choice1Of3 sa -> Choice1Of3 sa, Equinox.CosmosStore.Core.Log.InternalMetrics.dump + | Choice2Of3 sa -> Choice2Of3 sa, Equinox.DynamoStore.Core.Log.InternalMetrics.dump + | Choice3Of3 sa -> Choice3Of3 sa, (fun _ -> ()) let group, startFromTail, maxItems = p.GetResult ConsumerGroupName, p.Contains FromTail, p.TryGetResult MaxItems match maxItems with None -> () | Some bs -> Log.Information("ChangeFeed Max items Count {changeFeedMaxItems}", bs) if startFromTail then Log.Warning("ChangeFeed (If new projector group) Skipping projection of all existing events.") @@ -334,7 +342,7 @@ module Project = let source = let nullFilter _ = true match storeArgs with - | Choice1Of2 sa -> + | Choice1Of3 sa -> let monitored = sa.MonitoredContainer() let leases = sa.ConnectLeases() let parseFeedDoc = Propulsion.CosmosStore.EquinoxSystemTextJsonParser.enumStreamEvents nullFilter @@ -342,7 +350,7 @@ module Project = Propulsion.CosmosStore.CosmosStoreSource.Start ( Log.Logger, monitored, leases, group, observer, startFromTail = startFromTail, ?maxItems = maxItems, ?lagReportFreq = sa.MaybeLogLagInterval) - | Choice2Of2 sa -> + | Choice2Of3 sa -> let (indexStore, indexFilter), maybeHydrate = sa.MonitoringParams() let checkpoints = let cache = Equinox.Cache (appName, sizeMb = 1) @@ -358,6 +366,14 @@ module Project = checkpoints, sink, loadMode, startFromTail = startFromTail, storeLog = Log.forMetrics, ?trancheIds = indexFilter ).Start() + | Choice3Of3 sa -> + let checkpoints = sa.CreateCheckpointStore(group) + let trancheIds, client = sa.CreateClient() + Propulsion.MessageDb.MessageDbSource( + Log.Logger, stats.StatsInterval, + client, defaultArg maxItems 100, TimeSpan.FromSeconds 0.5, + checkpoints, sink, trancheIds + ).Start() let work = [ Async.AwaitKeyboardInterruptAsTaskCanceledException() sink.AwaitWithStopOnCancellation() @@ -379,7 +395,7 @@ let main argv = let c = Args.Configuration(Environment.GetEnvironmentVariable >> Option.ofObj) try match a.GetSubCommand() with | Init a -> CosmosInit.aux (c, a) |> Async.Ignore |> Async.RunSynchronously - | InitPg a -> MessageDb.Arguments(c, a).CreateCheckpointStore() |> Async.RunSynchronously + | InitPg a -> MessageDb.Arguments(c, a).CreateCheckpointStoreTable() |> Async.RunSynchronously | Checkpoint a -> Checkpoints.readOrOverride (c, a) |> Async.RunSynchronously | Index a -> Indexer.run (c, a) |> Async.RunSynchronously | Project a -> Project.run (c, a) |> Async.RunSynchronously From 9a5e3cc25b09e50889a15008728227489adc98df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 21:41:00 -0500 Subject: [PATCH 33/44] fix --- README.md | 2 +- src/Propulsion.MessageDb/MessageDbSource.fs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e5028fb8..1fd7766d 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The ubiquitous `Serilog` dependency is solely on the core module, not any sinks. - CosmosDB: Initialize `-aux` Container for ChangeFeedProcessor - CosmosDB/DynamoStore/EventStoreDB/Feed/SqlStreamStore: adjust checkpoints - - CosmosDB/DynamoStore/EventStoreDB: walk change feeds/indexes and/or project to Kafka + - CosmosDB/DynamoStore/EventStoreDB/MessageDb: walk change feeds/indexes and/or project to Kafka - DynamoStore: validate and/or reindex DynamoStore Index - MessageDb: Initialize a checkpoints table in a Postgres Database diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index a0fcc6e4..303e1259 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -26,6 +26,7 @@ type MessageDbCategoryClient(connectionString) = eventId = reader.GetGuid(4), ?correlationId = readNullableString 5, ?causationId = readNullableString 6, + context = reader.GetInt64(9), timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))) struct(StreamName.parse streamName, event) @@ -46,6 +47,8 @@ type MessageDbCategoryClient(connectionString) = use! reader = command.ExecuteReaderAsync(ct) let events = [| while reader.Read() do yield parseRow reader |] + checkpoint <- match Array.tryLast events with Some (_, ev) -> unbox ev.Context | None -> checkpoint + return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { use! conn = connect ct From 4eb0a9ea2ccfe8ee240d1fd7b8a5b83cd1bbca00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 21:45:47 -0500 Subject: [PATCH 34/44] fix tests --- tests/Propulsion.MessageDb.Integration/Tests.fs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index 8cb37e86..d5fa093f 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -43,8 +43,8 @@ let writeMessagesToCategory category = task { let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger let consumerGroup = $"{Guid.NewGuid():N}" - let category1 = $"{Guid.NewGuid():N}" - let category2 = $"{Guid.NewGuid():N}" + let category1 = TrancheId.parse $"{Guid.NewGuid():N}" + let category2 = TrancheId.parse $"{Guid.NewGuid():N}" do! writeMessagesToCategory category1 |> Async.AwaitTaskCorrect do! writeMessagesToCategory category2 |> Async.AwaitTaskCorrect let reader = MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") @@ -62,7 +62,10 @@ let ``It processes events for a category`` () = async { stop.contents() return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } use sink = Propulsion.Streams.Default.Config.Start(log, 2, 2, handle, stats, TimeSpan.FromMinutes 1) - let source = MessageDbSource(log, TimeSpan.FromMinutes 1, reader, 1000, TimeSpan.FromMilliseconds 100, checkpoints, sink, [| category1; category2 |]) + let source = MessageDbSource( + log, TimeSpan.FromMinutes 1, + reader, 1000, TimeSpan.FromMilliseconds 100, + checkpoints, sink, [| category1; category2 |]) use src = source.Start() // who says you can't do backwards referencing in F# stop.contents <- src.Stop From f7e105e5181bf99ee53495ae427fdb1b42c980c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 21:47:21 -0500 Subject: [PATCH 35/44] rename --- src/Propulsion.MessageDb/MessageDbSource.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 303e1259..a0387bb0 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -75,7 +75,7 @@ module private Impl = type MessageDbSource ( log : Serilog.ILogger, statsInterval, - reader: MessageDbCategoryClient, batchSize, tailSleepInterval, + client: MessageDbCategoryClient, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, trancheIds, // Override default start position to be at the tail of the index. Default: Replay all events. @@ -84,11 +84,11 @@ type MessageDbSource inherit Propulsion.Feed.Core.TailingFeedSource ( log, statsInterval, defaultArg sourceId FeedSourceId.wellKnownId, tailSleepInterval, checkpoints, ( if startFromTail <> Some true then None - else Some (Impl.readTailPositionForTranche reader)), + else Some (Impl.readTailPositionForTranche client)), sink, (fun req -> asyncSeq { let sw = Stopwatch.StartNew() - let! b = Impl.readBatch batchSize reader req + let! b = Impl.readBatch batchSize client req yield sw.Elapsed, b }), string) new (log, statsInterval, connectionString, batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail, ?sourceId) = From 70b0143a68a55f66e3a083e90b46d9d19ed62840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 21:56:00 -0500 Subject: [PATCH 36/44] add separate arg for checkpoint conn string --- tools/Propulsion.Tool/Args.fs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index af8d2650..4f3c0722 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -266,25 +266,28 @@ module MessageDb = open Configuration.MessageDb type [] Parameters = | [] ConnectionString of string + | [] CheckpointConnectionString of string | [] Schema of string | [] Tranches of Propulsion.Feed.TrancheId interface IArgParserTemplate with member a.Usage = a |> function - | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" - | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" - | Tranches _ -> "The message-db categories to load" + | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" + | CheckpointConnectionString _ -> "Connection string used for the checkpoint store. If not specified, defaults to the connection string argument" + | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" + | Tranches _ -> "The message-db categories to load" type Arguments(c : Configuration, p : ParseResults) = let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) + let checkpointConn = p.TryGetResult CheckpointConnectionString |> Option.defaultValue conn let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) member x.CreateClient() = Array.ofList (p.GetResults Tranches), Propulsion.MessageDb.MessageDbCategoryClient(conn) member x.CreateCheckpointStore(group) = - Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(conn, schema, group, TimeSpan.FromSeconds 5.) + Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(checkpointConn, schema, group, TimeSpan.FromSeconds 5.) member x.CreateCheckpointStoreTable() = async { let log = Log.Logger - let connStringWithoutPassword = NpgsqlConnectionStringBuilder(conn, Password = null) + let connStringWithoutPassword = NpgsqlConnectionStringBuilder(checkpointConn, Password = null) log.Information("Authenticating with postgres using {connectionString}", connStringWithoutPassword.ToString()) log.Information("Creating checkpoints table as {table}", $"{schema}.{Propulsion.MessageDb.ReaderCheckpoint.table}") let checkpointStore = x.CreateCheckpointStore("nil") From 0fd95eb0ef1e766296f993f28abc8cf714fe1f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 22:03:53 -0500 Subject: [PATCH 37/44] The big Mdb rename --- tools/Propulsion.Tool/Args.fs | 10 +++++----- tools/Propulsion.Tool/Program.fs | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 4f3c0722..b495f154 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -30,7 +30,7 @@ module Configuration = let [] BROKER = "PROPULSION_KAFKA_BROKER" let [] TOPIC = "PROPULSION_KAFKA_TOPIC" - module MessageDb = + module Mdb = let [] CONNECTION_STRING = "MDB_CONNECTION_STRING" let [] SCHEMA = "MDB_SCHEMA" @@ -55,8 +55,8 @@ type Configuration(tryGet : string -> string option) = member x.KafkaBroker = x.get Configuration.Kafka.BROKER member x.KafkaTopic = x.get Configuration.Kafka.TOPIC - member x.MdbConnectionString = x.get Configuration.MessageDb.CONNECTION_STRING - member x.MdbSchema = x.get Configuration.MessageDb.SCHEMA + member x.MdbConnectionString = x.get Configuration.Mdb.CONNECTION_STRING + member x.MdbSchema = x.get Configuration.Mdb.SCHEMA module Cosmos = @@ -262,8 +262,8 @@ module Dynamo = let context = DynamoStoreContext.create indexReadClient.Value Propulsion.Feed.ReaderCheckpoint.DynamoStore.create storeLog (group, checkpointInterval) (context, cache) -module MessageDb = - open Configuration.MessageDb +module Mdb = + open Configuration.Mdb type [] Parameters = | [] ConnectionString of string | [] CheckpointConnectionString of string diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index b585efeb..43dbedbf 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -16,7 +16,7 @@ type Parameters = | [] VerboseConsole | [] VerboseStore | [] Init of ParseResults - | [] InitPg of ParseResults + | [] InitPg of ParseResults | [] Index of ParseResults | [] Checkpoint of ParseResults | [] Project of ParseResults @@ -112,23 +112,23 @@ and [] KafkaParameters = | [] Broker of string | [] Cosmos of ParseResults | [] Dynamo of ParseResults - | [] MessageDb of ParseResults + | [] Mdb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Topic _ -> "Specify target topic. Default: Use $env:PROPULSION_KAFKA_TOPIC" | Broker _ -> "Specify target broker. Default: Use $env:PROPULSION_KAFKA_BROKER" | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." - | MessageDb _ -> "Specify MessageDb parameters." + | Mdb _ -> "Specify MessageDb parameters." and [] StatsParameters = | [] Cosmos of ParseResults | [] Dynamo of ParseResults - | [] MessageDb of ParseResults + | [] Mdb of ParseResults interface IArgParserTemplate with member a.Usage = a |> function | Cosmos _ -> "Specify CosmosDB parameters." | Dynamo _ -> "Specify DynamoDB parameters." - | MessageDb _ -> "Specify MessageDb parameters." + | Mdb _ -> "Specify MessageDb parameters." let [] appName = "propulsion-tool" @@ -282,7 +282,7 @@ module Project = match p.GetSubCommand() with | KafkaParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) | KafkaParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) - | KafkaParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) + | KafkaParameters.Mdb p -> Choice3Of3 (Args.Mdb.Arguments (c, p)) | x -> missingArg $"unexpected subcommand %A{x}" type StatsArguments(c, p : ParseResults) = @@ -290,7 +290,7 @@ module Project = match p.GetSubCommand() with | StatsParameters.Cosmos p -> Choice1Of3 (Args.Cosmos.Arguments (c, p)) | StatsParameters.Dynamo p -> Choice2Of3 (Args.Dynamo.Arguments (c, p)) - | StatsParameters.MessageDb p -> Choice3Of3 (Args.MessageDb.Arguments (c, p)) + | StatsParameters.Mdb p -> Choice3Of3 (Args.Mdb.Arguments (c, p)) type Arguments(c, p : ParseResults) = member val IdleDelay = TimeSpan.FromMilliseconds 10. @@ -395,7 +395,7 @@ let main argv = let c = Args.Configuration(Environment.GetEnvironmentVariable >> Option.ofObj) try match a.GetSubCommand() with | Init a -> CosmosInit.aux (c, a) |> Async.Ignore |> Async.RunSynchronously - | InitPg a -> MessageDb.Arguments(c, a).CreateCheckpointStoreTable() |> Async.RunSynchronously + | InitPg a -> Mdb.Arguments(c, a).CreateCheckpointStoreTable() |> Async.RunSynchronously | Checkpoint a -> Checkpoints.readOrOverride (c, a) |> Async.RunSynchronously | Index a -> Indexer.run (c, a) |> Async.RunSynchronously | Project a -> Project.run (c, a) |> Async.RunSynchronously From 2e4b6eb582c4bbd95cf46d1ded9aae86d73cc396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Thu, 17 Nov 2022 22:23:27 -0500 Subject: [PATCH 38/44] add a readme specifically for messagedb, mostly for myself --- .../Propulsion.MessageDb.fsproj | 1 + src/Propulsion.MessageDb/Readme.md | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/Propulsion.MessageDb/Readme.md diff --git a/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj index 07514a13..3d4bde9c 100644 --- a/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj +++ b/src/Propulsion.MessageDb/Propulsion.MessageDb.fsproj @@ -12,6 +12,7 @@ + diff --git a/src/Propulsion.MessageDb/Readme.md b/src/Propulsion.MessageDb/Readme.md new file mode 100644 index 00000000..0aaf7e6f --- /dev/null +++ b/src/Propulsion.MessageDb/Readme.md @@ -0,0 +1,49 @@ +# Propulsion.MessageDb + +This project houses a Propulsion source for [MessageDb](http://docs.eventide-project.org/user-guide/message-db/). + +## Quickstart + +The smallest possible sample looks like this, it is intended to give an overview of how the different pieces relate. +For a more production ready example to take a look at [jets' templates](https://github.com/jet/dotnet-templates) + +```fsharp +let quickStart log stats trancheIds handle = async { + // The group is used as a key to store and retrieve checkpoints + let groupName = "MyGroup" + // The checkpoint store will receive the highest version + // that has been handled and flushes it to the + // table on an interval + let checkpoints = ReaderCheckpoint.CheckpointStore("Host=localhost; Port=5433; Username=postgres; Password=postgres", "public", groupName, TimeSpan.FromSeconds 10) + // Creates the checkpoint table in the schema + // You can also create this manually + do! checkpoints.CreateSchemaIfNotExists() + + let client = MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") + let maxReadAhead = 100 + let maxConcurrentStreams = 2 + use sink = + Propulsion.Streams.Default.Config.Start( + log, maxReadAhead, maxConcurrentStreams, + handle, stats, TimeSpan.FromMinutes 1) + + use src = + MessageDbSource( + log, statsInterval = TimeSpan.FromMinutes 1, + client, batchSize = 1000, + // Controls the time to wait once fully caught up + // before requesting a new batch of events + tailSleepInterval = TimeSpan.FromMilliseconds 100, + checkpoints, sink, + // tranche is equivalent to a message-db category + trancheIds + ).Start() + + do! src.AwaitShutdown() } + +let handle struct(stream, evts: StreamSpan<_>) = async { + // process the events + return struct (Propulsion.Streams.SpanResult.AllProcessed, ()) } + +quickStart Log.Logger (createStats ()) [| category |] handle +``` From 9e580183e057a7e4a9243e0f894f4176477cb656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 11:23:25 -0500 Subject: [PATCH 39/44] review --- src/Propulsion.MessageDb/MessageDbSource.fs | 9 ++++----- src/Propulsion.MessageDb/Readme.md | 8 +++++--- .../Propulsion.MessageDb.Integration.fsproj | 2 +- tests/Propulsion.MessageDb.Integration/Tests.fs | 4 ++-- tools/Propulsion.Tool/Args.fs | 5 ++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index a0387bb0..e87ebf56 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -3,14 +3,13 @@ namespace Propulsion.MessageDb open FSharp.Control open FsCodec open FsCodec.Core -open Npgsql open NpgsqlTypes open Propulsion.Feed +open Propulsion.Feed.Core +open Propulsion.Internal open System open System.Data.Common open System.Diagnostics -open Propulsion.Feed.Core -open Propulsion.Internal type MessageDbCategoryClient(connectionString) = @@ -77,7 +76,7 @@ type MessageDbSource ( log : Serilog.ILogger, statsInterval, client: MessageDbCategoryClient, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, - trancheIds, + categories, // Override default start position to be at the tail of the index. Default: Replay all events. ?startFromTail, ?sourceId) = @@ -96,7 +95,7 @@ type MessageDbSource batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail=startFromTail, ?sourceId=sourceId) abstract member ListTranches : unit -> Async - default _.ListTranches() = async { return trancheIds } + default _.ListTranches() = async { return categories |> Array.map TrancheId.parse } abstract member Pump : unit -> Async default x.Pump() = base.Pump(x.ListTranches) diff --git a/src/Propulsion.MessageDb/Readme.md b/src/Propulsion.MessageDb/Readme.md index 0aaf7e6f..d7c6cd14 100644 --- a/src/Propulsion.MessageDb/Readme.md +++ b/src/Propulsion.MessageDb/Readme.md @@ -8,7 +8,7 @@ The smallest possible sample looks like this, it is intended to give an overview For a more production ready example to take a look at [jets' templates](https://github.com/jet/dotnet-templates) ```fsharp -let quickStart log stats trancheIds handle = async { +let quickStart log stats categories handle = async { // The group is used as a key to store and retrieve checkpoints let groupName = "MyGroup" // The checkpoint store will receive the highest version @@ -35,8 +35,10 @@ let quickStart log stats trancheIds handle = async { // before requesting a new batch of events tailSleepInterval = TimeSpan.FromMilliseconds 100, checkpoints, sink, - // tranche is equivalent to a message-db category - trancheIds + // An array of message-db categories to subscribe to + // Propulsion guarantees that events within streams are + // handled in order, it makes no guarantees across streams (Even within categories) + categories ).Start() do! src.AwaitShutdown() } diff --git a/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj index a5bef440..28a4f670 100644 --- a/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj +++ b/tests/Propulsion.MessageDb.Integration/Propulsion.MessageDb.Integration.fsproj @@ -12,7 +12,7 @@ - + diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index d5fa093f..cbe003b1 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -43,8 +43,8 @@ let writeMessagesToCategory category = task { let ``It processes events for a category`` () = async { let log = Serilog.Log.Logger let consumerGroup = $"{Guid.NewGuid():N}" - let category1 = TrancheId.parse $"{Guid.NewGuid():N}" - let category2 = TrancheId.parse $"{Guid.NewGuid():N}" + let category1 = $"{Guid.NewGuid():N}" + let category2 = $"{Guid.NewGuid():N}" do! writeMessagesToCategory category1 |> Async.AwaitTaskCorrect do! writeMessagesToCategory category2 |> Async.AwaitTaskCorrect let reader = MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index b495f154..8ad86b95 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -1,7 +1,6 @@ module Propulsion.Tool.Args open Argu -open Npgsql open Serilog open System @@ -264,6 +263,7 @@ module Dynamo = module Mdb = open Configuration.Mdb + open Npgsql type [] Parameters = | [] ConnectionString of string | [] CheckpointConnectionString of string @@ -292,5 +292,4 @@ module Mdb = log.Information("Creating checkpoints table as {table}", $"{schema}.{Propulsion.MessageDb.ReaderCheckpoint.table}") let checkpointStore = x.CreateCheckpointStore("nil") do! checkpointStore.CreateSchemaIfNotExists() - log.Information("Table created") - } + log.Information("Table created") } From e17d446725d5d4b26579e69e57c9d5765efdf140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 11:26:55 -0500 Subject: [PATCH 40/44] command text in style --- src/Propulsion.MessageDb/MessageDbSource.fs | 14 ++++---- src/Propulsion.MessageDb/ReaderCheckpoint.fs | 37 ++++++++------------ 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index e87ebf56..514ea19f 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -31,12 +31,11 @@ type MessageDbCategoryClient(connectionString) = struct(StreamName.parse streamName, event) member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { use! conn = connect ct - let command = conn.CreateCommand( - CommandText = "select position, type, data, metadata, id::uuid, - (metadata::jsonb->>'$correlationId')::text, - (metadata::jsonb->>'$causationId')::text, - time, stream_name, global_position - from get_category_messages(@Category, @Position, @BatchSize);") + let command = conn.CreateCommand(CommandText = "select position, type, data, metadata, id::uuid, + (metadata::jsonb->>'$correlationId')::text, + (metadata::jsonb->>'$causationId')::text, + time, stream_name, global_position + from get_category_messages(@Category, @Position, @BatchSize);") command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore @@ -51,8 +50,7 @@ type MessageDbCategoryClient(connectionString) = return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { use! conn = connect ct - let command = conn.CreateCommand() - command.CommandText <- "select max(global_position) from messages where category(stream_name) = @Category;" + let command = conn.CreateCommand(CommandText = "select max(global_position) from messages where category(stream_name) = @Category;") command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore use! reader = command.ExecuteReaderAsync(ct) diff --git a/src/Propulsion.MessageDb/ReaderCheckpoint.fs b/src/Propulsion.MessageDb/ReaderCheckpoint.fs index 2a5cfe2c..e20fc2f4 100644 --- a/src/Propulsion.MessageDb/ReaderCheckpoint.fs +++ b/src/Propulsion.MessageDb/ReaderCheckpoint.fs @@ -9,26 +9,20 @@ open Propulsion.Infrastructure let table = "propulsion_checkpoint" let createIfNotExists (conn : NpgsqlConnection, schema: string) = - let cmd = conn.CreateCommand() - cmd.CommandText <- $" - create table if not exists {schema}.{table} ( - source text not null, - tranche text not null, - consumer_group text not null, - position bigint not null, - primary key (source, tranche, consumer_group) - ) - " + let cmd = conn.CreateCommand(CommandText = $"create table if not exists {schema}.{table} ( + source text not null, + tranche text not null, + consumer_group text not null, + position bigint not null, + primary key (source, tranche, consumer_group));") cmd.ExecuteNonQueryAsync() |> Async.AwaitTaskCorrect |> Async.Ignore let commitPosition (conn : NpgsqlConnection, schema: string) source tranche (consumerGroup : string) (position : int64) = async { - let cmd = conn.CreateCommand() - cmd.CommandText <- - $"insert into {schema}.{table}(source, tranche, consumer_group, position) - values (@Source, @Tranche, @ConsumerGroup, @Position) - on conflict (source, tranche, consumer_group) - do update set position = @Position;" + let cmd = conn.CreateCommand(CommandText = $"insert into {schema}.{table}(source, tranche, consumer_group, position) + values (@Source, @Tranche, @ConsumerGroup, @Position) + on conflict (source, tranche, consumer_group) + do update set position = @Position;") cmd.Parameters.AddWithValue("Source", NpgsqlDbType.Text, SourceId.toString source) |> ignore cmd.Parameters.AddWithValue("Tranche", NpgsqlDbType.Text, TrancheId.toString tranche) |> ignore cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore @@ -38,13 +32,10 @@ let commitPosition (conn : NpgsqlConnection, schema: string) source tranche (con do! cmd.ExecuteNonQueryAsync(ct) |> Async.AwaitTaskCorrect |> Async.Ignore } let tryGetPosition (conn : NpgsqlConnection, schema : string) source tranche (consumerGroup : string) = async { - let cmd = conn.CreateCommand() - cmd.CommandText <- - $"select position from {schema}.{table} - where source = @Source - and tranche = @Tranche - and consumer_group = @ConsumerGroup" - + let cmd = conn.CreateCommand(CommandText = $"select position from {schema}.{table} + where source = @Source + and tranche = @Tranche + and consumer_group = @ConsumerGroup") cmd.Parameters.AddWithValue("Source", NpgsqlDbType.Text, SourceId.toString source) |> ignore cmd.Parameters.AddWithValue("Tranche", NpgsqlDbType.Text, TrancheId.toString tranche) |> ignore cmd.Parameters.AddWithValue("ConsumerGroup", NpgsqlDbType.Text, consumerGroup) |> ignore From 171e2a20bbcd00aea958927c72d0b434129f0cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 11:46:35 -0500 Subject: [PATCH 41/44] move the client into a core module --- src/Propulsion.MessageDb/MessageDbSource.fs | 92 ++++++++++--------- .../Propulsion.MessageDb.Integration/Tests.fs | 2 +- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/Propulsion.MessageDb/MessageDbSource.fs b/src/Propulsion.MessageDb/MessageDbSource.fs index 514ea19f..5e02322b 100644 --- a/src/Propulsion.MessageDb/MessageDbSource.fs +++ b/src/Propulsion.MessageDb/MessageDbSource.fs @@ -12,51 +12,53 @@ open System.Data.Common open System.Diagnostics -type MessageDbCategoryClient(connectionString) = - let connect = Npgsql.connect connectionString - let parseRow (reader: DbDataReader) = - let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) - let streamName = reader.GetString(8) - let event = TimelineEvent.Create( - index = reader.GetInt64(0), - eventType = reader.GetString(1), - data = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 2)), - meta = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 3)), - eventId = reader.GetGuid(4), - ?correlationId = readNullableString 5, - ?causationId = readNullableString 6, - context = reader.GetInt64(9), - timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))) - - struct(StreamName.parse streamName, event) - member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { - use! conn = connect ct - let command = conn.CreateCommand(CommandText = "select position, type, data, metadata, id::uuid, - (metadata::jsonb->>'$correlationId')::text, - (metadata::jsonb->>'$causationId')::text, - time, stream_name, global_position - from get_category_messages(@Category, @Position, @BatchSize);") - command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore - command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore - command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore - - let mutable checkpoint = fromPositionInclusive - - use! reader = command.ExecuteReaderAsync(ct) - let events = [| while reader.Read() do yield parseRow reader |] - - checkpoint <- match Array.tryLast events with Some (_, ev) -> unbox ev.Context | None -> checkpoint - - return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } - member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { - use! conn = connect ct - let command = conn.CreateCommand(CommandText = "select max(global_position) from messages where category(stream_name) = @Category;") - command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore - - use! reader = command.ExecuteReaderAsync(ct) - return if reader.Read() then reader.GetInt64(0) else 0L } +module Core = + type MessageDbCategoryClient(connectionString) = + let connect = Npgsql.connect connectionString + let parseRow (reader: DbDataReader) = + let readNullableString idx = if reader.IsDBNull(idx) then None else Some (reader.GetString idx) + let streamName = reader.GetString(8) + let event = TimelineEvent.Create( + index = reader.GetInt64(0), + eventType = reader.GetString(1), + data = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 2)), + meta = ReadOnlyMemory(Text.Encoding.UTF8.GetBytes(reader.GetString 3)), + eventId = reader.GetGuid(4), + ?correlationId = readNullableString 5, + ?causationId = readNullableString 6, + context = reader.GetInt64(9), + timestamp = DateTimeOffset(DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))) + + struct(StreamName.parse streamName, event) + member _.ReadCategoryMessages(category: TrancheId, fromPositionInclusive: int64, batchSize: int, ct) = task { + use! conn = connect ct + let command = conn.CreateCommand(CommandText = "select position, type, data, metadata, id::uuid, + (metadata::jsonb->>'$correlationId')::text, + (metadata::jsonb->>'$causationId')::text, + time, stream_name, global_position + from get_category_messages(@Category, @Position, @BatchSize);") + command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore + command.Parameters.AddWithValue("Position", NpgsqlDbType.Bigint, fromPositionInclusive) |> ignore + command.Parameters.AddWithValue("BatchSize", NpgsqlDbType.Bigint, int64 batchSize) |> ignore + + let mutable checkpoint = fromPositionInclusive + + use! reader = command.ExecuteReaderAsync(ct) + let events = [| while reader.Read() do yield parseRow reader |] + + checkpoint <- match Array.tryLast events with Some (_, ev) -> unbox ev.Context | None -> checkpoint + + return { checkpoint = Position.parse checkpoint; items = events; isTail = events.Length = 0 } } + member _.ReadCategoryLastVersion(category: TrancheId, ct) = task { + use! conn = connect ct + let command = conn.CreateCommand(CommandText = "select max(global_position) from messages where category(stream_name) = @Category;") + command.Parameters.AddWithValue("Category", NpgsqlDbType.Text, TrancheId.toString category) |> ignore + + use! reader = command.ExecuteReaderAsync(ct) + return if reader.Read() then reader.GetInt64(0) else 0L } module private Impl = + open Core open Propulsion.Infrastructure // AwaitTaskCorrect let readBatch batchSize (store : MessageDbCategoryClient) (category, pos) : Async> = async { @@ -72,7 +74,7 @@ module private Impl = type MessageDbSource ( log : Serilog.ILogger, statsInterval, - client: MessageDbCategoryClient, batchSize, tailSleepInterval, + client: Core.MessageDbCategoryClient, batchSize, tailSleepInterval, checkpoints : Propulsion.Feed.IFeedCheckpointStore, sink : Propulsion.Streams.Default.Sink, categories, // Override default start position to be at the tail of the index. Default: Replay all events. @@ -89,7 +91,7 @@ type MessageDbSource yield sw.Elapsed, b }), string) new (log, statsInterval, connectionString, batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail, ?sourceId) = - MessageDbSource(log, statsInterval, MessageDbCategoryClient(connectionString), + MessageDbSource(log, statsInterval, Core.MessageDbCategoryClient(connectionString), batchSize, tailSleepInterval, checkpoints, sink, trancheIds, ?startFromTail=startFromTail, ?sourceId=sourceId) abstract member ListTranches : unit -> Async diff --git a/tests/Propulsion.MessageDb.Integration/Tests.fs b/tests/Propulsion.MessageDb.Integration/Tests.fs index cbe003b1..de960f1b 100644 --- a/tests/Propulsion.MessageDb.Integration/Tests.fs +++ b/tests/Propulsion.MessageDb.Integration/Tests.fs @@ -47,7 +47,7 @@ let ``It processes events for a category`` () = async { let category2 = $"{Guid.NewGuid():N}" do! writeMessagesToCategory category1 |> Async.AwaitTaskCorrect do! writeMessagesToCategory category2 |> Async.AwaitTaskCorrect - let reader = MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") + let reader = Core.MessageDbCategoryClient("Host=localhost; Database=message_store; Port=5433; Username=message_store; Password=;") let checkpoints = ReaderCheckpoint.CheckpointStore("Host=localhost; Database=message_store; Port=5433; Username=postgres; Password=postgres", "public", $"TestGroup{consumerGroup}", TimeSpan.FromSeconds 10) do! checkpoints.CreateSchemaIfNotExists() let stats = { new Propulsion.Streams.Stats<_>(log, TimeSpan.FromMinutes 1, TimeSpan.FromMinutes 1) From 52065729d20e361c906f40200890ac3de627263e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 13:48:03 -0500 Subject: [PATCH 42/44] fix missing core naming --- tools/Propulsion.Tool/Args.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 8ad86b95..91dcf530 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -281,7 +281,7 @@ module Mdb = let checkpointConn = p.TryGetResult CheckpointConnectionString |> Option.defaultValue conn let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) - member x.CreateClient() = Array.ofList (p.GetResults Tranches), Propulsion.MessageDb.MessageDbCategoryClient(conn) + member x.CreateClient() = Array.ofList (p.GetResults Tranches), Propulsion.MessageDb.Core.MessageDbCategoryClient(conn) member x.CreateCheckpointStore(group) = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(checkpointConn, schema, group, TimeSpan.FromSeconds 5.) From 22a4df55f61ea9c8068f805513b42e1b5ea174c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 14:50:51 -0500 Subject: [PATCH 43/44] rename tranches to categories for mdb args --- tools/Propulsion.Tool/Args.fs | 6 +++--- tools/Propulsion.Tool/Program.fs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 91dcf530..75e592ac 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -268,20 +268,20 @@ module Mdb = | [] ConnectionString of string | [] CheckpointConnectionString of string | [] Schema of string - | [] Tranches of Propulsion.Feed.TrancheId + | [] Categories of string interface IArgParserTemplate with member a.Usage = a |> function | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" | CheckpointConnectionString _ -> "Connection string used for the checkpoint store. If not specified, defaults to the connection string argument" | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" - | Tranches _ -> "The message-db categories to load" + | Categories _ -> "The message-db categories to load" type Arguments(c : Configuration, p : ParseResults) = let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) let checkpointConn = p.TryGetResult CheckpointConnectionString |> Option.defaultValue conn let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) - member x.CreateClient() = Array.ofList (p.GetResults Tranches), Propulsion.MessageDb.Core.MessageDbCategoryClient(conn) + member x.CreateClient() = Array.ofList (p.GetResults Categories), Propulsion.MessageDb.Core.MessageDbCategoryClient(conn) member x.CreateCheckpointStore(group) = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(checkpointConn, schema, group, TimeSpan.FromSeconds 5.) diff --git a/tools/Propulsion.Tool/Program.fs b/tools/Propulsion.Tool/Program.fs index 43dbedbf..73c56125 100644 --- a/tools/Propulsion.Tool/Program.fs +++ b/tools/Propulsion.Tool/Program.fs @@ -368,11 +368,11 @@ module Project = ).Start() | Choice3Of3 sa -> let checkpoints = sa.CreateCheckpointStore(group) - let trancheIds, client = sa.CreateClient() + let categories, client = sa.CreateClient() Propulsion.MessageDb.MessageDbSource( Log.Logger, stats.StatsInterval, client, defaultArg maxItems 100, TimeSpan.FromSeconds 0.5, - checkpoints, sink, trancheIds + checkpoints, sink, categories ).Start() let work = [ Async.AwaitKeyboardInterruptAsTaskCanceledException() From f004c718b78b2f4c9b76bdaeb1f832aaa3a235b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Einar=20Nor=C3=B0fj=C3=B6r=C3=B0?= Date: Fri, 18 Nov 2022 14:59:50 -0500 Subject: [PATCH 44/44] yes, it should be category --- tools/Propulsion.Tool/Args.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/Propulsion.Tool/Args.fs b/tools/Propulsion.Tool/Args.fs index 75e592ac..303a128f 100644 --- a/tools/Propulsion.Tool/Args.fs +++ b/tools/Propulsion.Tool/Args.fs @@ -268,20 +268,20 @@ module Mdb = | [] ConnectionString of string | [] CheckpointConnectionString of string | [] Schema of string - | [] Categories of string + | [] Category of string interface IArgParserTemplate with member a.Usage = a |> function | ConnectionString _ -> $"Connection string for the postgres database housing message-db. (Optional if environment variable {CONNECTION_STRING} is defined)" | CheckpointConnectionString _ -> "Connection string used for the checkpoint store. If not specified, defaults to the connection string argument" | Schema _ -> $"Schema that should contain the checkpoints table Optional if environment variable {SCHEMA} is defined" - | Categories _ -> "The message-db categories to load" + | Category _ -> "The message-db categories to load" type Arguments(c : Configuration, p : ParseResults) = let conn = p.TryGetResult ConnectionString |> Option.defaultWith (fun () -> c.MdbConnectionString) let checkpointConn = p.TryGetResult CheckpointConnectionString |> Option.defaultValue conn let schema = p.TryGetResult Schema |> Option.defaultWith (fun () -> c.MdbSchema) - member x.CreateClient() = Array.ofList (p.GetResults Categories), Propulsion.MessageDb.Core.MessageDbCategoryClient(conn) + member x.CreateClient() = Array.ofList (p.GetResults Category), Propulsion.MessageDb.Core.MessageDbCategoryClient(conn) member x.CreateCheckpointStore(group) = Propulsion.MessageDb.ReaderCheckpoint.CheckpointStore(checkpointConn, schema, group, TimeSpan.FromSeconds 5.)