Skip to content
Permalink
Browse files

Sql Stream Store Support (#168)

  • Loading branch information
Rajivhost authored and bartelink committed Nov 6, 2019
1 parent 5fdf827 commit 4d3375389ad2ca24318a302e8ffd562194cf4342
@@ -10,6 +10,7 @@ The `Unreleased` section name is replaced by the expected version of next releas

### Added

- `SqlStreamStore`: Full support for Microsoft Sql Server, MySQL and Postgres using [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore) [#168](https://github.com/jet/equinox/pull/168) [@rajivhost](https://github.com/rajivhost)
- `Cosmos`: Exposed a `Connector.CreateClient` for interop with V2 ChangeFeedProcessor and `Propulsion.Cosmos` [#171](https://github.com/jet/equinox/pull/171)
- `MemoryStore`: Supports custom Codec logic (can use `FsCodec.Box.Codec` as default) [#173](https://github.com/jet/equinox/pull/173)

@@ -20,6 +20,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".project", ".project", "{7E
LICENSE = LICENSE
README.md = README.md
SECURITY.md = SECURITY.md
docker-compose.yml = docker-compose.yml
EndProjectSection
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Domain", "samples\Store\Domain\Domain.fsproj", "{37B4A45F-039E-4515-8A84-D242DDE12D22}"
@@ -58,6 +59,20 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Tutorial", "samples\Tutoria
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Tools.TestHarness", "tools\Equinox.Tools.TestHarness\Equinox.Tools.TestHarness.fsproj", "{C2EE7D8E-6982-41AD-ADCC-E6F586E93524}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore", "src\Equinox.SqlStreamStore\Equinox.SqlStreamStore.fsproj", "{A2ABD57E-0C40-4E29-8EA2-9FB21DBD4029}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.MsSql", "src\Equinox.SqlStreamStore.MsSql\Equinox.SqlStreamStore.MsSql.fsproj", "{11B0C20E-EE3E-44E6-9BCC-F2299AE216A2}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.Postgres", "src\Equinox.SqlStreamStore.Postgres\Equinox.SqlStreamStore.Postgres.fsproj", "{ACD2B8C4-A3C6-4A44-9A8A-7135937E932E}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.MySql", "src\Equinox.SqlStreamStore.MySql\Equinox.SqlStreamStore.MySql.fsproj", "{610464A7-6C30-45DE-8DBC-8B0B832B9EAE}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.MsSql.Integration", "tests\Equinox.SqlStreamStore.MsSql.Integration\Equinox.SqlStreamStore.MsSql.Integration.fsproj", "{761A8CCF-20D8-47DF-BB8E-52541B08A708}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.MySql.Integration", "tests\Equinox.SqlStreamStore.MySql.Integration\Equinox.SqlStreamStore.MySql.Integration.fsproj", "{0BEC1726-EF41-446F-932D-CB2B3FE6C5B9}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.SqlStreamStore.Postgres.Integration", "tests\Equinox.SqlStreamStore.Postgres.Integration\Equinox.SqlStreamStore.Postgres.Integration.fsproj", "{CDA2F063-B089-4452-8421-82A4EC57B13B}"
EndProject
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.Core", "src\Equinox.Core\Equinox.Core.fsproj", "{3021659A-5CA4-4E06-AF00-2457ED3F105B}"
EndProject
Global
@@ -134,6 +149,34 @@ Global
{C2EE7D8E-6982-41AD-ADCC-E6F586E93524}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C2EE7D8E-6982-41AD-ADCC-E6F586E93524}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C2EE7D8E-6982-41AD-ADCC-E6F586E93524}.Release|Any CPU.Build.0 = Release|Any CPU
{A2ABD57E-0C40-4E29-8EA2-9FB21DBD4029}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2ABD57E-0C40-4E29-8EA2-9FB21DBD4029}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2ABD57E-0C40-4E29-8EA2-9FB21DBD4029}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2ABD57E-0C40-4E29-8EA2-9FB21DBD4029}.Release|Any CPU.Build.0 = Release|Any CPU
{11B0C20E-EE3E-44E6-9BCC-F2299AE216A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{11B0C20E-EE3E-44E6-9BCC-F2299AE216A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{11B0C20E-EE3E-44E6-9BCC-F2299AE216A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{11B0C20E-EE3E-44E6-9BCC-F2299AE216A2}.Release|Any CPU.Build.0 = Release|Any CPU
{ACD2B8C4-A3C6-4A44-9A8A-7135937E932E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{ACD2B8C4-A3C6-4A44-9A8A-7135937E932E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{ACD2B8C4-A3C6-4A44-9A8A-7135937E932E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{ACD2B8C4-A3C6-4A44-9A8A-7135937E932E}.Release|Any CPU.Build.0 = Release|Any CPU
{610464A7-6C30-45DE-8DBC-8B0B832B9EAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{610464A7-6C30-45DE-8DBC-8B0B832B9EAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{610464A7-6C30-45DE-8DBC-8B0B832B9EAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{610464A7-6C30-45DE-8DBC-8B0B832B9EAE}.Release|Any CPU.Build.0 = Release|Any CPU
{761A8CCF-20D8-47DF-BB8E-52541B08A708}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{761A8CCF-20D8-47DF-BB8E-52541B08A708}.Debug|Any CPU.Build.0 = Debug|Any CPU
{761A8CCF-20D8-47DF-BB8E-52541B08A708}.Release|Any CPU.ActiveCfg = Release|Any CPU
{761A8CCF-20D8-47DF-BB8E-52541B08A708}.Release|Any CPU.Build.0 = Release|Any CPU
{0BEC1726-EF41-446F-932D-CB2B3FE6C5B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0BEC1726-EF41-446F-932D-CB2B3FE6C5B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0BEC1726-EF41-446F-932D-CB2B3FE6C5B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0BEC1726-EF41-446F-932D-CB2B3FE6C5B9}.Release|Any CPU.Build.0 = Release|Any CPU
{CDA2F063-B089-4452-8421-82A4EC57B13B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CDA2F063-B089-4452-8421-82A4EC57B13B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CDA2F063-B089-4452-8421-82A4EC57B13B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CDA2F063-B089-4452-8421-82A4EC57B13B}.Release|Any CPU.Build.0 = Release|Any CPU
{3021659A-5CA4-4E06-AF00-2457ED3F105B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3021659A-5CA4-4E06-AF00-2457ED3F105B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3021659A-5CA4-4E06-AF00-2457ED3F105B}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -13,6 +13,7 @@ Some aspects of the implementation are distilled from [`Jet.com` systems dating
- [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db) - contains code dating back to 2016, however [the storage model](DOCUMENTATION.md#Cosmos-Storage-Model) was arrived at based on intensive benchmarking (squash-merged in [#42](https://github.com/jet/equinox/pull/42)).
- [EventStore](https://eventstore.org/) - this codebase itself has been in production since 2017 (see commit history), with key elements dating back to approx 2016.
- `MemoryStore`: In-memory store (volatile, for unit or integration test purposes). Fulfils the full contract Equinox imposes on a store, but without I/O costs [(it's ~100 LOC wrapping a `ConcurrentDictionary`)](https://github.com/jet/equinox/blob/master/src/Equinox.MemoryStore/MemoryStore.fs), and the ability to [take serialization/deserialization cost out of the picture](https://github.com/jet/FsCodec#boxcodec).
- [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore): Bindings for the powerful and widely used SQL-backed Event Storage system. [See SqlStreamStore docs](https://sqlstreamstore.readthedocs.io/en/latest/#introduction). :pray: [@rajivhost](https://gihub.com/rajivhost)

# Features

@@ -70,6 +71,10 @@ The components within this repository are delivered as multi-targeted Nuget pack
- `Equinox.MemoryStore` [![MemoryStore NuGet](https://img.shields.io/nuget/v/Equinox.MemoryStore.svg)](https://www.nuget.org/packages/Equinox.MemoryStore/): In-memory store for integration testing/performance baselining/providing out-of-the-box zero dependency storage for examples. ([depends](https://www.fuget.org/packages/Equinox.MemoryStore) on `Equinox.Core`)
- `Equinox.EventStore` [![EventStore NuGet](https://img.shields.io/nuget/v/Equinox.EventStore.svg)](https://www.nuget.org/packages/Equinox.EventStore/): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.EventStore) on `Equinox.Core`, `EventStore.Client[Api.NetCore] >= 5.0.1`, `FSharp.Control.AsyncSeq`)
- `Equinox.Cosmos` [![Cosmos NuGet](https://img.shields.io/nuget/v/Equinox.Cosmos.svg)](https://www.nuget.org/packages/Equinox.Cosmos/): Production-strength Azure CosmosDb Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.Cosmos) on `Equinox.Core`, `Microsoft.Azure.DocumentDb[.Core] >= 2.2`, `FsCodec.NewtonsoftJson`, `FSharp.Control.AsyncSeq`)
- `Equinox.SqlStreamStore` [![SqlStreamStore NuGet](https://img.shields.io/nuget/v/Equinox.SqlStreamStore.svg)](https://www.nuget.org/packages/Equinox.SqlStreamStore/): Production-strength [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore) Adapter derived from `Equinox.EventStore` - provides core facilities (but does not connect to a specific database; see sibling `SqlStreamStore`.* packages). ([depends](https://www.fuget.org/packages/Equinox.SqlStreamStore) on `Equinox.Core`, `FsCodec`, `SqlStreamStore >= 1.2.0-beta.6`, `FSharp.Control.AsyncSeq`)
- `Equinox.SqlStreamStore.MsSql` [![MsSql NuGet](https://img.shields.io/nuget/v/Equinox.SqlStreamStore.MsSql.svg)](https://www.nuget.org/packages/Equinox.SqlStreamStore.MsSql/): [SqlStreamStore.MsSql](https://sqlstreamstore.readthedocs.io/en/latest/sqlserver) Sql Server `Connector` implementation for `Equinox.SqlStreamStore` package). ([depends](https://www.fuget.org/packages/Equinox.SqlStreamStore.MsSql) on `Equinox.SqlStreamStore`, `SqlStreamStore.MsSql >= 1.2.0-beta.6`)
- `Equinox.SqlStreamStore.MySql` [![MySql NuGet](https://img.shields.io/nuget/v/Equinox.SqlStreamStore.MySql.svg)](https://www.nuget.org/packages/Equinox.SqlStreamStore.MySql/): `SqlStreamStore.MySql` MySQL Í`Connector` implementation for `Equinox.SqlStreamStore` package). ([depends](https://www.fuget.org/packages/Equinox.SqlStreamStore.MySql) on `Equinox.SqlStreamStore`, `SqlStreamStore.MySql >= 1.2.0-beta.6`)
- `Equinox.SqlStreamStore.Postgres` [![Postgres NuGet](https://img.shields.io/nuget/v/Equinox.SqlStreamStore.Postgres.svg)](https://www.nuget.org/packages/Equinox.SqlStreamStore.Postgres/): [SqlStreamStore.Postgres](https://sqlstreamstore.readthedocs.io/en/latest/postgres) PostgreSQL `Connector` implementation for `Equinox.SqlStreamStore` package). ([depends](https://www.fuget.org/packages/Equinox.SqlStreamStore.Postgres) on `Equinox.SqlStreamStore`, `SqlStreamStore.Postgres >= 1.2.0-beta.6`)

### Projection libraries

@@ -296,6 +301,38 @@ While Equinox is implemented in F#, and F# is a great fit for writing event-sour
dotnet run -- -t topic0 -g consumer1
```

<a name="sqlstreamstore"></a>
8. Use [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore)

The SqlStreamStore consists of:

- being able to supply `ms`, `my`, `pg` flag to `eqx run`, e.g. `eqx run -t cart -f 50 -d 5 -C -U ms -c "sqlserverconnectionstring" -s schema`
- being able to supply `ms`, `my`, `pg` flag to Web sample, e.g. `dotnet run -p samples/Web/ -- my -c "mysqlconnectionstring"`
- being able to supply `ms`, `my`, `pg` flag to new `eqx config` command e.g. `eqx config pg -c "postgresconnectionstring" -u p "usercredentialsNotToBeLogged" -s schema`

```powershell
cd ~/code/equinox
# set up the DB/schema
& dotnet run -f netcoreapp2.1 -p tools/Equinox.Tool -- config pg -c "connectionstring" -p "u=un;p=password" -s "schema"
# run a benchmark
& dotnet run -c Release -f netcoreapp2.1 -p tools/Equinox.Tool -- run -t saveforlater -f 50 -d 5 -C -U pg -c "connectionstring" -p "u=un;p=password" -s "schema"
# run the webserver, -A to autocreate schema on connection
& dotnet run -p samples/Web/ -- my -c "mysqlconnectionstring" -A
#############################
# TODO - NOTE NOT YET RELEASED
##############################
# set up the DB/schema
& eqx config pg -c "connectionstring" -p "u=un;p=password" -s "schema"
# run a benchmark
& eqx run -t saveforlater -f 50 -d 5 -C -U pg -c "connectionstring" -p "u=un;p=password" -s "schema"
```

### BENCHMARKS

A key facility of this repo is being able to run load tests, either in process against a nominated store, or via HTTP to a nominated instance of `samples/Web` ASP.NET Core host app. The following test suites are implemented at present:
@@ -409,9 +446,13 @@ For EventStore, the tests assume a running local instance configured as follows

### Provisioning CosmosDb (when not using -sc)

dotnet run -f netcoreapp2.1 -p tools/Equinox.Tool -- init -ru 1000 `
dotnet run -f netcoreapp2.1 -p tools/Equinox.Tool -- init -ru 400 `
cosmos -s $env:EQUINOX_COSMOS_CONNECTION -d $env:EQUINOX_COSMOS_DATABASE -c $env:EQUINOX_COSMOS_CONTAINER

### Provisioning SqlStreamStore

There's a `docker-compose.yml` file in the root, so installing `docker-compose` and then running `docker-compose up` rigs local `equinox-mssql`, `equinox-mysql` and `equinox-postgres` servers and databases at known ports. _NOTE The `Equinox.SqlStreamStore.*.Integration` suites currently assume this is in place and will otherwise fail_.

## DEPROVISIONING

### Deprovisioning (aka nuking) EventStore data resulting from tests to reset baseline
@@ -445,7 +486,9 @@ All non-alpha releases derive from tagged commits on `master`. The tag defines t
- [Provision](provisioning):
- Start Local EventStore running in simulated cluster mode
- Set Environment variables X 3 for a CosmosDb database and container (you might need to `eqx init`)
- Run `./build.ps1` in Powershell (or Powershell Core on mach via `brew install cask pwsh`)
- `docker-compose up` to start 3 servers for the `SqlStreamStore.*.Integration` test suites
- [NB `SqlStreamStore.MsSql` has not been tested yet](https://github.com/jet/equinox/issues/175) :see_no_evil: **
- Run `./build.ps1` in PowerShell (or PowerShell Core on MacOS via `brew install cask pwsh`)
- [CHANGELOG](CHANGELOG.md) should be up to date
- commit should be tagged (remember to do `git push --tags` when pushing)
@@ -511,4 +554,4 @@ Yes; [`Counter.fsx` in th Tutorial project in this repo](https://github.com/jet/
# FURTHER READING
See [`DOCUMENTATION.md`](DOCUMENTATION.md)
See [`DOCUMENTATION.md`](DOCUMENTATION.md)
@@ -19,6 +19,10 @@
<Exec Command="dotnet pack src/Equinox.Cosmos $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.EventStore $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.MemoryStore $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.SqlStreamStore $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.SqlStreamStore.MsSql $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.SqlStreamStore.MySql $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack src/Equinox.SqlStreamStore.Postgres $(Cfg) $(PackOptions)" />
<Exec Command="dotnet pack tools/Equinox.Tools.TestHarness $(Cfg) $(PackOptions)" />
<Exec Command='dotnet publish tools/Equinox.Tool $(Cfg) -f net461 -o "$(ThisDirAbsolute)/bin/equinox-tool/net461" ' />
<Exec Command="dotnet pack tools/Equinox.Tool $(Cfg) $(PackOptions) /p:PackAsTool=true" />
@@ -0,0 +1,36 @@
version: '3.7'

services:
equinox-postgres:
container_name: equinox-postgres
image: postgres
restart: unless-stopped
environment:
- PG_TRUST_LOCALNET=true
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=
- POSTGRES_DB=EQUINOX_TEST_DB
ports:
- "5432:5432"

equinox-mssql:
container_name: equinox-mssql
image: microsoft/mssql-server-linux:2017-latest
restart: unless-stopped
ports:
- "1433:1433"
environment:
- SA_PASSWORD=!Passw0rd
- ACCEPT_EULA=Y

equinox-mysql:
container_name: equinox-mysql
image: mysql:5.6
restart: unless-stopped
ports:
- "3306:3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=1
- MYSQL_DATABASE=EQUINOX_TEST_DB
# it _almost_ (but doesnt) works with the default mysql using the following
# command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci']
@@ -14,6 +14,10 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\Equinox.SqlStreamStore.MsSql\Equinox.SqlStreamStore.MsSql.fsproj" />
<ProjectReference Include="..\..\src\Equinox.SqlStreamStore.MySql\Equinox.SqlStreamStore.MySql.fsproj" />
<ProjectReference Include="..\..\src\Equinox.SqlStreamStore.Postgres\Equinox.SqlStreamStore.Postgres.fsproj" />
<ProjectReference Include="..\..\src\Equinox.SqlStreamStore\Equinox.SqlStreamStore.fsproj" />
<ProjectReference Include="..\..\src\Equinox\Equinox.fsproj" />
<ProjectReference Include="..\..\src\Equinox.Cosmos\Equinox.Cosmos.fsproj" />
<ProjectReference Include="..\..\src\Equinox.EventStore\Equinox.EventStore.fsproj" />
@@ -20,6 +20,9 @@ type StreamResolver(storage) =
Equinox.EventStore.Resolver<'event,'state,_>(context, codec, fold, initial, ?caching = caching, ?access = accessStrategy).Resolve
| Storage.StorageConfig.Memory store ->
Equinox.MemoryStore.Resolver(store, codec, fold, initial).Resolve
| Storage.StorageConfig.Sql (context, caching, unfolds) ->
let accessStrategy = if unfolds then Equinox.SqlStreamStore.AccessStrategy.RollingSnapshots snapshot |> Some else None
Equinox.SqlStreamStore.Resolver<'event,'state,_>(context, codec, fold, initial, ?caching = caching, ?access = accessStrategy).Resolve

type ServiceBuilder(storageConfig, handlerLog) =
let resolver = StreamResolver(storageConfig)

0 comments on commit 4d33753

Please sign in to comment.
You can’t perform that action at this time.