diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml new file mode 100644 index 0000000..5726606 --- /dev/null +++ b/.github/workflows/code_quality.yml @@ -0,0 +1,25 @@ +name: Code quality + +on: + push: + branches: + - main + +jobs: + code-quality: + name: Code quality + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Run go vet + run: go vet ./... + + - name: Unit tests + run: go test ./... -v -short diff --git a/Makefile b/Makefile index 1579936..a08056e 100755 --- a/Makefile +++ b/Makefile @@ -4,31 +4,43 @@ env-up: test: docker exec research-online-redis-go-app go test ./... -v -count=1 +bench-go-sequence: + docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Go' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-go-10x-1000000x-sequence.txt + bench-redis-sequence: - docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-redis-10x-10000x-sequence.txt + docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-redis-10x-1000000x-sequence.txt bench-keydb-sequence: - docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-keydb-10x-10000x-sequence.txt + docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-keydb-10x-1000000x-sequence.txt bench-dragonflydb-sequence: - docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-dragonflydb-10x-10000x-sequence.txt + docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-dragonflydb-10x-1000000x-sequence.txt + +bench-go-parallel: + docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Go' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-go-10x-1000000x-parallel.txt bench-redis-parallel: - docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-redis-10x-10000x-parallel.txt + docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-redis-10x-1000000x-parallel.txt bench-keydb-parallel: - docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-keydb-10x-10000x-parallel.txt + docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-keydb-10x-1000000x-parallel.txt bench-dragonflydb-parallel: - docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-dragonflydb-10x-10000x-parallel.txt - -bench: bench-redis-sequence bench-keydb-sequence bench-dragonflydb-sequence bench-redis-parallel bench-keydb-parallel bench-dragonflydb-parallel - benchstat ./output/bench-redis-10x-10000x-sequence.txt - benchstat ./output/bench-keydb-10x-10000x-sequence.txt - benchstat ./output/bench-dragonflydb-10x-10000x-sequence.txt - benchstat ./output/bench-redis-10x-10000x-parallel.txt - benchstat ./output/bench-keydb-10x-10000x-parallel.txt - benchstat ./output/bench-dragonflydb-10x-10000x-parallel.txt + docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-dragonflydb-10x-1000000x-parallel.txt + +bench: + make bench-go-sequence bench-redis-sequence bench-keydb-sequence bench-dragonflydb-sequence + make bench-go-parallel bench-redis-parallel bench-keydb-parallel bench-dragonflydb-parallel + + benchstat ./output/bench-go-10x-1000000x-sequence.txt + benchstat ./output/bench-redis-10x-1000000x-sequence.txt + benchstat ./output/bench-keydb-10x-1000000x-sequence.txt + benchstat ./output/bench-dragonflydb-10x-1000000x-sequence.txt + + benchstat ./output/bench-go-10x-1000000x-parallel.txt + benchstat ./output/bench-redis-10x-1000000x-parallel.txt + benchstat ./output/bench-keydb-10x-1000000x-parallel.txt + benchstat ./output/bench-dragonflydb-10x-1000000x-parallel.txt bench-redis-memory-25m: docker exec research-online-redis-1 redis-cli flushall diff --git a/README.md b/README.md index 8dfa482..e1b09a3 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ https://dou.ua/forums/topic/35260/ # Databases | Name | Stars | Language | |---------------------------------------------------------|--------|----------------------------------------| -| [Redis](https://github.com/redis/redis) | 59100+ | [C](https://dou.ua/forums/tags/C/) | -| [KeyDB](https://github.com/Snapchat/KeyDB) | 7100+ | [C++](https://dou.ua/forums/tags/C++/) | -| [DragonflyDB](https://github.com/dragonflydb/dragonfly) | 18300+ | [C++](https://dou.ua/forums/tags/C++/) | +| [Redis](https://github.com/redis/redis) | 60900+ | [C](https://dou.ua/forums/tags/C/) | +| [KeyDB](https://github.com/Snapchat/KeyDB) | 7800+ | [C++](https://dou.ua/forums/tags/C++/) | +| [DragonflyDB](https://github.com/dragonflydb/dragonfly) | 20700+ | [C++](https://dou.ua/forums/tags/C++/) | # Data structure usage examples ### Hash @@ -230,3 +230,20 @@ make bench-dragonflydb-memory-10k-batch-10k # Star history of Redis vs KeyDB vs DragonflyDB [![Star History Chart](https://api.star-history.com/svg?repos=redis/redis,Snapchat/KeyDB,dragonflydb/dragonfly&type=Date)](https://star-history.com/#redis/redis&Snapchat/KeyDB&dragonflydb/dragonfly&Date) + +# Versions +```bash +docker pull redis:latest +docker pull eqalpha/keydb:latest +docker pull docker.dragonflydb.io/dragonflydb/dragonfly +``` +```bash +docker image inspect redis:latest --format '{{.RepoDigests}} {{.Size}}' +docker image inspect eqalpha/keydb:latest --format '{{.RepoDigests}} {{.Size}}' +docker image inspect docker.dragonflydb.io/dragonflydb/dragonfly --format '{{.RepoDigests}} {{.Size}}' +``` +| Database name | Docker image | Docker image size | +|---------------|-------------------------------------------------------------------------|-------------------| +| Redis | sha256:b0bdc1a83caf43f9eb74afca0fcfd6f09bea38bb87f6add4a858f06ef4617538 | 129.93 MB | +| KeyDB | sha256:c6c09ea6f80b073e224817e9b4a554db7f33362e8321c4084701884be72eed67 | 129.09 MB | +| DragonflyDB | sha256:73b995caf8fa8e3a00928ac5843864ba7f6a8b80ba959eff53386dd9cbb8b589 | 188.90 MB | diff --git a/go_online_storage.go b/go_online_storage.go new file mode 100644 index 0000000..0e14923 --- /dev/null +++ b/go_online_storage.go @@ -0,0 +1,61 @@ +package research_online_redis_go + +import ( + "context" + "sync" +) + +type GoOnlineStorage struct { + mu sync.Mutex + data map[int64]int64 +} + +func NewGoOnlineStorage() *GoOnlineStorage { + return &GoOnlineStorage{ + data: map[int64]int64{}, + } +} + +func (s *GoOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.data[pair.UserID] = pair.Timestamp + + return nil +} + +func (s *GoOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error { + s.mu.Lock() + defer s.mu.Unlock() + + for _, pair := range pairs { + s.data[pair.UserID] = pair.Timestamp + } + + return nil +} + +func (s *GoOnlineStorage) Count(ctx context.Context) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + return int64(len(s.data)), nil +} + +func (s *GoOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error) { + s.mu.Lock() + defer s.mu.Unlock() + + result := make([]UserOnlinePair, 0, len(s.data)) + for userID, timestamp := range s.data { + result = append(result, UserOnlinePair{ + UserID: userID, + Timestamp: timestamp, + }) + } + + s.data = map[int64]int64{} + + return result, nil +} diff --git a/go_online_storage_test.go b/go_online_storage_test.go new file mode 100644 index 0000000..2029733 --- /dev/null +++ b/go_online_storage_test.go @@ -0,0 +1,105 @@ +package research_online_redis_go + +import ( + "context" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGoOnlineStorage(t *testing.T) { + testMapOnlineStorage(t, NewGoOnlineStorage()) +} + +func BenchmarkGoOnlineStorage(b *testing.B) { + benchmarkMapOnlineStorage(b, NewGoOnlineStorage()) +} + +func testMapOnlineStorage( + t *testing.T, + storage OnlineStorage, +) { + t.Helper() + + ctx := context.Background() + + expected := []UserOnlinePair{ + { + UserID: 10000001, + Timestamp: 1679800725, + }, + { + UserID: 10000002, + Timestamp: 1679800730, + }, + { + UserID: 10000003, + Timestamp: 1679800735, + }, + } + + for _, pair := range expected { + err := storage.Store(ctx, pair) + + require.NoError(t, err) + } + + actualCount, err := storage.Count(ctx) + require.NoError(t, err) + require.Equal(t, int64(len(expected)), int64(actualCount)) + + actual, err := storage.GetAndClear(ctx) + require.NoError(t, err) + + requireUserOnlinePairsEqual(t, expected, actual) +} + +func benchmarkMapOnlineStorage( + b *testing.B, + storage OnlineStorage, +) { + b.Helper() + + ctx := context.Background() + + var ( + startTimestamp = time.Now().Unix() + startUserID = int64(1e7) + ) + + b.ResetTimer() + if os.Getenv("MODE") == "parallel" { + var ( + counter = int64(0) + ) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + index := atomic.AddInt64(&counter, 1) + + err := storage.Store(ctx, UserOnlinePair{ + UserID: startUserID + index, + Timestamp: startTimestamp + index, + }) + + require.NoError(b, err) + } + }) + } else { + for index := int64(0); index < int64(b.N); index++ { + err := storage.Store(ctx, UserOnlinePair{ + UserID: startUserID + index, + Timestamp: startTimestamp + index, + }) + + require.NoError(b, err) + } + } + + actualCount, err := storage.Count(ctx) + require.NoError(b, err) + require.Equal(b, int64(b.N), actualCount) +} diff --git a/hash_online_storage_test.go b/hash_online_storage_test.go index c280cc2..5b76fcf 100644 --- a/hash_online_storage_test.go +++ b/hash_online_storage_test.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" ) -var hashOnlineStorageConstructor onlineStorageConstructor = func(client *redis.Client) OnlineStorage { +var hashOnlineStorageConstructor = func(client *redis.Client) OnlineStorage { return NewHashOnlineStorage(client) } diff --git a/online_storage_test.go b/online_storage_test.go index bb2e5d6..75769b2 100644 --- a/online_storage_test.go +++ b/online_storage_test.go @@ -9,13 +9,19 @@ import ( "time" "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" ) -type onlineStorageConstructor func(client *redis.Client) OnlineStorage - -func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstructor) { +func testOnlineStorage( + t *testing.T, + addr string, + constructor func(client *redis.Client) OnlineStorage, +) { t.Helper() + if testing.Short() { + t.Skip() + } ctx := context.Background() @@ -24,7 +30,7 @@ func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstr require.NoError(t, client.FlushAll(ctx).Err()) - storage := newStorage(client) + storage := constructor(client) expected := []UserOnlinePair{ { @@ -57,8 +63,15 @@ func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstr requireUserOnlinePairsEqual(t, expected, actual) } -func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageConstructor) { +func benchmarkOnlineStorage( + b *testing.B, + addr string, + constructor func(client *redis.Client) OnlineStorage, +) { b.Helper() + if testing.Short() { + b.Skip() + } ctx := context.Background() @@ -67,11 +80,11 @@ func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageC require.NoError(b, client.FlushDB(ctx).Err()) - storage := newStorage(client) + storage := constructor(client) var ( expectedCount = int64(b.N) - startTimestamp = time.Now().Truncate(time.Hour).Unix() + startTimestamp = time.Now().Unix() startUserID = int64(1e7) ) @@ -110,8 +123,8 @@ func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageC for i := int64(0); i < batch; i++ { pairs[i] = UserOnlinePair{ - UserID: startUserID + index - i, - Timestamp: startTimestamp + index - i, + UserID: startUserID + index + i, + Timestamp: startTimestamp + index + i, } } diff --git a/redis_client_test.go b/redis_client_test.go index 385ec80..29b1dbf 100644 --- a/redis_client_test.go +++ b/redis_client_test.go @@ -9,28 +9,25 @@ import ( ) func TestRedisPing(t *testing.T) { - client := redis.NewClient(&redis.Options{ - Addr: "redis1:6379", - }) - - result, err := client.Ping(context.Background()).Result() - require.NoError(t, err) - require.Equal(t, "PONG", result) + testPing(t, "redis1:6379") } func TestKeydbPing(t *testing.T) { - client := redis.NewClient(&redis.Options{ - Addr: "keydb1:6379", - }) - - result, err := client.Ping(context.Background()).Result() - require.NoError(t, err) - require.Equal(t, "PONG", result) + testPing(t, "keydb1:6379") } func TestDragonflydbPing(t *testing.T) { + testPing(t, "dragonflydb1:6379") +} + +func testPing(t *testing.T, addr string) { + t.Helper() + if testing.Short() { + t.Skip() + } + client := redis.NewClient(&redis.Options{ - Addr: "dragonflydb1:6379", + Addr: addr, }) result, err := client.Ping(context.Background()).Result() diff --git a/set_online_storage_test.go b/set_online_storage_test.go index b03961b..9a0e262 100644 --- a/set_online_storage_test.go +++ b/set_online_storage_test.go @@ -7,10 +7,10 @@ import ( ) var ( - setOnlineStorageTestConstructor onlineStorageConstructor = func(client *redis.Client) OnlineStorage { + setOnlineStorageTestConstructor = func(client *redis.Client) OnlineStorage { return NewSetOnlineStorage(client, 1) } - setOnlineStorageBenchmarkConstructor onlineStorageConstructor = func(client *redis.Client) OnlineStorage { + setOnlineStorageBenchmarkConstructor = func(client *redis.Client) OnlineStorage { return NewSetOnlineStorage(client, 1800) } ) diff --git a/sorted_set_online_storage_test.go b/sorted_set_online_storage_test.go index fa18a27..cd44d2e 100644 --- a/sorted_set_online_storage_test.go +++ b/sorted_set_online_storage_test.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" ) -var sortedSetOnlineStorageConstructor onlineStorageConstructor = func(client *redis.Client) OnlineStorage { +var sortedSetOnlineStorageConstructor = func(client *redis.Client) OnlineStorage { return NewSortedSetOnlineStorage(client) }