Skip to content

Commit

Permalink
In memory (#6)
Browse files Browse the repository at this point in the history
* In memory

* prepare for new article https://dou.ua/forums/topic/44655/
  • Loading branch information
YaroslavPodorvanov committed Aug 6, 2023
1 parent 6a3094d commit 4a784e6
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 45 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/code_quality.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Code quality

on:
push:
branches:
- main

jobs:
code-quality:
name: Code quality
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@master

- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.20'

- name: Run go vet
run: go vet ./...

- name: Unit tests
run: go test ./... -v -short
40 changes: 26 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,43 @@ env-up:
test:
docker exec research-online-redis-go-app go test ./... -v -count=1

bench-go-sequence:
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Go' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-go-10x-1000000x-sequence.txt

bench-redis-sequence:
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-redis-10x-10000x-sequence.txt
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-redis-10x-1000000x-sequence.txt

bench-keydb-sequence:
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-keydb-10x-10000x-sequence.txt
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-keydb-10x-1000000x-sequence.txt

bench-dragonflydb-sequence:
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-dragonflydb-10x-10000x-sequence.txt
docker exec -e MODE=sequence research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-dragonflydb-10x-1000000x-sequence.txt

bench-go-parallel:
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Go' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-go-10x-1000000x-parallel.txt

bench-redis-parallel:
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-redis-10x-10000x-parallel.txt
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Redis(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-redis-10x-1000000x-parallel.txt

bench-keydb-parallel:
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-keydb-10x-10000x-parallel.txt
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Keydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-keydb-10x-1000000x-parallel.txt

bench-dragonflydb-parallel:
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=10000x -count=10 | tee ./output/bench-dragonflydb-10x-10000x-parallel.txt

bench: bench-redis-sequence bench-keydb-sequence bench-dragonflydb-sequence bench-redis-parallel bench-keydb-parallel bench-dragonflydb-parallel
benchstat ./output/bench-redis-10x-10000x-sequence.txt
benchstat ./output/bench-keydb-10x-10000x-sequence.txt
benchstat ./output/bench-dragonflydb-10x-10000x-sequence.txt
benchstat ./output/bench-redis-10x-10000x-parallel.txt
benchstat ./output/bench-keydb-10x-10000x-parallel.txt
benchstat ./output/bench-dragonflydb-10x-10000x-parallel.txt
docker exec -e MODE=parallel research-online-redis-go-app go test ./... -v -run=$$^ -bench='Dragonflydb(Hash|SortedSet|Set)' -benchmem -benchtime=1000000x -count=10 | tee ./output/bench-dragonflydb-10x-1000000x-parallel.txt

bench:
make bench-go-sequence bench-redis-sequence bench-keydb-sequence bench-dragonflydb-sequence
make bench-go-parallel bench-redis-parallel bench-keydb-parallel bench-dragonflydb-parallel

benchstat ./output/bench-go-10x-1000000x-sequence.txt
benchstat ./output/bench-redis-10x-1000000x-sequence.txt
benchstat ./output/bench-keydb-10x-1000000x-sequence.txt
benchstat ./output/bench-dragonflydb-10x-1000000x-sequence.txt

benchstat ./output/bench-go-10x-1000000x-parallel.txt
benchstat ./output/bench-redis-10x-1000000x-parallel.txt
benchstat ./output/bench-keydb-10x-1000000x-parallel.txt
benchstat ./output/bench-dragonflydb-10x-1000000x-parallel.txt

bench-redis-memory-25m:
docker exec research-online-redis-1 redis-cli flushall
Expand Down
23 changes: 20 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ https://dou.ua/forums/topic/35260/
# Databases
| Name | Stars | Language |
|---------------------------------------------------------|--------|----------------------------------------|
| [Redis](https://github.com/redis/redis) | 59100+ | [C](https://dou.ua/forums/tags/C/) |
| [KeyDB](https://github.com/Snapchat/KeyDB) | 7100+ | [C++](https://dou.ua/forums/tags/C++/) |
| [DragonflyDB](https://github.com/dragonflydb/dragonfly) | 18300+ | [C++](https://dou.ua/forums/tags/C++/) |
| [Redis](https://github.com/redis/redis) | 60900+ | [C](https://dou.ua/forums/tags/C/) |
| [KeyDB](https://github.com/Snapchat/KeyDB) | 7800+ | [C++](https://dou.ua/forums/tags/C++/) |
| [DragonflyDB](https://github.com/dragonflydb/dragonfly) | 20700+ | [C++](https://dou.ua/forums/tags/C++/) |

# Data structure usage examples
### Hash
Expand Down Expand Up @@ -230,3 +230,20 @@ make bench-dragonflydb-memory-10k-batch-10k

# Star history of Redis vs KeyDB vs DragonflyDB
[![Star History Chart](https://api.star-history.com/svg?repos=redis/redis,Snapchat/KeyDB,dragonflydb/dragonfly&type=Date)](https://star-history.com/#redis/redis&Snapchat/KeyDB&dragonflydb/dragonfly&Date)

# Versions
```bash
docker pull redis:latest
docker pull eqalpha/keydb:latest
docker pull docker.dragonflydb.io/dragonflydb/dragonfly
```
```bash
docker image inspect redis:latest --format '{{.RepoDigests}} {{.Size}}'
docker image inspect eqalpha/keydb:latest --format '{{.RepoDigests}} {{.Size}}'
docker image inspect docker.dragonflydb.io/dragonflydb/dragonfly --format '{{.RepoDigests}} {{.Size}}'
```
| Database name | Docker image | Docker image size |
|---------------|-------------------------------------------------------------------------|-------------------|
| Redis | sha256:b0bdc1a83caf43f9eb74afca0fcfd6f09bea38bb87f6add4a858f06ef4617538 | 129.93 MB |
| KeyDB | sha256:c6c09ea6f80b073e224817e9b4a554db7f33362e8321c4084701884be72eed67 | 129.09 MB |
| DragonflyDB | sha256:73b995caf8fa8e3a00928ac5843864ba7f6a8b80ba959eff53386dd9cbb8b589 | 188.90 MB |
61 changes: 61 additions & 0 deletions go_online_storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package research_online_redis_go

import (
"context"
"sync"
)

type GoOnlineStorage struct {
mu sync.Mutex
data map[int64]int64
}

func NewGoOnlineStorage() *GoOnlineStorage {
return &GoOnlineStorage{
data: map[int64]int64{},
}
}

func (s *GoOnlineStorage) Store(ctx context.Context, pair UserOnlinePair) error {
s.mu.Lock()
defer s.mu.Unlock()

s.data[pair.UserID] = pair.Timestamp

return nil
}

func (s *GoOnlineStorage) BatchStore(ctx context.Context, pairs []UserOnlinePair) error {
s.mu.Lock()
defer s.mu.Unlock()

for _, pair := range pairs {
s.data[pair.UserID] = pair.Timestamp
}

return nil
}

func (s *GoOnlineStorage) Count(ctx context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()

return int64(len(s.data)), nil
}

func (s *GoOnlineStorage) GetAndClear(ctx context.Context) ([]UserOnlinePair, error) {
s.mu.Lock()
defer s.mu.Unlock()

result := make([]UserOnlinePair, 0, len(s.data))
for userID, timestamp := range s.data {
result = append(result, UserOnlinePair{
UserID: userID,
Timestamp: timestamp,
})
}

s.data = map[int64]int64{}

return result, nil
}
105 changes: 105 additions & 0 deletions go_online_storage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package research_online_redis_go

import (
"context"
"os"
"sync/atomic"
"testing"
"time"

"github.com/stretchr/testify/require"
)

func TestGoOnlineStorage(t *testing.T) {
testMapOnlineStorage(t, NewGoOnlineStorage())
}

func BenchmarkGoOnlineStorage(b *testing.B) {
benchmarkMapOnlineStorage(b, NewGoOnlineStorage())
}

func testMapOnlineStorage(
t *testing.T,
storage OnlineStorage,
) {
t.Helper()

ctx := context.Background()

expected := []UserOnlinePair{
{
UserID: 10000001,
Timestamp: 1679800725,
},
{
UserID: 10000002,
Timestamp: 1679800730,
},
{
UserID: 10000003,
Timestamp: 1679800735,
},
}

for _, pair := range expected {
err := storage.Store(ctx, pair)

require.NoError(t, err)
}

actualCount, err := storage.Count(ctx)
require.NoError(t, err)
require.Equal(t, int64(len(expected)), int64(actualCount))

actual, err := storage.GetAndClear(ctx)
require.NoError(t, err)

requireUserOnlinePairsEqual(t, expected, actual)
}

func benchmarkMapOnlineStorage(
b *testing.B,
storage OnlineStorage,
) {
b.Helper()

ctx := context.Background()

var (
startTimestamp = time.Now().Unix()
startUserID = int64(1e7)
)

b.ResetTimer()
if os.Getenv("MODE") == "parallel" {
var (
counter = int64(0)
)

b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
index := atomic.AddInt64(&counter, 1)

err := storage.Store(ctx, UserOnlinePair{
UserID: startUserID + index,
Timestamp: startTimestamp + index,
})

require.NoError(b, err)
}
})
} else {
for index := int64(0); index < int64(b.N); index++ {
err := storage.Store(ctx, UserOnlinePair{
UserID: startUserID + index,
Timestamp: startTimestamp + index,
})

require.NoError(b, err)
}
}

actualCount, err := storage.Count(ctx)
require.NoError(b, err)
require.Equal(b, int64(b.N), actualCount)
}
2 changes: 1 addition & 1 deletion hash_online_storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/redis/go-redis/v9"
)

var hashOnlineStorageConstructor onlineStorageConstructor = func(client *redis.Client) OnlineStorage {
var hashOnlineStorageConstructor = func(client *redis.Client) OnlineStorage {
return NewHashOnlineStorage(client)
}

Expand Down
31 changes: 22 additions & 9 deletions online_storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,19 @@ import (
"time"

"github.com/redis/go-redis/v9"

"github.com/stretchr/testify/require"
)

type onlineStorageConstructor func(client *redis.Client) OnlineStorage

func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstructor) {
func testOnlineStorage(
t *testing.T,
addr string,
constructor func(client *redis.Client) OnlineStorage,
) {
t.Helper()
if testing.Short() {
t.Skip()
}

ctx := context.Background()

Expand All @@ -24,7 +30,7 @@ func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstr

require.NoError(t, client.FlushAll(ctx).Err())

storage := newStorage(client)
storage := constructor(client)

expected := []UserOnlinePair{
{
Expand Down Expand Up @@ -57,8 +63,15 @@ func testOnlineStorage(t *testing.T, addr string, newStorage onlineStorageConstr
requireUserOnlinePairsEqual(t, expected, actual)
}

func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageConstructor) {
func benchmarkOnlineStorage(
b *testing.B,
addr string,
constructor func(client *redis.Client) OnlineStorage,
) {
b.Helper()
if testing.Short() {
b.Skip()
}

ctx := context.Background()

Expand All @@ -67,11 +80,11 @@ func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageC

require.NoError(b, client.FlushDB(ctx).Err())

storage := newStorage(client)
storage := constructor(client)

var (
expectedCount = int64(b.N)
startTimestamp = time.Now().Truncate(time.Hour).Unix()
startTimestamp = time.Now().Unix()
startUserID = int64(1e7)
)

Expand Down Expand Up @@ -110,8 +123,8 @@ func benchmarkOnlineStorage(b *testing.B, addr string, newStorage onlineStorageC

for i := int64(0); i < batch; i++ {
pairs[i] = UserOnlinePair{
UserID: startUserID + index - i,
Timestamp: startTimestamp + index - i,
UserID: startUserID + index + i,
Timestamp: startTimestamp + index + i,
}
}

Expand Down
Loading

0 comments on commit 4a784e6

Please sign in to comment.