real world rocksdb tests/benchmarks

[baabeetaa]

• will do sifchain as it has highest traffic
• log the requests to file
• replay log file to test load

[baabeetaa]

use https://github.com/buger/goreplay

# write to file
./gor --input-raw :80 --output-file=requests.gor --output-file-append

# read from file
./gor --input-file requests.gor --input-file-max-wait 1ns --output-http "http://localhost"

[baabeetaa]

requests.gor file:

requests.gor.tar.gz

# on rocksdb node:

sh-5.1# ./gor --input-file requests.gor --input-file-max-wait 1ns --output-http "http://localhost"
2022/06/26 08:25:24 [PPID 15220 and PID 18880] Version:1.3.0
[DEBUG][elapsed 9.920591893s]: [INPUT-FILE] FileInput: end of file 'requests.gor'


# on goleveldb node:

sh-5.1# ./gor --input-file requests.gor --input-file-max-wait 1ns --output-http "http://localhost"
2022/06/26 08:17:21 [PPID 15220 and PID 18696] Version:1.3.0
[DEBUG][elapsed 10.017608967s]: [INPUT-FILE] FileInput: end of file 'requests.gor'

[baabeetaa]

try a benchmark tool: https://github.com/six-ddc/plow
cmd: $HOME/go/bin/plow http://127.0.0.1:26657 -c 10 -d 3m --body @file.json -T 'application/json' -m POST
chain:

    "sync_info": {
      "latest_block_hash": "9E9DF6F378BD787E1829C303DFE6D6AE8D40F79B288306EFA0D06EF2642DB29E",
      "latest_app_hash": "09D0E547E513834C3B80EFC5868E6A20FFE0A5740BA7A45E77201EE654B7C5C9",
      "latest_block_height": "7751881",
      "latest_block_time": "2022-07-18T03:45:56.210532394Z",
      "earliest_block_hash": "1F17FA2535933342E29E2E1E77DBAA3544D62A7FA5F349C203F5D5C640413CC6",
      "earliest_app_hash": "3245585720BA256828F1222EB98153248A89AAAB7CE7C6FF048DAF90ABE1BFD0",
      "earliest_block_height": "7524001",
      "earliest_block_time": "2022-07-02T02:16:55.105709728Z",
      "catching_up": false
    },

request:

# cat file.json
{
  "jsonrpc": "2.0",
  "id": "2229fc7c-534a-462d-9c90-fefad1726e24",
  "method": "abci_query",
  "params": {
    "path": "store/ibc/key",
    "data": "636C69656E74732F30372D74656E6465726D696E742D39322F636C69656E745374617465",
    "prove": false
  }
}

concurrency=10

goleveldb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 10 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 10 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed       3m0s
  Count      1844200
    2xx      1844200
  RPS      10245.553
  Reads    7.055MB/s
  Writes   3.566MB/s

Statistics   Min      Mean    StdDev     Max
  Latency    88µs    973µs    1.287ms  91.954ms
  RPS       9016.9  10248.93  492.18   12560.08

Latency Percentile:
  P50        P75      P90      P95      P99     P99.9     P99.99
  1.145ms  1.345ms  1.542ms  1.832ms  2.552ms  23.382ms  61.529ms

Latency Histogram:
  910µs     1716161  93.06%
  1.51ms     113524   6.16%
  3.143ms     13880   0.75%
  19.117ms      440   0.02%
  33.023ms      123   0.01%
  63.417ms       22   0.00%
  70.705ms       40   0.00%
  89.234ms       10   0.00%

rocksdb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 10 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 10 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed       3m0s
  Count      1737499
    2xx      1737499
  RPS       9652.769
  Reads    6.646MB/s
  Writes   3.360MB/s

Statistics    Min     Mean    StdDev    Max
  Latency    94µs    1.033ms  802µs   43.418ms
  RPS       8779.65  9655.4   341.47  10490.98

Latency Percentile:
  P50        P75      P90      P95      P99     P99.9    P99.99
  1.226ms  1.404ms  1.588ms  1.835ms  2.558ms  5.871ms  30.901ms

Latency Histogram:
  208µs      199741  11.50%
  1.037ms   1207923  69.52%
  1.426ms    291794  16.79%
  1.954ms     33817   1.95%
  3.221ms      3684   0.21%
  9.366ms       440   0.03%
  30.503ms       80   0.00%
  38.012ms       20   0.00%

concurrency=100

goleveldb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 100 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 100 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed        3m0s
  Count       3485914
    2xx       3485914
  RPS       19366.187
  Reads    13.335MB/s
  Writes    6.741MB/s

Statistics    Min       Mean    StdDev     Max
  Latency     95µs    5.161ms   2.299ms  91.591ms
  RPS       15503.18  19354.38  1768.38  24251.27

Latency Percentile:
  P50        P75      P90      P95      P99     P99.9     P99.99
  4.955ms  6.218ms  6.946ms  7.657ms  9.739ms  21.468ms  67.531ms

Latency Histogram:
  134µs       17015   0.49%
  4.889ms   2886940  82.82%
  6.272ms    490051  14.06%
  7.654ms     77157   2.21%
  11.195ms    13363   0.38%
  33.416ms      932   0.03%
  64.511ms      453   0.01%
  71.207ms        3   0.00%

rocksdb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 100 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 100 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed        3m0s
  Count       3169231
    2xx       3169231
  RPS       17606.779
  Reads    12.123MB/s
  Writes    6.129MB/s

Statistics    Min       Mean    StdDev     Max
  Latency    100µs    5.677ms   2.14ms   48.051ms
  RPS       14404.95  17606.71  1530.11  21870.52

Latency Percentile:
  P50        P75      P90      P95      P99      P99.9     P99.99
  5.373ms  6.645ms  7.714ms  8.832ms  11.799ms  29.197ms  43.263ms

Latency Histogram:
  152µs       17124   0.54%
  5.452ms   2837320  89.53%
  7.18ms     247001   7.79%
  9.978ms     59433   1.88%
  15.618ms     6986   0.22%
  30.964ms     1043   0.03%
  40.516ms      182   0.01%
  41.594ms      142   0.00%

concurrency=1000

goleveldb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 1000 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 1000 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed        3m0s
  Count       4171535
    2xx       4171535
  RPS       23175.151
  Reads    15.957MB/s
  Writes    8.069MB/s

Statistics    Min       Mean     StdDev      Max
  Latency     93µs    38.815ms  10.152ms  203.669ms
  RPS       17693.31  23175.15   1673.3   25979.56

Latency Percentile:
  P50         P75       P90      P95      P99       P99.9     P99.99
  38.078ms  41.942ms  47.09ms  51.17ms  80.621ms  102.572ms  124.489ms

Latency Histogram:
  461µs       113011   2.71%
  39.521ms   3966046  95.07%
  52.165ms     82565   1.98%
  79.689ms      8718   0.21%
  100.682ms     1137   0.03%
  108.174ms       40   0.00%
  145.561ms       17   0.00%
  203.669ms        1   0.00%

rocksdb:

# $HOME/go/bin/plow http://127.0.0.1:26657 -c 1000 -d 3m --body @file.json -T 'application/json' -m POST
Benchmarking http://127.0.0.1:26657 for 3m0s using 1000 connection(s).
@ Real-time charts is listening on http://[::]:18888

Summary:
  Elapsed        3m0s
  Count       3853730
    2xx       3853730
  RPS       21409.607
  Reads    14.742MB/s
  Writes    7.454MB/s

Statistics    Min       Mean    StdDev      Max
  Latency     99µs    42.02ms   10.49ms  185.613ms
  RPS       15591.45  21409.52  1568.34  24242.36

Latency Percentile:
  P50         P75       P90       P95       P99      P99.9     P99.99
  41.036ms  46.203ms  52.403ms  57.383ms  74.884ms  98.028ms  124.966ms

Latency Histogram:
  401µs        90078   2.34%
  42.928ms   3742543  97.11%
  54.471ms     15111   0.39%
  66.31ms       5475   0.14%
  91.941ms       409   0.01%
  115.183ms       81   0.00%
  120.07ms        25   0.00%
  129.417ms        8   0.00%