Skip to content

Commit

Permalink
Bug Fix: Saving big payloads by cache CouchDB (#1909)
Browse files Browse the repository at this point in the history 
* Bug Fix: Saving big payloads by cache CouchDB

Signed-off-by: przemyslaw <przemo.wasala@gmail.com>

* Update fastcache and use new API from fastcache

Signed-off-by: przemyslaw <przemo.wasala@gmail.com>

* Cache support only small payloads

Signed-off-by: przemyslaw <przemo.wasala@gmail.com>
Signed-off-by: manish <manish.sethi@gmail.com>
  • Loading branch information
@przemo22 @denyeart
przemo22 authored and denyeart committed Sep 24, 2020
1 parent fe684f8 commit 2e4527b
Show file tree
Hide file tree
Showing 40 changed files with 1,820 additions and 395 deletions.
15 changes: 11 additions & 4 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/cache.go
View file
Expand Up @@ -64,12 +64,12 @@ func (c *cache) getState(chainID, namespace, key string) (*CacheValue, error) {
}

cacheKey := constructCacheKey(chainID, namespace, key)
valBytes := cache.Get(nil, cacheKey)
if valBytes == nil {

if !cache.Has(cacheKey) {
return nil, nil
}

cacheValue := &CacheValue{}
valBytes := cache.Get(nil, cacheKey)
if err := proto.Unmarshal(valBytes, cacheValue); err != nil {
return nil, err
}
Expand All @@ -85,9 +85,15 @@ func (c *cache) putState(chainID, namespace, key string, cacheValue *CacheValue)

cacheKey := constructCacheKey(chainID, namespace, key)
valBytes, err := proto.Marshal(cacheValue)

if err != nil {
return err
}

if cache.Has(cacheKey) {
cache.Del(cacheKey)
}

cache.Set(cacheKey, valBytes)
return nil
}
Expand Down Expand Up @@ -126,11 +132,12 @@ func (c *cache) UpdateStates(chainID string, updates cacheUpdates) error {
cache.Del(cacheKey)
continue
}
if oldVal := cache.Get(nil, cacheKey); oldVal != nil {
if cache.Has(cacheKey) {
newValBytes, err := proto.Marshal(newVal)
if err != nil {
return err
}
cache.Del(cacheKey)
cache.Set(cacheKey, newValBytes)
}
}
Expand Down
62 changes: 62 additions & 0 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/cache_test.go
View file
Expand Up @@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
package statecouchdb

import (
"math/rand"
"testing"

"github.com/VictoriaMetrics/fastcache"
Expand Down Expand Up @@ -57,6 +58,67 @@ func TestGetPutState(t *testing.T) {
require.True(t, proto.Equal(expectedValue1, v))
}

func TestGetPutStateWithBigPayloadIfKeyDoesNotExist(t *testing.T) {
cache := newCache(32, sysNamespaces)

expectedValue := &CacheValue{Value: []byte("value")}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue))
v, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue, v))

// test PutState with BigPayload
token := make([]byte, (64*1024)+1)
rand.Read(token)
expectedValue1 := &CacheValue{Value: token}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue1))

v, err = cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
// actually bigPayloads are not saved in cache, should return nil/nothing
require.Nil(t, v)
}

func TestUpdateStatesWithSingleSmallAndSingleBigPayloads(t *testing.T) {
cache := newCache(32, sysNamespaces)

expectedValue1 := &CacheValue{Value: []byte("value1")}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue1))

expectedValue2 := &CacheValue{Value: []byte("value2")}
require.NoError(t, cache.putState("ch1", "ns1", "k2", expectedValue2))

v1, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue1, v1))

v2, err := cache.getState("ch1", "ns1", "k2")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue2, v2))

token := make([]byte, (64*1024)+1)
rand.Read(token)

expectedValue3 := &CacheValue{Value: []byte("value3")}
expectedValue4 := &CacheValue{Value: token}

updates := cacheUpdates{
"ns1": cacheKVs{
"k1": expectedValue3,
"k2": expectedValue4,
},
}
require.NoError(t, cache.UpdateStates("ch1", updates))

v3, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue3, v3))

v4, err := cache.getState("ch1", "ns1", "k2")
require.NoError(t, err)
require.Nil(t, v4)
}

func TestUpdateStates(t *testing.T) {
cache := newCache(32, sysNamespaces)

Expand Down
4 changes: 3 additions & 1 deletion go.mod
View file
Expand Up @@ -12,7 +12,7 @@ require (
github.com/Microsoft/hcsshim v0.8.6 // indirect
github.com/Shopify/sarama v1.20.1
github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect
github.com/VictoriaMetrics/fastcache v1.4.6
github.com/VictoriaMetrics/fastcache v1.5.7
github.com/VividCortex/gohistogram v1.0.0 // indirect
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
Expand All @@ -28,6 +28,8 @@ require (
github.com/fsouza/go-dockerclient v1.4.1
github.com/go-kit/kit v0.8.0
github.com/golang/protobuf v1.3.3
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-cmp v0.5.0 // indirect
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.7.2
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
Expand Down
21 changes: 10 additions & 11 deletions go.sum
View file
Expand Up @@ -13,30 +13,28 @@ github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiU
github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.20.1 h1:Bb0h3I++r4eX333Y0uZV2vwUXepJbt6ig05TUU1qt9I=
github.com/Shopify/sarama v1.20.1/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/VictoriaMetrics/fastcache v1.4.6 h1:cNywEjbbl77KfS20Nim0ukHMcBq9DGwOR5Pwazv2mw0=
github.com/VictoriaMetrics/fastcache v1.4.6/go.mod h1:FZV1r1HyPy1UHCTJ4vuuYJ1oDYM5SRPwOj9wlxbhog4=
github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/allegro/bigcache v1.1.1-0.20190116153254-84a0ff3f153c h1:JnxT952szw65TwPg7iz1iYVqEoikvX8mSkSgIhNr0Vo=
github.com/allegro/bigcache v1.1.1-0.20190116153254-84a0ff3f153c/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
Expand Down Expand Up @@ -103,14 +101,17 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
Expand Down Expand Up @@ -235,8 +236,6 @@ github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
Expand Down
20 changes: 0 additions & 20 deletions vendor/github.com/VictoriaMetrics/fastcache/.travis.yml
View file

This file was deleted.

2 changes: 1 addition & 1 deletion vendor/github.com/VictoriaMetrics/fastcache/README.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/github.com/VictoriaMetrics/fastcache/bigcache.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 41 additions & 10 deletions vendor/github.com/VictoriaMetrics/fastcache/fastcache.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 2e4527b

Please sign in to comment.