diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 69b76c6026..11185af6d0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,6 +132,7 @@ jobs: name: Run stable tests run: | docker run chainsafe/gossamer:test sh -c "make it-stable" + docker-rpc-tests: runs-on: ubuntu-latest steps: @@ -155,6 +156,7 @@ jobs: name: Run rpc tests run: | docker run chainsafe/gossamer:test sh -c "make it-rpc" + docker-stress-tests: runs-on: ubuntu-latest steps: @@ -178,6 +180,7 @@ jobs: name: Run stress run: | docker run chainsafe/gossamer:test sh -c "make it-stress" + docker-grandpa-tests: runs-on: ubuntu-latest steps: diff --git a/dot/core/interface.go b/dot/core/interface.go index c9760ae0ce..83fe0ab21f 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -60,6 +60,7 @@ type StorageState interface { TrieState(root *common.Hash) (*rtstorage.TrieState, error) StoreTrie(*rtstorage.TrieState, *types.Header) error GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) + GetStorage(root *common.Hash, key []byte) ([]byte, error) } // TransactionState is the interface for transaction state methods diff --git a/dot/core/service.go b/dot/core/service.go index f59f3bef92..1bac4a4f26 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -40,6 +40,9 @@ var ( logger log.Logger = log.New("pkg", "core") ) +// QueryKeyValueChanges represents the key-value data inside a block storage +type QueryKeyValueChanges map[string]string + // Service is an overhead layer that allows communication between the runtime, // BABE session, and network service. It deals with the validation of transactions // and blocks by calling their respective validation functions in the runtime. @@ -560,3 +563,58 @@ func (s *Service) GetMetadata(bhash *common.Hash) ([]byte, error) { rt.SetContextStorage(ts) return rt.Metadata() } + +// QueryStorage returns the key-value data by block based on `keys` params +// on every block starting `from` until `to` block, if `to` is not nil +func (s *Service) QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]QueryKeyValueChanges, error) { + if to == common.EmptyHash { + to = s.blockState.BestBlockHash() + } + + blocksToQuery, err := s.blockState.SubChain(from, to) + if err != nil { + return nil, err + } + + queries := make(map[common.Hash]QueryKeyValueChanges) + + for _, hash := range blocksToQuery { + changes, err := s.tryQueryStorage(hash, keys...) + if err != nil { + return nil, err + } + + queries[hash] = changes + } + + return queries, nil +} + +// tryQueryStorage will try to get all the `keys` inside the block's current state +func (s *Service) tryQueryStorage(block common.Hash, keys ...string) (QueryKeyValueChanges, error) { + stateRootHash, err := s.storageState.GetStateRootFromBlock(&block) + if err != nil { + return nil, err + } + + changes := make(QueryKeyValueChanges) + for _, k := range keys { + keyBytes, err := common.HexToBytes(k) + if err != nil { + return nil, err + } + + storedData, err := s.storageState.GetStorage(stateRootHash, keyBytes) + if err != nil { + return nil, err + } + + if storedData == nil { + continue + } + + changes[k] = common.BytesToHex(storedData) + } + + return changes, nil +} diff --git a/dot/core/service_test.go b/dot/core/service_test.go index 354552eaca..9d7a13ea63 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -33,8 +33,10 @@ import ( "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/extrinsic" + "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/runtime/wasmer" "github.com/ChainSafe/gossamer/lib/transaction" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" log "github.com/ChainSafe/log15" "github.com/stretchr/testify/mock" @@ -619,6 +621,173 @@ func TestService_HandleRuntimeChangesAfterCodeSubstitutes(t *testing.T) { require.NotEqualf(t, codeHashBefore, rt.GetCodeHash(), "expected different code hash after runtime update") // codeHash should change after runtime change } +func TestTryQueryStore_WhenThereIsDataToRetrieve(t *testing.T) { + s := NewTestService(t, nil) + storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil)) + + testKey, testValue := []byte("to"), []byte("0x1723712318238AB12312") + storageStateTrie.Set(testKey, testValue) + require.NoError(t, err) + + header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(), + common.Hash{}, big.NewInt(1), nil) + require.NoError(t, err) + + err = s.storageState.StoreTrie(storageStateTrie, header) + require.NoError(t, err) + + testBlock := &types.Block{ + Header: header, + Body: types.NewBody([]byte{}), + } + + err = s.blockState.AddBlock(testBlock) + require.NoError(t, err) + + blockhash := testBlock.Header.Hash() + hexKey := common.BytesToHex(testKey) + keys := []string{hexKey} + + changes, err := s.tryQueryStorage(blockhash, keys...) + require.NoError(t, err) + + require.Equal(t, changes[hexKey], common.BytesToHex(testValue)) +} + +func TestTryQueryStore_WhenDoesNotHaveDataToRetrieve(t *testing.T) { + s := NewTestService(t, nil) + storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil)) + require.NoError(t, err) + + header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(), + common.Hash{}, big.NewInt(1), nil) + require.NoError(t, err) + + err = s.storageState.StoreTrie(storageStateTrie, header) + require.NoError(t, err) + + testBlock := &types.Block{ + Header: header, + Body: types.NewBody([]byte{}), + } + + err = s.blockState.AddBlock(testBlock) + require.NoError(t, err) + + testKey := []byte("to") + blockhash := testBlock.Header.Hash() + hexKey := common.BytesToHex(testKey) + keys := []string{hexKey} + + changes, err := s.tryQueryStorage(blockhash, keys...) + require.NoError(t, err) + + require.Empty(t, changes) +} + +func TestTryQueryState_WhenDoesNotHaveStateRoot(t *testing.T) { + s := NewTestService(t, nil) + + header, err := types.NewHeader(s.blockState.GenesisHash(), common.Hash{}, common.Hash{}, big.NewInt(1), nil) + require.NoError(t, err) + + testBlock := &types.Block{ + Header: header, + Body: types.NewBody([]byte{}), + } + + err = s.blockState.AddBlock(testBlock) + require.NoError(t, err) + + testKey := []byte("to") + blockhash := testBlock.Header.Hash() + hexKey := common.BytesToHex(testKey) + keys := []string{hexKey} + + changes, err := s.tryQueryStorage(blockhash, keys...) + require.Error(t, err) + require.Nil(t, changes) +} + +func TestQueryStorate_WhenBlocksHasData(t *testing.T) { + keys := []string{ + common.BytesToHex([]byte("transfer.to")), + common.BytesToHex([]byte("transfer.from")), + common.BytesToHex([]byte("transfer.value")), + } + + s := NewTestService(t, nil) + + firstKey, firstValue := []byte("transfer.to"), []byte("some-address-herer") + firstBlock := createNewBlockAndStoreDataAtBlock( + t, s, firstKey, firstValue, s.blockState.GenesisHash(), 1, + ) + + secondKey, secondValue := []byte("transfer.from"), []byte("another-address-here") + secondBlock := createNewBlockAndStoreDataAtBlock( + t, s, secondKey, secondValue, firstBlock.Header.Hash(), 2, + ) + + thirdKey, thirdValue := []byte("transfer.value"), []byte("value-gigamegablaster") + thirdBlock := createNewBlockAndStoreDataAtBlock( + t, s, thirdKey, thirdValue, secondBlock.Header.Hash(), 3, + ) + + from := firstBlock.Header.Hash() + data, err := s.QueryStorage(from, common.Hash{}, keys...) + require.NoError(t, err) + require.Len(t, data, 3) + + require.Equal(t, data[firstBlock.Header.Hash()], QueryKeyValueChanges( + map[string]string{ + common.BytesToHex(firstKey): common.BytesToHex(firstValue), + }, + )) + + from = secondBlock.Header.Hash() + to := thirdBlock.Header.Hash() + + data, err = s.QueryStorage(from, to, keys...) + require.NoError(t, err) + require.Len(t, data, 2) + + require.Equal(t, data[secondBlock.Header.Hash()], QueryKeyValueChanges( + map[string]string{ + common.BytesToHex(secondKey): common.BytesToHex(secondValue), + }, + )) + require.Equal(t, data[thirdBlock.Header.Hash()], QueryKeyValueChanges( + map[string]string{ + common.BytesToHex(thirdKey): common.BytesToHex(thirdValue), + }, + )) +} + +func createNewBlockAndStoreDataAtBlock(t *testing.T, s *Service, key, value []byte, parentHash common.Hash, number int64) *types.Block { + t.Helper() + + storageStateTrie, err := storage.NewTrieState(trie.NewTrie(nil)) + storageStateTrie.Set(key, value) + require.NoError(t, err) + + header, err := types.NewHeader(parentHash, storageStateTrie.MustRoot(), + common.Hash{}, big.NewInt(number), nil) + require.NoError(t, err) + + err = s.storageState.StoreTrie(storageStateTrie, header) + require.NoError(t, err) + + testBlock := &types.Block{ + Header: header, + Body: types.NewBody([]byte{}), + } + + err = s.blockState.AddBlock(testBlock) + require.NoError(t, err) + + return testBlock +} + func TestDecodeSessionKeys(t *testing.T) { mockInstance := new(runtimemocks.MockInstance) mockInstance.On("DecodeSessionKeys", mock.AnythingOfType("[]uint8")).Return([]byte{}, nil).Once() diff --git a/dot/network/service.go b/dot/network/service.go index c85a682bff..ec9c130758 100644 --- a/dot/network/service.go +++ b/dot/network/service.go @@ -304,9 +304,16 @@ func (s *Service) collectNetworkMetrics() { } func (s *Service) logPeerCount() { + ticker := time.NewTicker(time.Second * 30) + defer ticker.Stop() + for { - logger.Debug("peer count", "num", s.host.peerCount(), "min", s.cfg.MinPeers, "max", s.cfg.MaxPeers) - time.Sleep(time.Second * 30) + select { + case <-ticker.C: + logger.Debug("peer count", "num", s.host.peerCount(), "min", s.cfg.MinPeers, "max", s.cfg.MaxPeers) + case <-s.ctx.Done(): + return + } } } diff --git a/dot/rpc/modules/api.go b/dot/rpc/modules/api.go index 36d497758b..48465bb676 100644 --- a/dot/rpc/modules/api.go +++ b/dot/rpc/modules/api.go @@ -3,6 +3,7 @@ package modules import ( "math/big" + "github.com/ChainSafe/gossamer/dot/core" "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" @@ -77,6 +78,7 @@ type CoreAPI interface { GetRuntimeVersion(bhash *common.Hash) (runtime.Version, error) HandleSubmittedExtrinsic(types.Extrinsic) error GetMetadata(bhash *common.Hash) ([]byte, error) + QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]core.QueryKeyValueChanges, error) DecodeSessionKeys(enc []byte) ([]byte, error) } diff --git a/dot/rpc/modules/mocks/core_api.go b/dot/rpc/modules/mocks/core_api.go index beda95a749..da382dbda6 100644 --- a/dot/rpc/modules/mocks/core_api.go +++ b/dot/rpc/modules/mocks/core_api.go @@ -3,7 +3,9 @@ package mocks import ( + core "github.com/ChainSafe/gossamer/dot/core" common "github.com/ChainSafe/gossamer/lib/common" + crypto "github.com/ChainSafe/gossamer/lib/crypto" mock "github.com/stretchr/testify/mock" @@ -126,3 +128,33 @@ func (_m *MockCoreAPI) HasKey(pubKeyStr string, keyType string) (bool, error) { func (_m *MockCoreAPI) InsertKey(kp crypto.Keypair) { _m.Called(kp) } + +// QueryStorage provides a mock function with given fields: from, to, keys +func (_m *MockCoreAPI) QueryStorage(from common.Hash, to common.Hash, keys ...string) (map[common.Hash]core.QueryKeyValueChanges, error) { + _va := make([]interface{}, len(keys)) + for _i := range keys { + _va[_i] = keys[_i] + } + var _ca []interface{} + _ca = append(_ca, from, to) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 map[common.Hash]core.QueryKeyValueChanges + if rf, ok := ret.Get(0).(func(common.Hash, common.Hash, ...string) map[common.Hash]core.QueryKeyValueChanges); ok { + r0 = rf(from, to, keys...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[common.Hash]core.QueryKeyValueChanges) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(common.Hash, common.Hash, ...string) error); ok { + r1 = rf(from, to, keys...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index ee22373ca5..f301b4e8ac 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -18,6 +18,7 @@ package modules import ( "encoding/hex" + "errors" "fmt" "net/http" "strings" @@ -85,9 +86,9 @@ type StateStorageRequest struct { // StateStorageQueryRangeRequest holds json fields type StateStorageQueryRangeRequest struct { - Keys []*common.Hash `json:"keys" validate:"required"` - StartBlock *common.Hash `json:"startBlock" validate:"required"` - Block *common.Hash `json:"block"` + Keys []string `json:"keys" validate:"required"` + StartBlock common.Hash `json:"startBlock" validate:"required"` + EndBlock common.Hash `json:"block"` } // StateStorageKeysQuery field to store storage keys @@ -129,8 +130,8 @@ type StateMetadataResponse string // StorageChangeSetResponse is the struct that holds the block and changes type StorageChangeSetResponse struct { - Block *common.Hash - Changes []KeyValueOption + Block *common.Hash `json:"block"` + Changes [][]string `json:"changes"` } // KeyValueOption struct holds json fields @@ -387,8 +388,32 @@ func (sm *StateModule) GetStorageSize(r *http.Request, req *StateStorageSizeRequ } // QueryStorage isn't implemented properly yet. -func (sm *StateModule) QueryStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *StorageChangeSetResponse) error { - // TODO implement change storage trie so that block hash parameter works (See issue #834) +func (sm *StateModule) QueryStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *[]StorageChangeSetResponse) error { + if req.StartBlock == common.EmptyHash { + return errors.New("the start block hash cannot be an empty value") + } + + changesByBlock, err := sm.coreAPI.QueryStorage(req.StartBlock, req.EndBlock, req.Keys...) + if err != nil { + return err + } + + response := make([]StorageChangeSetResponse, 0, len(changesByBlock)) + + for block, c := range changesByBlock { + var changes [][]string + + for key, value := range c { + changes = append(changes, []string{key, value}) + } + + response = append(response, StorageChangeSetResponse{ + Block: &block, + Changes: changes, + }) + } + + *res = response return nil } diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index ca8661e250..19a844d80e 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -17,6 +17,7 @@ package modules import ( "encoding/hex" + "errors" "fmt" "io/ioutil" "math/big" @@ -24,8 +25,11 @@ import ( "strings" "testing" + "github.com/ChainSafe/gossamer/dot/core" + "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -315,6 +319,60 @@ func TestStateModule_GetStorageSize(t *testing.T) { } } +func TestStateModule_QueryStorage(t *testing.T) { + t.Run("When starting block is empty", func(t *testing.T) { + module := new(StateModule) + req := new(StateStorageQueryRangeRequest) + + var res []StorageChangeSetResponse + err := module.QueryStorage(nil, req, &res) + require.Error(t, err, "the start block hash cannot be an empty value") + }) + + t.Run("When coreAPI QueryStorage returns error", func(t *testing.T) { + coreapimock := new(mocks.MockCoreAPI) + coreapimock.On("QueryStorage", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("common.Hash")).Return(nil, errors.New("problem while querying")) + + module := new(StateModule) + module.coreAPI = coreapimock + + req := new(StateStorageQueryRangeRequest) + req.StartBlock = common.NewHash([]byte{1, 2}) + + var res []StorageChangeSetResponse + err := module.QueryStorage(nil, req, &res) + require.Error(t, err) + coreapimock.AssertCalled(t, "QueryStorage", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("common.Hash")) + }) + + t.Run("When QueryStorage returns data", func(t *testing.T) { + blockhash := common.NewHash([]byte{123}) + + changes := map[common.Hash]core.QueryKeyValueChanges{ + blockhash: core.QueryKeyValueChanges(map[string]string{ + "0x80": "value", + "0x90": "another value", + }), + } + coreapimock := new(mocks.MockCoreAPI) + coreapimock.On("QueryStorage", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("common.Hash"), "0x90", "0x80").Return(changes, nil) + + module := new(StateModule) + module.coreAPI = coreapimock + + req := new(StateStorageQueryRangeRequest) + req.StartBlock = common.NewHash([]byte{1, 2}) + req.Keys = []string{"0x90", "0x80"} + + var res []StorageChangeSetResponse + err := module.QueryStorage(nil, req, &res) + require.NoError(t, err) + require.Len(t, res, 1) + + coreapimock.AssertCalled(t, "QueryStorage", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("common.Hash"), "0x90", "0x80") + }) +} + func TestStateModule_GetMetadata(t *testing.T) { t.Skip() // TODO: update expected_metadata sm, hash, _ := setupStateModule(t) diff --git a/tests/rpc/rpc_05-state_test.go b/tests/rpc/rpc_05-state_test.go index acf4769266..fc8e147796 100644 --- a/tests/rpc/rpc_05-state_test.go +++ b/tests/rpc/rpc_05-state_test.go @@ -88,8 +88,12 @@ func TestStateRPCResponseValidation(t *testing.T) { { description: "Test state_queryStorage", method: "state_queryStorage", - params: `[["0xf2794c22e353e9a839f12faab03a911bf68967d635641a7087e53f2bff1ecad3c6756fee45ec79ead60347fffb770bcdf0ec74da701ab3d6495986fe1ecc3027"], "0xa32c60dee8647b07435ae7583eb35cee606209a595718562dd4a486a07b6de15", null]`, - expected: modules.StorageChangeSetResponse{}, + params: fmt.Sprintf(`[["0xf2794c22e353e9a839f12faab03a911bf68967d635641a7087e53f2bff1ecad3c6756fee45ec79ead60347fffb770bcdf0ec74da701ab3d6495986fe1ecc3027"], "%s", null]`, blockHash), + expected: modules.StorageChangeSetResponse{ + Block: &blockHash, + Changes: [][]string{}, + }, + skip: true, }, { description: "Test valid block hash state_getRuntimeVersion",