From b170d62cede99a0fd49b4f73178f4dab0ddf3e9f Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Mon, 5 Jul 2021 14:14:05 -0600 Subject: [PATCH 1/7] integrate scale into dot/sync --- dot/sync/test_helpers.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dot/sync/test_helpers.go b/dot/sync/test_helpers.go index 4345afcb7f..b424b277c5 100644 --- a/dot/sync/test_helpers.go +++ b/dot/sync/test_helpers.go @@ -32,9 +32,9 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/runtime/wasmer" - "github.com/ChainSafe/gossamer/lib/scale" "github.com/ChainSafe/gossamer/lib/transaction" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/scale" log "github.com/ChainSafe/log15" "github.com/stretchr/testify/require" @@ -166,10 +166,11 @@ func BuildBlock(t *testing.T, instance runtime.Instance, parent *types.Header, e require.NoError(t, err) // decode inherent extrinsics - exts, err := scale.Decode(inherentExts, [][]byte{}) + var exts [][]byte + err = scale.Unmarshal(inherentExts, &exts) require.NoError(t, err) - inExt := exts.([][]byte) + inExt := exts var body *types.Body if ext != nil { @@ -191,7 +192,7 @@ func BuildBlock(t *testing.T, instance runtime.Instance, parent *types.Header, e // apply each inherent extrinsic for _, ext := range inExt { - in, err := scale.Encode(ext) //nolint + in, err := scale.Marshal(ext) //nolint require.NoError(t, err) ret, err := instance.ApplyExtrinsic(in) From e0b1b870644834ca1e3b3745dbad108d0e2bf410 Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Tue, 6 Jul 2021 08:38:05 -0600 Subject: [PATCH 2/7] integrate scale into dot/rpc --- dot/rpc/modules/state.go | 7 ++++--- dot/rpc/modules/system.go | 7 ++++--- dot/rpc/modules/system_test.go | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 33441bf24d..ee22373ca5 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -24,7 +24,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/scale" + "github.com/ChainSafe/gossamer/pkg/scale" ) // StateCallRequest holds json fields @@ -272,8 +272,9 @@ func (sm *StateModule) GetMetadata(r *http.Request, req *StateRuntimeMetadataQue return err } - decoded, err := scale.Decode(metadata, []byte{}) - *res = StateMetadataResponse(common.BytesToHex(decoded.([]byte))) + var decoded []byte + err = scale.Unmarshal(metadata, &decoded) + *res = StateMetadataResponse(common.BytesToHex(decoded)) return err } diff --git a/dot/rpc/modules/system.go b/dot/rpc/modules/system.go index a3885ad0b9..46a9b3e23a 100644 --- a/dot/rpc/modules/system.go +++ b/dot/rpc/modules/system.go @@ -24,7 +24,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto" - "github.com/ChainSafe/gossamer/lib/scale" + "github.com/ChainSafe/gossamer/pkg/scale" ctypes "github.com/centrifuge/go-substrate-rpc-client/v3/types" ) @@ -196,12 +196,13 @@ func (sm *SystemModule) AccountNextIndex(r *http.Request, req *StringRequest, re if err != nil { return err } - sdMeta, err := scale.Decode(rawMeta, []byte{}) + var sdMeta []byte + err = scale.Unmarshal(rawMeta, &sdMeta) if err != nil { return err } var metadata ctypes.Metadata - err = ctypes.DecodeFromBytes(sdMeta.([]byte), &metadata) + err = ctypes.DecodeFromBytes(sdMeta, &metadata) if err != nil { return err } diff --git a/dot/rpc/modules/system_test.go b/dot/rpc/modules/system_test.go index d47cf95bb2..9041f2433f 100644 --- a/dot/rpc/modules/system_test.go +++ b/dot/rpc/modules/system_test.go @@ -29,8 +29,8 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" - "github.com/ChainSafe/gossamer/lib/scale" "github.com/ChainSafe/gossamer/lib/transaction" + "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" ) @@ -302,7 +302,7 @@ func setupSystemModule(t *testing.T) *SystemModule { FreeFrozen common.Uint128 }{}, } - aliceAcctEncoded, err := scale.Encode(aliceAcctInfo) + aliceAcctEncoded, err := scale.Marshal(aliceAcctInfo) require.NoError(t, err) ts.Set(aliceAcctStoKey, aliceAcctEncoded) From 2bea2d7a0964e37923864172571765c2a8594ac6 Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Tue, 6 Jul 2021 10:08:51 -0600 Subject: [PATCH 3/7] scale into epoch --- dot/state/epoch.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/dot/state/epoch.go b/dot/state/epoch.go index 01dc6f45f3..9ac15170ea 100644 --- a/dot/state/epoch.go +++ b/dot/state/epoch.go @@ -26,7 +26,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/scale" + "github.com/ChainSafe/gossamer/pkg/scale" ) var ( @@ -221,7 +221,7 @@ func (s *EpochState) GetEpochForBlock(header *types.Header) (uint64, error) { func (s *EpochState) SetEpochData(epoch uint64, info *types.EpochData) error { raw := info.ToEpochDataRaw() - enc, err := scale.Encode(raw) + enc, err := scale.Marshal(raw) if err != nil { return err } @@ -235,18 +235,14 @@ func (s *EpochState) GetEpochData(epoch uint64) (*types.EpochData, error) { if err != nil { return nil, err } - - info, err := scale.Decode(enc, &types.EpochDataRaw{}) + + var info *types.EpochDataRaw + err = scale.Unmarshal(enc, &info) if err != nil { return nil, err } - raw, ok := info.(*types.EpochDataRaw) - if !ok { - return nil, errors.New("failed to decode raw epoch data") - } - - return raw.ToEpochData() + return info.ToEpochData() } // GetLatestEpochData returns the EpochData for the current epoch @@ -266,7 +262,7 @@ func (s *EpochState) HasEpochData(epoch uint64) (bool, error) { // SetConfigData sets the BABE config data for a given epoch func (s *EpochState) SetConfigData(epoch uint64, info *types.ConfigData) error { - enc, err := scale.Encode(info) + enc, err := scale.Marshal(info) if err != nil { return err } @@ -292,12 +288,13 @@ func (s *EpochState) GetConfigData(epoch uint64) (*types.ConfigData, error) { return nil, err } - info, err := scale.Decode(enc, new(types.ConfigData)) + var info *types.ConfigData + err = scale.Unmarshal(enc, &info) if err != nil { return nil, err } - return info.(*types.ConfigData), nil + return info, nil } // GetLatestConfigData returns the most recently set ConfigData From 1c37e610c0f897ea8e90c133dffde389f640b03c Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Tue, 6 Jul 2021 10:18:23 -0600 Subject: [PATCH 4/7] integrate scale into pruner --- dot/state/epoch.go | 2 +- dot/state/pruner/pruner.go | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/dot/state/epoch.go b/dot/state/epoch.go index 9ac15170ea..61fc210cb5 100644 --- a/dot/state/epoch.go +++ b/dot/state/epoch.go @@ -235,7 +235,7 @@ func (s *EpochState) GetEpochData(epoch uint64) (*types.EpochData, error) { if err != nil { return nil, err } - + var info *types.EpochDataRaw err = scale.Unmarshal(enc, &info) if err != nil { diff --git a/dot/state/pruner/pruner.go b/dot/state/pruner/pruner.go index b98ccf2843..52b16be004 100644 --- a/dot/state/pruner/pruner.go +++ b/dot/state/pruner/pruner.go @@ -7,7 +7,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/scale" + "github.com/ChainSafe/gossamer/pkg/scale" log "github.com/ChainSafe/log15" ) @@ -275,12 +275,12 @@ func (p *FullNode) start() { } func (p *FullNode) storeJournal(key *journalKey, jr *journalRecord) error { - encKey, err := scale.Encode(key) + encKey, err := scale.Marshal(key) if err != nil { return fmt.Errorf("failed to encode journal key block num %d: %w", key.blockNum, err) } - encRecord, err := scale.Encode(jr) + encRecord, err := scale.Marshal(jr) if err != nil { return fmt.Errorf("failed to encode journal record block num %d: %w", key.blockNum, err) } @@ -299,26 +299,28 @@ func (p *FullNode) loadDeathList() error { defer itr.Release() for itr.Next() { - jk, err := scale.Decode(itr.Key(), new(journalKey)) + var jk *journalKey + err := scale.Unmarshal(itr.Key(), &jk) if err != nil { return fmt.Errorf("failed to decode journal key %w", err) } - key := jk.(*journalKey) + key := jk val := itr.Value() - jr, err := scale.Decode(val, new(journalRecord)) + var jr *journalRecord + err = scale.Unmarshal(val, &jr) if err != nil { return fmt.Errorf("failed to decode journal record block num %d : %w", key.blockNum, err) } - p.addDeathRow(jr.(*journalRecord), key.blockNum) + p.addDeathRow(jr, key.blockNum) } return nil } func (p *FullNode) deleteJournalRecord(b chaindb.Batch, key *journalKey) error { - encKey, err := scale.Encode(key) + encKey, err := scale.Marshal(key) if err != nil { return err } @@ -332,7 +334,7 @@ func (p *FullNode) deleteJournalRecord(b chaindb.Batch, key *journalKey) error { } func (p *FullNode) storeLastPrunedIndex(blockNum int64) error { - encNum, err := scale.Encode(blockNum) + encNum, err := scale.Marshal(blockNum) if err != nil { return err } @@ -355,12 +357,13 @@ func (p *FullNode) getLastPrunedIndex() (int64, error) { return 0, err } - blockNum, err := scale.Decode(val, int64(0)) + var blockNum int64 + err = scale.Unmarshal(val, &blockNum) if err != nil { return 0, err } - return blockNum.(int64), nil + return blockNum, nil } func (p *FullNode) deleteKeys(b chaindb.Batch, nodesHash map[common.Hash]int64) error { From a2c85c0a9568baf6aab4b42a980bda20693c67ff Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Tue, 6 Jul 2021 10:57:42 -0600 Subject: [PATCH 5/7] integrate scale pkg into dot/state --- dot/state/grandpa.go | 14 ++++---------- dot/state/initialize.go | 1 + 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/dot/state/grandpa.go b/dot/state/grandpa.go index 20ff4488b9..343dc42f29 100644 --- a/dot/state/grandpa.go +++ b/dot/state/grandpa.go @@ -17,14 +17,13 @@ package state import ( - "bytes" "encoding/binary" "errors" "math/big" "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/lib/scale" + "github.com/ChainSafe/gossamer/pkg/scale" ) var ( @@ -88,7 +87,7 @@ func setIDChangeKey(setID uint64) []byte { // setAuthorities sets the authorities for a given setID func (s *GrandpaState) setAuthorities(setID uint64, authorities []*types.GrandpaVoter) error { - enc, err := scale.Encode(authorities) + enc, err := scale.Marshal(authorities) if err != nil { return err } @@ -103,13 +102,8 @@ func (s *GrandpaState) GetAuthorities(setID uint64) ([]*types.GrandpaVoter, erro return nil, err } - r := &bytes.Buffer{} - _, err = r.Write(enc) - if err != nil { - return nil, err - } - - v, err := types.DecodeGrandpaVoters(r) + var v types.GrandpaVoters + err = scale.Unmarshal(enc, &v) if err != nil { return nil, err } diff --git a/dot/state/initialize.go b/dot/state/initialize.go index e4a6afdac1..d086ffd2a8 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -149,6 +149,7 @@ func loadGrandpaAuthorities(t *trie.Trie) ([]*types.GrandpaVoter, error) { r := &bytes.Buffer{} _, _ = r.Write(authsRaw[1:]) + // TODO remove this function after scale integration into trie return types.DecodeGrandpaVoters(r) } From d396bc658f8ad11c3e5a27138246dcaa82bf6995 Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Tue, 6 Jul 2021 13:53:22 -0600 Subject: [PATCH 6/7] WIP/Fix digest encoding errors --- dot/state/grandpa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dot/state/grandpa.go b/dot/state/grandpa.go index 343dc42f29..32cc3d8ceb 100644 --- a/dot/state/grandpa.go +++ b/dot/state/grandpa.go @@ -102,7 +102,7 @@ func (s *GrandpaState) GetAuthorities(setID uint64) ([]*types.GrandpaVoter, erro return nil, err } - var v types.GrandpaVoters + var v []*types.GrandpaVoter err = scale.Unmarshal(enc, &v) if err != nil { return nil, err From e4e92f95ce2dc75b9ee03ea4281e7733fe832ee7 Mon Sep 17 00:00:00 2001 From: jimjbrettj Date: Wed, 7 Jul 2021 10:39:15 -0600 Subject: [PATCH 7/7] remove changes in dot/state to include in separate PR --- dot/rpc/modules/system_test.go | 4 ++-- dot/state/epoch.go | 21 ++++++++++++--------- dot/state/grandpa.go | 14 ++++++++++---- dot/state/initialize.go | 1 - dot/state/pruner/pruner.go | 25 +++++++++++-------------- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/dot/rpc/modules/system_test.go b/dot/rpc/modules/system_test.go index 593dcc0a7f..57fdc37316 100644 --- a/dot/rpc/modules/system_test.go +++ b/dot/rpc/modules/system_test.go @@ -33,9 +33,9 @@ import ( "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/wasmer" - "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/lib/transaction" "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/scale" log "github.com/ChainSafe/log15" "github.com/stretchr/testify/require" ) diff --git a/dot/state/epoch.go b/dot/state/epoch.go index 61fc210cb5..01dc6f45f3 100644 --- a/dot/state/epoch.go +++ b/dot/state/epoch.go @@ -26,7 +26,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/lib/scale" ) var ( @@ -221,7 +221,7 @@ func (s *EpochState) GetEpochForBlock(header *types.Header) (uint64, error) { func (s *EpochState) SetEpochData(epoch uint64, info *types.EpochData) error { raw := info.ToEpochDataRaw() - enc, err := scale.Marshal(raw) + enc, err := scale.Encode(raw) if err != nil { return err } @@ -236,13 +236,17 @@ func (s *EpochState) GetEpochData(epoch uint64) (*types.EpochData, error) { return nil, err } - var info *types.EpochDataRaw - err = scale.Unmarshal(enc, &info) + info, err := scale.Decode(enc, &types.EpochDataRaw{}) if err != nil { return nil, err } - return info.ToEpochData() + raw, ok := info.(*types.EpochDataRaw) + if !ok { + return nil, errors.New("failed to decode raw epoch data") + } + + return raw.ToEpochData() } // GetLatestEpochData returns the EpochData for the current epoch @@ -262,7 +266,7 @@ func (s *EpochState) HasEpochData(epoch uint64) (bool, error) { // SetConfigData sets the BABE config data for a given epoch func (s *EpochState) SetConfigData(epoch uint64, info *types.ConfigData) error { - enc, err := scale.Marshal(info) + enc, err := scale.Encode(info) if err != nil { return err } @@ -288,13 +292,12 @@ func (s *EpochState) GetConfigData(epoch uint64) (*types.ConfigData, error) { return nil, err } - var info *types.ConfigData - err = scale.Unmarshal(enc, &info) + info, err := scale.Decode(enc, new(types.ConfigData)) if err != nil { return nil, err } - return info, nil + return info.(*types.ConfigData), nil } // GetLatestConfigData returns the most recently set ConfigData diff --git a/dot/state/grandpa.go b/dot/state/grandpa.go index 32cc3d8ceb..20ff4488b9 100644 --- a/dot/state/grandpa.go +++ b/dot/state/grandpa.go @@ -17,13 +17,14 @@ package state import ( + "bytes" "encoding/binary" "errors" "math/big" "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/lib/scale" ) var ( @@ -87,7 +88,7 @@ func setIDChangeKey(setID uint64) []byte { // setAuthorities sets the authorities for a given setID func (s *GrandpaState) setAuthorities(setID uint64, authorities []*types.GrandpaVoter) error { - enc, err := scale.Marshal(authorities) + enc, err := scale.Encode(authorities) if err != nil { return err } @@ -102,8 +103,13 @@ func (s *GrandpaState) GetAuthorities(setID uint64) ([]*types.GrandpaVoter, erro return nil, err } - var v []*types.GrandpaVoter - err = scale.Unmarshal(enc, &v) + r := &bytes.Buffer{} + _, err = r.Write(enc) + if err != nil { + return nil, err + } + + v, err := types.DecodeGrandpaVoters(r) if err != nil { return nil, err } diff --git a/dot/state/initialize.go b/dot/state/initialize.go index d086ffd2a8..e4a6afdac1 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -149,7 +149,6 @@ func loadGrandpaAuthorities(t *trie.Trie) ([]*types.GrandpaVoter, error) { r := &bytes.Buffer{} _, _ = r.Write(authsRaw[1:]) - // TODO remove this function after scale integration into trie return types.DecodeGrandpaVoters(r) } diff --git a/dot/state/pruner/pruner.go b/dot/state/pruner/pruner.go index 52b16be004..b98ccf2843 100644 --- a/dot/state/pruner/pruner.go +++ b/dot/state/pruner/pruner.go @@ -7,7 +7,7 @@ import ( "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/lib/scale" log "github.com/ChainSafe/log15" ) @@ -275,12 +275,12 @@ func (p *FullNode) start() { } func (p *FullNode) storeJournal(key *journalKey, jr *journalRecord) error { - encKey, err := scale.Marshal(key) + encKey, err := scale.Encode(key) if err != nil { return fmt.Errorf("failed to encode journal key block num %d: %w", key.blockNum, err) } - encRecord, err := scale.Marshal(jr) + encRecord, err := scale.Encode(jr) if err != nil { return fmt.Errorf("failed to encode journal record block num %d: %w", key.blockNum, err) } @@ -299,28 +299,26 @@ func (p *FullNode) loadDeathList() error { defer itr.Release() for itr.Next() { - var jk *journalKey - err := scale.Unmarshal(itr.Key(), &jk) + jk, err := scale.Decode(itr.Key(), new(journalKey)) if err != nil { return fmt.Errorf("failed to decode journal key %w", err) } - key := jk + key := jk.(*journalKey) val := itr.Value() - var jr *journalRecord - err = scale.Unmarshal(val, &jr) + jr, err := scale.Decode(val, new(journalRecord)) if err != nil { return fmt.Errorf("failed to decode journal record block num %d : %w", key.blockNum, err) } - p.addDeathRow(jr, key.blockNum) + p.addDeathRow(jr.(*journalRecord), key.blockNum) } return nil } func (p *FullNode) deleteJournalRecord(b chaindb.Batch, key *journalKey) error { - encKey, err := scale.Marshal(key) + encKey, err := scale.Encode(key) if err != nil { return err } @@ -334,7 +332,7 @@ func (p *FullNode) deleteJournalRecord(b chaindb.Batch, key *journalKey) error { } func (p *FullNode) storeLastPrunedIndex(blockNum int64) error { - encNum, err := scale.Marshal(blockNum) + encNum, err := scale.Encode(blockNum) if err != nil { return err } @@ -357,13 +355,12 @@ func (p *FullNode) getLastPrunedIndex() (int64, error) { return 0, err } - var blockNum int64 - err = scale.Unmarshal(val, &blockNum) + blockNum, err := scale.Decode(val, int64(0)) if err != nil { return 0, err } - return blockNum, nil + return blockNum.(int64), nil } func (p *FullNode) deleteKeys(b chaindb.Batch, nodesHash map[common.Hash]int64) error {