[WIP] parity warp-like state sync #3594

ackratos · 2019-04-25T15:59:11Z

follow up of #3243
A new state sync proposal: https://docs.google.com/document/d/1npGTAa1qxe8EQZ1wG0a0Sip9t5oX2vYZNUDwr_LVRR4/edit?usp=sharing

Updated all relevant documentation in docs
Updated all code comments where relevant
Wrote tests
Updated CHANGELOG_PENDING.md

golangcibot · 2019-04-25T16:00:08Z

snapshot/reactor.go

+		app:          app,
+		config: 	  config,
+		pool:         pool,
+		stateSync:    config.StateSync,


config.StateSync undefined (type *config.Config has no field or method StateSync) (from typecheck)

golangcibot · 2019-04-25T16:00:08Z

snapshot/manager.go

+}
+
+func (mgr *SnapshotManager) restoreBlock(block *types.Block, seenCommit *types.Commit) {
+	mgr.blockStore.SetHeight(block.Height - 1)


mgr.blockStore.SetHeight undefined (type *blockchain.BlockStore has no field or method SetHeight) (from typecheck)

golangcibot · 2019-04-25T16:00:08Z

snapshot/manager.go

+}
+
+func (mgr *SnapshotManager) snapshotTMState() error {
+	if state := sm.LoadStateForHeight(mgr.stateDB, mgr.height); state != nil {


LoadStateForHeight not declared by package state (from typecheck)

golangcibot · 2019-04-25T16:00:08Z

node/node.go

@@ -351,6 +354,26 @@ func NewNode(config *cfg.Config,
 	evidenceReactor := evidence.NewEvidenceReactor(evidencePool)
 	evidenceReactor.SetLogger(evidenceLogger)

+	if state.Validators.Size() == 1 {
+		addr, _ := state.Validators.GetByIndex(0)
+		if bytes.Equal(privValidator.GetAddress(), addr) {


privValidator.GetAddress undefined (type types.PrivValidator has no field or method GetAddress) (from typecheck)

golangcibot · 2019-04-25T16:00:09Z

node/node.go

+	if state.Validators.Size() == 1 {
+		addr, _ := state.Validators.GetByIndex(0)
+		if bytes.Equal(privValidator.GetAddress(), addr) {
+			config.StateSync = false


config.StateSync undefined (type *config.Config has no field or method StateSync) (from typecheck)

golangcibot · 2019-04-25T16:00:09Z

blockchain/reactor.go

+			// numPending=0 total=0 outbound=1 inbound=0 height=6601 blocksSynced=0 poolHeight=6601 poolstarttime=2019-02-22T09:02:07.881888662Z poolMaxPeerHeight=6437 poolHeight=6601
+			// we need make sure blockstore has a block because when switch to consensus, it will verify the commit between block and state
+			// refer to `cs.blockStore.LoadSeenCommit(state.LastBlockHeight)`
+			if bcR.pool.IsCaughtUp() && (height == bcR.pool.initHeight || blocksSynced > 0) {


bcR.pool.initHeight undefined (type *BlockPool has no field or method initHeight) (from typecheck)

golangcibot · 2019-04-25T16:00:09Z

blockchain/reactor.go

+	if state != nil {
+		bcR.initialState = *state
+		bcR.pool.height = state.LastBlockHeight + 1
+		bcR.store.SetHeight(state.LastBlockHeight)


bcR.store.SetHeight undefined (type *BlockStore has no field or method SetHeight) (from typecheck)

golangcibot · 2019-04-25T16:00:09Z

snapshot/reactor.go

+	StateSyncChannel = byte(0x35)
+
+	// ====== move to config ======
+	stateSyncPriority = 10			// channel priority of state sync


File is not gofmt-ed with -s (from gofmt)

golangcibot · 2019-04-25T16:00:10Z

node/node.go

@@ -37,6 +37,7 @@ import (
 	grpccore "github.com/tendermint/tendermint/rpc/grpc"
 	rpcserver "github.com/tendermint/tendermint/rpc/lib/server"
 	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/snapshot"


File is not gofmt-ed with -s (from gofmt)

golangcibot · 2019-04-25T16:00:10Z

snapshot/pool.go

+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"github.com/golang/snappy"


File is not goimports-ed (from goimports)

golangcibot · 2019-04-26T02:41:56Z

snapshot/manager.go

+
+func (mgr *SnapshotManager) snapshotTMState() error {
+	if state := sm.LoadStateForHeight(mgr.stateDB, mgr.height); state != nil {
+		stateChunk := &abci.StateChunk{


composites: github.com/tendermint/tendermint/abci/types.StateChunk composite literal uses unkeyed fields (from govet)

golangcibot · 2019-04-26T02:41:57Z

snapshot/manager.go

+			return fmt.Errorf("block %d to be snapshoted is not expected %d", block.Height, mgr.height)
+		}
+		seenCommit := mgr.blockStore.LoadSeenCommit(mgr.height)
+		blockChunk := &abci.BlockChunk{


composites: github.com/tendermint/tendermint/abci/types.BlockChunk composite literal uses unkeyed fields (from govet)

golangcibot · 2019-04-26T02:41:57Z

snapshot/pool.go

+			if manifest.Version == abci.ManifestVersion {
+				return hash, &manifest, nil
+			} else {
+				return hash, nil, fmt.Errorf("snapshot manifest version mismatch", "expected", abci.ManifestVersion, "actual", manifest.Version)


printf: Errorf call has arguments but no formatting directives (from govet)

golangcibot · 2019-04-26T02:41:57Z

state/store.go

+const latestStateToKeep int64 = 1 << 20
+
+func calcStateKey(height int64) []byte {
+	return []byte(fmt.Sprintf("stateKey:%v", height % latestStateToKeep))


File is not gofmt-ed with -s (from gofmt)

golangcibot · 2019-04-29T07:12:57Z

snapshot/pool.go

+	mtx          sync.Mutex
+	manifest     *abci.Manifest
+	manifestHash abci.SHA256Sum
+	dummyHash abci.SHA256Sum


File is not gofmt-ed with -s (from gofmt)

golangcibot · 2019-05-08T06:14:50Z

abci/types/snapshot.go

+)
+
+const (
+	ManifestVersion int32 = 0


File is not gofmt-ed with -s (from gofmt)

tac0turtle · 2019-07-06T08:07:16Z

Will close this PR in favor of the discussion being made in #3769, @ackratos I see you have commented on the PR a bit, thank you for the contribution and helping drive state-sync forward. Any contributions that you can make will be invaluable.

ackratos · 2019-07-08T11:00:15Z

Will close this PR in favor of the discussion being made in #3769, @ackratos I see you have commented on the PR a bit, thank you for the contribution and helping drive state-sync forward. Any contributions that you can make will be invaluable.

It's my honor. But I still prefer to open this PR for people's reference. If you think this PR is kind of broken, I can rebase the latest develop branch. Or you would like to accept PR until discussion in #3769 reaches a conclusion?

ackratos requested review from ebuchman, melekes and xla as code owners April 25, 2019 15:59

ackratos mentioned this pull request Apr 25, 2019

[WIP] issue828 state syncing #3243

Closed

4 tasks

golangcibot reviewed Apr 25, 2019

View reviewed changes

ackratos mentioned this pull request Apr 25, 2019

sync: Sync current state without full replay for Applications #828

Closed

ackratos force-pushed the issue828_statesync2 branch from ad416ca to 1eb5484 Compare April 26, 2019 02:40

golangcibot reviewed Apr 26, 2019

View reviewed changes

ackratos force-pushed the issue828_statesync2 branch from 1eb5484 to ddf0316 Compare April 29, 2019 07:11

golangcibot reviewed Apr 29, 2019

View reviewed changes

ackratos mentioned this pull request May 6, 2019

[DEMO] parity warp-like state sync cosmos/cosmos-sdk#4275

Closed

5 tasks

ackratos force-pushed the issue828_statesync2 branch 2 times, most recently from ecd10ba to ffa7978 Compare May 8, 2019 06:13

golangcibot reviewed May 8, 2019

View reviewed changes

abci/types/snapshot.go

)

const (

ManifestVersion int32 = 0

Copy link

Contributor

golangcibot May 8, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

File is not gofmt-ed with -s (from gofmt)

ackratos force-pushed the issue828_statesync2 branch from ffa7978 to 1e649bf Compare May 8, 2019 06:32

parity warp-like state sync

40771ea

ackratos force-pushed the issue828_statesync2 branch from 1e649bf to 40771ea Compare May 8, 2019 08:12

tac0turtle changed the base branch from develop to master June 26, 2019 15:43

tac0turtle closed this Jul 6, 2019

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[WIP] parity warp-like state sync #3594

[WIP] parity warp-like state sync #3594

ackratos commented Apr 25, 2019 •

edited

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 25, 2019

golangcibot Apr 26, 2019

golangcibot Apr 26, 2019

golangcibot Apr 26, 2019

golangcibot Apr 26, 2019

golangcibot Apr 29, 2019

golangcibot May 8, 2019

tac0turtle commented Jul 6, 2019

ackratos commented Jul 8, 2019

[WIP] parity warp-like state sync #3594

[WIP] parity warp-like state sync #3594

Conversation

ackratos commented Apr 25, 2019 • edited

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

tac0turtle commented Jul 6, 2019

ackratos commented Jul 8, 2019

ackratos commented Apr 25, 2019 •

edited