storeHeight & stateHeight does not match

[baabeetaa]

NOTE[02-01|16:24:00] ABCI Handshake                           module=state appHeight=210486 appHash=0A2973AF199B84C07933D58A5A89422228FF4399
NOTE[02-01|16:24:00] ABCI Replay Blocks                       module=state appHeight=210486 storeHeight=210488 stateHeight=210487
panic: Paniced on a Sanity Check: Expected storeHeight (210488) and stateHeight (210487) to match.

goroutine 1 [running]:
panic(0x9a58c0, 0xc4203ceec0)
	/usr/local/go/src/runtime/panic.go:500 +0x1a1
github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common.PanicSanity(0x9a58c0, 0xc4203ceea0)
	/root/GOPATH/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common/errors.go:26 +0xe0
github.com/tendermint/tendermint/state.(*Handshaker).ReplayBlocks(0xc4200b7500, 0xc4201898a0, 0x14, 0x14, 0x33636, 0xea4f60, 0xc4203cea50, 0x14, 0x14)
	/root/GOPATH/src/github.com/tendermint/tendermint/state/execution.go:400 +0xda2
github.com/tendermint/tendermint/state.(*Handshaker).Handshake(0xc4200b7500, 0xea62a0, 0xc42007e1b0, 0x0, 0x0)
	/root/GOPATH/src/github.com/tendermint/tendermint/state/execution.go:329 +0x483
github.com/tendermint/tendermint/proxy.(*multiAppConn).OnStart(0xc42007e1b0, 0xc420189320, 0x15)
	/root/GOPATH/src/github.com/tendermint/tendermint/proxy/multi_app_conn.go:100 +0x206
github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common.(*BaseService).Start(0xc42007e1b0, 0x0, 0x0, 0x0)
	/root/GOPATH/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common/service.go:96 +0x546
github.com/tendermint/tendermint/node.NewNode(0xea6e20, 0xc420109ce0, 0xc420081ea0, 0xe99640, 0xc4201531d0, 0x0)
	/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:68 +0x483
github.com/tendermint/tendermint/node.NewNodeDefault(0xea6e20, 0xc420109ce0, 0x1)
	/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:51 +0xba
github.com/tendermint/tendermint/node.RunNode(0xea6e20, 0xc420109ce0)
	/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:350 +0x529
main.main()
	/root/GOPATH/src/github.com/tendermint/tendermint/cmd/tendermint/main.go:42 +0x373

[baabeetaa]

how to fix?

// Replay all blocks after blockHeight and ensure the result matches the current state.
func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnConsensus proxy.AppConnConsensus) error {

	storeBlockHeight := h.store.Height()
	stateBlockHeight := h.state.LastBlockHeight
	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)

	if storeBlockHeight == 0 {
		return nil
	} else if storeBlockHeight < appBlockHeight {
		// if the app is ahead, there's nothing we can do
		return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}

	} else if storeBlockHeight == appBlockHeight {
		// We ran Commit, but if we crashed before state.Save(),
		// load the intermediate state and update the state.AppHash.
		// NOTE: If ABCI allowed rollbacks, we could just replay the
		// block even though it's been committed
		stateAppHash := h.state.AppHash
		lastBlockAppHash := h.store.LoadBlock(storeBlockHeight).AppHash

		if bytes.Equal(stateAppHash, appHash) {
			// we're all synced up
			log.Debug("ABCI RelpayBlocks: Already synced")
		} else if bytes.Equal(stateAppHash, lastBlockAppHash) {
			// we crashed after commit and before saving state,
			// so load the intermediate state and update the hash
			h.state.LoadIntermediate()
			h.state.AppHash = appHash
			log.Debug("ABCI RelpayBlocks: Loaded intermediate state and updated state.AppHash")
		} else {
			PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash))

		}
		return nil

	} else if storeBlockHeight == appBlockHeight+1 &&
		storeBlockHeight == stateBlockHeight+1 {
		// We crashed after saving the block
		// but before Commit (both the state and app are behind),
		// so just replay the block

		// check that the lastBlock.AppHash matches the state apphash
		block := h.store.LoadBlock(storeBlockHeight)
		if !bytes.Equal(block.Header.AppHash, appHash) {
			return ErrLastStateMismatch{storeBlockHeight, block.Header.AppHash, appHash}
		}

		blockMeta := h.store.LoadBlockMeta(storeBlockHeight)

		h.nBlocks += 1
		var eventCache types.Fireable // nil

		// replay the latest block
		return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, MockMempool{})
	} else if storeBlockHeight != stateBlockHeight {
		// unless we failed before committing or saving state (previous 2 case),
		// the store and state should be at the same height!
		PanicSanity(Fmt("Expected storeHeight (%d) and stateHeight (%d) to match.", storeBlockHeight, stateBlockHeight))
	} else {
		// store is more than one ahead,
		// so app wants to replay many blocks

		// replay all blocks starting with appBlockHeight+1
		var eventCache types.Fireable // nil

		// TODO: use stateBlockHeight instead and let the consensus state
		// do the replay

		var appHash []byte
		for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
			h.nBlocks += 1
			block := h.store.LoadBlock(i)
			_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
			if err != nil {
				log.Warn("Error executing block on proxy app", "height", i, "err", err)
				return err
			}
			// Commit block, get hash back
			res := appConnConsensus.CommitSync()
			if res.IsErr() {
				log.Warn("Error in proxyAppConn.CommitSync", "error", res)
				return res
			}
			if res.Log != "" {
				log.Info("Commit.Log: " + res.Log)
			}
			appHash = res.Data
		}
		if !bytes.Equal(h.state.AppHash, appHash) {
			return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
		}
		return nil
	}
	return nil
}

[baabeetaa]

tendermint/tendermint#388

Need to stop TM before glogchain.
This bug is annoying, if app stop for some reason (got panic ) then the chain got corrupted. Have to synch again!

Dont know if combining TM + Glog to single binary is helpful?

[faddat]

In my mind the ideal form is

Binary One: (tm + glog)
Binary Two: (webapp)

Instead of getting our repositories all strange-ified, I also figure if we are doing the grand refactor then we could also create:

github.com/dawn-network/webfront
github.com/dawn-network/desktopqtclient
github.com/dawn-network/fluttermobile