cli: allow node operator to rollback last state (backport #7033) (#7080)

CHANGELOG_PENDING.md

@@ -21,9 +21,10 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 ### FEATURES
 
 - [\#6982](https://github.com/tendermint/tendermint/pull/6982) tendermint binary has built-in suppport for running the end to end application (with state sync support) (@cmwaters).
+- [cli] [#7033](https://github.com/tendermint/tendermint/pull/7033) Add a `rollback` command to rollback to the previous tendermint state in the event of non-determinstic app hash or reverting an upgrade.
 
 ### IMPROVEMENTS
 
 ### BUG FIXES
 
-- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair).
+- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair).

cmd/tendermint/commands/rollback.go

@@ -0,0 +1,69 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	dbm "github.com/tendermint/tm-db"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+)
+
+var RollbackStateCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "rollback tendermint state by one height",
+	Long: `
+A state rollback is performed to recover from an incorrect application state transition,
+when Tendermint has persisted an incorrect app hash and is thus unable to make
+progress. Rollback overwrites a state at height n with the state at height n - 1.
+The application should also roll back to height n - 1. No blocks are removed, so upon
+restarting Tendermint the transactions in block n will be re-executed against the
+application.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		height, hash, err := RollbackState(config)
+		if err != nil {
+			return fmt.Errorf("failed to rollback state: %w", err)
+		}
+
+		fmt.Printf("Rolled back state to height %d and hash %v", height, hash)
+		return nil
+	},
+}
+
+// RollbackState takes the state at the current height n and overwrites it with the state
+// at height n - 1. Note state here refers to tendermint state not application state.
+// Returns the latest state height and app hash alongside an error if there was one.
+func RollbackState(config *cfg.Config) (int64, []byte, error) {
+	// use the parsed config to load the block and state store
+	blockStore, stateStore, err := loadStateAndBlockStore(config)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	// rollback the last state
+	return state.Rollback(blockStore, stateStore)
+}
+
+func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) {
+	dbType := dbm.BackendType(config.DBBackend)
+
+	// Get BlockStore
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	// Get StateStore
+	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	stateStore := state.NewStore(stateDB)
+
+	return blockStore, stateStore, nil
+}

cmd/tendermint/main.go

@@ -27,6 +27,7 @@ func main() {
 		cmd.ShowNodeIDCmd,
 		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd,
+		cmd.RollbackStateCmd,
 		debug.DebugCmd,
 		cli.NewCompletionCmd(rootCmd, true),
 	)

state/rollback.go

@@ -0,0 +1,89 @@
+package state
+
+import (
+	"errors"
+	"fmt"
+
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	tmversion "github.com/tendermint/tendermint/proto/tendermint/version"
+	"github.com/tendermint/tendermint/version"
+)
+
+// Rollback overwrites the current Tendermint state (height n) with the most
+// recent previous state (height n - 1).
+// Note that this function does not affect application state.
+func Rollback(bs BlockStore, ss Store) (int64, []byte, error) {
+	invalidState, err := ss.Load()
+	if err != nil {
+		return -1, nil, err
+	}
+	if invalidState.IsEmpty() {
+		return -1, nil, errors.New("no state found")
+	}
+
+	rollbackHeight := invalidState.LastBlockHeight
+	rollbackBlock := bs.LoadBlockMeta(rollbackHeight)
+	if rollbackBlock == nil {
+		return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight)
+	}
+
+	previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	previousParams, err := ss.LoadConsensusParams(rollbackHeight)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	valChangeHeight := invalidState.LastHeightValidatorsChanged
+	// this can only happen if the validator set changed since the last block
+	if valChangeHeight > rollbackHeight {
+		valChangeHeight = rollbackHeight
+	}
+
+	paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged
+	// this can only happen if params changed from the last block
+	if paramsChangeHeight > rollbackHeight {
+		paramsChangeHeight = rollbackHeight
+	}
+
+	// build the new state from the old state and the prior block
+	rolledBackState := State{
+		Version: tmstate.Version{
+			Consensus: tmversion.Consensus{
+				Block: version.BlockProtocol,
+				App:   previousParams.Version.AppVersion,
+			},
+			Software: version.TMCoreSemVer,
+		},
+		// immutable fields
+		ChainID:       invalidState.ChainID,
+		InitialHeight: invalidState.InitialHeight,
+
+		LastBlockHeight: invalidState.LastBlockHeight - 1,
+		LastBlockID:     rollbackBlock.Header.LastBlockID,
+		LastBlockTime:   rollbackBlock.Header.Time,
+
+		NextValidators:              invalidState.Validators,
+		Validators:                  invalidState.LastValidators,
+		LastValidators:              previousValidatorSet,
+		LastHeightValidatorsChanged: valChangeHeight,
+
+		ConsensusParams:                  previousParams,
+		LastHeightConsensusParamsChanged: paramsChangeHeight,
+
+		LastResultsHash: rollbackBlock.Header.LastResultsHash,
+		AppHash:         rollbackBlock.Header.AppHash,
+	}
+
+	// persist the new state. This overrides the invalid one. NOTE: this will also
+	// persist the validator set and consensus params over the existing structures,
+	// but both should be the same
+	if err := ss.Save(rolledBackState); err != nil {
+		return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err)
+	}
+
+	return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil
+}