Regen and save historical states as default (#5544)

* Better error
* Use copy
* Conflict
* Fixed a bug on using pre state. Added a LRU cache for saving pre state.
* Removed interaction menu. Ran Gazelle
* Fixed delete state to continue than exit
* Gazelle
* Merge branch 'master' of github.com:prysmaticlabs/prysm into regen-default
* Merge refs/heads/master into regen-default
* Added a warning message
* Merge branch 'regen-default' of github.com:prysmaticlabs/prysm into regen-default
* Merge refs/heads/master into regen-default

beacon-chain/db/kv/BUILD.bazel

@@ -36,7 +36,6 @@ go_library(
         "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/cmd:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
@@ -46,6 +45,7 @@ go_library(
         "@com_github_ferranbt_fastssz//:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_golang_snappy//:go_default_library",
+        "@com_github_hashicorp_golang_lru//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",

beacon-chain/db/kv/check_historical_state.go

@@ -4,8 +4,8 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/shared/cmd"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	log "github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -30,22 +30,8 @@ func (kv *Store) HistoricalStatesDeleted(ctx context.Context) error {
 		return err
 	}
 
-	regenHistoricalStatesConfirmed := false
-	var err error
 	if historicalStateDeleted {
-		actionText := "--disable-new-state-mgmt was previously used and historical states cannot be found. To proceed without using the flag, the db will need " +
-			"to generate and re-save historical states. This process may take a while, - do you want to proceed? (Y/N)"
-		deniedText := "Historical states will not be generated. Please continue using --disable-new-state-mgmt"
-
-		regenHistoricalStatesConfirmed, err = cmd.ConfirmAction(actionText, deniedText)
-		if err != nil {
-			return err
-		}
-
-		if !regenHistoricalStatesConfirmed {
-			return errors.New("exiting... please use --disable-new-state-mgmt")
-		}
-
+		log.Warn("Regenerating and saving historical states. This may take a while.")
 		if err := kv.regenHistoricalStates(ctx); err != nil {
 			return errors.Wrap(err, "could not regenerate historical states, please retry")
 		}

beacon-chain/db/kv/regen_historical_states.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	transition "github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	log "github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
@@ -54,6 +56,10 @@ func (kv *Store) regenHistoricalStates(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	cacheState, err := lru.New(int(params.BeaconConfig().SlotsPerEpoch) * 2)
+	if err != nil {
+		return err
+	}
 	for i := lastArchivedIndex; i <= lastSavedBlockArchivedIndex; i++ {
 		targetSlot := startSlot + slotsPerArchivedPoint
 		filter := filters.NewFilter().SetStartSlot(startSlot + 1).SetEndSlot(targetSlot)
@@ -68,23 +74,44 @@ func (kv *Store) regenHistoricalStates(ctx context.Context) error {
 				if blocks[i].Block.Slot == 0 {
 					continue
 				}
-				currentState, err = regenHistoricalStateTransition(ctx, currentState, blocks[i])
+
+				var preState *stateTrie.BeaconState
+				item, ok := cacheState.Get(bytesutil.ToBytes32(blocks[i].Block.ParentRoot))
+				if !ok {
+					preState, err = kv.State(ctx, bytesutil.ToBytes32(blocks[i].Block.ParentRoot))
+					if err != nil {
+						return err
+					}
+				} else {
+					preState = item.(*stateTrie.BeaconState).Copy()
+				}
+				if preState == nil {
+					return errors.New("pre state can't be nil")
+				}
+
+				currentState, err = regenHistoricalStateTransition(ctx, preState.Copy(), blocks[i])
+				if err != nil {
+					return errors.Wrap(err, "could not regenerate historical state transition")
+				}
+
+				r, err := ssz.HashTreeRoot(blocks[i].Block)
 				if err != nil {
 					return err
 				}
+				cacheState.Add(r, currentState)
 			}
 		}
 		if targetSlot > currentState.Slot() {
 			currentState, err = regenHistoricalStateProcessSlots(ctx, currentState, targetSlot)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "could not regenerate historical process slot")
 			}
 		}
 
 		if len(blocks) > 0 {
 			// Save the historical root, state and highest index to the DB.
 			if helpers.IsEpochStart(currentState.Slot()) && currentState.Slot()%slotsPerArchivedPoint == 0 && blocks[len(blocks)-1].Block.Slot&slotsPerArchivedPoint == 0 {
-				if err := kv.saveArchivedInfo(ctx, currentState, blocks, i); err != nil {
+				if err := kv.saveArchivedInfo(ctx, currentState.Copy(), blocks, i); err != nil {
 					return err
 				}
 				log.WithFields(log.Fields{
@@ -94,6 +121,10 @@ func (kv *Store) regenHistoricalStates(ctx context.Context) error {
 		}
 		startSlot += slotsPerArchivedPoint
 	}
+
+	// Flush the cache, the cached states never be used again.
+	cacheState.Purge()
+
 	return nil
 }
 

beacon-chain/state/stategen/migrate.go

@@ -82,7 +82,10 @@ func (s *State) MigrateToCold(ctx context.Context, finalizedSlot uint64, finaliz
 			// could cause issue switching back.
 			if s.beaconDB.HasState(ctx, r) && r != finalizedRoot {
 				if err := s.beaconDB.DeleteState(ctx, r); err != nil {
-					return err
+					// For whatever reason if node is unable to delete a state due to
+					// state is finalized, it is more reasonable to continue than to exit.
+					log.Warnf("Unable to delete state during migration: %v", err)
+					continue
 				}
 				log.WithFields(logrus.Fields{
 					"slot": stateSummary.Slot,