Replay blocks and generate state without sig verification (#4943)

* New file

* Add transition_stategen.go

* Update ProcessBlockForStateRoot

* Feature flags

* Fixed tests

* Gaz

* Make them private

* E2e flags

* Gazelle

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

beacon-chain/core/state/transition.go

@@ -176,7 +176,7 @@ func CalculateStateRoot(
 	}
 
 	// Execute per block transition.
-	state, err = computeStateRoot(ctx, state, signed)
+	state, err = ProcessBlockForStateRoot(ctx, state, signed)
 	if err != nil {
 		return [32]byte{}, errors.Wrap(err, "could not process block")
 	}
@@ -640,9 +640,9 @@ func ProcessEpochPrecompute(ctx context.Context, state *stateTrie.BeaconState) (
 	return state, nil
 }
 
-// computeStateRoot computes the state root of the block without verifying proposer signature
-// and randao.
-func computeStateRoot(
+// ProcessBlockForStateRoot processes the state for state root computation. It skips proposer signature
+// and randao signature verifications.
+func ProcessBlockForStateRoot(
 	ctx context.Context,
 	state *stateTrie.BeaconState,
 	signed *ethpb.SignedBeaconBlock,

beacon-chain/core/state/transition_fuzz_test.go

@@ -187,7 +187,7 @@ func TestFuzzProcessEpochPrecompute_1000(t *testing.T) {
 	}
 }
 
-func TestFuzzcomputeStateRoot_1000(t *testing.T) {
+func TestFuzzProcessBlockForStateRoot_1000(t *testing.T) {
 	ctx := context.Background()
 	state := &stateTrie.BeaconState{}
 	sb := &ethpb.SignedBeaconBlock{}
@@ -196,7 +196,7 @@ func TestFuzzcomputeStateRoot_1000(t *testing.T) {
 	for i := 0; i < 1000; i++ {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(sb)
-		s, err := computeStateRoot(ctx, state, sb)
+		s, err := ProcessBlockForStateRoot(ctx, state, sb)
 		if err != nil && s != nil {
 			t.Fatalf("state should be nil on err. found: %v on error: %v for signed block: %v", s, err, sb)
 		}

beacon-chain/stategen/BUILD.bazel

@@ -14,7 +14,10 @@ go_library(
         "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//shared/bytesutil:go_default_library",
+        "//shared/featureconfig:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 

beacon-chain/stategen/replay.go

@@ -2,13 +2,17 @@ package stategen
 
 import (
 	"context"
-	"errors"
+	"fmt"
 
+	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	transition "github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"go.opencensus.io/trace"
 )
 
 // ReplayBlocks replays the input blocks on the input state until the target slot is reached.
@@ -17,17 +21,31 @@ func (s *State) ReplayBlocks(ctx context.Context, state *state.BeaconState, sign
 	// The input block list is sorted in decreasing slots order.
 	if len(signed) > 0 {
 		for i := len(signed) - 1; i >= 0; i-- {
-			state, err = transition.ExecuteStateTransitionNoVerifyAttSigs(ctx, state, signed[i])
-			if err != nil {
-				return nil, err
+			if featureconfig.Get().EnableStateGenSigVerify {
+				state, err = transition.ExecuteStateTransition(ctx, state, signed[i])
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				state, err = executeStateTransitionStateGen(ctx, state, signed[i])
+				if err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
 
 	// If there is skip slots at the end.
-	state, err = transition.ProcessSlots(ctx, state, targetSlot)
-	if err != nil {
-		return nil, err
+	if featureconfig.Get().EnableStateGenSigVerify {
+		state, err = transition.ProcessSlots(ctx, state, targetSlot)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		state, err = processSlotsStateGen(ctx, state, targetSlot)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return state, nil
@@ -78,3 +96,77 @@ func (s *State) LoadBlocks(ctx context.Context, startSlot uint64, endSlot uint64
 
 	return filteredBlocks, nil
 }
+
+// executeStateTransitionStateGen applies state transition on input historical state and block for state gen usages.
+// There's no signature verification involved given state gen only works with stored block and state in DB.
+// If the objects are already in stored in DB, one can omit redundant signature checks and ssz hashing calculations.
+// WARNING: This method should not be used on an unverified new block.
+func executeStateTransitionStateGen(
+	ctx context.Context,
+	state *stateTrie.BeaconState,
+	signed *ethpb.SignedBeaconBlock,
+) (*stateTrie.BeaconState, error) {
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+
+	ctx, span := trace.StartSpan(ctx, "beacon-chain.ChainService.ExecuteStateTransitionStateGen")
+	defer span.End()
+	var err error
+
+	// Execute per slots transition.
+	// Given this is for state gen, a node uses the version process slots without skip slots cache.
+	state, err = processSlotsStateGen(ctx, state, signed.Block.Slot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not process slot")
+	}
+
+	// Execute per block transition.
+	// Given this is for state gen, a node only cares about the post state without proposer
+	// and randao signature verifications.
+	state, err = transition.ProcessBlockForStateRoot(ctx, state, signed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not process block")
+	}
+
+	return state, nil
+}
+
+// processSlotsStateGen to process old slots for state gen usages.
+// There's no skip slot cache involved given state gen only works with already stored block and state in DB.
+// WARNING: This method should not be used for future slot.
+func processSlotsStateGen(ctx context.Context, state *stateTrie.BeaconState, slot uint64) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "beacon-chain.ChainService.ProcessSlotsStateGen")
+	defer span.End()
+	if state == nil {
+		return nil, errors.New("nil state")
+	}
+
+	if state.Slot() > slot {
+		err := fmt.Errorf("expected state.slot %d < slot %d", state.Slot(), slot)
+		return nil, err
+	}
+
+	if state.Slot() == slot {
+		return state, nil
+	}
+
+	for state.Slot() < slot {
+		state, err := transition.ProcessSlot(ctx, state)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not process slot")
+		}
+		if transition.CanProcessEpoch(state) {
+			state, err = transition.ProcessEpochPrecompute(ctx, state)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not process epoch with optimizations")
+			}
+		}
+		state.SetSlot(state.Slot() + 1)
+	}
+
+	return state, nil
+}

shared/featureconfig/config.go

@@ -43,6 +43,7 @@ type Flags struct {
 	DisableUpdateHeadPerAttestation            bool   // DisableUpdateHeadPerAttestation will disabling update head on per attestation basis.
 	EnableByteMempool                          bool   // EnaableByteMempool memory management.
 	EnableDomainDataCache                      bool   // EnableDomainDataCache caches validator calls to DomainData per epoch.
+	EnableStateGenSigVerify                    bool   // EnableStateGenSigVerify verifies proposer and randao signatures during state gen.
 
 	// DisableForkChoice disables using LMD-GHOST fork choice to update
 	// the head of the chain based on attestations and instead accepts any valid received block
@@ -144,7 +145,10 @@ func ConfigureBeaconChain(ctx *cli.Context) {
 		log.Warn("Enabling experimental memory management for beacon state")
 		cfg.EnableByteMempool = true
 	}
-
+	if ctx.GlobalBool(enableStateGenSigVerify.Name) {
+		log.Warn("Enabling sig verify for state gen")
+		cfg.EnableStateGenSigVerify = true
+	}
 	Init(cfg)
 }
 

shared/featureconfig/flags.go

@@ -97,6 +97,11 @@ var (
 		Usage: "Enable caching of domain data requests per epoch. This feature reduces the total " +
 			"calls to the beacon node for each assignment.",
 	}
+	enableStateGenSigVerify = cli.BoolFlag{
+		Name: "enable-state-gen-sig-verify",
+		Usage: "Enable signature verification for state gen. This feature increases the cost to generate a historical state," +
+			"the resulting state is signature verified.",
+	}
 )
 
 // Deprecated flags list.
@@ -205,8 +210,8 @@ var (
 		Hidden: true,
 	}
 	deprecatedInitSyncCacheStateFlag = cli.BoolFlag{
-		Name: "initial-sync-cache-state",
-		Usage: deprecatedUsage,
+		Name:   "initial-sync-cache-state",
+		Usage:  deprecatedUsage,
 		Hidden: true,
 	}
 )
@@ -268,6 +273,7 @@ var BeaconChainFlags = append(deprecatedFlags, []cli.Flag{
 	disableStrictAttestationPubsubVerificationFlag,
 	disableUpdateHeadPerAttestation,
 	enableByteMempool,
+	enableStateGenSigVerify,
 }...)
 
 // E2EBeaconChainFlags contains a list of the beacon chain feature flags to be tested in E2E.
@@ -279,4 +285,5 @@ var E2EBeaconChainFlags = []string{
 	"--enable-eth1-data-vote-cache",
 	"--proto-array-forkchoice",
 	"--enable-byte-mempool",
+	"--enable-state-gen-sig-verify",
 }