Added hard fork transition support to Erigon-CL. (ledgerwatch#7088)

cl/cltypes/eth1_header.go

@@ -36,6 +36,18 @@ func NewEth1Header(version clparams.StateVersion) *Eth1Header {
 	return &Eth1Header{version: version}
 }
 
+// Capella converts the header to capella version.
+func (e *Eth1Header) Capella() {
+	e.version = clparams.CapellaVersion
+	e.WithdrawalsRoot = libcommon.Hash{}
+}
+
+func (e *Eth1Header) IsZero() bool {
+	return e.ParentHash == libcommon.Hash{} && e.FeeRecipient == libcommon.Address{} && e.StateRoot == libcommon.Hash{} &&
+		e.ReceiptsRoot == libcommon.Hash{} && e.LogsBloom == types.Bloom{} && e.PrevRandao == libcommon.Hash{} && e.BlockNumber == 0 &&
+		e.GasLimit == 0 && e.GasUsed == 0 && e.Time == 0 && len(e.Extra) == 0 && e.BaseFeePerGas == [32]byte{} && e.BlockHash == libcommon.Hash{} && e.TransactionsRoot == libcommon.Hash{}
+}
+
 // Encodes header data partially. used to not dupicate code across Eth1Block and Eth1Header.
 func (h *Eth1Header) encodeHeaderMetadataForSSZ(dst []byte, extraDataOffset int) ([]byte, error) {
 	buf := dst

cmd/ef-tests-cl/consensus_tests/handlers.go

@@ -51,6 +51,9 @@ var sanitySlots = "sanity/slots"
 // random
 var random = "random/random"
 
+// transitionCore
+var transitionCore = "transition/core"
+
 // Stays here bc debugging >:-(
 func placeholderTest() error {
 	fmt.Println("hallo")
@@ -80,8 +83,9 @@ var handlers map[string]testFunc = map[string]testFunc{
 	path.Join(operationsDivision, caseVoluntaryExit):                     operationVoluntaryExitHandler,
 	path.Join(operationsDivision, caseWithdrawal):                        operationWithdrawalHandler,
 	path.Join(operationsDivision, caseBlsChange):                         operationSignedBlsChangeHandler,
-	sanityBlocks: testSanityFunction,
-	sanitySlots:  testSanityFunctionSlot,
-	finality:     finalityTestFunction,
-	random:       testSanityFunction, // Same as sanity handler.
+	transitionCore: transitionTestFunction,
+	sanityBlocks:   testSanityFunction,
+	sanitySlots:    testSanityFunctionSlot,
+	finality:       finalityTestFunction,
+	random:         testSanityFunction, // Same as sanity handler.
 }

cmd/ef-tests-cl/consensus_tests/transition.go

@@ -0,0 +1,87 @@
+package consensustests
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/transition"
+	"gopkg.in/yaml.v2"
+)
+
+type transitionMeta struct {
+	ForkEpoch uint64 `yaml:"fork_epoch"`
+}
+
+func transitionTestFunction(context testContext) error {
+	metaBytes, err := os.ReadFile("meta.yaml")
+	if err != nil {
+		return err
+	}
+	meta := transitionMeta{}
+	if err := yaml.Unmarshal(metaBytes, &meta); err != nil {
+		return err
+	}
+	contextPrev := context
+	contextPrev.version--
+	testState, err := decodeStateFromFile(contextPrev, "pre.ssz_snappy")
+	if err != nil {
+		return err
+	}
+
+	expectedState, err := decodeStateFromFile(context, "post.ssz_snappy")
+	if err != nil {
+		return err
+	}
+	switch context.version {
+	case clparams.AltairVersion:
+		testState.BeaconConfig().AltairForkEpoch = meta.ForkEpoch
+	case clparams.BellatrixVersion:
+		testState.BeaconConfig().BellatrixForkEpoch = meta.ForkEpoch
+	case clparams.CapellaVersion:
+		testState.BeaconConfig().CapellaForkEpoch = meta.ForkEpoch
+	}
+	startSlot := testState.Slot()
+	blockIndex := 0
+	for {
+		testSlot, err := testBlockSlot(blockIndex)
+		if err != nil {
+			return err
+		}
+		var block *cltypes.SignedBeaconBlock
+		if testSlot/clparams.MainnetBeaconConfig.SlotsPerEpoch >= meta.ForkEpoch {
+			block, err = testBlock(context, blockIndex)
+			if err != nil {
+				return err
+			}
+		} else {
+			block, err = testBlock(contextPrev, blockIndex)
+			if err != nil {
+				return err
+			}
+		}
+
+		if block == nil {
+			break
+		}
+
+		blockIndex++
+
+		if err := transition.TransitionState(testState, block, true); err != nil {
+			return fmt.Errorf("cannot transition state: %s. slot=%d. start_slot=%d", err, block.Block.Slot, startSlot)
+		}
+	}
+	expectedRoot, err := expectedState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	haveRoot, err := testState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	if haveRoot != expectedRoot {
+		return fmt.Errorf("mismatching state roots")
+	}
+	return nil
+}

cmd/ef-tests-cl/consensus_tests/utils.go

@@ -16,7 +16,8 @@ func decodeStateFromFile(context testContext, filepath string) (*state.BeaconSta
 	if err != nil {
 		return nil, err
 	}
-	testState := state.New(&clparams.MainnetBeaconConfig)
+	config := clparams.MainnetBeaconConfig
+	testState := state.New(&config)
 	if err := utils.DecodeSSZSnappyWithVersion(testState, sszSnappy, int(context.version)); err != nil {
 		return nil, err
 	}
@@ -53,3 +54,39 @@ func testBlocks(context testContext) ([]*cltypes.SignedBeaconBlock, error) {
 	}
 	return blocks, err
 }
+
+func testBlock(context testContext, index int) (*cltypes.SignedBeaconBlock, error) {
+	var blockBytes []byte
+	var err error
+	blockBytes, err = os.ReadFile(fmt.Sprintf("blocks_%d.ssz_snappy", index))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	blk := &cltypes.SignedBeaconBlock{}
+	if err = utils.DecodeSSZSnappyWithVersion(blk, blockBytes, int(context.version)); err != nil {
+		return nil, err
+	}
+
+	return blk, nil
+}
+
+func testBlockSlot(index int) (uint64, error) {
+	var blockBytes []byte
+	var err error
+	blockBytes, err = os.ReadFile(fmt.Sprintf("blocks_%d.ssz_snappy", index))
+	if os.IsNotExist(err) {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+
+	blockBytes, err = utils.DecompressSnappy(blockBytes)
+	if err != nil {
+		return 0, err
+	}
+	return ssz.UnmarshalUint64SSZ(blockBytes[100:108]), nil
+}

cmd/ef-tests-cl/setup.sh

@@ -1,4 +1,5 @@
 git clone https://github.com/ethereum/consensus-spec-tests
+cd consensus-spec-tests && git lfs pull && cd ..
 mv consensus-spec-tests/tests .
 rm -rf consensus-spec-tests
 rm -rf tests/minimal #these ones are useless

cmd/erigon-cl/core/state/accessors.go

@@ -4,8 +4,10 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math"
 	"sort"
 
+	"github.com/Giulio2002/bls"
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -471,7 +473,7 @@ func (b *BeaconState) ValidatorChurnLimit() uint64 {
 
 // Check whether a merge transition is complete by verifying the presence of a valid execution payload header.
 func (b *BeaconState) IsMergeTransitionComplete() bool {
-	return b.latestExecutionPayloadHeader.StateRoot != libcommon.Hash{}
+	return !b.latestExecutionPayloadHeader.IsZero()
 }
 
 // Compute the Unix timestamp at the specified slot number.
@@ -540,3 +542,52 @@ func (b *BeaconState) ExpectedWithdrawals() []*types.Withdrawal {
 	// Return the withdrawals slice
 	return withdrawals
 }
+func (b *BeaconState) ComputeNextSyncCommittee() (*cltypes.SyncCommittee, error) {
+	beaconConfig := b.beaconConfig
+	optimizedHashFunc := utils.OptimizedKeccak256()
+	epoch := b.Epoch() + 1
+	//math.MaxUint8
+	activeValidatorIndicies := b.GetActiveValidatorsIndices(epoch)
+	activeValidatorCount := uint64(len(activeValidatorIndicies))
+	seed := b.GetSeed(epoch, beaconConfig.DomainSyncCommittee)
+	i := uint64(0)
+	syncCommitteePubKeys := make([][48]byte, 0, cltypes.SyncCommitteeSize)
+	preInputs := b.ComputeShuffledIndexPreInputs(seed)
+	for len(syncCommitteePubKeys) < cltypes.SyncCommitteeSize {
+		shuffledIndex, err := b.ComputeShuffledIndex(i%activeValidatorCount, activeValidatorCount, seed, preInputs, optimizedHashFunc)
+		if err != nil {
+			return nil, err
+		}
+		candidateIndex := activeValidatorIndicies[shuffledIndex]
+		// Compute random byte.
+		buf := make([]byte, 8)
+		binary.LittleEndian.PutUint64(buf, i/32)
+		input := append(seed[:], buf...)
+		randomByte := uint64(utils.Keccak256(input)[i%32])
+		// retrieve validator.
+		validator, err := b.ValidatorAt(int(candidateIndex))
+		if err != nil {
+			return nil, err
+		}
+		if validator.EffectiveBalance*math.MaxUint8 >= beaconConfig.MaxEffectiveBalance*randomByte {
+			syncCommitteePubKeys = append(syncCommitteePubKeys, validator.PublicKey)
+		}
+		i++
+	}
+	// Format public keys.
+	formattedKeys := make([][]byte, cltypes.SyncCommitteeSize)
+	for i := range formattedKeys {
+		formattedKeys[i] = make([]byte, 48)
+		copy(formattedKeys[i], syncCommitteePubKeys[i][:])
+	}
+	aggregatePublicKeyBytes, err := bls.AggregatePublickKeys(formattedKeys)
+	if err != nil {
+		return nil, err
+	}
+	var aggregate [48]byte
+	copy(aggregate[:], aggregatePublicKeyBytes)
+	return &cltypes.SyncCommittee{
+		PubKeys:            syncCommitteePubKeys,
+		AggregatePublicKey: aggregate,
+	}, nil
+}

cmd/erigon-cl/core/state/setters.go

@@ -25,6 +25,9 @@ func (b *BeaconState) SetSlot(slot uint64) {
 	b.touchedLeaves[SlotLeafIndex] = true
 	b.slot = slot
 	b.proposerIndex = nil
+	if b.slot%b.beaconConfig.SlotsPerEpoch == 0 {
+		b.totalActiveBalanceCache = nil
+	}
 }
 
 func (b *BeaconState) SetFork(fork *cltypes.Fork) {

cmd/erigon-cl/core/state/upgrade.go

@@ -0,0 +1,94 @@
+package state
+
+import (
+	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cl/utils"
+)
+
+func (b *BeaconState) UpgradeToAltair() error {
+	b.previousStateRoot = libcommon.Hash{}
+	epoch := b.Epoch()
+	// update version
+	b.fork.Epoch = epoch
+	b.fork.CurrentVersion = utils.Uint32ToBytes4(b.beaconConfig.AltairForkVersion)
+	// Process new fields
+	b.previousEpochParticipation = make(cltypes.ParticipationFlagsList, len(b.validators))
+	b.currentEpochParticipation = make(cltypes.ParticipationFlagsList, len(b.validators))
+	b.inactivityScores = make([]uint64, len(b.validators))
+	// Change version
+	b.version = clparams.AltairVersion
+	// Fill in previous epoch participation from the pre state's pending attestations
+	for _, attestation := range b.previousEpochAttestations {
+		flags, err := b.GetAttestationParticipationFlagIndicies(attestation.Data, attestation.InclusionDelay)
+		if err != nil {
+			return err
+		}
+		indicies, err := b.GetAttestingIndicies(attestation.Data, attestation.AggregationBits, false)
+		if err != nil {
+			return err
+		}
+		for _, index := range indicies {
+			for _, flagIndex := range flags {
+				b.previousEpochParticipation[index].Add(int(flagIndex))
+			}
+		}
+	}
+	b.previousEpochAttestations = nil
+	// Process sync committees
+	var err error
+	if b.currentSyncCommittee, err = b.ComputeNextSyncCommittee(); err != nil {
+		return err
+	}
+	if b.nextSyncCommittee, err = b.ComputeNextSyncCommittee(); err != nil {
+		return err
+	}
+	// Update the state root cache
+	b.touchedLeaves[ForkLeafIndex] = true
+	b.touchedLeaves[PreviousEpochParticipationLeafIndex] = true
+	b.touchedLeaves[CurrentEpochParticipationLeafIndex] = true
+	b.touchedLeaves[InactivityScoresLeafIndex] = true
+	b.touchedLeaves[CurrentSyncCommitteeLeafIndex] = true
+	b.touchedLeaves[NextSyncCommitteeLeafIndex] = true
+
+	return nil
+}
+
+func (b *BeaconState) UpgradeToBellatrix() error {
+	b.previousStateRoot = libcommon.Hash{}
+	epoch := b.Epoch()
+	// update version
+	b.fork.Epoch = epoch
+	b.fork.PreviousVersion = b.fork.CurrentVersion
+	b.fork.CurrentVersion = utils.Uint32ToBytes4(b.beaconConfig.BellatrixForkVersion)
+	b.latestExecutionPayloadHeader = cltypes.NewEth1Header(clparams.BellatrixVersion)
+	// Update the state root cache
+	b.touchedLeaves[ForkLeafIndex] = true
+	b.touchedLeaves[LatestExecutionPayloadHeaderLeafIndex] = true
+	b.version = clparams.BellatrixVersion
+	return nil
+}
+
+func (b *BeaconState) UpgradeToCapella() error {
+	b.previousStateRoot = libcommon.Hash{}
+	epoch := b.Epoch()
+	// update version
+	b.fork.Epoch = epoch
+	b.fork.PreviousVersion = b.fork.CurrentVersion
+	b.fork.CurrentVersion = utils.Uint32ToBytes4(b.beaconConfig.CapellaForkVersion)
+	// Update the payload header.
+	b.latestExecutionPayloadHeader.Capella()
+	// Set new fields
+	b.nextWithdrawalIndex = 0
+	b.nextWithdrawalValidatorIndex = 0
+	b.historicalSummaries = nil
+	// Update the state root cache
+	b.touchedLeaves[ForkLeafIndex] = true
+	b.touchedLeaves[LatestExecutionPayloadHeaderLeafIndex] = true
+	b.touchedLeaves[NextWithdrawalIndexLeafIndex] = true
+	b.touchedLeaves[NextWithdrawalValidatorIndexLeafIndex] = true
+	b.touchedLeaves[HistoricalSummariesLeafIndex] = true
+	b.version = clparams.CapellaVersion
+	return nil
+}

cmd/erigon-cl/core/transition/block_transition.go

@@ -136,5 +136,5 @@ func ProcessExecutionPayload(state *state.BeaconState, payload *cltypes.Eth1Bloc
 }
 
 func executionEnabled(state *state.BeaconState, payload *cltypes.Eth1Block) bool {
-	return (!state.IsMergeTransitionComplete() && payload.StateRoot != libcommon.Hash{}) || state.IsMergeTransitionComplete()
+	return (!state.IsMergeTransitionComplete() && payload.BlockHash != libcommon.Hash{}) || state.IsMergeTransitionComplete()
 }