Kintsugi ssz (#9867)

beacon-chain/blockchain/process_block.go

@@ -632,10 +632,6 @@ func validTerminalPowBlock(transitionBlock *powchain.ExecutionBlock, transitionP
 }
 
 func executionPayloadToExecutableData(payload *ethpb.ExecutionPayload) *catalyst.ExecutableDataV1 {
-	txs := make([][]byte, len(payload.Transactions))
-	for i, t := range payload.Transactions {
-		txs[i] = t.GetOpaqueTransaction()
-	}
 	baseFeePerGas := new(big.Int)
 	// TODO_MERGE: The conversion from 32bytes to big int is broken. This assumes base fee per gas in single digit
 	baseFeePerGas.SetBytes([]byte{payload.BaseFeePerGas[0]})
@@ -654,6 +650,6 @@ func executionPayloadToExecutableData(payload *ethpb.ExecutionPayload) *catalyst
 		Timestamp:     payload.Timestamp,
 		ExtraData:     payload.ExtraData,
 		BaseFeePerGas: baseFeePerGas,
-		Transactions:  txs,
+		Transactions:  payload.Transactions,
 	}
 }

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_execution_payload.go

@@ -108,7 +108,6 @@ func (vs *Server) getExecutionPayload(ctx context.Context, slot types.Slot) (*et
 		FinalizedBlockHash: common.BytesToHash(finalizedBlockHash),
 	}
 	p := catalyst.PayloadAttributesV1{
-		ParentHash:   common.BytesToHash(parentHash),
 		Timestamp:    uint64(t.Unix()),
 		Random:       common.BytesToHash(random),
 		FeeRecipient: params.BeaconConfig().FeeRecipient,
@@ -126,13 +125,6 @@ func (vs *Server) getExecutionPayload(ctx context.Context, slot types.Slot) (*et
 }
 
 func executableDataToExecutionPayload(ed *catalyst.ExecutableDataV1) *ethpb.ExecutionPayload {
-	txs := make([]*ethpb.Transaction, len(ed.Transactions))
-	for i, t := range ed.Transactions {
-		txs[i] = &ethpb.Transaction{
-			TransactionOneof: &ethpb.Transaction_OpaqueTransaction{OpaqueTransaction: t},
-		}
-	}
-
 	return &ethpb.ExecutionPayload{
 		ParentHash:    bytesutil.PadTo(ed.ParentHash.Bytes(), 32),
 		Coinbase:      bytesutil.PadTo(ed.Coinbase.Bytes(), 20),
@@ -147,7 +139,7 @@ func executableDataToExecutionPayload(ed *catalyst.ExecutableDataV1) *ethpb.Exec
 		ExtraData:     ed.ExtraData,
 		BaseFeePerGas: bytesutil.PadTo(ed.BaseFeePerGas.Bytes(), 32),
 		BlockHash:     bytesutil.PadTo(ed.BlockHash.Bytes(), 32),
-		Transactions:  txs,
+		Transactions:  ed.Transactions,
 	}
 }
 

beacon-chain/sync/validate_beacon_blocks_test.go

@@ -1074,17 +1074,10 @@ func TestValidateBeaconBlockPubSub_ValidExecutionPayload(t *testing.T) {
 	msg.Block.Body.ExecutionPayload.GasLimit = 11
 	msg.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("blockHash"), 32)
 	msg.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte("parentHash"), 32)
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 1"),
-		},
-	})
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 2"),
-		},
-	})
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 1"))
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 2"))
 	msg.Signature, err = signing.ComputeDomainAndSign(beaconState, 0, msg.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx])
+	require.NoError(t, err)
 
 	stateGen := stategen.New(db)
 	chainService := &mock.ChainService{Genesis: time.Unix(presentTime-int64(params.BeaconConfig().SecondsPerSlot), 0),
@@ -1154,17 +1147,10 @@ func TestValidateBeaconBlockPubSub_InvalidPayloadTimestamp(t *testing.T) {
 	msg.Block.Body.ExecutionPayload.GasLimit = 11
 	msg.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("blockHash"), 32)
 	msg.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte("parentHash"), 32)
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 1"),
-		},
-	})
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 2"),
-		},
-	})
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 1"))
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 2"))
 	msg.Signature, err = signing.ComputeDomainAndSign(beaconState, 0, msg.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx])
+	require.NoError(t, err)
 
 	stateGen := stategen.New(db)
 	chainService := &mock.ChainService{Genesis: time.Unix(presentTime-int64(params.BeaconConfig().SecondsPerSlot), 0),
@@ -1233,17 +1219,10 @@ func TestValidateBeaconBlockPubSub_InvalidPayloadGasUsed(t *testing.T) {
 	msg.Block.Body.ExecutionPayload.GasLimit = 11
 	msg.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("blockHash"), 32)
 	msg.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte("parentHash"), 32)
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 1"),
-		},
-	})
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 2"),
-		},
-	})
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 1"))
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 2"))
 	msg.Signature, err = signing.ComputeDomainAndSign(beaconState, 0, msg.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx])
+	require.NoError(t, err)
 
 	stateGen := stategen.New(db)
 	chainService := &mock.ChainService{Genesis: time.Unix(presentTime-int64(params.BeaconConfig().SecondsPerSlot), 0),
@@ -1312,17 +1291,10 @@ func TestValidateBeaconBlockPubSub_InvalidParentHashInPayload(t *testing.T) {
 	msg.Block.Body.ExecutionPayload.GasLimit = 11
 	msg.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("blockHash"), 32)
 	msg.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte("InvalidHash"), 32)
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 1"),
-		},
-	})
-	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, &ethpb.Transaction{
-		TransactionOneof: &ethpb.Transaction_OpaqueTransaction{
-			OpaqueTransaction: []byte("transaction 2"),
-		},
-	})
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 1"))
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 2"))
 	msg.Signature, err = signing.ComputeDomainAndSign(beaconState, 0, msg.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx])
+	require.NoError(t, err)
 
 	stateGen := stategen.New(db)
 	chainService := &mock.ChainService{Genesis: time.Unix(presentTime-int64(params.BeaconConfig().SecondsPerSlot), 0),
@@ -1359,8 +1331,8 @@ func TestValidateBeaconBlockPubSub_InvalidParentHashInPayload(t *testing.T) {
 	}
 
 	// set the max payload size to small value to test
-	params.BeaconConfig().MaxExecutionTransactions = 1
-	params.BeaconConfig().MaxBytesPerOpaqueTransaction = 9
+	//params.BeaconConfig().MaxExecutionTransactions = 1
+	//params.BeaconConfig().MaxBytesPerOpaqueTransaction = 9
 
 	res, err := r.validateBeaconBlockPubSub(ctx, "", m)
 	require.NotNil(t, err)

deps.bzl

@@ -839,7 +839,7 @@ def prysm_deps():
     go_repository(
         name = "com_github_ferranbt_fastssz",
         importpath = "github.com/ferranbt/fastssz",  # keep
-        commit = "0d1e4983580aad82cecd4dfb7d908b2f4f60ae02",  # keep
+        commit = "fa514f0ef27e963d281ecebff240b36169c2b9e3",  # keep
         remote = "https://github.com/kasey/fastssz",  # keep
         nofuzz = True,
         replace = None,  # keep

encoding/ssz/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//crypto/hash:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "@com_github_ferranbt_fastssz//:go_default_library",
         "@com_github_minio_sha256_simd//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",

encoding/ssz/htrutils.go

@@ -3,6 +3,7 @@ package ssz
 import (
 	"bytes"
 	"encoding/binary"
+	ssz "github.com/ferranbt/fastssz"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -91,78 +92,33 @@ func SlashingsRoot(slashings []uint64) ([32]byte, error) {
 	return BitwiseMerkleize(hash.CustomSHA256Hasher(), slashingChunks, uint64(len(slashingChunks)), uint64(len(slashingChunks)))
 }
 
-func TransactionsRoot(txs []*ethpb.Transaction) ([32]byte, error) {
-	hasher := hash.CustomSHA256Hasher()
-	listMarshaling := make([][]byte, 0)
-	for _, txn := range txs {
-		rt, err := txn.HashTreeRoot()
-		if err != nil {
-			return [32]byte{}, err
-		}
-		listMarshaling = append(listMarshaling, rt[:])
-	}
-
-	bytesRoot, err := BitwiseMerkleize(hasher, listMarshaling, uint64(len(listMarshaling)), params.BeaconConfig().MaxTransactionsPerPayload)
-	if err != nil {
-		return [32]byte{}, errors.Wrap(err, "could not compute  merkleization")
-	}
-	bytesRootBuf := new(bytes.Buffer)
-	if err := binary.Write(bytesRootBuf, binary.LittleEndian, uint64(len(txs))); err != nil {
-		return [32]byte{}, errors.Wrap(err, "could not marshal length")
-	}
-	bytesRootBufRoot := make([]byte, 32)
-	copy(bytesRootBufRoot, bytesRootBuf.Bytes())
-	return MixInLength(bytesRoot, bytesRootBufRoot), nil
-}
-
-func transactionRoot(tx []byte) ([32]byte, error) {
-	hasher := hash.CustomSHA256Hasher()
-	chunkedRoots, err := packChunks(tx)
-	if err != nil {
-		return [32]byte{}, err
-	}
+const (
+	maxBytesPerTransaction = 1073741824
+	maxTransactionsPerPayload = 1048576
+)
 
-	maxLength := (params.BeaconConfig().MaxBytesPerTransaction + 31) / 32
-	bytesRoot, err := BitwiseMerkleize(hasher, chunkedRoots, uint64(len(chunkedRoots)), maxLength)
-	if err != nil {
-		return [32]byte{}, errors.Wrap(err, "could not compute merkleization")
+// TransactionsRoot computes the HTR for the Transactions property of the ExecutionPayload
+// The code was largely copy/pasted from the code generated to compute the HTR of the entire
+// ExecutionPayload.
+func TransactionsRoot(txs [][]byte) ([32]byte, error) {
+	var root [32]byte
+	hh := ssz.DefaultHasherPool.Get()
+	defer ssz.DefaultHasherPool.Put(hh)
+	idx := hh.Index()
+	num := uint64(len(txs))
+	if num > maxTransactionsPerPayload {
+		return root, ssz.ErrIncorrectListSize
 	}
-	bytesRootBuf := new(bytes.Buffer)
-	if err := binary.Write(bytesRootBuf, binary.LittleEndian, uint64(len(tx))); err != nil {
-		return [32]byte{}, errors.Wrap(err, "could not marshal length")
-	}
-	bytesRootBufRoot := make([]byte, 32)
-	copy(bytesRootBufRoot, bytesRootBuf.Bytes())
-	return MixInLength(bytesRoot, bytesRootBufRoot), nil
-}
-
-// Pack a given byte array into chunks. It'll pad the last chunk with zero bytes if
-// it does not have length bytes per chunk.
-func packChunks(bytes []byte) ([][]byte, error) {
-	numItems := len(bytes)
-	var chunks [][]byte
-	for i := 0; i < numItems; i += 32 {
-		j := i + 32
-		// We create our upper bound index of the chunk, if it is greater than numItems,
-		// we set it as numItems itself.
-		if j > numItems {
-			j = numItems
+	for _, elem := range txs {
+		elemIndx := hh.Index()
+		byteLen := uint64(len(elem))
+		if byteLen > maxBytesPerTransaction {
+			return root, ssz.ErrIncorrectListSize
 		}
-		// We create chunks from the list of items based on the
-		// indices determined above.
-		chunks = append(chunks, bytes[i:j])
-	}
-
-	if len(chunks) == 0 {
-		return chunks, nil
+		hh.Append(elem)
+		hh.MerkleizeWithMixin(elemIndx, byteLen, (maxBytesPerTransaction+31)/32)
 	}
+	hh.MerkleizeWithMixin(idx, num, maxTransactionsPerPayload)
 
-	// Right-pad the last chunk with zero bytes if it does not
-	// have length bytes.
-	lastChunk := chunks[len(chunks)-1]
-	for len(lastChunk) < 32 {
-		lastChunk = append(lastChunk, 0)
-	}
-	chunks[len(chunks)-1] = lastChunk
-	return chunks, nil
+	return hh.HashRoot()
 }

go.mod

@@ -131,4 +131,5 @@ replace github.com/json-iterator/go => github.com/prestonvanloon/go v1.1.7-0.201
 // See https://github.com/prysmaticlabs/grpc-gateway/issues/2
 replace github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/prysmaticlabs/grpc-gateway/v2 v2.3.1-0.20210702154020-550e1cd83ec1
 
-replace github.com/ferranbt/fastssz => github.com/kasey/fastssz v0.0.0-20211006152949-0d1e4983580a
+// fa514f0ef27e963d281ecebff240b36169c2b9e3
+replace github.com/ferranbt/fastssz => github.com/kasey/fastssz v0.0.0-20211108224242-fa514f0ef27e

go.sum

@@ -631,8 +631,8 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559 h1:0VWDXPNE0brOek1Q8bLfzKkvOzwbQE/snjGojlCr8CY=
 github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
-github.com/kasey/fastssz v0.0.0-20211006152949-0d1e4983580a h1:BhK6+Jl8pl6LA7z8IVi+Z1TPYDd7tCp1owwrj9jCSRU=
-github.com/kasey/fastssz v0.0.0-20211006152949-0d1e4983580a/go.mod h1:S8yiDeAXy8f88W4Ul+0dBMPx49S05byYbmZD6Uv94K4=
+github.com/kasey/fastssz v0.0.0-20211108224242-fa514f0ef27e h1:eLhcqPvSCiBeryzyEJ5SwNE6jzJFxDTb2Hph0HoOs9Y=
+github.com/kasey/fastssz v0.0.0-20211108224242-fa514f0ef27e/go.mod h1:S8yiDeAXy8f88W4Ul+0dBMPx49S05byYbmZD6Uv94K4=
 github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca h1:qNtd6alRqd3qOdPrKXMZImV192ngQ0WSh1briEO33Tk=
 github.com/kevinms/leakybucket-go v0.0.0-20200115003610-082473db97ca/go.mod h1:ph+C5vpnCcQvKBwJwKLTK3JLNGnBXYlG7m7JjoC/zYA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=