Merge branch 'develop' into feature-add-canonical-to-blocks-api

WORKSPACE

@@ -210,6 +210,21 @@ http_archive(
     url = "https://github.com/kubernetes/repo-infra/archive/6537f2101fb432b679f3d103ee729dd8ac5d30a0.tar.gz",
 )
 
+http_archive(
+    name = "eip3076_spec_tests",
+    build_file_content = """
+filegroup(
+    name = "test_data",
+    srcs = glob([
+        "**/*.json",
+    ]),
+    visibility = ["//visibility:public"],
+)
+    """,
+    sha256 = "91434d5fd5e1c6eb7b0174fed2afe25e09bddf00e1e4c431db931b2cee4e7773",
+    url = "https://github.com/eth2-clients/slashing-protection-interchange-tests/archive/b8413ca42dc92308019d0d4db52c87e9e125c4e9.tar.gz",
+)
+
 http_archive(
     name = "eth2_spec_tests_general",
     build_file_content = """

beacon-chain/blockchain/head_test.go

@@ -51,9 +51,7 @@ func TestSaveHead_Different(t *testing.T) {
 
 	cachedRoot, err := service.HeadRoot(context.Background())
 	require.NoError(t, err)
-	if !bytes.Equal(cachedRoot, newRoot[:]) {
-		t.Error("Head did not change")
-	}
+	assert.DeepEqual(t, cachedRoot, newRoot[:], "Head did not change")
 	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock(), "Head did not change")
 	assert.DeepEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
 }

beacon-chain/blockchain/process_attestation.go

@@ -7,6 +7,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/timeutils"
@@ -19,7 +20,7 @@ var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
 
 // onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
 // it to the DB. As a stateless function, this does not hold nor delay attestation based on the spec descriptions.
-// The delay is handled by the caller in `processAttestation`.
+// The delay is handled by the caller in `processAttestations`.
 //
 // Spec pseudocode definition:
 //   def on_attestation(store: Store, attestation: Attestation) -> None:
@@ -82,11 +83,18 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) error
 		return err
 	}
 
-	// Use the target state to validate attestation and calculate the committees.
-	indexedAtt, err := s.verifyAttestationIndices(ctx, baseState, a)
+	// Use the target state to verify attesting indices are valid.
+	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
 	if err != nil {
 		return err
 	}
+	indexedAtt, err := attestationutil.ConvertToIndexed(ctx, a, committee)
+	if err != nil {
+		return err
+	}
+	if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
+		return err
+	}
 
 	// Note that signature verification is ignored here because it was performed in sync's validation pipeline:
 	// validate_aggregate_proof.go and validate_beacon_attestation.go

beacon-chain/blockchain/process_attestation_helpers.go

@@ -10,7 +10,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/mputil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -102,19 +101,3 @@ func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.Attestation
 	}
 	return nil
 }
-
-// verifyAttestationIndices validates input attestation has valid attesting indices.
-func (s *Service) verifyAttestationIndices(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
-	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
-	if err != nil {
-		return nil, err
-	}
-	indexedAtt, err := attestationutil.ConvertToIndexed(ctx, a, committee)
-	if err != nil {
-		return nil, err
-	}
-	if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
-		return nil, err
-	}
-	return indexedAtt, nil
-}

beacon-chain/blockchain/receive_attestation.go

@@ -101,8 +101,8 @@ func (s *Service) VerifyFinalizedConsistency(ctx context.Context, root []byte) e
 	return nil
 }
 
-// This processes attestations from the attestation pool to account for validator votes and fork choice.
-func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
+// This routine processes fork choice attestations from the pool to account for validator votes and fork choice.
+func (s *Service) processAttestationsRoutine(subscribedToStateEvents chan struct{}) {
 	// Wait for state to be initialized.
 	stateChannel := make(chan *feed.Event, 1)
 	stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
@@ -124,41 +124,45 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 		case <-s.ctx.Done():
 			return
 		case <-st.C():
-			ctx := s.ctx
-			atts := s.attPool.ForkchoiceAttestations()
-			for _, a := range atts {
-				// Based on the spec, don't process the attestation until the subsequent slot.
-				// This delays consideration in the fork choice until their slot is in the past.
-				// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
-				nextSlot := a.Data.Slot + 1
-				if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
-					continue
-				}
-
-				hasState := s.beaconDB.HasStateSummary(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				if !(hasState && hasBlock) {
-					continue
-				}
-
-				if err := s.attPool.DeleteForkchoiceAttestation(a); err != nil {
-					log.WithError(err).Error("Could not delete fork choice attestation in pool")
-				}
-
-				if !helpers.VerifyCheckpointEpoch(a.Data.Target, s.genesisTime) {
-					continue
-				}
-
-				if err := s.ReceiveAttestationNoPubsub(ctx, a); err != nil {
-					log.WithFields(logrus.Fields{
-						"slot":             a.Data.Slot,
-						"committeeIndex":   a.Data.CommitteeIndex,
-						"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
-						"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
-						"aggregationCount": a.AggregationBits.Count(),
-					}).WithError(err).Warn("Could not receive attestation in chain service")
-				}
-			}
+			s.processAttestations(s.ctx)
+		}
+	}
+}
+
+// This processes fork choice attestations from the pool to account for validator votes and fork choice.
+func (s *Service) processAttestations(ctx context.Context) {
+	atts := s.attPool.ForkchoiceAttestations()
+	for _, a := range atts {
+		// Based on the spec, don't process the attestation until the subsequent slot.
+		// This delays consideration in the fork choice until their slot is in the past.
+		// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
+		nextSlot := a.Data.Slot + 1
+		if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+			continue
+		}
+
+		hasState := s.beaconDB.HasStateSummary(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+		hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+		if !(hasState && hasBlock) {
+			continue
+		}
+
+		if err := s.attPool.DeleteForkchoiceAttestation(a); err != nil {
+			log.WithError(err).Error("Could not delete fork choice attestation in pool")
+		}
+
+		if !helpers.VerifyCheckpointEpoch(a.Data.Target, s.genesisTime) {
+			continue
+		}
+
+		if err := s.ReceiveAttestationNoPubsub(ctx, a); err != nil {
+			log.WithFields(logrus.Fields{
+				"slot":             a.Data.Slot,
+				"committeeIndex":   a.Data.CommitteeIndex,
+				"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
+				"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
+				"aggregationCount": a.AggregationBits.Count(),
+			}).WithError(err).Warn("Could not process attestation for fork choice")
 		}
 	}
 }

beacon-chain/blockchain/receive_attestation_test.go

@@ -7,11 +7,17 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/prysmaticlabs/prysm/shared/timeutils"
+	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
 func TestAttestationCheckPtState_FarFutureSlot(t *testing.T) {
@@ -81,3 +87,34 @@ func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
 	err = service.VerifyLmdFfgConsistency(context.Background(), a)
 	require.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
 }
+
+func TestProcessAttestations_Ok(t *testing.T) {
+	hook := logTest.NewGlobal()
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB:        beaconDB,
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+		StateGen:        stategen.New(beaconDB),
+		AttPool:         attestations.NewPool(),
+	}
+	service, err := NewService(ctx, cfg)
+	service.genesisTime = timeutils.Now().Add(-1 * time.Duration(params.BeaconConfig().SecondsPerSlot) * time.Second)
+	require.NoError(t, err)
+	genesisState, pks := testutil.DeterministicGenesisState(t, 64)
+	require.NoError(t, genesisState.SetGenesisTime(uint64(timeutils.Now().Unix())-params.BeaconConfig().SecondsPerSlot))
+	require.NoError(t, service.saveGenesisData(ctx, genesisState))
+	atts, err := testutil.GenerateAttestations(genesisState, pks, 1, 0, false)
+	require.NoError(t, err)
+	tRoot := bytesutil.ToBytes32(atts[0].Data.Target.Root)
+	copied := genesisState.Copy()
+	copied, err = state.ProcessSlots(ctx, copied, 1)
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveState(ctx, copied, tRoot))
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(ctx, 0, tRoot, tRoot, tRoot, 1, 1))
+	require.NoError(t, service.attPool.SaveForkchoiceAttestations(atts))
+	service.processAttestations(ctx)
+	require.Equal(t, 0, len(service.attPool.ForkchoiceAttestations()))
+	require.LogsDoNotContain(t, hook, "Could not process attestation for fork choice")
+}

beacon-chain/blockchain/service.go

@@ -263,7 +263,7 @@ func (s *Service) Start() {
 		}()
 	}
 
-	go s.processAttestation(attestationProcessorSubscribed)
+	go s.processAttestationsRoutine(attestationProcessorSubscribed)
 }
 
 // processChainStartTime initializes a series of deposits from the ChainStart deposits in the eth1

beacon-chain/core/blocks/randao_test.go

@@ -1,7 +1,6 @@
 package blocks_test
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"testing"
@@ -64,13 +63,7 @@ func TestProcessRandao_SignatureVerifiesAndUpdatesLatestStateMixes(t *testing.T)
 	require.NoError(t, err, "Unexpected error processing block randao")
 	currentEpoch := helpers.CurrentEpoch(beaconState)
 	mix := newState.RandaoMixes()[currentEpoch%params.BeaconConfig().EpochsPerHistoricalVector]
-
-	if bytes.Equal(mix, params.BeaconConfig().ZeroHash[:]) {
-		t.Errorf(
-			"Expected empty signature to be overwritten by randao reveal, received %v",
-			params.BeaconConfig().EmptySignature,
-		)
-	}
+	assert.DeepNotEqual(t, params.BeaconConfig().ZeroHash[:], mix, "Expected empty signature to be overwritten by randao reveal")
 }
 
 func TestRandaoSignatureSet_OK(t *testing.T) {

beacon-chain/core/state/skip_slot_cache_test.go

@@ -1,7 +1,6 @@
 package state_test
 
 import (
-	"bytes"
 	"context"
 	"sync"
 	"testing"
@@ -11,6 +10,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sszutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
@@ -113,9 +113,8 @@ func TestSkipSlotCache_ConcurrentMixup(t *testing.T) {
 
 	tmp1, err := state.ProcessSlots(context.Background(), expected1.Copy(), problemSlot+1)
 	require.NoError(t, err)
-	if gotRoot := tmp1.StateRoots()[problemSlot]; !bytes.Equal(gotRoot, expectedRoot1[:]) {
-		t.Fatalf("state roots for chain 1 are bad, expected root doesn't match: %x <> %x", gotRoot, expectedRoot1[:])
-	}
+	gotRoot := tmp1.StateRoots()[problemSlot]
+	require.DeepEqual(t, expectedRoot1[:], gotRoot, "State roots for chain 1 are bad, expected root doesn't match")
 
 	expected2, err := state.ProcessSlots(context.Background(), state2.Copy(), problemSlot)
 	require.NoError(t, err)
@@ -125,9 +124,8 @@ func TestSkipSlotCache_ConcurrentMixup(t *testing.T) {
 
 	tmp2, err := state.ProcessSlots(context.Background(), expected2.Copy(), problemSlot+1)
 	require.NoError(t, err)
-	if gotRoot := tmp2.StateRoots()[problemSlot]; !bytes.Equal(gotRoot, expectedRoot2[:]) {
-		t.Fatalf("state roots for chain 2 are bad, expected root doesn't match %x <> %x", gotRoot, expectedRoot2[:])
-	}
+	gotRoot = tmp2.StateRoots()[problemSlot]
+	require.DeepEqual(t, expectedRoot2[:], gotRoot, "State roots for chain 2 are bad, expected root doesn't match")
 
 	var wg sync.WaitGroup
 	wg.Add(len(setups))
@@ -139,13 +137,9 @@ func TestSkipSlotCache_ConcurrentMixup(t *testing.T) {
 		roots := outState.StateRoots()
 		gotRoot := roots[problemSlot]
 		if i%2 == 0 {
-			if !bytes.Equal(gotRoot, expectedRoot1[:]) {
-				t.Errorf("unexpected root on chain 1, item %3d: %x", i, gotRoot)
-			}
+			assert.DeepEqual(t, expectedRoot1[:], gotRoot, "Unexpected root on chain 1")
 		} else {
-			if !bytes.Equal(gotRoot, expectedRoot2[:]) {
-				t.Errorf("unexpected root on chain 2, item %3d: %x", i, gotRoot)
-			}
+			assert.DeepEqual(t, expectedRoot2[:], gotRoot, "Unexpected root on chain 2")
 		}
 		wg.Done()
 	}

beacon-chain/core/state/transition_test.go

@@ -1,7 +1,6 @@
 package state_test
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"fmt"
@@ -101,9 +100,7 @@ func TestExecuteStateTransition_FullProcess(t *testing.T) {
 
 	mix, err := beaconState.RandaoMixAtIndex(1)
 	require.NoError(t, err)
-	if bytes.Equal(mix, oldMix) {
-		t.Errorf("Did not expect new and old randao mix to equal, %#x == %#x", mix, oldMix)
-	}
+	assert.DeepNotEqual(t, oldMix, mix, "Did not expect new and old randao mix to equal")
 }
 
 func TestExecuteStateTransitionNoVerify_FullProcess(t *testing.T) {

beacon-chain/db/kv/migration_archived_index_test.go

@@ -1,9 +1,7 @@
 package kv
 
 import (
-	"bytes"
 	"context"
-	"fmt"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
@@ -34,9 +32,7 @@ func Test_migrateArchivedIndex(t *testing.T) {
 			eval: func(t *testing.T, db *bbolt.DB) {
 				err := db.View(func(tx *bbolt.Tx) error {
 					v := tx.Bucket(archivedRootBucket).Get(bytesutil.Uint64ToBytesLittleEndian(2048))
-					if !bytes.Equal(v, []byte("foo")) {
-						return fmt.Errorf("did not receive correct data for key 2048, wanted 'foo' got %s", v)
-					}
+					assert.DeepEqual(t, []byte("foo"), v, "Did not receive correct data for key 2048")
 					return nil
 				})
 				assert.NoError(t, err)
@@ -66,9 +62,8 @@ func Test_migrateArchivedIndex(t *testing.T) {
 			eval: func(t *testing.T, db *bbolt.DB) {
 				err := db.View(func(tx *bbolt.Tx) error {
 					k := uint64(2048)
-					if v := tx.Bucket(stateSlotIndicesBucket).Get(bytesutil.Uint64ToBytesBigEndian(k)); !bytes.Equal(v, []byte("foo")) {
-						return fmt.Errorf("did not receive correct data for key %d, wanted 'foo' got %v", k, v)
-					}
+					v := tx.Bucket(stateSlotIndicesBucket).Get(bytesutil.Uint64ToBytesBigEndian(k))
+					assert.DeepEqual(t, []byte("foo"), v, "Did not receive correct data for key %d", k)
 					return nil
 				})
 				assert.NoError(t, err)