Add back archival endpoint ListBeaconCommittees with fallback (#5518)

* Add fallback
* Add back the old tests
* Merge refs/heads/master into list-beacon-committee-fallback
* Merge refs/heads/master into list-beacon-committee-fallback
* Merge refs/heads/master into list-beacon-committee-fallback
* Merge refs/heads/master into list-beacon-committee-fallback
* Merge refs/heads/master into list-beacon-committee-fallback
* Merge refs/heads/master into list-beacon-committee-fallback

beacon-chain/rpc/beacon/attestations_test.go

@@ -1010,6 +1010,7 @@ func TestServer_StreamIndexedAttestations_ContextCanceled(t *testing.T) {
 }
 
 func TestServer_StreamIndexedAttestations_OK(t *testing.T) {
+	params.UseMainnetConfig()
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 	exitRoutine := make(chan bool)
@@ -1019,11 +1020,18 @@ func TestServer_StreamIndexedAttestations_OK(t *testing.T) {
 
 	numValidators := 64
 	headState, privKeys := testutil.DeterministicGenesisState(t, uint64(numValidators))
-	randaoMixes := make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector)
-	for i := 0; i < len(randaoMixes); i++ {
-		randaoMixes[i] = make([]byte, 32)
+	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := db.SaveBlock(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	gRoot, err := ssz.HashTreeRoot(b.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SaveGenesisBlockRoot(ctx, gRoot); err != nil {
+		t.Fatal(err)
 	}
-	if err := headState.SetRandaoMixes(randaoMixes); err != nil {
+	if err := db.SaveState(ctx, headState, gRoot); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1116,6 +1124,7 @@ func TestServer_StreamIndexedAttestations_OK(t *testing.T) {
 		},
 		AttestationNotifier:         chainService.OperationNotifier(),
 		CollectedAttestationsBuffer: make(chan []*ethpb.Attestation, 1),
+		StateGen:                    stategen.New(db, cache.NewStateSummaryCache()),
 	}
 
 	mockStream := mockRPC.NewMockBeaconChain_StreamIndexedAttestationsServer(ctrl)

beacon-chain/rpc/beacon/committees.go

@@ -6,6 +6,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -20,31 +21,55 @@ func (bs *Server) ListBeaconCommittees(
 	req *ethpb.ListCommitteesRequest,
 ) (*ethpb.BeaconCommittees, error) {
 
-	var requestingGenesis bool
-	var startSlot uint64
-	headSlot := bs.GenesisTimeFetcher.CurrentSlot()
+	currentSlot := bs.GenesisTimeFetcher.CurrentSlot()
+	var requestedSlot uint64
 	switch q := req.QueryFilter.(type) {
 	case *ethpb.ListCommitteesRequest_Epoch:
-		startSlot = helpers.StartSlot(q.Epoch)
+		requestedSlot = helpers.StartSlot(q.Epoch)
 	case *ethpb.ListCommitteesRequest_Genesis:
-		requestingGenesis = q.Genesis
-		if !requestingGenesis {
-			startSlot = headSlot
-		}
+		requestedSlot = 0
 	default:
-		startSlot = headSlot
+		requestedSlot = currentSlot
 	}
-	committees, activeIndices, err := bs.retrieveCommitteesForEpoch(ctx, helpers.SlotToEpoch(startSlot))
-	if err != nil {
+
+	requestedEpoch := helpers.SlotToEpoch(requestedSlot)
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+	if requestedEpoch > currentEpoch {
 		return nil, status.Errorf(
-			codes.Internal,
-			"Could not retrieve committees for epoch %d: %v",
-			helpers.SlotToEpoch(startSlot),
-			err,
+			codes.InvalidArgument,
+			"Cannot retrieve information for an future epoch, current epoch %d, requesting %d",
+			currentEpoch,
+			requestedEpoch,
 		)
 	}
+
+	committees := make(map[uint64]*ethpb.BeaconCommittees_CommitteesList)
+	activeIndices := make([]uint64, 0)
+	var err error
+	if featureconfig.Get().DisableNewStateMgmt {
+		committees, activeIndices, err = bs.retrieveCommitteesForEpochUsingOldArchival(ctx, requestedEpoch)
+		if err != nil {
+			return nil, status.Errorf(
+				codes.Internal,
+				"Could not retrieve committees for epoch %d: %v",
+				requestedEpoch,
+				err,
+			)
+		}
+	} else {
+		committees, activeIndices, err = bs.retrieveCommitteesForEpoch(ctx, requestedEpoch)
+		if err != nil {
+			return nil, status.Errorf(
+				codes.Internal,
+				"Could not retrieve committees for epoch %d: %v",
+				requestedEpoch,
+				err,
+			)
+		}
+	}
+
 	return &ethpb.BeaconCommittees{
-		Epoch:                helpers.SlotToEpoch(startSlot),
+		Epoch:                requestedEpoch,
 		Committees:           committees,
 		ActiveValidatorCount: uint64(len(activeIndices)),
 	}, nil
@@ -53,6 +78,36 @@ func (bs *Server) ListBeaconCommittees(
 func (bs *Server) retrieveCommitteesForEpoch(
 	ctx context.Context,
 	epoch uint64,
+) (map[uint64]*ethpb.BeaconCommittees_CommitteesList, []uint64, error) {
+	startSlot := helpers.StartSlot(epoch)
+	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	if err != nil {
+		return nil, nil, status.Error(codes.Internal, "Could not get state")
+	}
+	seed, err := helpers.Seed(requestedState, epoch, params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return nil, nil, status.Error(codes.Internal, "Could not get seed")
+	}
+	activeIndices, err := helpers.ActiveValidatorIndices(requestedState, epoch)
+	if err != nil {
+		return nil, nil, status.Error(codes.Internal, "Could not get active indices")
+	}
+
+	committeesListsBySlot, err := computeCommittees(startSlot, activeIndices, seed)
+	if err != nil {
+		return nil, nil, status.Errorf(
+			codes.InvalidArgument,
+			"Could not compute committees for epoch %d: %v",
+			helpers.SlotToEpoch(startSlot),
+			err,
+		)
+	}
+	return committeesListsBySlot, activeIndices, nil
+}
+
+func (bs *Server) retrieveCommitteesForEpochUsingOldArchival(
+	ctx context.Context,
+	epoch uint64,
 ) (map[uint64]*ethpb.BeaconCommittees_CommitteesList, []uint64, error) {
 	var attesterSeed [32]byte
 	var activeIndices []uint64

beacon-chain/rpc/beacon/committees_test.go

@@ -9,12 +9,16 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -27,23 +31,30 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 	helpers.ClearCache()
 
 	numValidators := 128
+	ctx := context.Background()
 	headState := setupActiveValidators(t, db, numValidators)
 
-	randaoMixes := make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector)
-	for i := 0; i < len(randaoMixes); i++ {
-		randaoMixes[i] = make([]byte, 32)
-	}
-	if err := headState.SetRandaoMixes(randaoMixes); err != nil {
-		t.Fatal(err)
-	}
-
 	m := &mock.ChainService{
-		State:   headState,
 		Genesis: roughtime.Now().Add(time.Duration(-1*int64((headState.Slot()*params.BeaconConfig().SecondsPerSlot))) * time.Second),
 	}
 	bs := &Server{
 		HeadFetcher:        m,
 		GenesisTimeFetcher: m,
+		StateGen:           stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := db.SaveBlock(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	gRoot, err := ssz.HashTreeRoot(b.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SaveGenesisBlockRoot(ctx, gRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SaveState(ctx, headState, gRoot); err != nil {
+		t.Fatal(err)
 	}
 
 	activeIndices, err := helpers.ActiveValidatorIndices(headState, 0)
@@ -76,6 +87,10 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 }
 
 func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
+	fc := featureconfig.Get()
+	fc.DisableNewStateMgmt = true
+	featureconfig.Init(fc)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 	helpers.ClearCache()
@@ -146,6 +161,10 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 }
 
 func TestServer_ListBeaconCommittees_FromArchive(t *testing.T) {
+	fc := featureconfig.Get()
+	fc.DisableNewStateMgmt = true
+	featureconfig.Init(fc)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 	helpers.ClearCache()