Skip to content

Commit

Permalink
Change database *Index() to use slice (#4466)
Browse files Browse the repository at this point in the history 
* Change database *Index() to use slice

* Remove underscore from helper name
  • Loading branch information
@mcdee @prestonvanloon
mcdee authored and prestonvanloon committed Jan 9, 2020
1 parent fc38a04 commit 3839f57
Show file tree
Hide file tree
Showing 25 changed files with 187 additions and 153 deletions.
2 changes: 1 addition & 1 deletion beacon-chain/blockchain/forkchoice/process_block.go
View file
Expand Up @@ -332,7 +332,7 @@ func (s *Store) saveNewValidators(ctx context.Context, preStateValidatorCount in
if preStateValidatorCount != postStateValidatorCount {
for i := preStateValidatorCount; i < postStateValidatorCount; i++ {
pubKey := postState.Validators[i].PublicKey
if err := s.db.SaveValidatorIndex(ctx, bytesutil.ToBytes48(pubKey), uint64(i)); err != nil {
if err := s.db.SaveValidatorIndex(ctx, pubKey, uint64(i)); err != nil {
return errors.Wrapf(err, "could not save activated validator: %d", i)
}
log.WithFields(logrus.Fields{
Expand Down
12 changes: 7 additions & 5 deletions beacon-chain/blockchain/forkchoice/process_block_test.go
View file
Expand Up @@ -119,20 +119,22 @@ func TestStore_SaveNewValidators(t *testing.T) {
store := NewForkChoiceService(ctx, db)
preCount := 2 // validators 0 and validators 1
s := &pb.BeaconState{Validators: []*ethpb.Validator{
{PublicKey: []byte{0}}, {PublicKey: []byte{1}},
{PublicKey: []byte{2}}, {PublicKey: []byte{3}},
{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}},
{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}},
}}
if err := store.saveNewValidators(ctx, preCount, s); err != nil {
t.Fatal(err)
}

if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{2})) {
if !db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) {
t.Error("Wanted validator saved in db")
}
if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{3})) {
if !db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}) {
t.Error("Wanted validator saved in db")
}
if db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{1})) {
if db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) {
t.Error("validator not suppose to be saved in db")
}
}
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/blockchain/service.go
View file
Expand Up @@ -294,7 +294,7 @@ func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock,
// This gets called when beacon chain is first initialized to save validator indices and pubkeys in db
func (s *Service) saveGenesisValidators(ctx context.Context, state *pb.BeaconState) error {
for i, v := range state.Validators {
if err := s.beaconDB.SaveValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey), uint64(i)); err != nil {
if err := s.beaconDB.SaveValidatorIndex(ctx, v.PublicKey, uint64(i)); err != nil {
return errors.Wrapf(err, "could not save validator index: %d", i)
}
}
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/blockchain/service_test.go
View file
Expand Up @@ -259,7 +259,7 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
}

for _, v := range s.Validators {
if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey)) {
if !db.HasValidatorIndex(ctx, v.PublicKey) {
t.Errorf("Validator %s missing from db", hex.EncodeToString(v.PublicKey))
}
}
Expand Down
8 changes: 4 additions & 4 deletions beacon-chain/db/iface/interface.go
View file
Expand Up @@ -44,10 +44,10 @@ type Database interface {
SaveGenesisBlockRoot(ctx context.Context, blockRoot [32]byte) error
IsFinalizedBlock(ctx context.Context, blockRoot [32]byte) bool
// Validator related methods.
ValidatorIndex(ctx context.Context, publicKey [48]byte) (uint64, bool, error)
HasValidatorIndex(ctx context.Context, publicKey [48]byte) bool
DeleteValidatorIndex(ctx context.Context, publicKey [48]byte) error
SaveValidatorIndex(ctx context.Context, publicKey [48]byte, validatorIdx uint64) error
ValidatorIndex(ctx context.Context, publicKey []byte) (uint64, bool, error)
HasValidatorIndex(ctx context.Context, publicKey []byte) bool
DeleteValidatorIndex(ctx context.Context, publicKey []byte) error
SaveValidatorIndex(ctx context.Context, publicKey []byte, validatorIdx uint64) error
// State related methods.
State(ctx context.Context, blockRoot [32]byte) (*ethereum_beacon_p2p_v1.BeaconState, error)
HeadState(ctx context.Context) (*ethereum_beacon_p2p_v1.BeaconState, error)
Expand Down
8 changes: 4 additions & 4 deletions beacon-chain/db/kafka/passthrough.go
View file
Expand Up @@ -87,17 +87,17 @@ func (e Exporter) DeleteBlocks(ctx context.Context, blockRoots [][32]byte) error
}

// ValidatorIndex -- passthrough.
func (e Exporter) ValidatorIndex(ctx context.Context, publicKey [48]byte) (uint64, bool, error) {
func (e Exporter) ValidatorIndex(ctx context.Context, publicKey []byte) (uint64, bool, error) {
return e.db.ValidatorIndex(ctx, publicKey)
}

// HasValidatorIndex -- passthrough.
func (e Exporter) HasValidatorIndex(ctx context.Context, publicKey [48]byte) bool {
func (e Exporter) HasValidatorIndex(ctx context.Context, publicKey []byte) bool {
return e.db.HasValidatorIndex(ctx, publicKey)
}

// DeleteValidatorIndex -- passthrough.
func (e Exporter) DeleteValidatorIndex(ctx context.Context, publicKey [48]byte) error {
func (e Exporter) DeleteValidatorIndex(ctx context.Context, publicKey []byte) error {
return e.db.DeleteValidatorIndex(ctx, publicKey)
}

Expand Down Expand Up @@ -212,7 +212,7 @@ func (e Exporter) SaveGenesisBlockRoot(ctx context.Context, blockRoot [32]byte)
}

// SaveValidatorIndex -- passthrough.
func (e Exporter) SaveValidatorIndex(ctx context.Context, publicKey [48]byte, validatorIdx uint64) error {
func (e Exporter) SaveValidatorIndex(ctx context.Context, publicKey []byte, validatorIdx uint64) error {
return e.db.SaveValidatorIndex(ctx, publicKey, validatorIdx)
}

Expand Down
32 changes: 20 additions & 12 deletions beacon-chain/db/kv/validators.go
View file
Expand Up @@ -5,20 +5,25 @@ import (
"encoding/binary"

"github.com/boltdb/bolt"
"github.com/pkg/errors"
"github.com/prysmaticlabs/prysm/shared/params"
"go.opencensus.io/trace"
)

// ValidatorIndex by public key.
func (k *Store) ValidatorIndex(ctx context.Context, publicKey [48]byte) (uint64, bool, error) {
func (k *Store) ValidatorIndex(ctx context.Context, publicKey []byte) (uint64, bool, error) {
if len(publicKey) != params.BeaconConfig().BLSPubkeyLength {
return 0, false, errors.New("incorrect key length")
}
// Return latest validatorIndex from cache if it exists.
if v, ok := k.validatorIndexCache.Get(string(publicKey[:])); v != nil && ok {
if v, ok := k.validatorIndexCache.Get(string(publicKey)); v != nil && ok {
return v.(uint64), true, nil
}
var validatorIdx uint64
var ok bool
err := k.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(validatorsBucket)
enc := bkt.Get(publicKey[:])
enc := bkt.Get(publicKey)
if enc == nil {
return nil
}
Expand All @@ -31,42 +36,45 @@ func (k *Store) ValidatorIndex(ctx context.Context, publicKey [48]byte) (uint64,
}

// HasValidatorIndex verifies if a validator's index by public key exists in the db.
func (k *Store) HasValidatorIndex(ctx context.Context, publicKey [48]byte) bool {
func (k *Store) HasValidatorIndex(ctx context.Context, publicKey []byte) bool {
ctx, span := trace.StartSpan(ctx, "BeaconDB.HasValidatorIndex")
defer span.End()
if v, ok := k.validatorIndexCache.Get(string(publicKey[:])); v != nil && ok {
if v, ok := k.validatorIndexCache.Get(string(publicKey)); v != nil && ok {
return true
}
exists := false
// #nosec G104. Always returns nil.
k.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(validatorsBucket)
exists = bkt.Get(publicKey[:]) != nil
exists = bkt.Get(publicKey) != nil
return nil
})
return exists
}

// DeleteValidatorIndex clears a validator index from the db by the validator's public key.
func (k *Store) DeleteValidatorIndex(ctx context.Context, publicKey [48]byte) error {
func (k *Store) DeleteValidatorIndex(ctx context.Context, publicKey []byte) error {
ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteValidatorIndex")
defer span.End()
return k.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(validatorsBucket)
k.validatorIndexCache.Del(string(publicKey[:]))
return bucket.Delete(publicKey[:])
k.validatorIndexCache.Del(string(publicKey))
return bucket.Delete(publicKey)
})
}

// SaveValidatorIndex by public key in the db.
func (k *Store) SaveValidatorIndex(ctx context.Context, publicKey [48]byte, validatorIdx uint64) error {
func (k *Store) SaveValidatorIndex(ctx context.Context, publicKey []byte, validatorIdx uint64) error {
if len(publicKey) != params.BeaconConfig().BLSPubkeyLength {
return errors.New("incorrect key length")
}
ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveValidatorIndex")
defer span.End()
return k.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(validatorsBucket)
buf := uint64ToBytes(validatorIdx)
k.validatorIndexCache.Set(string(publicKey[:]), validatorIdx, int64(len(buf)))
return bucket.Put(publicKey[:], buf)
k.validatorIndexCache.Set(string(publicKey), validatorIdx, int64(len(buf)))
return bucket.Put(publicKey, buf)
})
}

Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/db/kv/validators_test.go
View file
Expand Up @@ -9,7 +9,7 @@ func TestStore_ValidatorIndexCRUD(t *testing.T) {
db := setupDB(t)
defer teardownDB(t, db)
validatorIdx := uint64(100)
pubKey := [48]byte{1, 2, 3, 4}
pubKey := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4}
ctx := context.Background()
_, ok, err := db.ValidatorIndex(ctx, pubKey)
if err != nil {
Expand Down
1 change: 0 additions & 1 deletion beacon-chain/interop-cold-start/BUILD.bazel
View file
Expand Up @@ -15,7 +15,6 @@ go_library(
"//beacon-chain/powchain:go_default_library",
"//proto/beacon/p2p/v1:go_default_library",
"//shared:go_default_library",
"//shared/bytesutil:go_default_library",
"//shared/interop:go_default_library",
"//shared/stateutil:go_default_library",
"@com_github_pkg_errors//:go_default_library",
Expand Down
3 changes: 1 addition & 2 deletions beacon-chain/interop-cold-start/service.go
View file
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
"github.com/prysmaticlabs/prysm/shared"
"github.com/prysmaticlabs/prysm/shared/bytesutil"
"github.com/prysmaticlabs/prysm/shared/interop"
"github.com/prysmaticlabs/prysm/shared/stateutil"
)
Expand Down Expand Up @@ -171,7 +170,7 @@ func (s *Service) saveGenesisState(ctx context.Context, genesisState *pb.BeaconS
}

for i, v := range genesisState.Validators {
if err := s.beaconDB.SaveValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey), uint64(i)); err != nil {
if err := s.beaconDB.SaveValidatorIndex(ctx, v.PublicKey, uint64(i)); err != nil {
return errors.Wrapf(err, "could not save validator index: %d", i)
}
s.chainStartDeposits[i] = &ethpb.Deposit{
Expand Down
1 change: 0 additions & 1 deletion beacon-chain/rpc/aggregator/BUILD.bazel
View file
Expand Up @@ -13,7 +13,6 @@ go_library(
"//beacon-chain/p2p:go_default_library",
"//beacon-chain/sync:go_default_library",
"//proto/beacon/rpc/v1:go_default_library",
"//shared/bytesutil:go_default_library",
"@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
"@com_github_sirupsen_logrus//:go_default_library",
"@io_opencensus_go//trace:go_default_library",
Expand Down
3 changes: 1 addition & 2 deletions beacon-chain/rpc/aggregator/server.go
View file
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
"github.com/prysmaticlabs/prysm/beacon-chain/sync"
pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
"github.com/prysmaticlabs/prysm/shared/bytesutil"
"github.com/sirupsen/logrus"
"go.opencensus.io/trace"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -44,7 +43,7 @@ func (as *Server) SubmitAggregateAndProof(ctx context.Context, req *pb.Aggregati
return nil, status.Errorf(codes.Unavailable, "Syncing to latest head, not ready to respond")
}

validatorIndex, exists, err := as.BeaconDB.ValidatorIndex(ctx, bytesutil.ToBytes48(req.PublicKey))
validatorIndex, exists, err := as.BeaconDB.ValidatorIndex(ctx, req.PublicKey)
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not get validator index from DB: %v", err)
}
Expand Down
24 changes: 16 additions & 8 deletions beacon-chain/rpc/aggregator/server_test.go
View file
Expand Up @@ -2,6 +2,7 @@ package aggregator

import (
"context"
"encoding/binary"
"reflect"
"strings"
"testing"
Expand All @@ -27,6 +28,13 @@ func init() {
params.OverrideBeaconConfig(params.MinimalSpecConfig())
}

// pubKey is a helper to generate a well-formed public key.
func pubKey(i uint64) []byte {
pubKey := make([]byte, params.BeaconConfig().BLSPubkeyLength)
binary.LittleEndian.PutUint64(pubKey, uint64(i))
return pubKey
}

func TestSubmitAggregateAndProof_Syncing(t *testing.T) {
db := dbutil.SetupDB(t)
defer dbutil.TeardownDB(t, db)
Expand Down Expand Up @@ -64,10 +72,10 @@ func TestSubmitAggregateAndProof_CantFindValidatorIndex(t *testing.T) {

priv := bls.RandKey()
sig := priv.Sign([]byte{'A'}, 0)
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal()}
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey(3)}
wanted := "Could not locate validator index in DB"
if _, err := aggregatorServer.SubmitAggregateAndProof(ctx, req); !strings.Contains(err.Error(), wanted) {
t.Error("Did not receive wanted error")
t.Errorf("Did not receive wanted error: expected %v, received %v", wanted, err.Error())
}
}

Expand All @@ -89,8 +97,8 @@ func TestSubmitAggregateAndProof_IsAggregator(t *testing.T) {

priv := bls.RandKey()
sig := priv.Sign([]byte{'A'}, 0)
pubKey := [48]byte{'A'}
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey[:]}
pubKey := pubKey(1)
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey}
if err := aggregatorServer.BeaconDB.SaveValidatorIndex(ctx, pubKey, 100); err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -127,8 +135,8 @@ func TestSubmitAggregateAndProof_AggregateOk(t *testing.T) {

priv := bls.RandKey()
sig := priv.Sign([]byte{'B'}, 0)
pubKey := [48]byte{'B'}
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey[:]}
pubKey := pubKey(2)
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey}
if err := aggregatorServer.BeaconDB.SaveValidatorIndex(ctx, pubKey, 100); err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -180,8 +188,8 @@ func TestSubmitAggregateAndProof_AggregateNotOk(t *testing.T) {

priv := bls.RandKey()
sig := priv.Sign([]byte{'B'}, 0)
pubKey := [48]byte{'B'}
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey[:]}
pubKey := pubKey(2)
req := &pb.AggregationRequest{CommitteeIndex: 1, SlotSignature: sig.Marshal(), PublicKey: pubKey}
if err := aggregatorServer.BeaconDB.SaveValidatorIndex(ctx, pubKey, 100); err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/rpc/beacon/assignments.go
View file
Expand Up @@ -60,7 +60,7 @@ func (bs *Server) ListValidatorAssignments(

// Filter out assignments by public keys.
for _, pubKey := range req.PublicKeys {
index, ok, err := bs.BeaconDB.ValidatorIndex(ctx, bytesutil.ToBytes48(pubKey))
index, ok, err := bs.BeaconDB.ValidatorIndex(ctx, pubKey)
if err != nil {
return nil, status.Errorf(codes.Internal, "Could not retrieve validator index: %v", err)
}
Expand Down

0 comments on commit 3839f57

Please sign in to comment.