forked from prysmaticlabs/prysm
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Change slasher DB structure to mirror beacon-chains (prysmaticlabs#4848)
* Add interface and move slashing types to /types package * WIP restructure to match beacon chain DB * Fix build * Fix comment * Fix comments * fix comments for sure * Use wrapper function for evict * Remove unused * Update slasher/db/kv/kv.go * Update slasher/db/testing/BUILD.bazel * Update slasher/db/types/BUILD.bazel * Update slasher/db/types/types.go Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
- Loading branch information
1 parent
8288aa5
commit 320031e
Showing
35 changed files
with
714 additions
and
468 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package db | ||
|
|
||
| import "github.com/prysmaticlabs/prysm/slasher/db/iface" | ||
|
|
||
| // ReadOnlyDatabase exposes the Slasher's DB read only functions for all slasher related buckets. | ||
| type ReadOnlyDatabase = iface.ReadOnlyDatabase | ||
|
|
||
| // WriteAccessDatabase exposes the Slasher's DB writing functions for all slasher related buckets. | ||
| type WriteAccessDatabase = iface.WriteAccessDatabase | ||
|
|
||
| // FullAccessDatabase exposes Slasher's DB write and read functions for all slasher related buckets. | ||
| type FullAccessDatabase = iface.FullAccessDatabase | ||
|
|
||
| // Database defines the necessary methods for the Slasher's DB which may be implemented by any | ||
| // key-value or relational database in practice. This is the full database interface which should | ||
| // not be used often. Prefer a more restrictive interface in this package. | ||
| type Database = iface.Database |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,137 +1,10 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "os" | ||
| "path" | ||
| "time" | ||
|
|
||
| "github.com/boltdb/bolt" | ||
| "github.com/dgraph-io/ristretto" | ||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
| "github.com/prysmaticlabs/prysm/slasher/db/kv" | ||
| ) | ||
|
|
||
| var log = logrus.WithField("prefix", "slasherDB") | ||
|
|
||
| var databaseFileName = "slasher.db" | ||
|
|
||
| var d *Store | ||
|
|
||
| // Store defines an implementation of the Prysm Database interface | ||
| // using BoltDB as the underlying persistent kv-store for eth2. | ||
| type Store struct { | ||
| db *bolt.DB | ||
| databasePath string | ||
| spanCache *ristretto.Cache | ||
| spanCacheEnabled bool | ||
| } | ||
|
|
||
| // Config options for the slasher db. | ||
| type Config struct { | ||
| // SpanCacheEnabled use span cache to detect surround slashing. | ||
| SpanCacheEnabled bool | ||
| cacheItems int64 | ||
| maxCacheSize int64 | ||
| } | ||
|
|
||
| // Close closes the underlying boltdb database. | ||
| func (db *Store) Close() error { | ||
| return db.db.Close() | ||
| } | ||
|
|
||
| func (db *Store) update(fn func(*bolt.Tx) error) error { | ||
| return db.db.Update(fn) | ||
| } | ||
| func (db *Store) batch(fn func(*bolt.Tx) error) error { | ||
| return db.db.Batch(fn) | ||
| } | ||
| func (db *Store) view(fn func(*bolt.Tx) error) error { | ||
| return db.db.View(fn) | ||
| } | ||
|
|
||
| // NewDB initializes a new DB. | ||
| func NewDB(dirPath string, cfg *Config) (*Store, error) { | ||
| var err error | ||
| d, err = NewKVStore(dirPath, cfg) | ||
| return d, err | ||
| } | ||
|
|
||
| // ClearDB removes any previously stored data at the configured data directory. | ||
| func (db *Store) ClearDB() error { | ||
| if _, err := os.Stat(db.databasePath); os.IsNotExist(err) { | ||
| return nil | ||
| } | ||
| return os.Remove(db.databasePath) | ||
| } | ||
|
|
||
| // DatabasePath at which this database writes files. | ||
| func (db *Store) DatabasePath() string { | ||
| return db.databasePath | ||
| } | ||
|
|
||
| func createBuckets(tx *bolt.Tx, buckets ...[]byte) error { | ||
| for _, bucket := range buckets { | ||
| if _, err := tx.CreateBucketIfNotExists(bucket); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // NewKVStore initializes a new boltDB key-value store at the directory | ||
| // path specified, creates the kv-buckets based on the schema, and stores | ||
| // an open connection db object as a property of the Store struct. | ||
| func NewKVStore(dirPath string, cfg *Config) (*Store, error) { | ||
| if err := os.MkdirAll(dirPath, 0700); err != nil { | ||
| return nil, err | ||
| } | ||
| datafile := path.Join(dirPath, databaseFileName) | ||
| boltDB, err := bolt.Open(datafile, 0600, &bolt.Options{Timeout: 1 * time.Second}) | ||
| if err != nil { | ||
| if err == bolt.ErrTimeout { | ||
| return nil, errors.New("cannot obtain database lock, database may be in use by another process") | ||
| } | ||
| return nil, err | ||
| } | ||
| if cfg.cacheItems == 0 { | ||
| cfg.cacheItems = 20000 | ||
| } | ||
| if cfg.maxCacheSize == 0 { | ||
| cfg.maxCacheSize = 2 << 30 //(2GB) | ||
| } | ||
| spanCache, err := ristretto.NewCache(&ristretto.Config{ | ||
| NumCounters: cfg.cacheItems, // number of keys to track frequency of (10M). | ||
| MaxCost: cfg.maxCacheSize, // maximum cost of cache. | ||
| BufferItems: 64, // number of keys per Get buffer. | ||
| OnEvict: saveToDB, | ||
| }) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "failed to start span cache") | ||
| } | ||
| kv := &Store{db: boltDB, databasePath: datafile, spanCache: spanCache, spanCacheEnabled: cfg.SpanCacheEnabled} | ||
|
|
||
| if err := kv.db.Update(func(tx *bolt.Tx) error { | ||
| return createBuckets( | ||
| tx, | ||
| historicIndexedAttestationsBucket, | ||
| historicBlockHeadersBucket, | ||
| compressedIdxAttsBucket, | ||
| validatorsPublicKeysBucket, | ||
| validatorsMinMaxSpanBucket, | ||
| slashingBucket, | ||
| ) | ||
| }); err != nil { | ||
| return nil, err | ||
| } | ||
| return kv, err | ||
| } | ||
|
|
||
| // Size returns the db size in bytes. | ||
| func (db *Store) Size() (int64, error) { | ||
| var size int64 | ||
| err := db.db.View(func(tx *bolt.Tx) error { | ||
| size = tx.Size() | ||
| return nil | ||
| }) | ||
| return size, err | ||
| func NewDB(dirPath string, cfg *kv.Config) (*kv.Store, error) { | ||
| return kv.NewKVStore(dirPath, cfg) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package db | ||
|
|
||
| import "github.com/prysmaticlabs/prysm/slasher/db/kv" | ||
|
|
||
| var _ = Database(&kv.Store{}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_library") | ||
|
|
||
| go_library( | ||
| name = "go_default_library", | ||
| srcs = ["interface.go"], | ||
| importpath = "github.com/prysmaticlabs/prysm/slasher/db/iface", | ||
| visibility = ["//slasher/db:__subpackages__"], | ||
| deps = [ | ||
| "//proto/slashing:go_default_library", | ||
| "//slasher/db/types:go_default_library", | ||
| "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", | ||
| ], | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package iface | ||
|
|
||
| import ( | ||
| "io" | ||
|
|
||
| ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" | ||
| slashpb "github.com/prysmaticlabs/prysm/proto/slashing" | ||
| "github.com/prysmaticlabs/prysm/slasher/db/types" | ||
| ) | ||
|
|
||
| // ReadOnlyDatabase represents a read only database with functions that do not modify the DB. | ||
| type ReadOnlyDatabase interface { | ||
| // AttesterSlashing related methods. | ||
| AttesterSlashings(status types.SlashingStatus) ([]*ethpb.AttesterSlashing, error) | ||
| DeleteAttesterSlashing(attesterSlashing *ethpb.AttesterSlashing) error | ||
| HasAttesterSlashing(slashing *ethpb.AttesterSlashing) (bool, types.SlashingStatus, error) | ||
| GetLatestEpochDetected() (uint64, error) | ||
|
|
||
| // BlockHeader related methods. | ||
| BlockHeaders(epoch uint64, validatorID uint64) ([]*ethpb.SignedBeaconBlockHeader, error) | ||
| HasBlockHeader(epoch uint64, validatorID uint64) bool | ||
|
|
||
| // IndexedAttestations related methods. | ||
| IdxAttsForTargetFromID(targetEpoch uint64, validatorID uint64) ([]*ethpb.IndexedAttestation, error) | ||
| IdxAttsForTarget(targetEpoch uint64) ([]*ethpb.IndexedAttestation, error) | ||
| LatestIndexedAttestationsTargetEpoch() (uint64, error) | ||
| LatestValidatorIdx() (uint64, error) | ||
| DoubleVotes(validatorIdx uint64, dataRoot []byte, origAtt *ethpb.IndexedAttestation) ([]*ethpb.AttesterSlashing, error) | ||
| HasIndexedAttestation(targetEpoch uint64, validatorID uint64) (bool, error) | ||
|
|
||
| // MinMaxSpan related methods. | ||
| ValidatorSpansMap(validatorIdx uint64) (*slashpb.EpochSpanMap, error) | ||
|
|
||
| // ProposerSlashing related methods. | ||
| ProposalSlashingsByStatus(status types.SlashingStatus) ([]*ethpb.ProposerSlashing, error) | ||
| HasProposerSlashing(slashing *ethpb.ProposerSlashing) (bool, types.SlashingStatus, error) | ||
|
|
||
| // Validator Index -> Pubkey related methods. | ||
| ValidatorPubKey(validatorID uint64) ([]byte, error) | ||
| } | ||
|
|
||
| // WriteAccessDatabase represents a write access database with only functions that can modify the DB. | ||
| type WriteAccessDatabase interface { | ||
| // AttesterSlashing related methods. | ||
| SaveAttesterSlashing(status types.SlashingStatus, slashing *ethpb.AttesterSlashing) error | ||
| SaveAttesterSlashings(status types.SlashingStatus, slashings []*ethpb.AttesterSlashing) error | ||
| SetLatestEpochDetected(epoch uint64) error | ||
|
|
||
| // BlockHeader related methods. | ||
| SaveBlockHeader(epoch uint64, validatorID uint64, blockHeader *ethpb.SignedBeaconBlockHeader) error | ||
| DeleteBlockHeader(epoch uint64, validatorID uint64, blockHeader *ethpb.SignedBeaconBlockHeader) error | ||
| PruneBlockHistory(currentEpoch uint64, pruningEpochAge uint64) error | ||
|
|
||
| // IndexedAttestations related methods. | ||
| SaveIndexedAttestation(idxAttestation *ethpb.IndexedAttestation) error | ||
| DeleteIndexedAttestation(idxAttestation *ethpb.IndexedAttestation) error | ||
| PruneAttHistory(currentEpoch uint64, pruningEpochAge uint64) error | ||
|
|
||
| // MinMaxSpan related methods. | ||
| SaveValidatorSpansMap(validatorIdx uint64, spanMap *slashpb.EpochSpanMap) error | ||
| SaveCachedSpansMaps() error | ||
| DeleteValidatorSpanMap(validatorIdx uint64) error | ||
|
|
||
| // ProposerSlashing related methods. | ||
| DeleteProposerSlashing(slashing *ethpb.ProposerSlashing) error | ||
| SaveProposerSlashing(status types.SlashingStatus, slashing *ethpb.ProposerSlashing) error | ||
| SaveProposerSlashings(status types.SlashingStatus, slashings []*ethpb.ProposerSlashing) error | ||
|
|
||
| // Validator Index -> Pubkey related methods. | ||
| SavePubKey(validatorID uint64, pubKey []byte) error | ||
| DeletePubKey(validatorID uint64) error | ||
| } | ||
|
|
||
| // FullAccessDatabase represents a full access database with only DB interaction functions. | ||
| type FullAccessDatabase interface { | ||
| ReadOnlyDatabase | ||
| WriteAccessDatabase | ||
| } | ||
|
|
||
| // Database represents a full access database with the proper DB helper functions. | ||
| type Database interface { | ||
| io.Closer | ||
| FullAccessDatabase | ||
|
|
||
| DatabasePath() string | ||
| ClearDB() error | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") | ||
|
|
||
| go_library( | ||
| name = "go_default_library", | ||
| srcs = [ | ||
| "attester_slashings.go", | ||
| "block_header.go", | ||
| "indexed_attestations.go", | ||
| "kv.go", | ||
| "min_max_span.go", | ||
| "proposer_slashings.go", | ||
| "schema.go", | ||
| "validator_id_pubkey.go", | ||
| ], | ||
| importpath = "github.com/prysmaticlabs/prysm/slasher/db/kv", | ||
| visibility = ["//slasher:__subpackages__"], | ||
| deps = [ | ||
| "//proto/slashing:go_default_library", | ||
| "//shared/bytesutil:go_default_library", | ||
| "//shared/hashutil:go_default_library", | ||
| "//shared/params:go_default_library", | ||
| "//slasher/db/types:go_default_library", | ||
| "@com_github_boltdb_bolt//:go_default_library", | ||
| "@com_github_dgraph_io_ristretto//:go_default_library", | ||
| "@com_github_gogo_protobuf//proto:go_default_library", | ||
| "@com_github_pkg_errors//:go_default_library", | ||
| "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", | ||
| "@com_github_sirupsen_logrus//:go_default_library", | ||
| ], | ||
| ) | ||
|
|
||
| go_test( | ||
| name = "go_default_test", | ||
| srcs = [ | ||
| "attester_slashings_test.go", | ||
| "block_header_test.go", | ||
| "indexed_attestations_test.go", | ||
| "kv_test.go", | ||
| "min_max_span_test.go", | ||
| "proposer_slashings_test.go", | ||
| "validator_id_pubkey_test.go", | ||
| ], | ||
| embed = [":go_default_library"], | ||
| deps = [ | ||
| "//proto/slashing:go_default_library", | ||
| "//shared/testutil:go_default_library", | ||
| "//slasher/db/types:go_default_library", | ||
| "//slasher/flags:go_default_library", | ||
| "@com_github_gogo_protobuf//proto:go_default_library", | ||
| "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", | ||
| "@com_github_urfave_cli//:go_default_library", | ||
| ], | ||
| ) |
Oops, something went wrong.