Change slasher DB structure to mirror beacon-chains

slasher/db/BUILD.bazel

@@ -3,51 +3,20 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
-        "attester_slashings.go",
-        "block_header.go",
+        "alias.go",
         "db.go",
-        "indexed_attestations.go",
-        "min_max_span.go",
-        "proposer_slashings.go",
-        "schema.go",
-        "setup_db.go",
-        "validator_id_pubkey.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/slasher/db",
     visibility = ["//slasher:__subpackages__"],
     deps = [
-        "//proto/slashing:go_default_library",
-        "//shared/bytesutil:go_default_library",
-        "//shared/hashutil:go_default_library",
-        "//shared/params:go_default_library",
-        "//slasher/flags:go_default_library",
-        "@com_github_boltdb_bolt//:go_default_library",
-        "@com_github_dgraph_io_ristretto//:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
+        "//slasher/db/iface:go_default_library",
+        "//slasher/db/kv:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = [
-        "attester_slashings_test.go",
-        "block_header_test.go",
-        "indexed_attestations_test.go",
-        "min_max_span_test.go",
-        "proposer_slashings_test.go",
-        "setup_db_test.go",
-        "validator_id_pubkey_test.go",
-    ],
+    srcs = ["db_test.go"],
     embed = [":go_default_library"],
-    deps = [
-        "//proto/slashing:go_default_library",
-        "//slasher/flags:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
-        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-    ],
+    deps = ["//slasher/db/kv:go_default_library"],
 )

slasher/db/alias.go

@@ -0,0 +1,17 @@
+package db
+
+import "github.com/prysmaticlabs/prysm/slasher/db/iface"
+
+// ReadOnlyDatabase exposes the Slasher's DB read only functions for all slasher related buckets.
+type ReadOnlyDatabase = iface.ReadOnlyDatabase
+
+// WriteAccessDatabase exposes the Slasher's DB writing functions for all slasher related buckets.
+type WriteAccessDatabase = iface.WriteAccessDatabase
+
+// FullAccessDatabase exposes Slasher's DB write and read functions for all slasher related buckets.
+type FullAccessDatabase = iface.FullAccessDatabase
+
+// Database defines the necessary methods for the Slasher's DB which may be implemented by any
+// key-value or relational database in practice. This is the full database interface which should
+// not be used often. Prefer a more restrictive interface in this package.
+type Database = iface.Database

slasher/db/db.go

@@ -1,137 +1,10 @@
 package db
 
 import (
-	"os"
-	"path"
-	"time"
-
-	"github.com/boltdb/bolt"
-	"github.com/dgraph-io/ristretto"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	"github.com/prysmaticlabs/prysm/slasher/db/kv"
 )
 
-var log = logrus.WithField("prefix", "slasherDB")
-
-var databaseFileName = "slasher.db"
-
-var d *Store
-
-// Store defines an implementation of the Prysm Database interface
-// using BoltDB as the underlying persistent kv-store for eth2.
-type Store struct {
-	db               *bolt.DB
-	databasePath     string
-	spanCache        *ristretto.Cache
-	spanCacheEnabled bool
-}
-
-// Config options for the slasher db.
-type Config struct {
-	// SpanCacheEnabled use span cache to detect surround slashing.
-	SpanCacheEnabled bool
-	cacheItems       int64
-	maxCacheSize     int64
-}
-
-// Close closes the underlying boltdb database.
-func (db *Store) Close() error {
-	return db.db.Close()
-}
-
-func (db *Store) update(fn func(*bolt.Tx) error) error {
-	return db.db.Update(fn)
-}
-func (db *Store) batch(fn func(*bolt.Tx) error) error {
-	return db.db.Batch(fn)
-}
-func (db *Store) view(fn func(*bolt.Tx) error) error {
-	return db.db.View(fn)
-}
-
 // NewDB initializes a new DB.
-func NewDB(dirPath string, cfg *Config) (*Store, error) {
-	var err error
-	d, err = NewKVStore(dirPath, cfg)
-	return d, err
-}
-
-// ClearDB removes any previously stored data at the configured data directory.
-func (db *Store) ClearDB() error {
-	if _, err := os.Stat(db.databasePath); os.IsNotExist(err) {
-		return nil
-	}
-	return os.Remove(db.databasePath)
-}
-
-// DatabasePath at which this database writes files.
-func (db *Store) DatabasePath() string {
-	return db.databasePath
-}
-
-func createBuckets(tx *bolt.Tx, buckets ...[]byte) error {
-	for _, bucket := range buckets {
-		if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// NewKVStore initializes a new boltDB key-value store at the directory
-// path specified, creates the kv-buckets based on the schema, and stores
-// an open connection db object as a property of the Store struct.
-func NewKVStore(dirPath string, cfg *Config) (*Store, error) {
-	if err := os.MkdirAll(dirPath, 0700); err != nil {
-		return nil, err
-	}
-	datafile := path.Join(dirPath, databaseFileName)
-	boltDB, err := bolt.Open(datafile, 0600, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		if err == bolt.ErrTimeout {
-			return nil, errors.New("cannot obtain database lock, database may be in use by another process")
-		}
-		return nil, err
-	}
-	if cfg.cacheItems == 0 {
-		cfg.cacheItems = 20000
-	}
-	if cfg.maxCacheSize == 0 {
-		cfg.maxCacheSize = 2 << 30 //(2GB)
-	}
-	spanCache, err := ristretto.NewCache(&ristretto.Config{
-		NumCounters: cfg.cacheItems,   // number of keys to track frequency of (10M).
-		MaxCost:     cfg.maxCacheSize, // maximum cost of cache.
-		BufferItems: 64,               // number of keys per Get buffer.
-		OnEvict:     saveToDB,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to start span cache")
-	}
-	kv := &Store{db: boltDB, databasePath: datafile, spanCache: spanCache, spanCacheEnabled: cfg.SpanCacheEnabled}
-
-	if err := kv.db.Update(func(tx *bolt.Tx) error {
-		return createBuckets(
-			tx,
-			historicIndexedAttestationsBucket,
-			historicBlockHeadersBucket,
-			compressedIdxAttsBucket,
-			validatorsPublicKeysBucket,
-			validatorsMinMaxSpanBucket,
-			slashingBucket,
-		)
-	}); err != nil {
-		return nil, err
-	}
-	return kv, err
-}
-
-// Size returns the db size in bytes.
-func (db *Store) Size() (int64, error) {
-	var size int64
-	err := db.db.View(func(tx *bolt.Tx) error {
-		size = tx.Size()
-		return nil
-	})
-	return size, err
+func NewDB(dirPath string, cfg *kv.Config) (*kv.Store, error) {
+	return kv.NewKVStore(dirPath, cfg)
 }

slasher/db/db_test.go

@@ -0,0 +1,5 @@
+package db
+
+import "github.com/prysmaticlabs/prysm/slasher/db/kv"
+
+var _ = Database(&kv.Store{})

slasher/db/iface/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["interface.go"],
+    importpath = "github.com/prysmaticlabs/prysm/slasher/db/iface",
+    visibility = ["//slasher/db:__subpackages__"],
+    deps = [
+        "//proto/slashing:go_default_library",
+        "//slasher/db/types:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+    ],
+)

slasher/db/iface/interface.go

@@ -0,0 +1,87 @@
+package iface
+
+import (
+	"io"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	slashpb "github.com/prysmaticlabs/prysm/proto/slashing"
+	"github.com/prysmaticlabs/prysm/slasher/db/types"
+)
+
+// ReadOnlyDatabase represents a read only database with functions that do not modify the DB.
+type ReadOnlyDatabase interface {
+	// AttesterSlashing related methods.
+	AttesterSlashings(status types.SlashingStatus) ([]*ethpb.AttesterSlashing, error)
+	DeleteAttesterSlashing(attesterSlashing *ethpb.AttesterSlashing) error
+	HasAttesterSlashing(slashing *ethpb.AttesterSlashing) (bool, types.SlashingStatus, error)
+	GetLatestEpochDetected() (uint64, error)
+
+	// BlockHeader related methods.
+	BlockHeaders(epoch uint64, validatorID uint64) ([]*ethpb.SignedBeaconBlockHeader, error)
+	HasBlockHeader(epoch uint64, validatorID uint64) bool
+
+	// IndexedAttestations related methods.
+	IdxAttsForTargetFromID(targetEpoch uint64, validatorID uint64) ([]*ethpb.IndexedAttestation, error)
+	IdxAttsForTarget(targetEpoch uint64) ([]*ethpb.IndexedAttestation, error)
+	LatestIndexedAttestationsTargetEpoch() (uint64, error)
+	LatestValidatorIdx() (uint64, error)
+	DoubleVotes(validatorIdx uint64, dataRoot []byte, origAtt *ethpb.IndexedAttestation) ([]*ethpb.AttesterSlashing, error)
+	HasIndexedAttestation(targetEpoch uint64, validatorID uint64) (bool, error)
+
+	// MinMaxSpan related methods.
+	ValidatorSpansMap(validatorIdx uint64) (*slashpb.EpochSpanMap, error)
+
+	// ProposerSlashing related methods.
+	ProposalSlashingsByStatus(status types.SlashingStatus) ([]*ethpb.ProposerSlashing, error)
+	HasProposerSlashing(slashing *ethpb.ProposerSlashing) (bool, types.SlashingStatus, error)
+
+	// Validator Index -> Pubkey related methods.
+	ValidatorPubKey(validatorID uint64) ([]byte, error)
+}
+
+// WriteAccessDatabase represents a write access database with only functions that can modify the DB.
+type WriteAccessDatabase interface {
+	// AttesterSlashing related methods.
+	SaveAttesterSlashing(status types.SlashingStatus, slashing *ethpb.AttesterSlashing) error
+	SaveAttesterSlashings(status types.SlashingStatus, slashings []*ethpb.AttesterSlashing) error
+	SetLatestEpochDetected(epoch uint64) error
+
+	// BlockHeader related methods.
+	SaveBlockHeader(epoch uint64, validatorID uint64, blockHeader *ethpb.SignedBeaconBlockHeader) error
+	DeleteBlockHeader(epoch uint64, validatorID uint64, blockHeader *ethpb.SignedBeaconBlockHeader) error
+	PruneBlockHistory(currentEpoch uint64, pruningEpochAge uint64) error
+
+	// IndexedAttestations related methods.
+	SaveIndexedAttestation(idxAttestation *ethpb.IndexedAttestation) error
+	DeleteIndexedAttestation(idxAttestation *ethpb.IndexedAttestation) error
+	PruneAttHistory(currentEpoch uint64, pruningEpochAge uint64) error
+
+	// MinMaxSpan related methods.
+	SaveValidatorSpansMap(validatorIdx uint64, spanMap *slashpb.EpochSpanMap) error
+	SaveCachedSpansMaps() error
+	DeleteValidatorSpanMap(validatorIdx uint64) error
+
+	// ProposerSlashing related methods.
+	DeleteProposerSlashing(slashing *ethpb.ProposerSlashing) error
+	SaveProposerSlashing(status types.SlashingStatus, slashing *ethpb.ProposerSlashing) error
+	SaveProposerSlashings(status types.SlashingStatus, slashings []*ethpb.ProposerSlashing) error
+
+	// Validator Index -> Pubkey related methods.
+	SavePubKey(validatorID uint64, pubKey []byte) error
+	DeletePubKey(validatorID uint64) error
+}
+
+// FullAccessDatabase represents a full access database with only DB interaction functions.
+type FullAccessDatabase interface {
+	ReadOnlyDatabase
+	WriteAccessDatabase
+}
+
+// Database represents a full access database with the proper DB helper functions.
+type Database interface {
+	io.Closer
+	FullAccessDatabase
+
+	DatabasePath() string
+	ClearDB() error
+}

slasher/db/kv/BUILD.bazel

@@ -0,0 +1,53 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "attester_slashings.go",
+        "block_header.go",
+        "indexed_attestations.go",
+        "kv.go",
+        "min_max_span.go",
+        "proposer_slashings.go",
+        "schema.go",
+        "validator_id_pubkey.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/slasher/db/kv",
+    visibility = ["//slasher:__subpackages__"],
+    deps = [
+        "//proto/slashing:go_default_library",
+        "//shared/bytesutil:go_default_library",
+        "//shared/hashutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//slasher/db/types:go_default_library",
+        "@com_github_boltdb_bolt//:go_default_library",
+        "@com_github_dgraph_io_ristretto//:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "attester_slashings_test.go",
+        "block_header_test.go",
+        "indexed_attestations_test.go",
+        "kv_test.go",
+        "min_max_span_test.go",
+        "proposer_slashings_test.go",
+        "validator_id_pubkey_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//proto/slashing:go_default_library",
+        "//shared/testutil:go_default_library",
+        "//slasher/db/types:go_default_library",
+        "//slasher/flags:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+        "@com_github_urfave_cli//:go_default_library",
+    ],
+)