Add in Slasher Metrics

beacon-chain/operations/slashings/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "doc.go",
+        "metrics.go",
         "service.go",
         "types.go",
     ],
@@ -14,6 +15,8 @@ go_library(
         "//beacon-chain/state:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )

beacon-chain/operations/slashings/metrics.go

@@ -0,0 +1,45 @@
+package slashings
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	numPendingAttesterSlashings = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "num_pending_attester_slashings",
+			Help: "Number of pending attester slashings in the pool",
+		},
+	)
+	numAttesterSlashingsIncluded = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "num_attester_slashings_included",
+			Help: "Number of attester slashings included in blocks",
+		},
+	)
+	attesterSlashingReattempts = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "attester_slashing_reattempts",
+			Help: "Times an attester slashing for an already slashed validator is received",
+		},
+	)
+	numPendingProposerSlashings = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "num_pending_proposer_slashings",
+			Help: "Number of pending proposer slashings in the pool",
+		},
+	)
+	numProposerSlashingsIncluded = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "num_proposer_slashings_included",
+			Help: "Number of proposer slashings included in blocks",
+		},
+	)
+	proposerSlashingReattempts = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "proposer_slashing_reattempts",
+			Help: "Times a proposer slashing for an already slashed validator is received",
+		},
+	)
+)

beacon-chain/operations/slashings/service.go

@@ -27,6 +27,9 @@ func (p *Pool) PendingAttesterSlashings() []*ethpb.AttesterSlashing {
 	p.lock.RLock()
 	defer p.lock.RUnlock()
 
+	// Update prom metric.
+	numPendingAttesterSlashings.Set(float64(len(p.pendingAttesterSlashing)))
+
 	included := make(map[uint64]bool)
 	pending := make([]*ethpb.AttesterSlashing, 0, params.BeaconConfig().MaxAttesterSlashings)
 	for i, slashing := range p.pendingAttesterSlashing {
@@ -52,6 +55,10 @@ func (p *Pool) PendingAttesterSlashings() []*ethpb.AttesterSlashing {
 func (p *Pool) PendingProposerSlashings() []*ethpb.ProposerSlashing {
 	p.lock.RLock()
 	defer p.lock.RUnlock()
+
+	// Update prom metric.
+	numPendingProposerSlashings.Set(float64(len(p.pendingProposerSlashing)))
+
 	pending := make([]*ethpb.ProposerSlashing, 0, params.BeaconConfig().MaxProposerSlashings)
 	for i, slashing := range p.pendingProposerSlashing {
 		if i >= int(params.BeaconConfig().MaxProposerSlashings) {
@@ -79,6 +86,7 @@ func (p *Pool) InsertAttesterSlashing(state *beaconstate.BeaconState, slashing *
 		// has been recently included in the pool of slashings, do not process this new
 		// slashing.
 		if !ok {
+			attesterSlashingReattempts.Inc()
 			return fmt.Errorf("validator at index %d cannot be slashed", val)
 		}
 
@@ -119,6 +127,7 @@ func (p *Pool) InsertProposerSlashing(state *beaconstate.BeaconState, slashing *
 	// has been recently included in the pool of slashings, do not process this new
 	// slashing.
 	if !ok {
+		proposerSlashingReattempts.Inc()
 		return fmt.Errorf("validator at index %d cannot be slashed", idx)
 	}
 
@@ -154,6 +163,7 @@ func (p *Pool) MarkIncludedAttesterSlashing(as *ethpb.AttesterSlashing) {
 			p.pendingAttesterSlashing = append(p.pendingAttesterSlashing[:i], p.pendingAttesterSlashing[i+1:]...)
 		}
 		p.included[val] = true
+		numAttesterSlashingsIncluded.Inc()
 	}
 }
 
@@ -170,6 +180,7 @@ func (p *Pool) MarkIncludedProposerSlashing(ps *ethpb.ProposerSlashing) {
 		p.pendingProposerSlashing = append(p.pendingProposerSlashing[:i], p.pendingProposerSlashing[i+1:]...)
 	}
 	p.included[ps.ProposerIndex] = true
+	numProposerSlashingsIncluded.Inc()
 }
 
 // this function checks a few items about a validator before proceeding with inserting

slasher/beaconclient/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "chain_data.go",
         "historical_data_retrieval.go",
+        "metrics.go",
         "receivers.go",
         "service.go",
         "submit.go",
@@ -20,6 +21,8 @@ go_library(
         "@com_github_grpc_ecosystem_go_grpc_middleware//tracing/opentracing:go_default_library",
         "@com_github_grpc_ecosystem_go_grpc_prometheus//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//plugin/ocgrpc:go_default_library",

slasher/beaconclient/metrics.go

@@ -0,0 +1,13 @@
+package beaconclient
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	slasherNumAttestationsReceived = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "slasher_num_attestations_received",
+		Help: "The # of attestations received by slasher",
+	})
+)

slasher/beaconclient/receivers.go

@@ -79,7 +79,7 @@ func (bs *Service) collectReceivedAttestations(ctx context.Context) {
 	defer span.End()
 
 	var atts []*ethpb.IndexedAttestation
-	ticker := time.NewTicker(2*time.Second)
+	ticker := time.NewTicker(2 * time.Second)
 	for {
 		select {
 		case <-ticker.C:
@@ -95,6 +95,7 @@ func (bs *Service) collectReceivedAttestations(ctx context.Context) {
 				continue
 			}
 			log.Infof("%d attestations for slot %d saved to slasher DB", len(collectedAtts), collectedAtts[0].Data.Slot)
+			slasherNumAttestationsReceived.Add(float64(len(collectedAtts)))
 
 			// After saving, we send the received attestation over the attestation feed.
 			for _, att := range collectedAtts {
@@ -108,4 +109,4 @@ func (bs *Service) collectReceivedAttestations(ctx context.Context) {
 			return
 		}
 	}
-}
+}

slasher/cache/span_cache.go

@@ -10,7 +10,7 @@ import (
 var (
 	// epochSpansCacheSize defines the max number of epoch spans the cache can hold.
 	epochSpansCacheSize = 256
-	// Metrics
+	// Metrics for the span cache.
 	epochSpansCacheHit = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "epoch_spans_cache_hit",
 		Help: "The total number of cache hits on the epoch spans cache.",

slasher/db/kv/BUILD.bazel

@@ -25,6 +25,8 @@ go_library(
         "@com_github_boltdb_bolt//:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",

slasher/db/kv/kv.go

@@ -108,6 +108,7 @@ func NewKVStore(dirPath string, cfg *Config) (*Store, error) {
 	}); err != nil {
 		return nil, err
 	}
+
 	return kv, err
 }
 

slasher/db/kv/spanner.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/boltdb/bolt"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/slasher/detection/attestations/types"
@@ -18,6 +20,21 @@ import (
 var highestObservedEpoch uint64
 var lowestObservedEpoch = params.BeaconConfig().FarFutureEpoch
 
+var (
+	slasherLowestObservedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "slasher_lowest_observed_epoch",
+		Help: "The lowest epoch number seen by slasher",
+	})
+	slasherHighestObservedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "slasher_highest_observed_epoch",
+		Help: "The highest epoch number seen by slasher",
+	})
+	epochSpansCacheEvictions = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "epoch_spans_cache_evictions",
+		Help: "The number of cache evictions seen by slasher",
+	})
+)
+
 // This function defines a function which triggers upon a span map being
 // evicted from the cache. It allows us to persist the span map by the epoch value
 // to the database itself in the validatorsMinMaxSpanBucket.
@@ -45,6 +62,7 @@ func persistSpanMapsOnEviction(db *Store) func(key interface{}, value interface{
 					return err
 				}
 			}
+			epochSpansCacheEvictions.Inc()
 			return nil
 		})
 		if err != nil {
@@ -315,9 +333,11 @@ func (db *Store) findOrLoadEpochInCache(ctx context.Context, epoch uint64) (map[
 
 func setObservedEpochs(epoch uint64) {
 	if epoch > highestObservedEpoch {
+		slasherHighestObservedEpoch.Set(float64(epoch))
 		highestObservedEpoch = epoch
 	}
 	if epoch < lowestObservedEpoch {
+		slasherLowestObservedEpoch.Set(float64(epoch))
 		lowestObservedEpoch = epoch
 	}
 }

slasher/detection/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "detect.go",
         "listeners.go",
+        "metrics.go",
         "service.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/slasher/detection",
@@ -19,6 +20,8 @@ go_library(
         "//slasher/detection/attestations/types:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",

slasher/detection/attestations/BUILD.bazel

@@ -14,6 +14,8 @@ go_library(
         "//slasher/db:go_default_library",
         "//slasher/detection/attestations/iface:go_default_library",
         "//slasher/detection/attestations/types:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],

slasher/detection/attestations/spanner.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -13,6 +15,17 @@ import (
 	"go.opencensus.io/trace"
 )
 
+var (
+	latestMinSpanDistanceObserved = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "latest_min_span_distance_observed",
+		Help: "The latest distance between target - source observed for min spans",
+	})
+	latestMaxSpanDistanceObserved = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "latest_max_span_distance_observed",
+		Help: "The latest distance between target - source observed for max spans",
+	})
+)
+
 // We look back 128 epochs when updating min/max spans
 // for incoming attestations.
 // TODO(#5040): Remove lookback and handle min spans properly.
@@ -110,7 +123,6 @@ func (s *SpanDetector) DetectSlashingsForAttestation(
 		}
 	}
 
-
 	// Clear out any duplicate results.
 	keys := make(map[[32]byte]bool)
 	var detectionList []*types.DetectionResult
@@ -191,6 +203,7 @@ func (s *SpanDetector) updateMinSpan(ctx context.Context, att *ethpb.IndexedAtte
 	if int(lowestEpoch) <= 0 {
 		lowestEpoch = 0
 	}
+	latestMinSpanDistanceObserved.Set(float64(att.Data.Target.Epoch - att.Data.Source.Epoch))
 
 	for epoch := source - 1; epoch >= lowestEpoch; epoch-- {
 		spanMap, err := s.slasherDB.EpochSpansMap(ctx, epoch)
@@ -232,6 +245,7 @@ func (s *SpanDetector) updateMaxSpan(ctx context.Context, att *ethpb.IndexedAtte
 	defer traceSpan.End()
 	source := att.Data.Source.Epoch
 	target := att.Data.Target.Epoch
+	latestMaxSpanDistanceObserved.Set(float64(source - target))
 	valIndices := make([]uint64, len(att.AttestingIndices))
 	copy(valIndices, att.AttestingIndices)
 	for epoch := source + 1; epoch < target; epoch++ {

slasher/detection/detect.go

@@ -76,6 +76,7 @@ func (ds *Service) detectDoubleVote(
 		}
 
 		if isDoubleVote(incomingAtt, att) {
+			doubleVotesDetected.Inc()
 			return &ethpb.AttesterSlashing{
 				Attestation_1: incomingAtt,
 				Attestation_2: att,
@@ -114,11 +115,13 @@ func (ds *Service) detectSurroundVotes(
 		// Slashings must be submitted as the incoming attestation surrounding the saved attestation.
 		// So we swap the order if needed.
 		if isSurrounding(incomingAtt, att) {
+			surroundingVotesDetected.Inc()
 			return &ethpb.AttesterSlashing{
 				Attestation_1: incomingAtt,
 				Attestation_2: att,
 			}, nil
 		} else if isSurrounded(incomingAtt, att) {
+			surroundedVotesDetected.Inc()
 			return &ethpb.AttesterSlashing{
 				Attestation_1: att,
 				Attestation_2: incomingAtt,

slasher/detection/metrics.go

@@ -0,0 +1,25 @@
+package detection
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	doubleProposalsDetected = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "double_proposals_detected",
+		Help: "The # of double propose slashable events detected",
+	})
+	doubleVotesDetected = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "double_votes_detected",
+		Help: "The # of double vote slashable events detected",
+	})
+	surroundingVotesDetected = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "surrounding_votes_detected",
+		Help: "The # of surrounding slashable events detected",
+	})
+	surroundedVotesDetected = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "surrounded_votes_detected",
+		Help: "The # of surrounded slashable events detected",
+	})
+)