Report unhealthy if we think we are out of sync (#4796)

* Report unhealthy if we think we are out of sync
* gofmt
* Merge refs/heads/master into out-of-sync-unhealthy

beacon-chain/p2p/peers/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/p2p/peers",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
+        "//beacon-chain/core/helpers:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/roughtime:go_default_library",
@@ -21,6 +22,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/params:go_default_library",
         "@com_github_libp2p_go_libp2p_core//network:go_default_library",
         "@com_github_libp2p_go_libp2p_peer//:go_default_library",
         "@com_github_multiformats_go_multiaddr//:go_default_library",

beacon-chain/p2p/peers/status.go

@@ -28,6 +28,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	ma "github.com/multiformats/go-multiaddr"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
@@ -392,3 +393,16 @@ func (p *Status) fetch(pid peer.ID) *peerStatus {
 	}
 	return p.status[pid]
 }
+
+// CurrentEpoch returns the highest reported epoch amongst peers.
+func (p *Status) CurrentEpoch() uint64 {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	var highestSlot uint64
+	for _, ps := range p.status {
+		if ps.chainState.HeadSlot > highestSlot {
+			highestSlot = ps.chainState.HeadSlot
+		}
+	}
+	return helpers.SlotToEpoch(highestSlot)
+}

beacon-chain/p2p/peers/status_test.go

@@ -11,6 +11,7 @@ import (
 	ma "github.com/multiformats/go-multiaddr"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p/peers"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestStatus(t *testing.T) {
@@ -458,6 +459,30 @@ func TestBestFinalized_returnsMaxValue(t *testing.T) {
 	}
 }
 
+func TestStatus_CurrentEpoch(t *testing.T) {
+	maxBadResponses := 2
+	p := peers.NewStatus(maxBadResponses)
+	// Peer 1
+	pid1 := addPeer(t, p, peers.PeerConnected)
+	p.SetChainState(pid1, &pb.Status{
+		HeadSlot: params.BeaconConfig().SlotsPerEpoch * 4,
+	})
+	// Peer 2
+	pid2 := addPeer(t, p, peers.PeerConnected)
+	p.SetChainState(pid2, &pb.Status{
+		HeadSlot: params.BeaconConfig().SlotsPerEpoch * 5,
+	})
+	// Peer 3
+	pid3 := addPeer(t, p, peers.PeerConnected)
+	p.SetChainState(pid3, &pb.Status{
+		HeadSlot: params.BeaconConfig().SlotsPerEpoch * 4,
+	})
+
+	if p.CurrentEpoch() != 5 {
+		t.Fatalf("Expected current epoch to be 5, got %d", p.CurrentEpoch())
+	}
+}
+
 // addPeer is a helper to add a peer with a given connection state)
 func addPeer(t *testing.T, p *peers.Status, state peers.PeerConnectionState) peer.ID {
 	// Set up some peers with different states

beacon-chain/sync/service.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
 	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
@@ -111,8 +112,16 @@ func (r *Service) Stop() error {
 
 // Status of the currently running regular sync service.
 func (r *Service) Status() error {
-	if r.chainStarted && r.initialSync.Syncing() {
-		return errors.New("waiting for initial sync")
+	if r.chainStarted {
+		if r.initialSync.Syncing() {
+			return errors.New("waiting for initial sync")
+		}
+		// If our head slot is on a previous epoch and our peers are reporting their head block are
+		// in the most recent epoch, then we might be out of sync.
+		if headEpoch := helpers.SlotToEpoch(r.chain.HeadSlot()); headEpoch < helpers.SlotToEpoch(r.chain.CurrentSlot())-1 &&
+			headEpoch < r.p2p.Peers().CurrentEpoch()-1 {
+			return errors.New("out of sync")
+		}
 	}
 	return nil
 }