Merge pull request #2009 from ElrondNetwork/EN-6839-dump-goroutines

En 6839 dump goroutines

cmd/node/main.go

@@ -25,12 +25,14 @@ import (
 	"github.com/ElrondNetwork/elrond-go/consensus/round"
 	"github.com/ElrondNetwork/elrond-go/core"
 	"github.com/ElrondNetwork/elrond-go/core/accumulator"
+	"github.com/ElrondNetwork/elrond-go/core/alarm"
 	"github.com/ElrondNetwork/elrond-go/core/check"
 	"github.com/ElrondNetwork/elrond-go/core/indexer"
 	"github.com/ElrondNetwork/elrond-go/core/random"
 	"github.com/ElrondNetwork/elrond-go/core/serviceContainer"
 	"github.com/ElrondNetwork/elrond-go/core/statistics"
 	"github.com/ElrondNetwork/elrond-go/core/throttler"
+	"github.com/ElrondNetwork/elrond-go/core/watchdog"
 	"github.com/ElrondNetwork/elrond-go/crypto"
 	"github.com/ElrondNetwork/elrond-go/crypto/signing/mcl"
 	"github.com/ElrondNetwork/elrond-go/data"
@@ -2047,6 +2049,12 @@ func createNode(
 		return nil, err
 	}
 
+	alarmScheduler := alarm.NewAlarmScheduler()
+	watchdogTimer, err := watchdog.NewWatchdog(alarmScheduler, chanStopNodeProcess)
+	if err != nil {
+		return nil, err
+	}
+
 	peerDenialEvaluator, err := blackList.NewPeerDenialEvaluator(
 		network.PeerBlackListHandler,
 		network.PkTimeCache,
@@ -2122,6 +2130,7 @@ func createNode(
 		node.WithNodeStopChannel(chanStopNodeProcess),
 		node.WithApiTransactionByHashThrottler(apiTxsByHashThrottler),
 		node.WithPeerHonestyHandler(peerHonestyHandler),
+		node.WithWatchdogTimer(watchdogTimer),
 	)
 	if err != nil {
 		return nil, errors.New("error creating node: " + err.Error())

consensus/chronology/chronology.go

@@ -24,6 +24,9 @@ var log = logger.GetOrCreate("consensus/chronology")
 // srBeforeStartRound defines the state which exist before the start of the round
 const srBeforeStartRound = -1
 
+const numRoundsToWaitBeforeSignalingChronologyStuck = 10
+const chronologyAlarmID = "chronology"
+
 // chronology defines the data needed by the chronology
 type chronology struct {
 	genesisTime time.Time
@@ -38,20 +41,23 @@ type chronology struct {
 	mutSubrounds     sync.RWMutex
 	appStatusHandler core.AppStatusHandler
 	cancelFunc       func()
+
+	watchdog core.WatchdogTimer
 }
 
 // NewChronology creates a new chronology object
 func NewChronology(
 	genesisTime time.Time,
 	rounder consensus.Rounder,
 	syncTimer ntp.SyncTimer,
+	watchdog core.WatchdogTimer,
 ) (*chronology, error) {
 
 	err := checkNewChronologyParams(
 		rounder,
 		syncTimer,
+		watchdog,
 	)
-
 	if err != nil {
 		return nil, err
 	}
@@ -61,6 +67,7 @@ func NewChronology(
 		rounder:          rounder,
 		syncTimer:        syncTimer,
 		appStatusHandler: statusHandler.NewNilStatusHandler(),
+		watchdog:         watchdog,
 	}
 
 	chr.subroundId = srBeforeStartRound
@@ -74,6 +81,7 @@ func NewChronology(
 func checkNewChronologyParams(
 	rounder consensus.Rounder,
 	syncTimer ntp.SyncTimer,
+	watchdog core.WatchdogTimer,
 ) error {
 
 	if check.IfNil(rounder) {
@@ -82,6 +90,9 @@ func checkNewChronologyParams(
 	if check.IfNil(syncTimer) {
 		return ErrNilSyncTimer
 	}
+	if check.IfNil(watchdog) {
+		return ErrNilWatchdog
+	}
 
 	return nil
 }
@@ -118,6 +129,9 @@ func (chr *chronology) RemoveAllSubrounds() {
 
 // StartRounds actually starts the chronology and calls the DoWork() method of the subroundHandlers loaded
 func (chr *chronology) StartRounds() {
+	watchdogAlarmDuration := chr.rounder.TimeDuration() * numRoundsToWaitBeforeSignalingChronologyStuck
+	chr.watchdog.SetDefault(watchdogAlarmDuration, chronologyAlarmID)
+
 	var ctx context.Context
 	ctx, chr.cancelFunc = context.WithCancel(context.Background())
 	go chr.startRounds(ctx)
@@ -169,6 +183,7 @@ func (chr *chronology) updateRound() {
 	chr.rounder.UpdateRound(chr.genesisTime, chr.syncTimer.CurrentTime())
 
 	if oldRoundIndex != chr.rounder.Index() {
+		chr.watchdog.Reset(chronologyAlarmID)
 		msg := fmt.Sprintf("ROUND %d BEGINS (%d)", chr.rounder.Index(), chr.rounder.TimeStamp().Unix())
 		log.Debug(display.Headline(msg, chr.syncTimer.FormattedCurrentTime(), "#"))
 		logger.SetCorrelationRound(chr.rounder.Index())

consensus/chronology/chronology_test.go

@@ -36,6 +36,7 @@ func TestChronology_NewChronologyNilRounderShouldFail(t *testing.T) {
 		genesisTime,
 		nil,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	assert.Nil(t, chr)
@@ -50,12 +51,28 @@ func TestChronology_NewChronologyNilSyncerShouldFail(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		nil,
+		&mock.WatchdogMock{},
 	)
 
 	assert.Nil(t, chr)
 	assert.Equal(t, err, chronology.ErrNilSyncTimer)
 }
 
+func TestChronology_NewChronologyNilWatchdogShouldFail(t *testing.T) {
+	t.Parallel()
+	rounderMock := &mock.RounderMock{}
+	genesisTime := time.Now()
+	chr, err := chronology.NewChronology(
+		genesisTime,
+		rounderMock,
+		&mock.SyncTimerMock{},
+		nil,
+	)
+
+	assert.Nil(t, chr)
+	assert.Equal(t, err, chronology.ErrNilWatchdog)
+}
+
 func TestChronology_NewChronologyShouldWork(t *testing.T) {
 	t.Parallel()
 	rounderMock := &mock.RounderMock{}
@@ -65,6 +82,7 @@ func TestChronology_NewChronologyShouldWork(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	assert.Nil(t, err)
@@ -80,6 +98,7 @@ func TestChronology_AddSubroundShouldWork(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.AddSubround(initSubroundHandlerMock())
@@ -98,6 +117,7 @@ func TestChronology_RemoveAllSubroundsShouldReturnEmptySubroundHandlersArray(t *
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.AddSubround(initSubroundHandlerMock())
@@ -124,6 +144,7 @@ func TestChronology_StartRoundShouldReturnWhenRoundIndexIsNegative(t *testing.T)
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	srm := initSubroundHandlerMock()
@@ -143,6 +164,7 @@ func TestChronology_StartRoundShouldReturnWhenLoadSubroundHandlerReturnsNil(t *t
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	initSubroundHandlerMock()
@@ -161,6 +183,7 @@ func TestChronology_StartRoundShouldReturnWhenDoWorkReturnsFalse(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	srm := initSubroundHandlerMock()
@@ -181,6 +204,7 @@ func TestChronology_StartRoundShouldWork(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	srm := initSubroundHandlerMock()
@@ -203,6 +227,7 @@ func TestChronology_UpdateRoundShouldInitRound(t *testing.T) {
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	srm := initSubroundHandlerMock()
@@ -221,6 +246,7 @@ func TestChronology_LoadSubrounderShouldReturnNilWhenSubroundHandlerNotExists(t
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	assert.Nil(t, chr.LoadSubroundHandler(0))
@@ -235,6 +261,7 @@ func TestChronology_LoadSubrounderShouldReturnNilWhenIndexIsOutOfBound(t *testin
 		genesisTime,
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.AddSubround(initSubroundHandlerMock())
@@ -251,6 +278,7 @@ func TestChronology_InitRoundShouldNotSetSubroundWhenRoundIndexIsNegative(t *tes
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.AddSubround(initSubroundHandlerMock())
@@ -274,6 +302,7 @@ func TestChronology_InitRoundShouldSetSubroundWhenRoundIndexIsPositive(t *testin
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	sr := initSubroundHandlerMock()
@@ -291,6 +320,7 @@ func TestChronology_StartRoundShouldNotUpdateRoundWhenCurrentRoundIsNotFinished(
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.SetSubroundId(0)
@@ -308,6 +338,7 @@ func TestChronology_StartRoundShouldUpdateRoundWhenCurrentRoundIsFinished(t *tes
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	chr.SetSubroundId(-1)
@@ -325,6 +356,7 @@ func TestChronology_SetAppStatusHandlerWithNilValueShouldErr(t *testing.T) {
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 	err := chr.SetAppStatusHandler(nil)
 
@@ -340,6 +372,7 @@ func TestChronology_SetAppStatusHandlerWithOkValueShouldPass(t *testing.T) {
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	err := chr.SetAppStatusHandler(&mock.AppStatusHandlerMock{})
@@ -357,6 +390,7 @@ func TestChronology_CheckIfStatusHandlerWorks(t *testing.T) {
 		syncTimerMock.CurrentTime(),
 		rounderMock,
 		syncTimerMock,
+		&mock.WatchdogMock{},
 	)
 
 	err := chr.SetAppStatusHandler(&mock.AppStatusHandlerStub{

consensus/chronology/errors.go

@@ -12,3 +12,6 @@ var ErrNilSyncTimer = errors.New("sync timer is nil")
 
 // ErrNilAppStatusHandler is raised when the AppStatusHandler is nil when setting it
 var ErrNilAppStatusHandler = errors.New("nil AppStatusHandler")
+
+// ErrNilWatchdog signals that a nil watchdog has been provided
+var ErrNilWatchdog = errors.New("nil watchdog")

consensus/mock/watchdogMock.go

@@ -0,0 +1,30 @@
+package mock
+
+import (
+	"time"
+)
+
+// WatchdogMock -
+type WatchdogMock struct {
+}
+
+// Set -
+func (w *WatchdogMock) Set(callback func(alarmID string), duration time.Duration, alarmID string) {
+}
+
+// SetDefault -
+func (w *WatchdogMock) SetDefault(duration time.Duration, alarmID string) {
+}
+
+// Stop -
+func (w *WatchdogMock) Stop(alarmID string) {
+}
+
+// Reset -
+func (w *WatchdogMock) Reset(alarmID string) {
+}
+
+// IsInterfaceNil -
+func (w *WatchdogMock) IsInterfaceNil() bool {
+	return false
+}

consensus/spos/errors.go

@@ -19,9 +19,6 @@ var ErrNilPublicKey = errors.New("public key is nil")
 // ErrNilPrivateKey is raised when a valid private key was expected but nil was used
 var ErrNilPrivateKey = errors.New("private key is nil")
 
-// ErrNilPrevRandSeed is raised when the provided previous random seed is nil
-var ErrNilPrevRandSeed = errors.New("previous random seed is nil")
-
 // ErrNilConsensusData is raised when valid consensus data was expected but nil was received
 var ErrNilConsensusData = errors.New("consensus data is nil")
 
@@ -151,9 +148,6 @@ var ErrNilHeaderHash = errors.New("header hash is nil")
 // ErrNilBody is raised when an expected body is nil
 var ErrNilBody = errors.New("body is nil")
 
-// ErrInvalidDataToBroadcast is raised when invalid data is set to be broadcast
-var ErrInvalidDataToBroadcast = errors.New("data to broadcast is invalid")
-
 // ErrNilMetaHeader is raised when an expected meta header is nil
 var ErrNilMetaHeader = errors.New("meta header is nil")
 

consensus/spos/sposFactory/sposFactory.go

@@ -27,7 +27,13 @@ func GetSubroundsFactory(
 ) (spos.SubroundsFactory, error) {
 	switch consensusType {
 	case blsConsensusType:
-		subRoundFactoryBls, err := bls.NewSubroundsFactory(consensusDataContainer, consensusState, worker, chainID, currentPid)
+		subRoundFactoryBls, err := bls.NewSubroundsFactory(
+			consensusDataContainer,
+			consensusState,
+			worker,
+			chainID,
+			currentPid,
+		)
 		if err != nil {
 			return nil, err
 		}

core/alarm/alarm.go

@@ -173,6 +173,29 @@ func (as *alarmScheduler) Close() {
 	as.cancelFunc()
 }
 
+// Reset resets the alarm with the given id
+func (as *alarmScheduler) Reset(alarmID string) {
+	as.mutScheduledAlarms.RLock()
+	alarm, ok := as.scheduledAlarms[alarmID]
+	callback := alarm.callback
+	duration := alarm.initialDuration
+	as.mutScheduledAlarms.RUnlock()
+
+	if !ok {
+		return
+	}
+
+	evt := alarmEvent{
+		alarmID: alarmID,
+		alarm:   nil,
+		event:   cancel,
+	}
+
+	as.event <- evt
+
+	as.Add(callback, duration, alarmID)
+}
+
 // IsInterfaceNil returns true if interface is nil
 func (as *alarmScheduler) IsInterfaceNil() bool {
 	return as == nil