Skip to content

Commit

Permalink
Only print log once, 2 minutes after startup
Browse files Browse the repository at this point in the history
We don't want to spam users' logs with this message, so this just
waits 2 minutes and prints the log message once (if there are any
throttles).
  • Loading branch information
guseggert committed May 18, 2022
1 parent a7fdd7a commit 7db95ce
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 26 deletions.
40 changes: 19 additions & 21 deletions core/node/libp2p/rcmgr_logging.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ import (
)

type loggingResourceManager struct {
clock clock.Clock
logger *zap.SugaredLogger
delegate network.ResourceManager
logInterval time.Duration
clock clock.Clock
logger *zap.SugaredLogger
delegate network.ResourceManager
logAfter time.Duration

mut sync.Mutex
limitExceededErrs uint64
Expand All @@ -33,26 +33,24 @@ type loggingScope struct {
var _ network.ResourceManager = (*loggingResourceManager)(nil)

func (n *loggingResourceManager) start(ctx context.Context) {
logInterval := n.logInterval
if logInterval == 0 {
logInterval = 10 * time.Second
logAfter := n.logAfter
if logAfter == 0 {
logAfter = 2 * time.Minute
}
ticker := n.clock.Ticker(logInterval)
timer := n.clock.Timer(logAfter)
go func() {
defer ticker.Stop()
for {
select {
case <-ticker.C:
n.mut.Lock()
errs := n.limitExceededErrs
n.limitExceededErrs = 0
n.mut.Unlock()
if errs != 0 {
n.logger.Errorf("Resource limits were exceeded %d times, consider inspecting logs and raising the resource manager limits.", errs)
}
case <-ctx.Done():
return
defer timer.Stop()
select {
case <-timer.C:
n.mut.Lock()
errs := n.limitExceededErrs
n.limitExceededErrs = 0
n.mut.Unlock()
if errs != 0 {
n.logger.Errorf("Resource limits were exceeded %d times shortly after startup, consider inspecting logs and raising the resource manager limits.", errs)
}
case <-ctx.Done():
return
}
}()
}
Expand Down
10 changes: 5 additions & 5 deletions core/node/libp2p/rcmgr_logging_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ func TestLoggingResourceManager(t *testing.T) {
oCore, oLogs := observer.New(zap.ErrorLevel)
oLogger := zap.New(oCore)
lrm := &loggingResourceManager{
clock: clock,
logger: oLogger.Sugar(),
delegate: rm,
logInterval: 1 * time.Second,
clock: clock,
logger: oLogger.Sugar(),
delegate: rm,
logAfter: 1 * time.Second,
}

// 2 of these should result in resource limit exceeded errors and subsequent log messages
Expand All @@ -51,7 +51,7 @@ func TestLoggingResourceManager(t *testing.T) {
if oLogs.Len() == 0 {
continue
}
require.Equal(t, "Resource limits were exceeded 2 times, consider inspecting logs and raising the resource manager limits.", oLogs.All()[0].Message)
require.Equal(t, "Resource limits were exceeded 2 times shortly after startup, consider inspecting logs and raising the resource manager limits.", oLogs.All()[0].Message)
return
}
}
Expand Down

0 comments on commit 7db95ce

Please sign in to comment.