Log warning when request counters exceed a threshold

legacy/reqcounter/reqcounter.go

@@ -32,6 +32,7 @@ type RequestCounter struct {
 	Since      time.Time     // When the current request counter began recording requests.
 	Interval   time.Duration // The duration of time between each log.
 	Resettable bool          // Reset after logging.
+	Threshold  uint64        // When to warn of a high request count (only for resettable request counters).
 }
 
 // increment adds 1 to the request counter, signifying another call to GCR.
@@ -47,16 +48,25 @@ func (rc *RequestCounter) Flush() {
 	rc.Mutex.Lock()
 	defer rc.Mutex.Unlock()
 
-	// Log the number of requests within this measurement window.
-	msg := fmt.Sprintf("From %s to %s [%d min] there have been %d requests to GCR.", rc.Since.Format(TimestampFormat), Clock.Now().Format(TimestampFormat), rc.Interval/time.Minute, rc.Requests)
-	Debug(msg)
+	rc.log()
 
 	if rc.Resettable {
 		// Reset the request counter.
 		rc.reset()
 	}
 }
 
+// log the number of HTTP requests found. If the number of requests exceeds the
+// threshold, an additional warning message will be logged.
+func (rc *RequestCounter) log() {
+	msg := fmt.Sprintf("From %s to %s [%d min] there have been %d requests to GCR.", rc.Since.Format(TimestampFormat), Clock.Now().Format(TimestampFormat), rc.Interval/time.Minute, rc.Requests)
+	Debug(msg)
+	if rc.Requests > rc.Threshold && rc.Resettable {
+		msg = fmt.Sprintf("The threshold of %d requests has been surpassed.", rc.Threshold)
+		Warn(msg)
+	}
+}
+
 // reset clears the request counter and stamps the current time of reset.
 func (rc *RequestCounter) reset() {
 	rc.Requests = 0
@@ -124,6 +134,8 @@ var (
 	NetMonitor *NetworkMonitor
 	// Debug is defined to simplify testing of logrus.Debug calls.
 	Debug func(args ...interface{}) = logrus.Debug
+	// Warn is defined to simplify testing of logrus.Warn calls.
+	Warn func(args ...interface{}) = logrus.Warn
 	// Clock is defined to allow mocking of time functions.
 	Clock tw.Time = tw.RealTime{}
 )
@@ -141,13 +153,15 @@ func Init() {
 			Since:      Clock.Now(),
 			Interval:   QuotaWindowShort,
 			Resettable: true,
+			Threshold:  50000,
 		},
 		{
 			Mutex:      sync.Mutex{},
 			Requests:   0,
 			Since:      Clock.Now(),
 			Interval:   QuotaWindowLong,
 			Resettable: true,
+			Threshold:  1000000,
 		},
 		{
 			Mutex:      sync.Mutex{},

legacy/reqcounter/reqcounter_test.go

@@ -125,6 +125,31 @@ func TestFlush(t *testing.T) {
 	// Ensure the request counter did not reset.
 	require.Equal(t, uint64(66), requestCounter.Requests, "Calling Flush() reset the requests of a non-resettable request counter.")
 	require.True(t, defaultTime.Equal(requestCounter.Since), "Calling Flush() reset the timestamp of a non-resettable request counter.")
+
+	// Create a request counter that exceeded a threshold.
+	requestCounter = NewRequestCounter(600)
+	requestCounter.Threshold = 599
+	// Reset debug counter.
+	debugCalls = 0
+	// Mock logrus.Warn
+	warnCalls := 0
+	rc.Warn = func(args ...interface{}) {
+		warnCalls++
+	}
+	requestCounter.Flush()
+	// Ensure both logrus.Debug and logrus.Warn was called.
+	require.Equal(t, 1, debugCalls, "Flush() failed to trigger a debug statement after exceeding it's threshold.")
+	require.Equal(t, 1, warnCalls, "Flush() failed to trigger a warning statement after exceeding it's threshold.")
+
+	// Create a non-resetting request counter that exceeded a threshold.
+	requestCounter = NewRequestCounter(600)
+	requestCounter.Threshold = 599
+	requestCounter.Resettable = false
+	// Reset warn counter.
+	warnCalls = 0
+	requestCounter.Flush()
+	// Ensure non-resetting counters never warn on exceeding a threshold.
+	require.Equal(t, 0, warnCalls, "Flush() triggered a warning for exceeding the threshold of a non-resettable request counter.")
 }
 
 func TestRequestCounterIncrement(t *testing.T) {