Ignore/truncate metric messages that are bigger than a certain thresh…

…old to protect against clogging all of cloudprober due to a bug in one of the modules.

ORIGINAL_AUTHOR=Clyde-Xu <clydexu@google.com>
PiperOrigin-RevId: 164014062

cloudprober.go

@@ -30,6 +30,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/google/cloudprober/config"
 	"github.com/google/cloudprober/logger"
 	"github.com/google/cloudprober/metrics"
@@ -154,6 +155,12 @@ func (pr *Prober) Start(ctx context.Context) {
 		var em *metrics.EventMetrics
 		for {
 			em = <-dataChan
+			var s = em.String()
+			if len(s) > logger.MaxLogEntrySize {
+				glog.Warningf("Metric entry for timestamp %d dropped due to large size: %d", em.Timestamp, len(s))
+				continue
+			}
+
 			// Replicate the surfacer message to every surfacer we have
 			// registered. Note that s.Write() is expected to be
 			// non-blocking to avoid blocking of EventMetrics message

logger/logger.go

@@ -42,6 +42,9 @@ const (
 	// Regular Expression for all characters that are illegal for log names
 	//	Ref: https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/LogEntry
 	disapprovedRegExp = "[^A-Za-z0-9_/.-]"
+
+	// MaxLogEntrySize Max value of each log entry size
+	MaxLogEntrySize = 4096
 )
 
 // Logger implements a logger that logs messages to Google Cloud Logging. It provides a suite
@@ -144,8 +147,15 @@ func payloadToString(payload interface{}) string {
 // client is not initialized (e.g. if not running on GCE) or cloud logging fails for some reason,
 // it writes logs through the traditional logger.
 func (l *Logger) log(severity logging.Severity, payload interface{}) {
+	textPayload := payloadToString(payload)
+	if len(textPayload) > MaxLogEntrySize {
+		truncateMsg := "... (truncated)"
+		truncateMsgLen := len(truncateMsg)
+		textPayload = textPayload[:MaxLogEntrySize-truncateMsgLen] + truncateMsg
+		payload = textPayload
+	}
 	if l.logc == nil {
-		genericLog(2, severity, payloadToString(payload))
+		genericLog(2, severity, textPayload)
 		return
 	}
 	l.logger.Log(logging.Entry{