Expose better logs for dynamic config conversion errors (#4282)

common/dynamicconfig/collection.go

@@ -105,14 +105,12 @@ func NewCollection(client Client, logger log.Logger) *Collection {
 	}
 }
 
-func (c *Collection) logError(key Key, err error) {
+func (c *Collection) throttleLog() bool {
 	// TODO: This is a lot of unnecessary contention with little benefit. Consider using
 	// https://github.com/cespare/percpu here.
 	errCount := atomic.AddInt64(&c.errCount, 1)
-	if errCount%errCountLogThreshold == 0 {
-		// log only every 'x' errors to reduce mem allocs and to avoid log noise
-		c.logger.Debug("No such key in dynamic config, using default", tag.Key(key.String()), tag.Error(err))
-	}
+	// log only the first x errors and then one every x after that to reduce log noise
+	return errCount < errCountLogThreshold || errCount%errCountLogThreshold == 0
 }
 
 // GetIntProperty gets property and asserts that it's an integer
@@ -451,7 +449,9 @@ func matchAndConvert[T any](
 
 	val, matchErr := findMatch(cvs, defaultCVs, precedence)
 	if matchErr != nil {
-		c.logError(key, matchErr)
+		if c.throttleLog() {
+			c.logger.Debug("No such key in dynamic config, using default", tag.Key(key.String()), tag.Error(matchErr))
+		}
 		// couldn't find a constrained match, use default
 		val = defaultValue
 	}
@@ -460,12 +460,14 @@ func matchAndConvert[T any](
 	if convertErr != nil && matchErr == nil {
 		// We failed to convert the value to the desired type. Try converting the default. note
 		// that if matchErr != nil then val _is_ defaultValue and we don't have to try this again.
-		c.logError(key, convertErr)
+		if c.throttleLog() {
+			c.logger.Warn("Failed to convert value, using default", tag.Key(key.String()), tag.IgnoredValue(val), tag.Error(convertErr))
+		}
 		typedVal, convertErr = converter(defaultValue)
 	}
 	if convertErr != nil {
 		// If we can't convert the default, that's a bug in our code, use Warn level.
-		c.logger.Warn("can't convert default value (this is a bug; fix server code)", tag.Key(key.String()), tag.Error(matchErr))
+		c.logger.Warn("Can't convert default value (this is a bug; fix server code)", tag.Key(key.String()), tag.IgnoredValue(defaultValue), tag.Error(convertErr))
 		// Return typedVal anyway since we have to return something.
 	}
 	return typedVal