prober: Remove spam of changing probe status

CHANGELOG.md

@@ -24,6 +24,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Fixed
 
+- [#5051](https://github.com/thanos-io/thanos/pull/5051) Prober: Remove spam of changing probe status.
 - [#4918](https://github.com/thanos-io/thanos/pull/4918) Tracing: Fixing force tracing with Jaeger.
 - [#4879](https://github.com/thanos-io/thanos/pull/4879) Bucket verify: Fixed bug causing wrong number of blocks to be checked.
 - [#4908](https://github.com/thanos-io/thanos/pull/4908) UI: Show 'minus' icon and add tooltip when store min / max time is not available.

pkg/prober/intrumentation.go

@@ -4,6 +4,8 @@
 package prober
 
 import (
+	"sync"
+
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -13,8 +15,9 @@ import (
 )
 
 const (
-	ready   = "ready"
-	healthy = "healthy"
+	ready    = "ready"
+	notReady = "not-ready"
+	healthy  = "healthy"
 )
 
 // InstrumentationProbe stores instrumentation state of Probe.
@@ -23,15 +26,17 @@ type InstrumentationProbe struct {
 	component component.Component
 	logger    log.Logger
 
-	status *prometheus.GaugeVec
+	statusMetric *prometheus.GaugeVec
+	mu           sync.Mutex
+	statusString string
 }
 
 // NewInstrumentation returns InstrumentationProbe records readiness and healthiness for given component.
 func NewInstrumentation(component component.Component, logger log.Logger, reg prometheus.Registerer) *InstrumentationProbe {
 	p := InstrumentationProbe{
 		component: component,
 		logger:    logger,
-		status: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+		statusMetric: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
 			Name:        "status",
 			Help:        "Represents status (0 indicates failure, 1 indicates success) of the component.",
 			ConstLabels: map[string]string{"component": component.String()},
@@ -44,24 +49,34 @@ func NewInstrumentation(component component.Component, logger log.Logger, reg pr
 
 // Ready records the component status when Ready is called, if combined with other Probes.
 func (p *InstrumentationProbe) Ready() {
-	p.status.WithLabelValues(ready).Set(1)
-	level.Info(p.logger).Log("msg", "changing probe status", "status", "ready")
+	p.statusMetric.WithLabelValues(ready).Set(1)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.statusString != ready {
+		level.Info(p.logger).Log("msg", "changing probe status", "status", ready)
+		p.statusString = ready
+	}
 }
 
 // NotReady records the component status when NotReady is called, if combined with other Probes.
 func (p *InstrumentationProbe) NotReady(err error) {
-	p.status.WithLabelValues(ready).Set(0)
-	level.Warn(p.logger).Log("msg", "changing probe status", "status", "not-ready", "reason", err)
+	p.statusMetric.WithLabelValues(ready).Set(0)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.statusString != notReady {
+		level.Warn(p.logger).Log("msg", "changing probe status", "status", notReady, "reason", err)
+		p.statusString = notReady
+	}
 }
 
 // Healthy records the component status when Healthy is called, if combined with other Probes.
 func (p *InstrumentationProbe) Healthy() {
-	p.status.WithLabelValues(healthy).Set(1)
+	p.statusMetric.WithLabelValues(healthy).Set(1)
 	level.Info(p.logger).Log("msg", "changing probe status", "status", "healthy")
 }
 
 // NotHealthy records the component status when NotHealthy is called, if combined with other Probes.
 func (p *InstrumentationProbe) NotHealthy(err error) {
-	p.status.WithLabelValues(healthy).Set(0)
+	p.statusMetric.WithLabelValues(healthy).Set(0)
 	level.Info(p.logger).Log("msg", "changing probe status", "status", "not-healthy", "reason", err)
 }