Spread probes more evenly across time.

Spread probes more evenly across time. This behavior can be disabled by a config option (disable_jitter) if required.

Impact of this change is huge: http://screen/ThmBPS658nt
Another VM: http://screen/iU5kpL7FAoP

PiperOrigin-RevId: 239342903

cloudprober.go

@@ -24,6 +24,7 @@ package cloudprober
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -66,6 +67,7 @@ type Prober struct {
 	Probes         map[string]*probes.ProbeInfo
 	Servers        []*servers.ServerInfo
 	c              *configpb.ProberConfig
+	l              *logger.Logger
 	rdsServer      *rdsserver.Server
 	rtcReporter    *rtcreporter.Reporter
 	surfacers      []*surfacers.SurfacerInfo
@@ -147,7 +149,8 @@ func (pr *Prober) init() error {
 
 	// Create a global logger. Each component gets its own logger on successful
 	// creation. For everything else, we use a global logger.
-	globalLogger, err := logger.NewCloudproberLog("global")
+	var err error
+	pr.l, err = logger.NewCloudproberLog("global")
 	if err != nil {
 		return fmt.Errorf("error in initializing global logger: %v", err)
 	}
@@ -167,7 +170,7 @@ func (pr *Prober) init() error {
 	}
 
 	// Initiliaze probes
-	pr.Probes, err = probes.Init(pr.c.GetProbe(), globalTargetsOpts, globalLogger, sysvars.Vars())
+	pr.Probes, err = probes.Init(pr.c.GetProbe(), globalTargetsOpts, pr.l, sysvars.Vars())
 	if err != nil {
 		return err
 	}
@@ -275,9 +278,46 @@ func (pr *Prober) start(ctx context.Context) {
 		go pr.rtcReporter.Start(ctx)
 	}
 
-	// Start probes, each in its own goroutines
+	if pr.c.GetDisableJitter() {
+		for _, p := range pr.Probes {
+			go p.Start(ctx, dataChan)
+		}
+		return
+	}
+	pr.startProbesWithJitter(ctx, dataChan)
+}
+
+// startProbesWithJitter try to space out probes over time, as much as possible,
+// without making it too complicated. We arrange probes into interval buckets -
+// all probes with the same interval will be part of the same bucket, and we
+// then spread out probes within that interval by introducing a delay of
+// interval / len(probes) between probes. We also introduce a random jitter
+// between different interval buckets.
+func (pr *Prober) startProbesWithJitter(ctx context.Context, dataChan chan *metrics.EventMetrics) {
+	// Seed random number generator.
+	rand.Seed(time.Now().UnixNano())
+
+	// Make interval -> [probe1, probe2, probe3..] map
+	intervalBuckets := make(map[time.Duration][]*probes.ProbeInfo)
 	for _, p := range pr.Probes {
-		go p.Start(ctx, dataChan)
+		intervalBuckets[p.Options.Interval] = append(intervalBuckets[p.Options.Interval], p)
+	}
+
+	for interval, probeInfos := range intervalBuckets {
+		go func(interval time.Duration, probeInfos []*probes.ProbeInfo) {
+			// Introduce a random jitter between interval buckets.
+			randomDelayMsec := rand.Int63n(int64(interval.Seconds() * 1000))
+			time.Sleep(time.Duration(randomDelayMsec) * time.Millisecond)
+
+			interProbeDelay := interval / time.Duration(len(probeInfos))
+
+			// Spread out probes evenly with an interval bucket.
+			for _, p := range probeInfos {
+				pr.l.Info("Starting probe: ", p.Name)
+				go p.Start(ctx, dataChan)
+				time.Sleep(interProbeDelay)
+			}
+		}(interval, probeInfos)
 	}
 }
 

config/proto/config.proto

@@ -45,6 +45,11 @@ message ProberConfig {
   // variable CLOUDPROBER_HOST.
   optional string host = 101;
 
+  // Probes are staggered across time to avoid executing all of them at the
+  // same time. This behavior can be disabled by setting the following option
+  // to true.
+  optional bool disable_jitter = 102 [default = false];
+
   // How often to export system variables. To learn more about system variables:
   // http://godoc.org/github.com/google/cloudprober/sysvars.
   optional int32 sysvars_interval_msec = 97 [default = 10000];

probes/probes.go

@@ -75,6 +75,7 @@ type Probe interface {
 // ProbeInfo encapsulates the probe and associated information.
 type ProbeInfo struct {
 	Probe
+	Options       *options.Options
 	Name          string
 	Type          string
 	Interval      string
@@ -150,6 +151,7 @@ func Init(probeProtobufs []*configpb.ProbeDef, globalTargetsOpts *targetspb.Glob
 
 		probeInfo := &ProbeInfo{
 			Probe:       probe,
+			Options:     opts,
 			Name:        p.GetName(),
 			Type:        p.GetType().String(),
 			Interval:    opts.Interval.String(),