perf: adds another direct upstream to improve performance when scrape…

…r is overloaded (#636)

pkg/cli/server.go

@@ -36,18 +36,19 @@ type Server struct {
 }
 
 type serverService struct {
-	config           *config.Server
-	logger           *logrus.Logger
-	controller       *server.Controller
-	storage          *storage.Storage
-	directUpstream   *direct.Direct
-	analyticsService *analytics.Service
-	selfProfiling    *agent.ProfileSession
-	debugReporter    *debug.Reporter
-	healthController *health.Controller
-	adminServer      *admin.Server
-	discoveryManager *discovery.Manager
-	scrapeManager    *scrape.Manager
+	config               *config.Server
+	logger               *logrus.Logger
+	controller           *server.Controller
+	storage              *storage.Storage
+	directUpstream       *direct.Direct
+	directScrapeUpstream *direct.Direct
+	analyticsService     *analytics.Service
+	selfProfiling        *agent.ProfileSession
+	debugReporter        *debug.Reporter
+	healthController     *health.Controller
+	adminServer          *admin.Server
+	discoveryManager     *discovery.Manager
+	scrapeManager        *scrape.Manager
 
 	stopped chan struct{}
 	done    chan struct{}
@@ -124,6 +125,7 @@ func newServerService(c *config.Server) (*serverService, error) {
 	svc.healthController = health.NewController(svc.logger, time.Minute, diskPressure)
 	svc.debugReporter = debug.NewReporter(svc.logger, svc.storage, prometheus.DefaultRegisterer)
 	svc.directUpstream = direct.New(svc.storage, metricsExporter)
+	svc.directScrapeUpstream = direct.New(svc.storage, metricsExporter)
 
 	if !svc.config.NoSelfProfiling {
 		svc.selfProfiling, _ = agent.NewSession(agent.SessionConfig{
@@ -155,7 +157,7 @@ func newServerService(c *config.Server) (*serverService, error) {
 
 	svc.scrapeManager = scrape.NewManager(
 		svc.logger.WithField("component", "scrape-manager"),
-		svc.directUpstream)
+		svc.directScrapeUpstream)
 
 	if !c.AnalyticsOptOut {
 		svc.analyticsService = analytics.NewService(c, svc.storage, svc.controller)
@@ -187,6 +189,7 @@ func (svc *serverService) Start() error {
 
 	svc.healthController.Start()
 	svc.directUpstream.Start()
+	svc.directScrapeUpstream.Start()
 
 	if !svc.config.NoSelfProfiling {
 		if err := svc.selfProfiling.Start(); err != nil {
@@ -255,6 +258,7 @@ func (svc *serverService) stop() {
 
 	svc.logger.Debug("stopping upstream")
 	svc.directUpstream.Stop()
+	svc.directScrapeUpstream.Stop()
 	svc.logger.Debug("stopping storage")
 	if err := svc.storage.Close(); err != nil {
 		svc.logger.WithError(err).Error("storage close")