Add method to change log level at runtime

In order to avoid having to restart the agent just to change the log
level, add a method (POST to /logger with "debug" or "default") to
change the level at runtime.

Signed-off-by: Marcelo E. Magallon <marcelo.magallon@grafana.com>

cmd/synthetic-monitoring-agent/http.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -26,9 +28,10 @@ type MuxOpts struct {
 		prometheus.Registerer
 		prometheus.Gatherer
 	}
-	isReady           *readynessHandler
-	disconnectEnabled bool
-	pprofEnabled      bool
+	isReady               *readynessHandler
+	changeLogLevelEnabled bool
+	disconnectEnabled     bool
+	pprofEnabled          bool
 }
 
 func NewMux(opts MuxOpts) *Mux {
@@ -49,6 +52,10 @@ func NewMux(opts MuxOpts) *Mux {
 	isReady.Store(false)
 	router.Handle("/ready", opts.isReady)
 
+	if opts.changeLogLevelEnabled {
+		router.Handle("/logger", loggerHandler(opts.Logger))
+	}
+
 	// disconnect this agent from the API
 	if opts.disconnectEnabled {
 		router.Handle("/disconnect", http.HandlerFunc(disconnectHandler))
@@ -187,3 +194,38 @@ func disconnectHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	fmt.Fprintln(w, "disconnecting this agent from the API for 1 minute.")
 }
+
+func loggerHandler(logger zerolog.Logger) http.Handler {
+	defaultLevel := logger.GetLevel()
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+			return
+		}
+
+		level, err := io.ReadAll(&io.LimitedReader{R: r.Body, N: 10})
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Consume the rest of the request if there's anything there.
+		_, _ = io.Copy(io.Discard, r.Body)
+
+		_ = r.Body.Close()
+
+		switch strings.TrimSpace(strings.ToLower(string(level))) {
+		case "debug":
+			zerolog.SetGlobalLevel(zerolog.DebugLevel)
+			logger.Warn().Stringer("level", zerolog.DebugLevel).Msg("changed log level")
+
+		case "default":
+			zerolog.SetGlobalLevel(defaultLevel)
+			logger.Warn().Stringer("level", defaultLevel).Msg("changed log level")
+
+		default:
+			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		}
+	})
+}

cmd/synthetic-monitoring-agent/main.go

@@ -36,17 +36,19 @@ func run(args []string, stdout io.Writer) error {
 	flags := flag.NewFlagSet(filepath.Base(args[0]), flag.ExitOnError)
 
 	var (
-		features          = feature.NewCollection()
-		debug             = flags.Bool("debug", false, "debug output (enables verbose)")
-		verbose           = flags.Bool("verbose", false, "verbose logging")
-		reportVersion     = flags.Bool("version", false, "report version and exit")
-		grpcApiServerAddr = flags.String("api-server-address", "localhost:4031", "GRPC API server address")
-		grpcInsecure      = flags.Bool("api-insecure", false, "Don't use TLS with connections to GRPC API")
-		apiToken          = flags.String("api-token", "", "synthetic monitoring probe authentication token")
-		enableDisconnect  = flags.Bool("enable-disconnect", false, "enable HTTP /disconnect endpoint")
-		enablePProf       = flags.Bool("enable-pprof", false, "exposes profiling data via HTTP /debug/pprof/ endpoint")
-		httpListenAddr    = flags.String("listen-address", "localhost:4050", "listen address")
-		k6URI             = flags.String("k6-uri", "k6", "how to run k6 (path or URL)")
+		features             = feature.NewCollection()
+		devMode              = flags.Bool("dev", false, "turn on all development flags")
+		debug                = flags.Bool("debug", false, "debug output (enables verbose)")
+		verbose              = flags.Bool("verbose", false, "verbose logging")
+		reportVersion        = flags.Bool("version", false, "report version and exit")
+		grpcApiServerAddr    = flags.String("api-server-address", "localhost:4031", "GRPC API server address")
+		grpcInsecure         = flags.Bool("api-insecure", false, "Don't use TLS with connections to GRPC API")
+		apiToken             = flags.String("api-token", "", "synthetic monitoring probe authentication token")
+		enableChangeLogLevel = flags.Bool("enable-change-log-level", false, "enable changing the log level at runtime")
+		enableDisconnect     = flags.Bool("enable-disconnect", false, "enable HTTP /disconnect endpoint")
+		enablePProf          = flags.Bool("enable-pprof", false, "exposes profiling data via HTTP /debug/pprof/ endpoint")
+		httpListenAddr       = flags.String("listen-address", "localhost:4050", "listen address")
+		k6URI                = flags.String("k6-uri", "k6", "how to run k6 (path or URL)")
 	)
 
 	flags.Var(&features, "features", "optional feature flags")
@@ -66,6 +68,13 @@ func run(args []string, stdout io.Writer) error {
 		return nil
 	}
 
+	if *devMode {
+		*debug = true
+		*enableChangeLogLevel = true
+		*enableDisconnect = true
+		*enablePProf = true
+	}
+
 	// If the token is provided on the command line, prefer that. Otherwise
 	// pull it from the environment variable SM_AGENT_API_TOKEN. If that's
 	// not available, fallback to API_TOKEN, which was the environment
@@ -111,6 +120,9 @@ func run(args []string, stdout io.Writer) error {
 		Str("commit", version.Commit()).
 		Str("buildstamp", version.Buildstamp()).
 		Str("features", features.String()).
+		Bool("change-log-level-enabled", *enableChangeLogLevel).
+		Bool("disconnect-enabled", *enableDisconnect).
+		Bool("pprof-enabled", *enablePProf).
 		Msg("starting")
 
 	if features.IsSet(feature.K6) {
@@ -141,11 +153,12 @@ func run(args []string, stdout io.Writer) error {
 	readynessHandler := NewReadynessHandler()
 
 	router := NewMux(MuxOpts{
-		Logger:            zl.With().Str("subsystem", "mux").Logger(),
-		PromRegisterer:    promRegisterer,
-		isReady:           readynessHandler,
-		disconnectEnabled: *enableDisconnect,
-		pprofEnabled:      *enablePProf,
+		Logger:                zl.With().Str("subsystem", "mux").Logger(),
+		PromRegisterer:        promRegisterer,
+		isReady:               readynessHandler,
+		changeLogLevelEnabled: *enableChangeLogLevel,
+		disconnectEnabled:     *enableDisconnect,
+		pprofEnabled:          *enablePProf,
 	})
 
 	httpConfig := http.Config{