serverboot: dynamic --log-level flag for the server binaries

cmd/ateapi/main.go

@@ -83,6 +83,7 @@ var (
 	drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.")
 
 	showVersion     = pflag.Bool("version", false, "Print version and exit.")
+	logLevelFlag    = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 	clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.")
 )
 
@@ -94,6 +95,9 @@ func main() {
 	}
 	ctx := context.Background()
 	serverboot.InitLogger()
+	if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
+		serverboot.Fatal(ctx, "Invalid --log-level", err)
+	}
 
 	// Kept separate from ctx so that in-progress work (clients, informers) is
 	// not cancelled the moment SIGTERM arrives. The drainOnShutdown

cmd/atelet/main.go

@@ -76,7 +76,8 @@ var (
 	localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000")
 	imageCacheDir                = pflag.String("image-cache-dir", ateompath.ImageCacheDir, "Directory for the node-local OCI image layer cache. Must be on the volume shared with the ateom pods (the cached layers are their overlay lowerdirs), and on a disk sized for both capacity and IOPS: unpack throughput is gated by the volume's IOPS.")
 
-	showVersion = pflag.Bool("version", false, "Print version and exit.")
+	showVersion  = pflag.Bool("version", false, "Print version and exit.")
+	logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 )
 
 func main() {
@@ -87,6 +88,9 @@ func main() {
 	}
 	ctx := context.Background()
 	serverboot.InitLogger()
+	if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
+		serverboot.Fatal(ctx, "Invalid --log-level", err)
+	}
 
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
 		ServiceName: "atelet",

cmd/atenet/internal/dns.go

@@ -20,10 +20,10 @@ import (
 	"log/slog"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
+	"github.com/agent-substrate/substrate/internal/serverboot"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -49,18 +49,10 @@ func NewDnsCmd() *cobra.Command {
 			ctx, cancel := context.WithCancel(cmd.Context())
 			defer cancel()
 
-			var level slog.Level
-			switch strings.ToLower(cfg.LogLevel) {
-			case "debug":
-				level = slog.LevelDebug
-			case "warn":
-				level = slog.LevelWarn
-			case "error":
-				level = slog.LevelError
-			default:
-				level = slog.LevelInfo
+			serverboot.InitLogger()
+			if err := serverboot.SetLogLevel(cfg.LogLevel); err != nil {
+				return err
 			}
-			slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})))
 
 			sigChan := make(chan os.Signal, 1)
 			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

cmd/atenet/internal/router/router.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -136,18 +135,10 @@ func (s *RouterServer) Run(ctx context.Context) error {
 	}
 	parkCfg := s.cfg.ParkedRequest.normalized()
 
-	var level slog.Level
-	switch strings.ToLower(s.cfg.LogLevel) {
-	case "debug":
-		level = slog.LevelDebug
-	case "warn":
-		level = slog.LevelWarn
-	case "error":
-		level = slog.LevelError
-	default:
-		level = slog.LevelInfo
+	serverboot.InitLogger()
+	if err := serverboot.SetLogLevel(s.cfg.LogLevel); err != nil {
+		return err
 	}
-	slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})))
 
 	// Tracing must be initialized before constructing the ateapi gRPC client
 	// below, because otelgrpc.NewClientHandler captures the global

cmd/ateom-gvisor/main.go

@@ -62,7 +62,8 @@ var (
 	atunnelEgressListenAddress = pflag.String("atunnel-egress-listen-address", "0.0.0.0:15001", "Address for transparently intercepted actor egress TCP")
 	atunnelEgressTrustBundle   = pflag.String("atunnel-egress-trust-bundle", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle for the egress gateway")
 
-	showVersion = pflag.Bool("version", false, "Print version and exit.")
+	showVersion  = pflag.Bool("version", false, "Print version and exit.")
+	logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 
 	reapLock sync.RWMutex
 )
@@ -91,8 +92,11 @@ func do(ctx context.Context) error {
 	defer cancel()
 
 	syncedWriter := actorlog.NewSyncedWriter(os.Stdout)
-	logger := slog.New(contextlogging.NewHandler(slog.NewJSONHandler(syncedWriter, nil)))
+	logger := slog.New(contextlogging.NewHandler(slog.NewJSONHandler(syncedWriter, &slog.HandlerOptions{Level: serverboot.LogLevel()})))
 	slog.SetDefault(logger)
+	if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
+		return err
+	}
 
 	slog.InfoContext(ctx, "ateom booting")
 

cmd/ateom-microvm/main.go

@@ -53,11 +53,12 @@ import (
 )
 
 var (
-	podUID      = flag.String("pod-uid", "", "The UID of the current pod")
-	chBinary    = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).")
-	kataConfig  = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.")
-	kataDebug   = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.")
-	showVersion = flag.Bool("version", false, "Print version and exit.")
+	podUID       = flag.String("pod-uid", "", "The UID of the current pod")
+	chBinary     = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).")
+	kataConfig   = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.")
+	kataDebug    = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.")
+	showVersion  = flag.Bool("version", false, "Print version and exit.")
+	logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 
 	atunnelListenAddress       = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS")
 	atunnelCredentialBundle    = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle for actor ingress HTTPS")
@@ -94,6 +95,9 @@ func do(ctx context.Context) error {
 	// interleave-corrupt each other's lines.
 	logWriter := actorlog.NewSyncedWriter(os.Stdout)
 	serverboot.InitLoggerWithWriter(logWriter)
+	if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
+		return err
+	}
 	slog.InfoContext(ctx, "ateom-microvm booting", slog.String("version", version.String()))
 
 	const serviceName = "ateom-microvm"

internal/serverboot/serverboot.go

@@ -52,9 +52,32 @@ func InitLogger() {
 // one synchronized writer between the runtime logger and a separate writer (e.g.
 // ateom's actor-log forwarder) so their lines don't interleave.
 func InitLoggerWithWriter(w io.Writer) {
-	slog.SetDefault(slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))))
+	slog.SetDefault(slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, &slog.HandlerOptions{Level: &logLevel}))))
 }
 
+// logLevel is the dynamic minimum level behind the serverboot loggers.
+// A LevelVar so SetLogLevel works before or after InitLogger.
+var logLevel slog.LevelVar
+
+// SetLogLevel sets the minimum level of the serverboot loggers from a flag
+// value: "debug", "info", "warn", or "error" (case-insensitive). Empty
+// means unset and leaves the current level unchanged, so configs that
+// never populate the field keep the default.
+func SetLogLevel(level string) error {
+	if level == "" {
+		return nil
+	}
+	if err := logLevel.UnmarshalText([]byte(level)); err != nil {
+		return fmt.Errorf("invalid log level %q (want debug, info, warn, or error): %w", level, err)
+	}
+	return nil
+}
+
+// LogLevel exposes the level behind the serverboot loggers, for binaries
+// that build their own handler but should still honor --log-level. A
+// Leveler (not the LevelVar) so SetLogLevel stays the only mutation path.
+func LogLevel() slog.Leveler { return &logLevel }
+
 // serviceInstanceID is generated once so the tracer and meter resources share it.
 var serviceInstanceID = uuid.NewString()
 

internal/serverboot/serverboot_test.go

@@ -15,7 +15,9 @@
 package serverboot
 
 import (
+	"bytes"
 	"context"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -156,3 +158,57 @@ func getCode(t *testing.T, mux *http.ServeMux, path string) int {
 	mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
 	return rec.Code
 }
+
+func TestSetLogLevel(t *testing.T) {
+	t.Cleanup(func() { logLevel.Set(slog.LevelInfo) })
+
+	// The untouched default must be exactly info: every existing deployment
+	// relies on this for "no behavior change without the flag".
+	if got := logLevel.Level(); got != slog.LevelInfo {
+		t.Fatalf("default log level = %v, want %v", got, slog.LevelInfo)
+	}
+
+	var buf bytes.Buffer
+	InitLoggerWithWriter(&buf)
+	t.Cleanup(InitLogger)
+
+	slog.Info("visible at default level")
+	if !strings.Contains(buf.String(), "visible at default level") {
+		t.Errorf("info line not emitted at default level: %s", buf.String())
+	}
+	buf.Reset()
+	slog.Debug("hidden at default level")
+	if buf.Len() != 0 {
+		t.Errorf("debug line emitted at default level: %s", buf.String())
+	}
+
+	if err := SetLogLevel("debug"); err != nil {
+		t.Fatalf("SetLogLevel(debug): %v", err)
+	}
+	slog.Debug("visible at debug")
+	if !strings.Contains(buf.String(), "visible at debug") {
+		t.Errorf("debug line not emitted after SetLogLevel(debug): %s", buf.String())
+	}
+
+	// Case-insensitive, and dynamic: raising the level silences info.
+	if err := SetLogLevel("WARN"); err != nil {
+		t.Fatalf("SetLogLevel(WARN): %v", err)
+	}
+	buf.Reset()
+	slog.Info("hidden at warn")
+	if buf.Len() != 0 {
+		t.Errorf("info line emitted at warn level: %s", buf.String())
+	}
+
+	if err := SetLogLevel("verbose"); err == nil {
+		t.Error("SetLogLevel accepted an invalid level")
+	}
+
+	// Empty means unset: no error, level unchanged.
+	if err := SetLogLevel(""); err != nil {
+		t.Errorf("SetLogLevel(\"\") = %v, want nil", err)
+	}
+	if got := logLevel.Level(); got != slog.LevelWarn {
+		t.Errorf("SetLogLevel(\"\") changed the level to %v", got)
+	}
+}

manifests/ate-install/kind/atelet/kustomization.yaml

@@ -35,6 +35,9 @@ patches:
               - --localhost-registry-replacement=kind-registry:5000
               - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem
               - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem
+              # Kind clusters are dev/CI: run atelet at debug so e2e suites can
+              # assert on per-item log lines. Production installs default to info.
+              - --log-level=debug
               env:
               - name: OTEL_EXPORTER_OTLP_ENDPOINT
                 value: http://opentelemetry-collector.otel-system.svc:4317