logging: change log level dynamically

cmd/tetragon/main.go

@@ -37,6 +37,7 @@ import (
 	"github.com/cilium/tetragon/pkg/sensors/base"
 	"github.com/cilium/tetragon/pkg/sensors/program"
 	"github.com/cilium/tetragon/pkg/server"
+	"github.com/cilium/tetragon/pkg/tgsyscall"
 	"github.com/cilium/tetragon/pkg/tracingpolicy"
 	"github.com/cilium/tetragon/pkg/unixlisten"
 	"github.com/cilium/tetragon/pkg/version"
@@ -124,7 +125,8 @@ func tetragonExecute() error {
 	defer cancel()
 
 	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, tgsyscall.SIGRTMIN_20,
+		tgsyscall.SIGRTMIN_21, tgsyscall.SIGRTMIN_22)
 
 	// Logging should always be bootstrapped first. Do not add any code above this!
 	if err := logger.SetupLogging(option.Config.LogOpts, option.Config.Debug); err != nil {
@@ -217,13 +219,39 @@ func tetragonExecute() error {
 		obs.RemovePrograms()
 	}()
 
-	go func() {
-		// if we receive a signal, call cancel so that contexts are finalized, which will
-		// leads to normally return from tetragonExecute().
-		s := <-sigs
-		log.Infof("Received signal %s, shutting down...", s)
-		cancel()
-	}()
+	defaultLevel := logger.GetLogLevel()
+	go func(obs *observer.Observer) {
+		for {
+			s := <-sigs
+			switch s {
+			case syscall.SIGINT, syscall.SIGTERM:
+				// if we receive a signal, call cancel so that contexts are finalized, which will
+				// leads to normally return from tetragonExecute().
+				log.Infof("Received signal %s, shutting down...", s)
+				cancel()
+				return
+			case tgsyscall.SIGRTMIN_20: // SIGRTMIN+20
+				currentLevel := logger.GetLogLevel()
+				if currentLevel == logrus.DebugLevel {
+					log.Infof("Received signal SIGRTMIN+20: LogLevel is already '%s'", currentLevel)
+				} else {
+					logger.SetLogLevel(logrus.DebugLevel)
+					log.Infof("Received signal SIGRTMIN+20: switching from LogLevel '%s' to '%s'", currentLevel, logger.GetLogLevel())
+				}
+			case tgsyscall.SIGRTMIN_21: // SIGRTMIN+21
+				currentLevel := logger.GetLogLevel()
+				if currentLevel == logrus.TraceLevel {
+					log.Infof("Received signal SIGRTMIN+21: LogLevel is already '%s'", currentLevel)
+				} else {
+					logger.SetLogLevel(logrus.TraceLevel)
+					log.Infof("Received signal SIGRTMIN+21: switching from LogLevel '%s' to '%s'", currentLevel, logger.GetLogLevel())
+				}
+			case tgsyscall.SIGRTMIN_22: // SIGRTMIN+22
+				logger.SetLogLevel(defaultLevel)
+				log.Infof("Received signal SIGRTMIN+22: resetting original LogLevel '%s'", logger.GetLogLevel())
+			}
+		}
+	}(obs)
 
 	// start sensor manager, and have it wait on sensorMgWait until we load
 	// the base sensor. note that this means that calling methods on the

docs/content/en/docs/tutorials/debugging-tetragon.md

@@ -0,0 +1,37 @@
+---
+title: "Debugging Tetragon"
+weight: 1
+icon: "overview"
+description: "Diagnosing Tetragon problems"
+---
+
+### Enable debug log level
+
+When debugging, it might be useful to change the log level. The default log level is controlled by the log-level option:
+
+* Enable debug level with `--log-level=debug`
+
+* Enable trace level with `--log-level=trace`
+
+### Change log level dynamically
+
+It is possible to change the log level dynamically by sending the corresponding signal to tetragon process.
+
+* Change log level to debug level by sending the `SIGRTMIN+20` signal to tetragon pid:
+
+  ```shell
+  sudo kill -s SIGRTMIN+20 $(pidof tetragon)
+  ```
+
+* Change log level to trace level by sending the `SIGRTMIN+21` signal to tetragon pid:
+
+  ```shell
+  sudo kill -s SIGRTMIN+21 $(pidof tetragon)
+  ```
+
+* To Restore the original log level send the `SIGRTMIN+22` signal to tetragon pid:
+
+  ```shell
+  sudo kill -s SIGRTMIN+22 $(pidof tetragon)
+  ```
+

pkg/logger/log.go

@@ -100,7 +100,7 @@ func GetLogLevel() logrus.Level {
 	return DefaultLogger.GetLevel()
 }
 
-func setLogLevel(logLevel logrus.Level) {
+func SetLogLevel(logLevel logrus.Level) {
 	DefaultLogger.SetLevel(logLevel)
 }
 
@@ -149,7 +149,7 @@ func SetupLogging(o LogOptions, debug bool) error {
 	if debug {
 		setLogLevelToDebug()
 	} else {
-		setLogLevel(o.getLogLevel())
+		SetLogLevel(o.getLogLevel())
 	}
 
 	// always suppress the default logger so libraries don't print things

pkg/tgsyscall/tgsyscall.go

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+/*
+ * This package contains low-level operating system privimitives
+ * that are missing from the the default syscall package
+ */
+
+package tgsyscall
+
+import (
+	"syscall"
+)
+
+const (
+	/*
+	 * The real time signals can be used for application-defined
+	 * purpose.
+	 *
+	 * We avoid using first rt signals to easy debugging in case
+	 * inspecting logs as the first rt signals are heavily used by
+	 * nptl implementation to manage threads.
+	 *
+	 * https://man7.org/linux/man-pages/man7/signal.7.html
+	 */
+	SIGRTMIN    = syscall.Signal(0x22) // SIGRTMIN
+	SIGRTMIN_20 = SIGRTMIN + 20        // SIGRTMIN+20 used to set log level to debug
+	SIGRTMIN_21 = SIGRTMIN + 21        // SIGRTMIN+21 used to set log level to trace
+	SIGRTMIN_22 = SIGRTMIN + 22        // SIGRTMIN+22 restore original configured log level
+)