fixes for policystatemetrics

pkg/metrics/policystatemetrics/policystatemetrics.go

@@ -6,22 +6,21 @@ package policystatemetrics
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/cilium/tetragon/api/v1/tetragon"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/metrics/consts"
 	"github.com/cilium/tetragon/pkg/observer"
-	"github.com/cilium/tetragon/pkg/sensors"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
 type policyStateCollector struct {
-	descriptor    *prometheus.Desc
-	sensorManager *sensors.Manager
+	descriptor *prometheus.Desc
 }
 
 func InitMetrics(registry *prometheus.Registry) {
-	registry.MustRegister(newPolicyStateCollector(observer.GetSensorManager()))
+	registry.MustRegister(newPolicyStateCollector())
 }
 
 func InitMetricsForDocs(registry *prometheus.Registry) {
@@ -30,14 +29,13 @@ func InitMetricsForDocs(registry *prometheus.Registry) {
 
 // This metric collector converts the output of ListTracingPolicies into a few
 // gauges metrics on collection. Thus, it needs a sensor manager to query.
-func newPolicyStateCollector(sensorManager *sensors.Manager) *policyStateCollector {
+func newPolicyStateCollector() *policyStateCollector {
 	return &policyStateCollector{
 		descriptor: prometheus.NewDesc(
 			prometheus.BuildFQName(consts.MetricsNamespace, "", "tracingpolicy_loaded"),
 			"The number of loaded tracing policy by state.",
 			[]string{"state"}, nil,
 		),
-		sensorManager: sensorManager,
 	}
 }
 
@@ -46,11 +44,16 @@ func (c *policyStateCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (c *policyStateCollector) Collect(ch chan<- prometheus.Metric) {
-	if c.sensorManager == nil {
+
+	sm := observer.GetSensorManager()
+	if sm == nil {
 		logger.GetLogger().Debug("failed retrieving the sensor manager: manager is nil")
 		return
 	}
-	list, err := c.sensorManager.ListTracingPolicies(context.Background())
+
+	ctx, cancel := context.WithTimeout(context.TODO(), 900*time.Millisecond)
+	defer cancel()
+	list, err := sm.ListTracingPolicies(ctx)
 	if err != nil {
 		logger.GetLogger().WithError(err).Warn("error listing tracing policies to collect policies state")
 		return
@@ -97,7 +100,7 @@ type policyStateZeroCollector struct {
 
 func newPolicyStateZeroCollector() prometheus.Collector {
 	return &policyStateZeroCollector{
-		policyStateCollector: *newPolicyStateCollector(nil),
+		policyStateCollector: *newPolicyStateCollector(),
 	}
 }
 

pkg/metrics/policystatemetrics/policystatemetrics_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/cilium/tetragon/pkg/observer"
 	tus "github.com/cilium/tetragon/pkg/testutils/sensors"
 	"github.com/cilium/tetragon/pkg/tracingpolicy"
 	"github.com/prometheus/client_golang/prometheus"
@@ -30,9 +31,16 @@ tetragon_tracingpolicy_loaded{state="load_error"} %d
 	}
 
 	reg := prometheus.NewRegistry()
+
+	// NB(kkourt): the policy state collector uses observer.GetSensorManager() to get the sensor
+	// manager because in the observer tests we only initialize metrics while the observer
+	// changes for every test (see:
+	// https://github.com/cilium/tetragon/blob/22eb995b19207ac0ced2dd83950ec8e8aedd122d/pkg/observer/observertesthelper/observer_test_helper.go#L272-L276)
 	manager := tus.GetTestSensorManager(context.TODO(), t).Manager
+	observer.SetSensorManager(manager)
+	t.Cleanup(observer.ResetSensorManager)
 
-	collector := newPolicyStateCollector(manager)
+	collector := newPolicyStateCollector()
 	reg.Register(collector)
 
 	err := manager.AddTracingPolicy(context.TODO(), &tracingpolicy.GenericTracingPolicy{

pkg/sensors/manager.go

@@ -249,9 +249,15 @@ func (h *Manager) ListTracingPolicies(ctx context.Context) (*tetragon.ListTracin
 		retChan: retc,
 	}
 
-	h.sensorCtl <- op
-	err := <-retc
-	return op.result, err
+	select {
+	case h.sensorCtl <- op:
+		err := <-retc
+		return op.result, err
+
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
 }
 
 func (h *Manager) RemoveSensor(ctx context.Context, sensorName string) error {