tetragon: Add events buffer queue metrics

Adding following metric counters for events buffer queue:

  ringbufqueuemetrics.Received - number of records received from perf event reader go routine
  ringbufqueuemetrics.Lost     - number of records lost because the RB Queue channel was full

Signed-off-by: Jiri Olsa <jolsa@kernel.org>

pkg/metrics/config/initmetrics.go

@@ -14,6 +14,7 @@ import (
 	pfmetrics "github.com/cilium/tetragon/pkg/metrics/policyfilter"
 	"github.com/cilium/tetragon/pkg/metrics/processexecmetrics"
 	"github.com/cilium/tetragon/pkg/metrics/ringbufmetrics"
+	"github.com/cilium/tetragon/pkg/metrics/ringbufqueuemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/syscallmetrics"
 	"github.com/cilium/tetragon/pkg/metrics/watchermetrics"
 	"github.com/cilium/tetragon/pkg/observer"
@@ -33,6 +34,7 @@ func InitAllMetrics(registry *prometheus.Registry) {
 	pfmetrics.InitMetrics(registry)
 	processexecmetrics.InitMetrics(registry)
 	ringbufmetrics.InitMetrics(registry)
+	ringbufqueuemetrics.InitMetrics(registry)
 	syscallmetrics.InitMetrics(registry)
 	watchermetrics.InitMetrics(registry)
 	observer.InitMetrics(registry)

pkg/metrics/ringbufqueuemetrics/ringbufqueuemetrics.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package ringbufqueuemetrics
+
+import (
+	"github.com/cilium/tetragon/pkg/metrics/consts"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	Received = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "ringbuf_queue_received_total",
+		Help:        "The total number of Tetragon events ring buffer queue received.",
+		ConstLabels: nil,
+	})
+	Lost = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "ringbuf_queue_lost_total",
+		Help:        "The total number of Tetragon events ring buffer queue lost.",
+		ConstLabels: nil,
+	})
+)
+
+func InitMetrics(registry *prometheus.Registry) {
+	registry.MustRegister(Received)
+	registry.MustRegister(Lost)
+}

pkg/observer/observer.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cilium/tetragon/pkg/metrics/errormetrics"
 	"github.com/cilium/tetragon/pkg/metrics/opcodemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/ringbufmetrics"
+	"github.com/cilium/tetragon/pkg/metrics/ringbufqueuemetrics"
 	"github.com/cilium/tetragon/pkg/option"
 	"github.com/cilium/tetragon/pkg/reader/notify"
 	"github.com/cilium/tetragon/pkg/sensors"
@@ -242,6 +243,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error {
 					case eventsQueue <- &record:
 					default:
 						// eventsQueue channel is full, drop the event
+						ringbufqueuemetrics.Lost.Inc()
 					}
 					k.recvCntr++
 					ringbufmetrics.PerfEventReceived.Inc()
@@ -263,6 +265,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error {
 			select {
 			case event := <-eventsQueue:
 				k.receiveEvent(event.RawSample)
+				ringbufqueuemetrics.Received.Inc()
 			case <-stopCtx.Done():
 				k.log.WithError(stopCtx.Err()).Infof("Listening for events completed.")
 				k.log.Debugf("Unprocessed events in RB queue: %d", len(eventsQueue))