tetragon: Add events buffer queue metrics

Adding following metric counters for events buffer queue:

  eventsQueueReceived - number of records received from perf event reader go routine
  eventsQueueLost     - number of records lost because the RBQueue channel was full

Signed-off-by: Jiri Olsa <jolsa@kernel.org>

pkg/metrics/eventsqueuemetrics/metrics.go

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package eventsqueuemetrics
+
+import (
+	"github.com/cilium/tetragon/pkg/metrics/consts"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	eventsQueueReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "eventsqueue_stats_received",
+		Help:        "The total number of Tetragon events queue received.",
+		ConstLabels: nil,
+	}, nil)
+	eventsQueueLost = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "eventsqueue_stats_lost",
+		Help:        "The total number of Tetragon events queue lost.",
+		ConstLabels: nil,
+	}, nil)
+)
+
+func InitMetrics(registry *prometheus.Registry) {
+	registry.MustRegister(eventsQueueReceived)
+	registry.MustRegister(eventsQueueLost)
+}
+
+func ReceivedInc() {
+	eventsQueueReceived.WithLabelValues().Inc()
+}
+
+func LostInc() {
+	eventsQueueLost.WithLabelValues().Inc()
+}

pkg/metrics/metrics.go

@@ -11,6 +11,7 @@ import (
 	"github.com/cilium/tetragon/pkg/metrics/errormetrics"
 	"github.com/cilium/tetragon/pkg/metrics/eventcachemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/eventmetrics"
+	"github.com/cilium/tetragon/pkg/metrics/eventsqueuemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/kprobemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/mapmetrics"
 	"github.com/cilium/tetragon/pkg/metrics/opcodemetrics"
@@ -34,6 +35,7 @@ func InitAllMetrics(registry *prometheus.Registry) {
 	pfmetrics.InitMetrics(registry)
 	processexecmetrics.InitMetrics(registry)
 	ringbufmetrics.InitMetrics(registry)
+	eventsqueuemetrics.InitMetrics(registry)
 	syscallmetrics.InitMetrics(registry)
 	watchermetrics.InitMetrics(registry)
 	observer.InitMetrics(registry)

pkg/observer/observer.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cilium/tetragon/pkg/bpf"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/metrics/errormetrics"
+	"github.com/cilium/tetragon/pkg/metrics/eventsqueuemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/opcodemetrics"
 	"github.com/cilium/tetragon/pkg/metrics/ringbufmetrics"
 	"github.com/cilium/tetragon/pkg/option"
@@ -242,6 +243,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error {
 					case eventsQueue <- &record:
 					default:
 						// eventsQueue channel is full, drop the event
+						eventsqueuemetrics.LostInc()
 					}
 					k.recvCntr++
 					ringbufmetrics.PerfEventReceived.Inc()
@@ -263,6 +265,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error {
 			select {
 			case event := <-eventsQueue:
 				k.receiveEvent(event.RawSample)
+				eventsqueuemetrics.ReceivedInc()
 			case <-stopCtx.Done():
 				k.log.WithError(stopCtx.Err()).Infof("Listening for events completed.")
 				k.log.Debugf("Unprocessed events in RB queue: %d", len(eventsQueue))