agent: define new flags to control Cilium's datapath events notificat…

…ions

This commit introduces three new configuration flags for the Cilium agent, allowing users to choose the bpf event types they want to expose to Cilium monitor and Hubble.
 - `--bpf-events-drop-enabled`                                   Expose 'drop' events for Cilium monitor and/or Hubble (default true)
 - `--bpf-events-policy-verdict-enabled`                         Expose 'policy verdict' events for Cilium monitor and/or Hubble (default true)
 - `--bpf-events-trace-enabled`                                  Expose 'trace' events for Cilium monitor and/or Hubble (default true)

The default values for these flags remain set to `true`, not changing the current behaviour.

In our case, we found particularly useful to disable the TraceNotification in order to reduce the CPU overhead on some of our nodes when Hubble is enabled as we were mostly interested into dropped packets.

Signed-off-by: Maxime Visonneau <maxime.visonneau@gmail.com>

Documentation/operations/performance/tuning.rst

@@ -216,6 +216,31 @@ In order to optimize for maximum performance, Hubble can be disabled:
              --namespace kube-system \\
              --set hubble.enabled=false
 
+You can also choose to stop exposing event types in which you
+are not interested. For instance if you are mainly interested in
+dropped traffic, you can disable "trace" events which will likely reduce
+the overall CPU consumption of the agent.
+
+.. tabs::
+
+    .. group-tab:: Cilium CLI
+
+       .. code-block:: shell-session
+
+           cilium config TraceNotification=disable
+
+    .. group-tab:: Helm
+
+       .. parsed-literal::
+
+           helm install cilium |CHART_RELEASE| \\
+             --namespace kube-system \\
+             --set bpf.events.trace.enabled=false
+
+.. warning::
+
+    Suppressing one or more event types will impact ``cilium monitor`` as well as Hubble observability capabilities, metrics and exports.
+
 MTU
 ===
 

daemon/cmd/daemon_main.go

@@ -344,6 +344,15 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableL7Proxy, defaults.EnableL7Proxy, "Enable L7 proxy for L7 policy enforcement")
 	option.BindEnv(vp, option.EnableL7Proxy)
 
+	flags.Bool(option.BPFEventsDropEnabled, defaults.BPFEventsDropEnabled, "Expose 'drop' events for Cilium monitor and/or Hubble")
+	option.BindEnv(vp, option.BPFEventsDropEnabled)
+
+	flags.Bool(option.BPFEventsPolicyVerdictEnabled, defaults.BPFEventsPolicyVerdictEnabled, "Expose 'policy verdict' events for Cilium monitor and/or Hubble")
+	option.BindEnv(vp, option.BPFEventsPolicyVerdictEnabled)
+
+	flags.Bool(option.BPFEventsTraceEnabled, defaults.BPFEventsTraceEnabled, "Expose 'trace' events for Cilium monitor and/or Hubble")
+	option.BindEnv(vp, option.BPFEventsTraceEnabled)
+
 	flags.Bool(option.EnableTracing, false, "Enable tracing while determining policy (debugging)")
 	option.BindEnv(vp, option.EnableTracing)
 
@@ -1303,9 +1312,9 @@ func initEnv(vp *viper.Viper) {
 
 	option.Config.Opts.SetBool(option.Debug, debugDatapath)
 	option.Config.Opts.SetBool(option.DebugLB, debugDatapath)
-	option.Config.Opts.SetBool(option.DropNotify, true)
-	option.Config.Opts.SetBool(option.TraceNotify, true)
-	option.Config.Opts.SetBool(option.PolicyVerdictNotify, true)
+	option.Config.Opts.SetBool(option.DropNotify, option.Config.BPFEventsDropEnabled)
+	option.Config.Opts.SetBool(option.PolicyVerdictNotify, option.Config.BPFEventsPolicyVerdictEnabled)
+	option.Config.Opts.SetBool(option.TraceNotify, option.Config.BPFEventsTraceEnabled)
 	option.Config.Opts.SetBool(option.PolicyTracing, option.Config.EnableTracing)
 	option.Config.Opts.SetBool(option.ConntrackAccounting, false)
 	option.Config.Opts.SetBool(option.ConntrackLocal, false)

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -414,6 +414,10 @@ data:
   bpf-lb-external-clusterip: {{ .Values.bpf.lbExternalClusterIP | quote }}
 {{- end }}
 
+  bpf-events-drop-enabled: {{ .Values.bpf.events.drop.enabled | quote }}
+  bpf-events-policy-verdict-enabled: {{ .Values.bpf.events.policyVerdict.enabled | quote }}
+  bpf-events-trace-enabled: {{ .Values.bpf.events.trace.enabled | quote }}
+
   # Pre-allocation of map entries allows per-packet latency to be reduced, at
   # the expense of up-front memory allocation for the entries in the maps. The
   # default value below will minimize memory usage in the default installation;

install/kubernetes/cilium/values.yaml.tmpl

@@ -407,6 +407,17 @@ bpf:
   # tracking table.
   # @default -- `262144`
   ctAnyMax: ~
+  # -- Control events generated by the Cilium datapath exposed to Cilium monitor and Hubble.
+  events:
+    drop:
+      # -- Enable drop events.
+      enabled: true
+    policyVerdict:
+      # -- Enable policy verdict events.
+      enabled: true
+    trace:
+      # -- Enable trace events.
+      enabled: true
   # -- Configure the maximum number of service entries in the
   # load balancer maps.
   lbMapMax: 65536

pkg/defaults/defaults.go

@@ -556,6 +556,15 @@ const (
 	// identity in a numeric identity. Values > 255 will decrease the number of
 	// allocatable identities.
 	MaxConnectedClusters = 255
+
+	// BPFEventsDropEnabled controls whether the Cilium datapath exposes "drop" events to Cilium monitor and Hubble.
+	BPFEventsDropEnabled = true
+
+	// BPFEventsPolicyVerdictEnabled controls whether the Cilium datapath exposes "policy verdict" events to Cilium monitor and Hubble.
+	BPFEventsPolicyVerdictEnabled = true
+
+	// BPFEventsTraceEnabled controls whether the Cilium datapath exposes "trace" events to Cilium monitor and Hubble.
+	BPFEventsTraceEnabled = true
 )
 
 var (

pkg/option/config.go

@@ -1197,6 +1197,15 @@ const (
 
 	// PolicyCIDRMatchMode defines the entities that CIDR selectors can reach
 	PolicyCIDRMatchMode = "policy-cidr-match-mode"
+
+	// BPFEventsDropEnabled defines the DropNotification setting for any endpoint
+	BPFEventsDropEnabled = "bpf-events-drop-enabled"
+
+	// BPFEventsPolicyVerdictEnabled defines the PolicyVerdictNotification setting for any endpoint
+	BPFEventsPolicyVerdictEnabled = "bpf-events-policy-verdict-enabled"
+
+	// BPFEventsTraceEnabled defines the TraceNotification setting for any endpoint
+	BPFEventsTraceEnabled = "bpf-events-trace-enabled"
 )
 
 // Default string arguments
@@ -2407,6 +2416,15 @@ type DaemonConfig struct {
 	BPFMapEventBuffersValidator func(val string) (string, error) `json:"-"`
 	bpfMapEventConfigs          BPFEventBufferConfigs
 
+	// BPFEventsDropEnabled controls whether the Cilium datapath exposes "drop" events to Cilium monitor and Hubble.
+	BPFEventsDropEnabled bool
+
+	// BPFEventsPolicyVerdictEnabled controls whether the Cilium datapath exposes "policy verdict" events to Cilium monitor and Hubble.
+	BPFEventsPolicyVerdictEnabled bool
+
+	// BPFEventsTraceEnabled  controls whether the Cilium datapath exposes "trace" events to Cilium monitor and Hubble.
+	BPFEventsTraceEnabled bool
+
 	// IPAMCiliumNodeUpdateRate is the maximum rate at which the CiliumNode custom
 	// resource is updated.
 	IPAMCiliumNodeUpdateRate time.Duration
@@ -2484,6 +2502,10 @@ var (
 		EnableK8sNetworkPolicy: defaults.EnableK8sNetworkPolicy,
 		PolicyCIDRMatchMode:    defaults.PolicyCIDRMatchMode,
 		MaxConnectedClusters:   defaults.MaxConnectedClusters,
+
+		BPFEventsDropEnabled:          defaults.BPFEventsDropEnabled,
+		BPFEventsPolicyVerdictEnabled: defaults.BPFEventsPolicyVerdictEnabled,
+		BPFEventsTraceEnabled:         defaults.BPFEventsTraceEnabled,
 	}
 )
 
@@ -3170,6 +3192,9 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) {
 	c.EnablePMTUDiscovery = vp.GetBool(EnablePMTUDiscovery)
 	c.IPv6NAT46x64CIDR = defaults.IPv6NAT46x64CIDR
 	c.IPAMCiliumNodeUpdateRate = vp.GetDuration(IPAMCiliumNodeUpdateRate)
+	c.BPFEventsDropEnabled = vp.GetBool(BPFEventsDropEnabled)
+	c.BPFEventsPolicyVerdictEnabled = vp.GetBool(BPFEventsPolicyVerdictEnabled)
+	c.BPFEventsTraceEnabled = vp.GetBool(BPFEventsTraceEnabled)
 
 	c.ServiceNoBackendResponse = vp.GetString(ServiceNoBackendResponse)
 	switch c.ServiceNoBackendResponse {