daemon: Add options to launch Hubble

Add options to launch Hubble with the following configuration parameters:

- hubble-listen-addresses:
    List of IP addresses for Hubble server to listen to. Hubble is disabled
    if this list is empty.
- hubble-flow-buffer-size:
    Maximum number of flows in Hubble's buffer.
- hubble-metrics-server:
    Address to serve Hubble metrics on. The metrics server is disabled if
    this string is empty.
- hubble-metrics:
    List of Hubble metrics to enable.

Closes #9925

Signed-off-by: Michi Mutsuzaki <michi@isovalent.com>

Documentation/cmdref/cilium-agent.md

@@ -86,6 +86,10 @@ cilium-agent [flags]
       --http-request-timeout uint                             Time after which a forwarded HTTP request is considered failed unless completed (in seconds); Use 0 for unlimited (default 3600)
       --http-retry-count uint                                 Number of retries performed after a forwarded request attempt fails (default 3)
       --http-retry-timeout uint                               Time after which a forwarded but uncompleted request is retried (connection failures are retried immediately); defaults to 0 (never)
+      --hubble-flow-buffer-size int                           Maximum number of flows in Hubble's buffer. (default 131071)
+      --hubble-listen-addresses strings                       List of IP addresses for Hubble server to listen to.
+      --hubble-metrics strings                                List of Hubble metrics to enable.
+      --hubble-metrics-server string                          Address to serve Hubble metrics on.
       --identity-allocation-mode string                       Method to use for identity allocation (default "kvstore")
       --identity-change-grace-period duration                 Time to wait before using new identity on endpoint identity change (default 5s)
       --install-iptables-rules                                Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading) (default true)

daemon/daemon_main.go

@@ -42,7 +42,9 @@ import (
 	"github.com/cilium/cilium/pkg/defaults"
 	"github.com/cilium/cilium/pkg/envoy"
 	"github.com/cilium/cilium/pkg/flowdebug"
+	"github.com/cilium/cilium/pkg/hubble"
 	"github.com/cilium/cilium/pkg/identity"
+	"github.com/cilium/cilium/pkg/ipcache"
 	"github.com/cilium/cilium/pkg/k8s"
 	"github.com/cilium/cilium/pkg/k8s/watchers"
 	"github.com/cilium/cilium/pkg/kvstore"
@@ -61,6 +63,9 @@ import (
 	"github.com/cilium/cilium/pkg/probe"
 	"github.com/cilium/cilium/pkg/version"
 
+	hubbleServe "github.com/cilium/hubble/cmd/serve"
+	"github.com/cilium/hubble/pkg/parser"
+	hubbleServer "github.com/cilium/hubble/pkg/server"
 	"github.com/go-openapi/loads"
 	gops "github.com/google/gops/agent"
 	"github.com/jessevdk/go-flags"
@@ -719,6 +724,18 @@ func init() {
 	flags.Bool(option.DisableCNPStatusUpdates, false, "Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with `cnp-node-status-gc=false` in cilium-operator)")
 	option.BindEnv(option.DisableCNPStatusUpdates)
 
+	flags.StringSlice(option.HubbleListenAddresses, []string{}, "List of IP addresses for Hubble server to listen to.")
+	option.BindEnv(option.HubbleListenAddresses)
+
+	flags.Int(option.HubbleFlowBufferSize, 131071, "Maximum number of flows in Hubble's buffer.")
+	option.BindEnv(option.HubbleFlowBufferSize)
+
+	flags.String(option.HubbleMetricsServer, "", "Address to serve Hubble metrics on.")
+	option.BindEnv(option.HubbleMetricsServer)
+
+	flags.StringSlice(option.HubbleMetrics, []string{}, "List of Hubble metrics to enable.")
+	option.BindEnv(option.HubbleMetrics)
+
 	viper.BindPFlags(flags)
 }
 
@@ -1305,6 +1322,7 @@ func runDaemon() {
 
 	bootstrapStats.overall.End(true)
 	bootstrapStats.updateMetrics()
+	d.launchHubble()
 
 	select {
 	case err := <-metricsErrs:
@@ -1566,3 +1584,29 @@ func initKubeProxyReplacementOptions() {
 		option.Config.EnableExternalIPs = false
 	}
 }
+
+func (d *Daemon) launchHubble() {
+	logger := logging.DefaultLogger.WithField(logfields.LogSubsys, "hubble")
+	addresses := option.Config.HubbleListenAddresses
+	if len(addresses) == 0 {
+		logger.Info("Hubble server is disabled")
+		return
+	}
+	epDNSGetter := hubble.NewLocalEndpointDNSGetter(d.endpointManager)
+	identityGetter := hubble.NewLocalIdentityGetter(d.identityAllocator)
+	ipGetter := hubble.NewLocalIPGetter(ipcache.IPIdentityCache)
+	serviceGetter := hubble.NewLocalServiceGetter(d.svc)
+	payloadParser, _ := parser.New(epDNSGetter, identityGetter, epDNSGetter, ipGetter, serviceGetter)
+	s := hubbleServer.NewLocalServer(payloadParser, option.Config.HubbleFlowBufferSize, logger)
+	go s.Start()
+	d.monitorAgent.GetMonitor().RegisterNewListener(context.TODO(), hubble.NewHubbleListener(s))
+	logger.WithField("addresses", addresses).Info("Starting Hubble server")
+	hubbleServe.Serve(logger, addresses, s)
+	if option.Config.HubbleMetricsServer != "" {
+		logger.WithFields(logrus.Fields{
+			"address": option.Config.HubbleMetricsServer,
+			"metrics": option.Config.HubbleMetrics,
+		}).Info("Starting Hubble Metrics server")
+		hubbleServe.EnableMetrics(log, option.Config.HubbleMetricsServer, option.Config.HubbleMetrics)
+	}
+}

pkg/option/config.go

@@ -669,6 +669,18 @@ const (
 
 	// EnableRemoteNodeIdentity enables use of the remote-node identity
 	EnableRemoteNodeIdentity = "enable-remote-node-identity"
+
+	// HubbleListenAddresses specifies addresses for Hubble server to listen to.
+	HubbleListenAddresses = "hubble-listen-addresses"
+
+	// HubbleFlowBufferSize specifies the maximum number of flows in Hubble's buffer.
+	HubbleFlowBufferSize = "hubble-flow-buffer-size"
+
+	// HubbleMetricsServer specifies the addresses to serve Hubble metrics on.
+	HubbleMetricsServer = "hubble-metrics-server"
+
+	// HubbleMetrics specifies enabled metrics and their configuration options.
+	HubbleMetrics = "hubble-metrics"
 )
 
 // Default string arguments
@@ -1344,6 +1356,18 @@ type DaemonConfig struct {
 
 	// EnableRemoteNodeIdentity enables use of the remote-node identity
 	EnableRemoteNodeIdentity bool
+
+	// HubbleListenAddresses specifies addresses for Hubble to listen to.
+	HubbleListenAddresses []string
+
+	// HubbleFlowBufferSize specifies the maximum number of flows in Hubble's buffer.
+	HubbleFlowBufferSize int
+
+	// HubbleMetricsServer specifies the addresses to serve Hubble metrics on.
+	HubbleMetricsServer string
+
+	// HubbleMetrics specifies enabled metrics and their configuration options.
+	HubbleMetrics []string
 }
 
 var (
@@ -1942,6 +1966,12 @@ func (c *DaemonConfig) Populate() {
 		}
 	}
 
+	// Hubble options.
+	c.HubbleListenAddresses = viper.GetStringSlice(HubbleListenAddresses)
+	c.HubbleFlowBufferSize = viper.GetInt(HubbleFlowBufferSize)
+	c.HubbleMetricsServer = viper.GetString(HubbleMetricsServer)
+	c.HubbleMetrics = viper.GetStringSlice(HubbleMetrics)
+
 	// Hidden options
 	c.ConfigFile = viper.GetString(ConfigFile)
 	c.HTTP403Message = viper.GetString(HTTP403Message)