diff --git a/Documentation/cmdref/cilium-agent.md b/Documentation/cmdref/cilium-agent.md index 9853f7f25da1..79e31ef864e4 100644 --- a/Documentation/cmdref/cilium-agent.md +++ b/Documentation/cmdref/cilium-agent.md @@ -86,6 +86,10 @@ cilium-agent [flags] --http-request-timeout uint Time after which a forwarded HTTP request is considered failed unless completed (in seconds); Use 0 for unlimited (default 3600) --http-retry-count uint Number of retries performed after a forwarded request attempt fails (default 3) --http-retry-timeout uint Time after which a forwarded but uncompleted request is retried (connection failures are retried immediately); defaults to 0 (never) + --hubble-flow-buffer-size int Maximum number of flows in Hubble's buffer. (default 131071) + --hubble-listen-addresses strings List of IP addresses for Hubble server to listen to. + --hubble-metrics strings List of Hubble metrics to enable. + --hubble-metrics-server string Address to serve Hubble metrics on. --identity-allocation-mode string Method to use for identity allocation (default "kvstore") --identity-change-grace-period duration Time to wait before using new identity on endpoint identity change (default 5s) --install-iptables-rules Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading) (default true) diff --git a/daemon/daemon_main.go b/daemon/daemon_main.go index c284f53a94aa..9ab2b8daa03a 100644 --- a/daemon/daemon_main.go +++ b/daemon/daemon_main.go @@ -42,7 +42,9 @@ import ( "github.com/cilium/cilium/pkg/defaults" "github.com/cilium/cilium/pkg/envoy" "github.com/cilium/cilium/pkg/flowdebug" + "github.com/cilium/cilium/pkg/hubble" "github.com/cilium/cilium/pkg/identity" + "github.com/cilium/cilium/pkg/ipcache" "github.com/cilium/cilium/pkg/k8s" "github.com/cilium/cilium/pkg/k8s/watchers" "github.com/cilium/cilium/pkg/kvstore" @@ -61,6 +63,9 @@ import ( "github.com/cilium/cilium/pkg/probe" "github.com/cilium/cilium/pkg/version" + hubbleServe "github.com/cilium/hubble/cmd/serve" + "github.com/cilium/hubble/pkg/parser" + hubbleServer "github.com/cilium/hubble/pkg/server" "github.com/go-openapi/loads" gops "github.com/google/gops/agent" "github.com/jessevdk/go-flags" @@ -719,6 +724,18 @@ func init() { flags.Bool(option.DisableCNPStatusUpdates, false, "Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with `cnp-node-status-gc=false` in cilium-operator)") option.BindEnv(option.DisableCNPStatusUpdates) + flags.StringSlice(option.HubbleListenAddresses, []string{}, "List of IP addresses for Hubble server to listen to.") + option.BindEnv(option.HubbleListenAddresses) + + flags.Int(option.HubbleFlowBufferSize, 131071, "Maximum number of flows in Hubble's buffer.") + option.BindEnv(option.HubbleFlowBufferSize) + + flags.String(option.HubbleMetricsServer, "", "Address to serve Hubble metrics on.") + option.BindEnv(option.HubbleMetricsServer) + + flags.StringSlice(option.HubbleMetrics, []string{}, "List of Hubble metrics to enable.") + option.BindEnv(option.HubbleMetrics) + viper.BindPFlags(flags) } @@ -1305,6 +1322,7 @@ func runDaemon() { bootstrapStats.overall.End(true) bootstrapStats.updateMetrics() + d.launchHubble() select { case err := <-metricsErrs: @@ -1566,3 +1584,29 @@ func initKubeProxyReplacementOptions() { option.Config.EnableExternalIPs = false } } + +func (d *Daemon) launchHubble() { + logger := logging.DefaultLogger.WithField(logfields.LogSubsys, "hubble") + addresses := option.Config.HubbleListenAddresses + if len(addresses) == 0 { + logger.Info("Hubble server is disabled") + return + } + epDNSGetter := hubble.NewLocalEndpointDNSGetter(d.endpointManager) + identityGetter := hubble.NewLocalIdentityGetter(d.identityAllocator) + ipGetter := hubble.NewLocalIPGetter(ipcache.IPIdentityCache) + serviceGetter := hubble.NewLocalServiceGetter(d.svc) + payloadParser, _ := parser.New(epDNSGetter, identityGetter, epDNSGetter, ipGetter, serviceGetter) + s := hubbleServer.NewLocalServer(payloadParser, option.Config.HubbleFlowBufferSize, logger) + go s.Start() + d.monitorAgent.GetMonitor().RegisterNewListener(context.TODO(), hubble.NewHubbleListener(s)) + logger.WithField("addresses", addresses).Info("Starting Hubble server") + hubbleServe.Serve(logger, addresses, s) + if option.Config.HubbleMetricsServer != "" { + logger.WithFields(logrus.Fields{ + "address": option.Config.HubbleMetricsServer, + "metrics": option.Config.HubbleMetrics, + }).Info("Starting Hubble Metrics server") + hubbleServe.EnableMetrics(log, option.Config.HubbleMetricsServer, option.Config.HubbleMetrics) + } +} diff --git a/pkg/option/config.go b/pkg/option/config.go index c7daae460b35..2b8564f3e304 100644 --- a/pkg/option/config.go +++ b/pkg/option/config.go @@ -669,6 +669,18 @@ const ( // EnableRemoteNodeIdentity enables use of the remote-node identity EnableRemoteNodeIdentity = "enable-remote-node-identity" + + // HubbleListenAddresses specifies addresses for Hubble server to listen to. + HubbleListenAddresses = "hubble-listen-addresses" + + // HubbleFlowBufferSize specifies the maximum number of flows in Hubble's buffer. + HubbleFlowBufferSize = "hubble-flow-buffer-size" + + // HubbleMetricsServer specifies the addresses to serve Hubble metrics on. + HubbleMetricsServer = "hubble-metrics-server" + + // HubbleMetrics specifies enabled metrics and their configuration options. + HubbleMetrics = "hubble-metrics" ) // Default string arguments @@ -1344,6 +1356,18 @@ type DaemonConfig struct { // EnableRemoteNodeIdentity enables use of the remote-node identity EnableRemoteNodeIdentity bool + + // HubbleListenAddresses specifies addresses for Hubble to listen to. + HubbleListenAddresses []string + + // HubbleFlowBufferSize specifies the maximum number of flows in Hubble's buffer. + HubbleFlowBufferSize int + + // HubbleMetricsServer specifies the addresses to serve Hubble metrics on. + HubbleMetricsServer string + + // HubbleMetrics specifies enabled metrics and their configuration options. + HubbleMetrics []string } var ( @@ -1942,6 +1966,12 @@ func (c *DaemonConfig) Populate() { } } + // Hubble options. + c.HubbleListenAddresses = viper.GetStringSlice(HubbleListenAddresses) + c.HubbleFlowBufferSize = viper.GetInt(HubbleFlowBufferSize) + c.HubbleMetricsServer = viper.GetString(HubbleMetricsServer) + c.HubbleMetrics = viper.GetStringSlice(HubbleMetrics) + // Hidden options c.ConfigFile = viper.GetString(ConfigFile) c.HTTP403Message = viper.GetString(HTTP403Message)