envoy: Add support for exposing Envoy Admin API

This is to support the capability to expose Envoy Admin API via new helm flags envoy.admin.{enabled,port} for both embedded and daemonset modes. Fixes: #30647 Signed-off-by: Tam Mach <tam.mach@cilium.io>
cilium · Feb 7, 2024 · d05be50 · d05be50
1 parent 4fd12d7
commit d05be50
Show file tree

Hide file tree

Showing 13 changed files with 178 additions and 0 deletions.
diff --git a/Documentation/helm-values.rst b/Documentation/helm-values.rst
diff --git a/Documentation/operations/system_requirements.rst b/Documentation/operations/system_requirements.rst
@@ -413,6 +413,7 @@ Port Range / Protocol    Description
 9890/tcp                 cilium-agent gops server (listening on 127.0.0.1)
 9891/tcp                 operator gops server (listening on 127.0.0.1)
 9893/tcp                 Hubble Relay gops server (listening on 127.0.0.1)
+9901/tcp                 cilium-envoy Admin API
 9962/tcp                 cilium-agent Prometheus metrics
 9963/tcp                 cilium-operator Prometheus metrics
 9964/tcp                 cilium-envoy Prometheus metrics

diff --git a/install/kubernetes/cilium/README.md b/install/kubernetes/cilium/README.md
diff --git a/install/kubernetes/cilium/files/cilium-envoy/configmap/bootstrap-config.json b/install/kubernetes/cilium/files/cilium-envoy/configmap/bootstrap-config.json
@@ -60,6 +60,61 @@
         ]
       },
       {{- end }}
+      {{- if and .Values.envoy.admin.enabled }}
+      {
+        "name": "envoy-admin-listener",
+        "address": {
+          "socket_address": {
+            "address": "0.0.0.0",
+            "port_value": {{ .Values.envoy.admin.port }}
+          }
+        },
+        "filter_chains": [
+          {
+            "filters": [
+              {
+                "name": "envoy.filters.network.http_connection_manager",
+                "typed_config": {
+                  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
+                  "stat_prefix": "envoy-admin-listener",
+                  "route_config": {
+                    "virtual_hosts": [
+                      {
+                        "name": "admin_route",
+                        "domains": [
+                          "*"
+                        ],
+                        "routes": [
+                          {
+                            "name": "admin_route",
+                            "match": {
+                              "prefix": "/"
+                            },
+                            "route": {
+                              "cluster": "/envoy-admin",
+                              "prefix_rewrite": "/"
+                            }
+                          }
+                        ]
+                      }
+                    ]
+                  },
+                  "http_filters": [
+                    {
+                      "name": "envoy.filters.http.router",
+                      "typed_config": {
+                        "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
+                      }
+                    }
+                  ],
+                  "stream_idle_timeout": "0s"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {{- end }}
       {
         "name": "envoy-health-listener",
         "address": {

diff --git a/install/kubernetes/cilium/templates/cilium-agent/daemonset.yaml b/install/kubernetes/cilium/templates/cilium-agent/daemonset.yaml
@@ -249,6 +249,12 @@ spec:
           hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}
           protocol: TCP
         {{- end }}
+        {{- if and .Values.envoy.admin.port (not .Values.envoy.enabled) }}
+        - name: envoy-admin
+          containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.admin.port }}
+          hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.admin.port }}
+          protocol: TCP
+        {{- end }}
         {{- end }}
         {{- if .Values.hubble.metrics.enabled }}
         - name: hubble-metrics

diff --git a/install/kubernetes/cilium/templates/cilium-configmap.yaml b/install/kubernetes/cilium/templates/cilium-configmap.yaml
@@ -228,6 +228,10 @@ data:
   {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled }}
   proxy-prometheus-port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}"
   {{- end }}
+
+  {{- if .Values.envoy.admin.port }}
+  proxy-admin-port: "{{ .Values.envoy.admin.port }}"
+  {{- end }}
 {{- end }}
 
 {{- if .Values.operator.prometheus.enabled }}

diff --git a/install/kubernetes/cilium/templates/cilium-envoy/daemonset.yaml b/install/kubernetes/cilium/templates/cilium-envoy/daemonset.yaml
@@ -158,6 +158,12 @@ spec:
           containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}
           hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}
           protocol: TCP
+        {{- if .Values.envoy.admin.port }}
+        - name: envoy-admin
+          containerPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.admin.port }}
+          hostPort: {{ .Values.proxy.prometheus.port | default .Values.envoy.admin.port }}
+          protocol: TCP
+        {{- end }}
         {{- end }}
         securityContext:
           {{- if .Values.envoy.securityContext.privileged }}

diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml
diff --git a/install/kubernetes/cilium/values.yaml.tmpl b/install/kubernetes/cilium/values.yaml.tmpl
@@ -1908,6 +1908,11 @@ envoy:
   # -- DNS policy for Cilium envoy pods.
   # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy
   dnsPolicy: ~
+  admin:
+    # -- Enable admin interface for cilium-envoy.
+    enabled: false
+    # -- Port for the admin interface.
+    port: 9901
   # -- Configure Cilium Envoy Prometheus options.
   # Note that some of these apply to either cilium-agent or cilium-envoy.
   prometheus:

diff --git a/pkg/envoy/secretsync_test.go b/pkg/envoy/secretsync_test.go
@@ -215,6 +215,10 @@ func (*fakeXdsServer) AddListener(name string, kind policy.L7ParserType, port ui
 	panic("unimplemented")
 }
 
+func (*fakeXdsServer) AddAdminListener(port uint16, wg *completion.WaitGroup) {
+	panic("unimplemented")
+}
+
 func (*fakeXdsServer) AddMetricsListener(port uint16, wg *completion.WaitGroup) {
 	panic("unimplemented")
 }

diff --git a/pkg/envoy/xds_server.go b/pkg/envoy/xds_server.go
@@ -73,6 +73,7 @@ const (
 	ingressClusterName    = "ingress-cluster"
 	ingressTLSClusterName = "ingress-cluster-tls"
 	metricsListenerName   = "envoy-prometheus-metrics-listener"
+	adminListenerName     = "envoy-admin-listener"
 )
 
 type Listener struct {
@@ -91,6 +92,8 @@ type Listener struct {
 type XDSServer interface {
 	// AddListener adds a listener to a running Envoy proxy.
 	AddListener(name string, kind policy.L7ParserType, port uint16, isIngress bool, mayUseOriginalSourceAddr bool, wg *completion.WaitGroup)
+	// AddAdminListener adds an Admin API listener to Envoy.
+	AddAdminListener(port uint16, wg *completion.WaitGroup)
 	// AddMetricsListener adds a prometheus metrics listener to Envoy.
 	AddMetricsListener(port uint16, wg *completion.WaitGroup)
 	// RemoveListener removes an existing Envoy Listener.
@@ -585,6 +588,71 @@ func GetLocalListenerAddresses(port uint16, ipv4, ipv6 bool) (*envoy_config_core
 	}, additionalAddress
 }
 
+func (s *xdsServer) AddAdminListener(port uint16, wg *completion.WaitGroup) {
+	if port == 0 {
+		return // 0 == disabled
+	}
+	log.WithField(logfields.Port, port).Debug("Envoy: AddMetricsListener")
+
+	s.addListener(adminListenerName, func() *envoy_config_listener.Listener {
+		hcmConfig := &envoy_config_http.HttpConnectionManager{
+			StatPrefix:       adminListenerName,
+			UseRemoteAddress: &wrapperspb.BoolValue{Value: true},
+			SkipXffAppend:    true,
+			HttpFilters: []*envoy_config_http.HttpFilter{{
+				Name: "envoy.filters.http.router",
+				ConfigType: &envoy_config_http.HttpFilter_TypedConfig{
+					TypedConfig: toAny(&envoy_extensions_filters_http_router_v3.Router{}),
+				},
+			}},
+			StreamIdleTimeout: &durationpb.Duration{}, // 0 == disabled
+			RouteSpecifier: &envoy_config_http.HttpConnectionManager_RouteConfig{
+				RouteConfig: &envoy_config_route.RouteConfiguration{
+					VirtualHosts: []*envoy_config_route.VirtualHost{{
+						Name:    "admin_listener_route",
+						Domains: []string{"*"},
+						Routes: []*envoy_config_route.Route{{
+							Match: &envoy_config_route.RouteMatch{
+								PathSpecifier: &envoy_config_route.RouteMatch_Prefix{Prefix: "/"},
+							},
+							Action: &envoy_config_route.Route_Route{
+								Route: &envoy_config_route.RouteAction{
+									ClusterSpecifier: &envoy_config_route.RouteAction_Cluster{
+										Cluster: adminClusterName,
+									},
+								},
+							},
+						}},
+					}},
+				},
+			},
+		}
+
+		listenerConf := &envoy_config_listener.Listener{
+			Name:    adminListenerName,
+			Address: getPublicListenerAddress(port, option.Config.IPv4Enabled(), option.Config.IPv6Enabled()),
+			FilterChains: []*envoy_config_listener.FilterChain{{
+				Filters: []*envoy_config_listener.Filter{{
+					Name: "envoy.filters.network.http_connection_manager",
+					ConfigType: &envoy_config_listener.Filter_TypedConfig{
+						TypedConfig: toAny(hcmConfig),
+					},
+				}},
+			}},
+		}
+
+		return listenerConf
+	}, wg, func(err error) {
+		if err != nil {
+			log.WithField(logfields.Port, port).WithError(err).Debug("Envoy: Adding admin listener failed")
+			// Remove the added listener in case of a failure
+			s.removeListener(adminListenerName, nil, false)
+		} else {
+			log.WithField(logfields.Port, port).Info("Envoy: Listening for Admin API")
+		}
+	}, false)
+}
+
 func (s *xdsServer) AddMetricsListener(port uint16, wg *completion.WaitGroup) {
 	if port == 0 {
 		return // 0 == disabled

diff --git a/pkg/envoy/xds_server_ondemand.go b/pkg/envoy/xds_server_ondemand.go
@@ -61,6 +61,13 @@ func (o *onDemandXdsStarter) startEmbeddedEnvoy(wg *completion.WaitGroup) error
 			// but then a failure to bind to the configured port would fail starting Envoy.
 			o.XDSServer.AddMetricsListener(uint16(option.Config.ProxyPrometheusPort), wg)
 		}
+
+		// Add Admin listener if the port is (properly) configured
+		if option.Config.ProxyAdminPort < 0 || option.Config.ProxyAdminPort > 65535 {
+			log.WithField(logfields.Port, option.Config.ProxyAdminPort).Error("Envoy: Invalid configured proxy-admin-port")
+		} else if option.Config.ProxyAdminPort != 0 {
+			o.XDSServer.AddAdminListener(uint16(option.Config.ProxyAdminPort), wg)
+		}
 	})
 
 	return startErr

diff --git a/pkg/option/config.go b/pkg/option/config.go
@@ -156,6 +156,9 @@ const (
 	// GopsPort is the TCP port for the gops server.
 	GopsPort = "gops-port"
 
+	// ProxyAdminPort specifies the port to serve Cilium Envoy Admin API on.
+	ProxyAdminPort = "proxy-admin-port"
+
 	// ProxyPrometheusPort specifies the port to serve Cilium host proxy metrics on.
 	ProxyPrometheusPort = "proxy-prometheus-port"
 
@@ -1606,6 +1609,9 @@ type DaemonConfig struct {
 	// agent for proxy configuration and access logging.
 	ProxyGID int
 
+	// ProxyAdminPort specifies the port to serve Envoy admin on.
+	ProxyAdminPort int
+
 	// ProxyPrometheusPort specifies the port to serve Envoy metrics on.
 	ProxyPrometheusPort int
 
@@ -3120,6 +3126,7 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) {
 	c.ProcFs = vp.GetString(ProcFs)
 	c.ProxyConnectTimeout = vp.GetInt(ProxyConnectTimeout)
 	c.ProxyGID = vp.GetInt(ProxyGID)
+	c.ProxyAdminPort = vp.GetInt(ProxyAdminPort)
 	c.ProxyPrometheusPort = vp.GetInt(ProxyPrometheusPort)
 	c.ProxyMaxRequestsPerConnection = vp.GetInt(ProxyMaxRequestsPerConnection)
 	c.ProxyMaxConnectionDuration = time.Duration(vp.GetInt64(ProxyMaxConnectionDuration))