Add flag to enforce mTLS on hubble relay clients

Fixes: #24265
Signed-off-by: Marek Chodor <mchodor@google.com>
Co-authored-by: ZSC <zacharysarah@users.noreply.github.com>

Documentation/operations/upgrade.rst

@@ -392,6 +392,10 @@ New Options
   ``tunnel=disabled``, now deprecated.
 * ``tunnel-protocol``: This option allows setting the tunneling protocol, in place
   of e.g., ``tunnel=vxlan``.
+* ``tls-relay-client-ca-files``: This option lets you provide a certificate authority (CA)
+  key and cert in Hubble Relay to authenticate Hubble Relay's clients with mTLS. When you provide a CA key and cert,
+  Hubble Relay enforces mTLS authentication on its clients (for example, Hubble CLI
+  client can't connect to Hubble Relay using ``--tls-allow-insecure``).
 
 Deprecated Options
 ~~~~~~~~~~~~~~~~~~
@@ -406,6 +410,11 @@ Deprecated Options
 * The ``cluster-pool-v2beta`` IPAM mode is deprecated and will be removed in v1.15.
   The functionality to dynamically allocate Pod CIDRs is now provided  by the
   more flexible ``multi-pool`` IPAM mode.
+* The following Hubble Relay options are deprecated and will be removed in v1.15:
+   * ``tls-client-cert-file`` (replaced with ``tls-hubble-client-cert-file``).
+   * ``tls-client-key-file`` (replaced with ``tls-hubble-client-key-file``).
+   * ``tls-server-cert-file`` (replaced with ``tls-relay-server-cert-file``).
+   * ``tls-server-key-file`` (replaced with ``tls-relay-server-key-file``).
 
 Deprecated Commands
 ~~~~~~~~~~~~~~~~~~~

hubble-relay/cmd/serve/serve.go

@@ -25,26 +25,31 @@ import (
 )
 
 const (
-	keyClusterName            = "cluster-name"
-	keyPprof                  = "pprof"
-	keyPprofAddress           = "pprof-address"
-	keyPprofPort              = "pprof-port"
-	keyGops                   = "gops"
-	keyGopsPort               = "gops-port"
-	keyDialTimeout            = "dial-timeout"
-	keyRetryTimeout           = "retry-timeout"
-	keyListenAddress          = "listen-address"
-	keyMetricsListenAddress   = "metrics-listen-address"
-	keyPeerService            = "peer-service"
-	keySortBufferMaxLen       = "sort-buffer-len-max"
-	keySortBufferDrainTimeout = "sort-buffer-drain-timeout"
-	keyTLSClientCertFile      = "tls-client-cert-file"
-	keyTLSClientKeyFile       = "tls-client-key-file"
-	keyTLSHubbleServerCAFiles = "tls-hubble-server-ca-files"
-	keyTLSClientDisabled      = "disable-client-tls"
-	keyTLSServerCertFile      = "tls-server-cert-file"
-	keyTLSServerKeyFile       = "tls-server-key-file"
-	keyTLSServerDisabled      = "disable-server-tls"
+	keyClusterName             = "cluster-name"
+	keyPprof                   = "pprof"
+	keyPprofAddress            = "pprof-address"
+	keyPprofPort               = "pprof-port"
+	keyGops                    = "gops"
+	keyGopsPort                = "gops-port"
+	keyDialTimeout             = "dial-timeout"
+	keyRetryTimeout            = "retry-timeout"
+	keyListenAddress           = "listen-address"
+	keyMetricsListenAddress    = "metrics-listen-address"
+	keyPeerService             = "peer-service"
+	keySortBufferMaxLen        = "sort-buffer-len-max"
+	keySortBufferDrainTimeout  = "sort-buffer-drain-timeout"
+	keyTLSHubbleClientCertFile = "tls-hubble-client-cert-file"
+	keyTLSClientCertFile       = "tls-client-cert-file" // Deprecated: replaced by keyTLSHubbleClientCertFile
+	keyTLSHubbleClientKeyFile  = "tls-hubble-client-key-file"
+	keyTLSClientKeyFile        = "tls-client-key-file" // Deprecated: replaced by keyTLSHubbleClientKeyFile
+	keyTLSHubbleServerCAFiles  = "tls-hubble-server-ca-files"
+	keyTLSClientDisabled       = "disable-client-tls"
+	keyTLSRelayServerCertFile  = "tls-relay-server-cert-file"
+	keyTLSServerCertFile       = "tls-server-cert-file" // Deprecated: replaced by keyTLSRelayServerCertFile
+	keyTLSRelayServerKeyFile   = "tls-relay-server-key-file"
+	keyTLSServerKeyFile        = "tls-server-key-file" // Deprecated: replaced by keyTLSRelayServerKeyFile
+	keyTLSRelayClientCAFiles   = "tls-relay-client-ca-files"
+	keyTLSServerDisabled       = "disable-server-tls"
 )
 
 // New creates a new serve command.
@@ -111,11 +116,23 @@ func New(vp *viper.Viper) *cobra.Command {
 		"",
 		"Path to the public key file for the client certificate to connect to Hubble server instances. The file must contain PEM encoded data.",
 	)
+	flags.MarkDeprecated(keyTLSClientCertFile, fmt.Sprintf("use --%s", keyTLSHubbleClientCertFile))
+	flags.String(
+		keyTLSHubbleClientCertFile,
+		"",
+		"Path to the public key file for the client certificate to connect to Hubble server instances. The file must contain PEM encoded data.",
+	)
 	flags.String(
 		keyTLSClientKeyFile,
 		"",
 		"Path to the private key file for the client certificate to connect to Hubble server instances. The file must contain PEM encoded data.",
 	)
+	flags.MarkDeprecated(keyTLSClientKeyFile, fmt.Sprintf("use --%s", keyTLSHubbleClientKeyFile))
+	flags.String(
+		keyTLSHubbleClientKeyFile,
+		"",
+		"Path to the private key file for the client certificate to connect to Hubble server instances. The file must contain PEM encoded data.",
+	)
 	flags.StringSlice(
 		keyTLSHubbleServerCAFiles,
 		[]string{},
@@ -126,11 +143,28 @@ func New(vp *viper.Viper) *cobra.Command {
 		"",
 		"Path to the public key file for the Hubble Relay server. The file must contain PEM encoded data.",
 	)
+	flags.MarkDeprecated(keyTLSServerCertFile, fmt.Sprintf("use --%s", keyTLSRelayServerCertFile))
+	flags.String(
+		keyTLSRelayServerCertFile,
+		"",
+		"Path to the public key file for the Hubble Relay server. The file must contain PEM encoded data.",
+	)
 	flags.String(
 		keyTLSServerKeyFile,
 		"",
 		"Path to the private key file for the Hubble Relay server. The file must contain PEM encoded data.",
 	)
+	flags.MarkDeprecated(keyTLSServerKeyFile, fmt.Sprintf("use --%s", keyTLSRelayServerKeyFile))
+	flags.String(
+		keyTLSRelayServerKeyFile,
+		"",
+		"Path to the private key file for the Hubble Relay server. The file must contain PEM encoded data.",
+	)
+	flags.StringSlice(
+		keyTLSRelayClientCAFiles,
+		[]string{},
+		"Paths to one or more public key files of the CA which sign certificates for Hubble Relay client instances.",
+	)
 	flags.Bool(
 		keyTLSClientDisabled,
 		false,
@@ -183,8 +217,8 @@ func runServe(vp *viper.Viper) error {
 		tlsClientConfig, err := certloader.NewWatchedClientConfig(
 			logger.WithField("config", "tls-to-hubble"),
 			vp.GetStringSlice(keyTLSHubbleServerCAFiles),
-			vp.GetString(keyTLSClientCertFile),
-			vp.GetString(keyTLSClientKeyFile),
+			hubbleClientCertFile(vp),
+			hubbleClientKeyFile(vp),
 		)
 		if err != nil {
 			return err
@@ -199,9 +233,9 @@ func runServe(vp *viper.Viper) error {
 	} else {
 		tlsServerConfig, err := certloader.NewWatchedServerConfig(
 			logger.WithField("config", "tls-server"),
-			nil, // no caFiles, mTLS is not supported for Relay clients yet.
-			vp.GetString(keyTLSServerCertFile),
-			vp.GetString(keyTLSServerKeyFile),
+			vp.GetStringSlice(keyTLSRelayClientCAFiles),
+			relayServerCertFile(vp),
+			relayServerKeyFile(vp),
 		)
 		if err != nil {
 			return err
@@ -247,3 +281,31 @@ func runServe(vp *viper.Viper) error {
 	}
 	return nil
 }
+
+func relayServerKeyFile(vp *viper.Viper) string {
+	if val := vp.GetString(keyTLSRelayServerKeyFile); val != "" {
+		return val
+	}
+	return vp.GetString(keyTLSServerKeyFile)
+}
+
+func relayServerCertFile(vp *viper.Viper) string {
+	if val := vp.GetString(keyTLSRelayServerCertFile); val != "" {
+		return val
+	}
+	return vp.GetString(keyTLSServerCertFile)
+}
+
+func hubbleClientKeyFile(vp *viper.Viper) string {
+	if val := vp.GetString(keyTLSHubbleClientKeyFile); val != "" {
+		return val
+	}
+	return vp.GetString(keyTLSClientKeyFile)
+}
+
+func hubbleClientCertFile(vp *viper.Viper) string {
+	if val := vp.GetString(keyTLSHubbleClientCertFile); val != "" {
+		return val
+	}
+	return vp.GetString(keyTLSClientCertFile)
+}

install/kubernetes/cilium/templates/hubble-relay/configmap.yaml

@@ -29,15 +29,18 @@ data:
     sort-buffer-len-max: {{ .Values.hubble.relay.sortBufferLenMax }}
     sort-buffer-drain-timeout: {{ .Values.hubble.relay.sortBufferDrainTimeout }}
     {{- if .Values.hubble.tls.enabled }}
-    tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
-    tls-client-key-file: /var/lib/hubble-relay/tls/client.key
+    tls-hubble-client-cert-file: /var/lib/hubble-relay/tls/client.crt
+    tls-hubble-client-key-file: /var/lib/hubble-relay/tls/client.key
     tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt
     {{- else }}
     disable-client-tls: true
     {{- end }}
     {{- if and .Values.hubble.tls.enabled .Values.hubble.relay.tls.server.enabled }}
-    tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt
-    tls-server-key-file: /var/lib/hubble-relay/tls/server.key
+    tls-relay-server-cert-file: /var/lib/hubble-relay/tls/server.crt
+    tls-relay-server-key-file: /var/lib/hubble-relay/tls/server.key
+    {{- if .Values.hubble.relay.tls.server.mtls }}
+    tls-relay-client-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt
+    {{- end }}
     {{- else }}
     disable-server-tls: true
     {{- end }}

install/kubernetes/cilium/values.yaml.tmpl

@@ -1220,6 +1220,10 @@ hubble:
         # When set to true, enable TLS on for Hubble Relay server
         # (ie: for clients connecting to the Hubble Relay API).
         enabled: false
+        # When set to true enforces mutual TLS between Hubble Relay server and its clients.
+        # False allow non-mutual TLS connections.
+        # This option has no effect when TLS is disabled.
+        mtls: false
         # These values need to be set manually if hubble.tls.auto.enabled is false.
         cert: ""
         key: ""