feat(transparent-proxy): deprecate argument 'redirect-inbound-port-v6' and introduce 'ip-family-mode'

UPGRADE.md

@@ -94,6 +94,15 @@ Example output of a zone ingress token:
 The option to configure authentication was deprecated and has been removed in release `2.7.x`. If you are still using `KUMA_DP_SERVER_AUTH_*`
 environment variables or `dpServer.auth.*` configuration, please migrate your configuration to use `dpServer.authn` before upgrade.
 
+### Deprecation of `--redirect-inbound-port-v6` flag and `runtime.kubernetes.injector.sidecarContainer.redirectPortInboundV6` configuration option.
+
+The `--redirect-inbound-port-v6` flag and the corresponding configuration option `runtime.kubernetes.injector.sidecarContainer.redirectPortInboundV6` are deprecated and will be removed in a future release of Kuma. These flags and configuration options were used to configure the port used for redirecting IPv6 traffic to Kuma.
+
+In the upcoming release, Kuma will redirect IPv6 traffic to the same port as IPv4 traffic (15006). This means that you no longer need to configure a separate port for IPv6 traffic. If you want to disable traffic redirection for IPv6 traffic, you can set `--ip-family-mode ipv4`. We have also added a new configuration option `runtime.kubernetes.injector.sidecarContainer.ipFamilyMode` to switch traffic redirection for IP families.
+
+We recommend that you update your configurations to use the new defaults for IPv6 traffic redirection. If you need to retain separate ports for IPv4 and IPv6 traffic, you can continue to use the deprecated flags and configuration options until they are removed.
+
+
 ## Upgrade to `2.6.x`
 
 ### Policy
@@ -173,17 +182,17 @@ With the release of Kuma 2.6.0, we've made some changes to the implementation of
 
 To ensure a smooth transition to Kuma 2.6.0, carefully review your existing configuration files and make necessary adjustments related to denied request responses and RBAC-related Envoy stats.
 
-### Deprecation of postgres driverName=postgres (lib/pq)
-
-The postgres driver `postgres` (lib/pq) is deprecated and will be removed in the future.
-Please migrate to the new postgres driver `pgx` by setting `DriverName=pgx` configuration option or `KUMA_STORE_POSTGRES_DRIVER_NAME=pgx` env variable.
-
-### Make format SI valid for bandwidth in MeshFaultInjection policy
+### Make SI format valid for bandwidth in MeshFaultInjection policy
 
 Prior to this upgrade `mbps` and `gbps` were used for units for parameter `conf.responseBandwidth.percentage`.
 These are not valid units according to the [International System of Units](https://en.wikipedia.org/wiki/International_System_of_Units) they are respectively corrected to `Gbps` and `Mbps` if using
 these invalid units convert them into `kbps` prior to upgrade to avoid invalid format.
 
+### Deprecation of postgres driverName=postgres (lib/pq)
+
+The postgres driver `postgres` (lib/pq) is deprecated and will be removed in the future.
+Please migrate to the new postgres driver `pgx` by setting `DriverName=pgx` configuration option or `KUMA_STORE_POSTGRES_DRIVER_NAME=pgx` env variable.
+
 ## Upgrade to `2.5.x`
 
 ### Transparent-proxy and CNI v1 removal

api/mesh/v1alpha1/dataplane.proto

@@ -220,6 +220,16 @@ message Dataplane {
 
     // TransparentProxying describes configuration for transparent proxying.
     message TransparentProxying {
+      enum IpFamilyMode {
+        // This value is to support backward compatibility and should not be used in new data plane objects.
+        UnSpecified = 0;
+        // Enables transparent proxying for both IPv4 and IPv6 traffic, This is the default.
+        DualStack = 1;
+        // Enables transparent proxying for IPv4 traffic only.
+        IPv4 = 2;
+        // Enables transparent proxying for IPv6 traffic only. This mode is to be supported in the future.
+        IPv6 = 3;
+      }
 
       // Port on which all inbound traffic is being transparently redirected.
       uint32 redirect_port_inbound = 1
@@ -245,6 +255,9 @@ message Dataplane {
       // can dramatically improve the performance of the mesh. If not specified,
       // all services in the mesh are reachable.
       repeated string reachable_services = 5;
+
+      // The IP family mode to enable for. Can be "IPv4" or "DualStack".
+      IpFamilyMode ip_family_mode  = 6;
     }
 
     // Gateway describes a configuration of the gateway of the data plane proxy.

app/cni/pkg/cni/annotations.go

@@ -1,6 +1,7 @@
 package cni
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -12,6 +13,8 @@ const (
 	defaultOutboundPort        = "15001"
 	defaultInboundPort         = "15006"
 	defaultInboundPortV6       = "15010"
+	zeroInboundPortV6          = "0"
+	defaultIPFamilyMode        = "unspecified"
 	defaultBuiltinDNSPort      = "15053"
 	defaultNoRedirectUID       = "5678"
 	defaultRedirectExcludePort = defaultProxyStatusPort
@@ -24,6 +27,7 @@ var annotationRegistry = map[string]*annotationParam{
 	"excludeOutboundPorts":        {"traffic.kuma.io/exclude-outbound-ports", defaultRedirectExcludePort, validatePortList},
 	"inboundPort":                 {"kuma.io/transparent-proxying-inbound-port", defaultInboundPort, validatePortList},
 	"inboundPortV6":               {"kuma.io/transparent-proxying-inbound-v6-port", defaultInboundPortV6, validatePortList},
+	"ipFamilyMode":                {"kuma.io/transparent-proxying-ip-family-mode", defaultIPFamilyMode, validateIpFamilyMode},
 	"outboundPort":                {"kuma.io/transparent-proxying-outbound-port", defaultOutboundPort, validatePortList},
 	"isGateway":                   {"kuma.io/gateway", "false", alwaysValidFunc},
 	"builtinDNS":                  {"kuma.io/builtin-dns", "false", alwaysValidFunc},
@@ -39,6 +43,7 @@ type IntermediateConfig struct {
 	targetPort                  string
 	inboundPort                 string
 	inboundPortV6               string
+	ipFamilyMode                string
 	noRedirectUID               string
 	excludeInboundPorts         string
 	excludeOutboundPorts        string
@@ -94,6 +99,20 @@ func validatePortList(ports string) error {
 	return nil
 }
 
+func validateIpFamilyMode(val string) error {
+	if val == "" {
+		return errors.New("value is empty")
+	}
+
+	validValues := []string{"dualstack", "ipv4", "ipv6"}
+	for _, valid := range validValues {
+		if valid == val {
+			return nil
+		}
+	}
+	return errors.New(fmt.Sprintf("value '%s' is not a valid IP family mode", val))
+}
+
 func getAnnotationOrDefault(name string, annotations map[string]string) (string, error) {
 	if _, ok := annotationRegistry[name]; !ok {
 		return "", errors.Errorf("no registered annotation with name %s", name)
@@ -117,6 +136,7 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig,
 	allFields := map[string]*string{
 		"outboundPort":                &intermediateConfig.targetPort,
 		"inboundPort":                 &intermediateConfig.inboundPort,
+		"ipFamilyMode":                &intermediateConfig.ipFamilyMode,
 		"inboundPortV6":               &intermediateConfig.inboundPortV6,
 		"excludeInboundPorts":         &intermediateConfig.excludeInboundPorts,
 		"excludeOutboundPorts":        &intermediateConfig.excludeOutboundPorts,
@@ -133,6 +153,8 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig,
 		}
 	}
 
+	// defaults to the ipv4 port if ipv6 port is not set
+	assignIPv6InboundRedirectPort(allFields)
 	return intermediateConfig, nil
 }
 
@@ -144,3 +166,23 @@ func mapAnnotation(annotations map[string]string, field *string, fieldName strin
 	*field = val
 	return nil
 }
+
+func assignIPv6InboundRedirectPort(allFields map[string]*string) {
+	v6PortFieldPointer := allFields["inboundPortV6"]
+	ipFamilyModeAnno := allFields["ipFamilyMode"]
+
+	if *ipFamilyModeAnno == defaultIPFamilyMode {
+		defaultIpMode := "dualstack"
+		// an existing pod can disable ipv6 by setting inboundPortV6 to 0, and they don't have ipFamilyMode set
+		if *v6PortFieldPointer == zeroInboundPortV6 {
+			defaultIpMode = "ipv4"
+		}
+		*ipFamilyModeAnno = defaultIpMode
+	}
+
+	if *ipFamilyModeAnno == "ipv4" {
+		*v6PortFieldPointer = "0"
+	} else if *v6PortFieldPointer == defaultInboundPortV6 {
+		*v6PortFieldPointer = *allFields["inboundPort"]
+	}
+}

app/cni/pkg/cni/annotations_test.go

@@ -13,6 +13,13 @@ var _ = Describe("NewIntermediateConfig", func() {
 		Expect(cfg.noRedirectUID).To(Equal(defaultNoRedirectUID))
 	})
 
+	It("should set ip family mode to dualstack by default", func() {
+		a := map[string]string{}
+		cfg, err := NewIntermediateConfig(a)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cfg.ipFamilyMode).To(Equal("dualstack"))
+	})
+
 	It("should override UID when annotation is specified", func() {
 		a := map[string]string{
 			"kuma.io/sidecar-uid": "1234",
@@ -21,4 +28,24 @@ var _ = Describe("NewIntermediateConfig", func() {
 		Expect(err).ToNot(HaveOccurred())
 		Expect(cfg.noRedirectUID).To(Equal("1234"))
 	})
+
+	It("should set inbound redirect port for ipv4 and ipv6", func() {
+		a := map[string]string{
+			"kuma.io/transparent-proxying-inbound-port": "1234",
+		}
+		cfg, err := NewIntermediateConfig(a)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cfg.inboundPort).To(Equal("1234"))
+		Expect(cfg.inboundPortV6).To(Equal("1234"))
+	})
+
+	It("should disable ipv6 inbound redirect when disabled by annotation", func() {
+		a := map[string]string{
+			"kuma.io/transparent-proxying-inbound-v6-port": "0",
+		}
+		cfg, err := NewIntermediateConfig(a)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cfg.ipFamilyMode).To(Equal("ipv4"))
+		Expect(cfg.inboundPortV6).To(Equal("0"))
+	})
 })

app/cni/pkg/cni/injector_linux.go

@@ -106,9 +106,15 @@ func mapToConfig(intermediateConfig *IntermediateConfig, logWriter *bufio.Writer
 	if err != nil {
 		return nil, err
 	}
-	inboundPortV6, err := convertToUint16("inbound port ipv6", intermediateConfig.inboundPortV6)
-	if err != nil {
-		return nil, err
+
+	var inboundPortV6 uint16
+	if intermediateConfig.ipFamilyMode == "ipv4" {
+		inboundPortV6 = 0
+	} else {
+		inboundPortV6, err = convertToUint16("inbound port ipv6", intermediateConfig.inboundPortV6)
+		if err != nil {
+			return nil, err
+		}
 	}
 	enableIpV6, err := transparentproxy.ShouldEnableIPv6(inboundPortV6)
 	if err != nil {
@@ -160,6 +166,6 @@ func GetEnabled(value string) (bool, error) {
 	case "disabled", "false":
 		return false, nil
 	default:
-		return false, errors.Errorf(`wrong value "%s", available values are: "enabled", "disabled"`, value)
+		return false, errors.Errorf(`wrong value "%s", available values are: "enabled", "disabled", "true", "false"`, value)
 	}
 }

app/kumactl/cmd/completion/testdata/bash.golden

@@ -5622,6 +5622,10 @@ _kumactl_install_transparent-proxy()
     two_word_flags+=("--exclude-outbound-udp-ports-for-uids")
     local_nonpersistent_flags+=("--exclude-outbound-udp-ports-for-uids")
     local_nonpersistent_flags+=("--exclude-outbound-udp-ports-for-uids=")
+    flags+=("--ip-family-mode=")
+    two_word_flags+=("--ip-family-mode")
+    local_nonpersistent_flags+=("--ip-family-mode")
+    local_nonpersistent_flags+=("--ip-family-mode=")
     flags+=("--kuma-dp-uid=")
     two_word_flags+=("--kuma-dp-uid")
     local_nonpersistent_flags+=("--kuma-dp-uid")

app/kumactl/cmd/install/install_transparent_proxy.go

@@ -23,6 +23,7 @@ type transparentProxyArgs struct {
 	Verbose                        bool
 	RedirectPortOutBound           string
 	RedirectInbound                bool
+	IpFamilyMode                   string
 	RedirectPortInBound            string
 	RedirectPortInBoundV6          string
 	ExcludeInboundPorts            string
@@ -53,12 +54,15 @@ type transparentProxyArgs struct {
 
 func newInstallTransparentProxy() *cobra.Command {
 	args := transparentProxyArgs{
-		DryRun:                         false,
-		Verbose:                        false,
-		RedirectPortOutBound:           "15001",
-		RedirectInbound:                true,
-		RedirectPortInBound:            "15006",
-		RedirectPortInBoundV6:          "15010",
+		DryRun:               false,
+		Verbose:              false,
+		RedirectPortOutBound: "15001",
+		RedirectInbound:      true,
+		RedirectPortInBound:  "15006",
+		// this argument is to be deprecated, it now defaults to the same port with ipv4 (instead of 15010)
+		// before deprecation, the user can still change it as needed
+		RedirectPortInBoundV6:          "15006",
+		IpFamilyMode:                   "dualstack",
 		ExcludeInboundPorts:            "",
 		ExcludeOutboundPorts:           "",
 		ExcludeOutboundTCPPortsForUIDs: []string{},
@@ -163,7 +167,14 @@ runuser -u kuma-dp -- \
 					_, _ = cmd.ErrOrStderr().Write([]byte("# [WARNING] --skip-dns-conntrack-zone-split will be ignored when --ebpf-enabled is being used\n"))
 				}
 			}
+
+			defaultCfg := config.DefaultConfig()
 			// Backward compatibility
+			if args.RedirectPortInBoundV6 != "" &&
+				args.RedirectPortInBoundV6 != fmt.Sprintf("%d", defaultCfg.Redirect.Inbound.Port) /* new default value, identical to ipv4 port */ &&
+				args.RedirectPortInBoundV6 != fmt.Sprintf("%d", defaultCfg.Redirect.Inbound.PortIPv6) /* old default value, dedicated for ipv6 */ {
+				_, _ = cmd.ErrOrStderr().Write([]byte("# [WARNING] flag --redirect-inbound-port-v6 is deprecated, use --redirect-inbound-port or --ip-family-mode ipv4 instead\n"))
+			}
 			if len(args.ExcludeOutboundPorts) > 0 && (len(args.ExcludeOutboundUDPPortsForUIDs) > 0 || len(args.ExcludeOutboundTCPPortsForUIDs) > 0) {
 				return errors.Errorf("--exclude-outbound-ports-for-uids set you can't use --exclude-outbound-tcp-ports-for-uids and --exclude-outbound-udp-ports-for-uids anymore")
 			}
@@ -190,10 +201,11 @@ runuser -u kuma-dp -- \
 
 	cmd.Flags().BoolVar(&args.DryRun, "dry-run", args.DryRun, "dry run")
 	cmd.Flags().BoolVar(&args.Verbose, "verbose", args.Verbose, "verbose")
+	cmd.Flags().StringVar(&args.IpFamilyMode, "ip-family-mode", args.IpFamilyMode, "The IP family mode to enable traffic redirection for. Can be 'dualstack' or 'ipv4'")
 	cmd.Flags().StringVar(&args.RedirectPortOutBound, "redirect-outbound-port", args.RedirectPortOutBound, "outbound port redirected to Envoy, as specified in dataplane's `networking.transparentProxying.redirectPortOutbound`")
 	cmd.Flags().BoolVar(&args.RedirectInbound, "redirect-inbound", args.RedirectInbound, "redirect the inbound traffic to the Envoy. Should be disabled for Gateway data plane proxies.")
 	cmd.Flags().StringVar(&args.RedirectPortInBound, "redirect-inbound-port", args.RedirectPortInBound, "inbound port redirected to Envoy, as specified in dataplane's `networking.transparentProxying.redirectPortInbound`")
-	cmd.Flags().StringVar(&args.RedirectPortInBoundV6, "redirect-inbound-port-v6", args.RedirectPortInBoundV6, "IPv6 inbound port redirected to Envoy, as specified in dataplane's `networking.transparentProxying.redirectPortInboundV6`")
+	cmd.Flags().StringVar(&args.RedirectPortInBoundV6, "redirect-inbound-port-v6", args.RedirectPortInBoundV6, "[DEPRECATED (use --redirect-inbound-port or --ip-family-mode ipv4)] IPv6 inbound port redirected to Envoy, as specified in dataplane's `networking.transparentProxying.redirectPortInboundV6`")
 	cmd.Flags().StringVar(&args.ExcludeInboundPorts, "exclude-inbound-ports", args.ExcludeInboundPorts, "a comma separated list of inbound ports to exclude from redirect to Envoy")
 	cmd.Flags().StringVar(&args.ExcludeOutboundPorts, "exclude-outbound-ports", args.ExcludeOutboundPorts, "a comma separated list of outbound ports to exclude from redirect to Envoy")
 	cmd.Flags().StringVar(&args.User, "kuma-dp-user", args.UID, "the user that will run kuma-dp")
@@ -260,6 +272,7 @@ func configureTransparentProxy(cmd *cobra.Command, args *transparentProxyArgs) e
 		RedirectInBound:           args.RedirectInbound,
 		RedirectPortInBound:       args.RedirectPortInBound,
 		RedirectPortInBoundV6:     args.RedirectPortInBoundV6,
+		IpFamilyMode:              args.IpFamilyMode,
 		ExcludeInboundPorts:       args.ExcludeInboundPorts,
 		ExcludeOutboundPorts:      args.ExcludeOutboundPorts,
 		ExcludedOutboundsForUIDs:  args.ExcludeOutboundPortsForUIDs,

app/kumactl/cmd/install/install_transparent_proxy_test.go

@@ -115,7 +115,7 @@ var _ = Describe("kumactl install transparent proxy", func() {
 			},
 			goldenFile: "install-transparent-proxy.dns.no-conntrack.golden.txt",
 		}),
-		Entry("should generate defaults with overrides", testCase{
+		Entry("should generate defaults with overrides and log deprecate", testCase{
 			extraArgs: []string{
 				"--kuma-dp-user", "root",
 				"--redirect-outbound-port", "12345",
@@ -124,6 +124,18 @@ var _ = Describe("kumactl install transparent proxy", func() {
 				"--exclude-outbound-ports", "2000,2001",
 				"--exclude-inbound-ports", "1000,1001",
 			},
+			errorMatcher: Equal("# [WARNING] flag --redirect-inbound-port-v6 is deprecated, use --redirect-inbound-port or --ip-family-mode ipv4 instead\n"),
+			goldenFile:   "install-transparent-proxy.overrides.golden.txt",
+		}),
+		Entry("should generate when ipv6 disabled", testCase{
+			extraArgs: []string{
+				"--kuma-dp-user", "root",
+				"--redirect-outbound-port", "12345",
+				"--redirect-inbound-port", "12346",
+				"--ip-family-mode", "ipv4",
+				"--exclude-outbound-ports", "2000,2001",
+				"--exclude-inbound-ports", "1000,1001",
+			},
 			goldenFile: "install-transparent-proxy.overrides.golden.txt",
 		}),
 		Entry("should generate defaults with outbound exclude ports", testCase{

docs/generated/raw/kuma-cp.yaml

@@ -269,7 +269,10 @@ runtime:
         # Redirect port for inbound traffic.
         redirectPortInbound: 15006 # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_INBOUND
         # Redirect port for inbound traffic.
-        redirectPortInboundV6: 15010 # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_INBOUND_V6
+        # Deprecated: Use redirectPortInbound or ipFamilyMode instead.
+        redirectPortInboundV6: 15006 # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_INBOUND_V6
+        # IP family mode enabled for traffic redirection, can be 'dualstack' or 'ipv4'
+        ipFamilyMode: dualstack # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_IP_FAMILY_MODE
         # Redirect port for outbound traffic.
         redirectPortOutbound: 15001 # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_OUTBOUND
         # User ID.

docs/generated/raw/protos/Dataplane.json

@@ -264,6 +264,27 @@
                     },
                     "type": "array",
                     "description": "List of reachable services (represented by the value of `kuma.io/service`) via transparent proxying. Setting an explicit list can dramatically improve the performance of the mesh. If not specified, all services in the mesh are reachable."
+                },
+                "ip_family_mode": {
+                    "enum": [
+                        "UnSpecified",
+                        0,
+                        "DualStack",
+                        1,
+                        "IPv4",
+                        2,
+                        "IPv6",
+                        3
+                    ],
+                    "oneOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "integer"
+                        }
+                    ],
+                    "title": "Ip Family Mode"
                 }
             },
             "additionalProperties": true,