Enable cross-subnet mode with Calico by default

docs/networking/calico.md

@@ -53,7 +53,7 @@ As of Calico version 3.17, in order to use IP-in-IP encapsulation, Calico must u
 ### Enable Cross-Subnet mode in Calico
 
 Calico supports a new option for both of its IP-in-IP and VXLAN encapsulation modes where traffic is only encapsulated
-when it’s destined to subnets with intermediate infrastructure lacking Calico route awareness—for example, across
+when it’s destined to subnets with intermediate infrastructure lacking Calico route awareness, for example, across
 heterogeneous public clouds or on AWS where traffic is crossing availability zones.
 
 With this mode, encapsulation is only [performed selectively](https://docs.projectcalico.org/v3.10/networking/vxlan-ipip#configure-ip-in-ip-encapsulation-for-only-cross-subnet-traffic).
@@ -92,6 +92,9 @@ or
 ```
 depending on which encapsulation mode you have selected.
 
+**Cross-subnet mode is the default mode in kOps 1.22+** for both IP-in-IP and VXLAN encapsulation.
+It can be disabled or adjusted by setting the `ipipMode`, `vxlanMode` and `awsSrcDstCheck` options.
+
 In AWS an IAM policy will be added to all nodes to allow Calico to execute `ec2:DescribeInstances` and `ec2:ModifyNetworkInterfaceAttribute`, as required when [awsSrcDstCheck](https://docs.projectcalico.org/reference/resources/felixconfig#spec) is set.
 For older versions of kOps, an addon controller ([k8s-ec2-srcdst](https://github.com/ottoyiu/k8s-ec2-srcdst))
 will be deployed as a Pod (which will be scheduled on one of the masters) to facilitate the disabling of said source/destination address checks.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3160,8 +3160,8 @@ spec:
                     properties:
                       awsSrcDstCheck:
                         description: 'AWSSrcDstCheck enables/disables ENI source/destination
-                          checks (AWS only) Options: DoNothing (default), Enable,
-                          or Disable'
+                          checks (AWS only) Options: Disable (default), Enable, or
+                          DoNothing'
                         type: string
                       bpfEnabled:
                         description: BPFEnabled enables the eBPF dataplane mode.
@@ -3204,8 +3204,8 @@ spec:
                         pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                         x-kubernetes-int-or-string: true
                       crossSubnet:
-                        description: CrossSubnet enables Calico's cross-subnet mode
-                          when set to true
+                        description: CrossSubnet is deprecated as of kOps 1.22 and
+                          has no effect
                         type: boolean
                       encapsulationMode:
                         description: 'EncapsulationMode specifies the network packet
@@ -3217,11 +3217,12 @@ spec:
                           or vxlan'
                         type: string
                       ipipMode:
-                        description: IPIPMode is the encapsulation mode to use for
-                          the default Calico IPv4 pool created at start up, determining
-                          when to use IP-in-IP encapsulation, conveyed to the "calico-node"
-                          daemon container via the CALICO_IPV4POOL_IPIP environment
-                          variable.
+                        description: 'IPIPMode determines when to use IP-in-IP encapsulation
+                          for the default Calico IPv4 pool. It is conveyed to the
+                          "calico-node" daemon container via the CALICO_IPV4POOL_IPIP
+                          environment variable. EncapsulationMode must be set to "ipip".
+                          Options: "CrossSubnet", "Always", or "Never". Default: "CrossSubnet"
+                          if EncapsulationMode is "ipip", "Never" otherwise.'
                         type: string
                       iptablesBackend:
                         description: 'IptablesBackend controls which variant of iptables
@@ -3297,6 +3298,14 @@ spec:
                         description: Version overrides the Calico container image
                           tag.
                         type: string
+                      vxlanMode:
+                        description: 'VXLANMode determines when to use VXLAN encapsulation
+                          for the default Calico IPv4 pool. It is conveyed to the
+                          "calico-node" daemon container via the CALICO_IPV4POOL_VXLAN
+                          environment variable. EncapsulationMode must be set to "vxlan".
+                          Options: "CrossSubnet", "Always", or "Never". Default: "CrossSubnet"
+                          if EncapsulationMode is "vxlan", "Never" otherwise.'
+                        type: string
                       wireguardEnabled:
                         description: 'WireguardEnabled enables WireGuard encryption
                           for all on-the-wire pod-to-pod traffic (default: false)'
@@ -3333,7 +3342,7 @@ spec:
                         type: boolean
                       disableTxChecksumOffloading:
                         description: DisableTxChecksumOffloading is deprecated as
-                          of kops 1.19 and has no effect
+                          of kOps 1.19 and has no effect.
                         type: boolean
                       iptablesBackend:
                         description: 'IptablesBackend controls which variant of iptables
@@ -3836,7 +3845,7 @@ spec:
                         type: string
                       disableTxChecksumOffloading:
                         description: DisableTxChecksumOffloading is deprecated as
-                          of kops 1.19 and has no effect
+                          of kOps 1.19 and has no effect.
                         type: boolean
                       iptablesResyncSeconds:
                         description: IptablesResyncSeconds sets resync period for
@@ -3871,8 +3880,8 @@ spec:
                     type: object
                   romana:
                     description: RomanaNetworkingSpec declares that we want Romana
-                      networking Romana is deprecated as of kops 1.18 and removed
-                      as of kops 1.19
+                      networking Romana is deprecated as of kOps 1.18 and removed
+                      as of kOps 1.19.
                     properties:
                       daemonServiceIP:
                         description: DaemonServiceIP is the Kubernetes Service IP

pkg/apis/kops/networking.go

@@ -95,7 +95,7 @@ type WeaveNetworkingSpec struct {
 type FlannelNetworkingSpec struct {
 	// Backend is the backend overlay type we want to use (vxlan or udp)
 	Backend string `json:"backend,omitempty"`
-	// DisableTxChecksumOffloading is deprecated as of kops 1.19 and has no effect
+	// DisableTxChecksumOffloading is deprecated as of kOps 1.19 and has no effect.
 	DisableTxChecksumOffloading bool `json:"disableTxChecksumOffloading,omitempty"`
 	// IptablesResyncSeconds sets resync period for iptables rules, in seconds
 	IptablesResyncSeconds *int32 `json:"iptablesResyncSeconds,omitempty"`
@@ -109,7 +109,7 @@ type CalicoNetworkingSpec struct {
 	Version string `json:"version,omitempty"`
 
 	// AWSSrcDstCheck enables/disables ENI source/destination checks (AWS only)
-	// Options: DoNothing (default), Enable, or Disable
+	// Options: Disable (default), Enable, or DoNothing
 	AWSSrcDstCheck string `json:"awsSrcDstCheck,omitempty"`
 	// BPFEnabled enables the eBPF dataplane mode.
 	BPFEnabled bool `json:"bpfEnabled,omitempty"`
@@ -132,17 +132,19 @@ type CalicoNetworkingSpec struct {
 	ChainInsertMode string `json:"chainInsertMode,omitempty"`
 	// CPURequest CPU request of Calico container. Default: 100m
 	CPURequest *resource.Quantity `json:"cpuRequest,omitempty"`
-	// CrossSubnet enables Calico's cross-subnet mode when set to true
-	CrossSubnet bool `json:"crossSubnet,omitempty"`
+	// CrossSubnet is deprecated as of kOps 1.22 and has no effect
+	CrossSubnet *bool `json:"crossSubnet,omitempty"`
 	// EncapsulationMode specifies the network packet encapsulation protocol for Calico to use,
 	// employing such encapsulation at the necessary scope per the related CrossSubnet field. In
 	// "ipip" mode, Calico will use IP-in-IP encapsulation as needed. In "vxlan" mode, Calico will
 	// encapsulate packets as needed using the VXLAN scheme.
 	// Options: ipip (default) or vxlan
 	EncapsulationMode string `json:"encapsulationMode,omitempty"`
-	// IPIPMode is the encapsulation mode to use for the default Calico IPv4 pool created at start
-	// up, determining when to use IP-in-IP encapsulation, conveyed to the "calico-node" daemon
-	// container via the CALICO_IPV4POOL_IPIP environment variable.
+	// IPIPMode determines when to use IP-in-IP encapsulation for the default Calico IPv4 pool.
+	// It is conveyed to the "calico-node" daemon container via the CALICO_IPV4POOL_IPIP
+	// environment variable. EncapsulationMode must be set to "ipip".
+	// Options: "CrossSubnet", "Always", or "Never".
+	// Default: "CrossSubnet" if EncapsulationMode is "ipip", "Never" otherwise.
 	IPIPMode string `json:"ipipMode,omitempty"`
 	// IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route
 	// between nodes.  This should be set when the host has multiple interfaces
@@ -183,6 +185,12 @@ type CalicoNetworkingSpec struct {
 	TyphaPrometheusMetricsPort int32 `json:"typhaPrometheusMetricsPort,omitempty"`
 	// TyphaReplicas is the number of replicas of Typha to deploy
 	TyphaReplicas int32 `json:"typhaReplicas,omitempty"`
+	// VXLANMode determines when to use VXLAN encapsulation for the default Calico IPv4 pool.
+	// It is conveyed to the "calico-node" daemon container via the CALICO_IPV4POOL_VXLAN
+	// environment variable. EncapsulationMode must be set to "vxlan".
+	// Options: "CrossSubnet", "Always", or "Never".
+	// Default: "CrossSubnet" if EncapsulationMode is "vxlan", "Never" otherwise.
+	VXLANMode string `json:"vxlanMode,omitempty"`
 	// WireguardEnabled enables WireGuard encryption for all on-the-wire pod-to-pod traffic
 	// (default: false)
 	WireguardEnabled bool `json:"wireguardEnabled,omitempty"`
@@ -203,7 +211,7 @@ type CanalNetworkingSpec struct {
 	// DisableFlannelForwardRules configures Flannel to NOT add the
 	// default ACCEPT traffic rules to the iptables FORWARD chain
 	DisableFlannelForwardRules bool `json:"disableFlannelForwardRules,omitempty"`
-	// DisableTxChecksumOffloading is deprecated as of kops 1.19 and has no effect
+	// DisableTxChecksumOffloading is deprecated as of kOps 1.19 and has no effect.
 	DisableTxChecksumOffloading bool `json:"disableTxChecksumOffloading,omitempty"`
 	// IptablesBackend controls which variant of iptables binary Felix uses
 	// Default: Auto (other options: Legacy, NFT)
@@ -238,7 +246,7 @@ type KuberouterNetworkingSpec struct {
 }
 
 // RomanaNetworkingSpec declares that we want Romana networking
-// Romana is deprecated as of kops 1.18 and removed as of kops 1.19
+// Romana is deprecated as of kOps 1.18 and removed as of kOps 1.19.
 type RomanaNetworkingSpec struct {
 	// DaemonServiceIP is the Kubernetes Service IP for the romana-daemon pod
 	DaemonServiceIP string `json:"daemonServiceIP,omitempty"`

pkg/apis/kops/v1alpha2/networking.go

@@ -95,7 +95,7 @@ type WeaveNetworkingSpec struct {
 type FlannelNetworkingSpec struct {
 	// Backend is the backend overlay type we want to use (vxlan or udp)
 	Backend string `json:"backend,omitempty"`
-	// DisableTxChecksumOffloading is deprecated as of kops 1.19 and has no effect
+	// DisableTxChecksumOffloading is deprecated as of kOps 1.19 and has no effect.
 	DisableTxChecksumOffloading bool `json:"disableTxChecksumOffloading,omitempty"`
 	// IptablesResyncSeconds sets resync period for iptables rules, in seconds
 	IptablesResyncSeconds *int32 `json:"iptablesResyncSeconds,omitempty"`
@@ -109,7 +109,7 @@ type CalicoNetworkingSpec struct {
 	Version string `json:"version,omitempty"`
 
 	// AWSSrcDstCheck enables/disables ENI source/destination checks (AWS only)
-	// Options: DoNothing (default), Enable, or Disable
+	// Options: Disable (default), Enable, or DoNothing
 	AWSSrcDstCheck string `json:"awsSrcDstCheck,omitempty"`
 	// BPFEnabled enables the eBPF dataplane mode.
 	BPFEnabled bool `json:"bpfEnabled,omitempty"`
@@ -132,17 +132,19 @@ type CalicoNetworkingSpec struct {
 	ChainInsertMode string `json:"chainInsertMode,omitempty"`
 	// CPURequest CPU request of Calico container. Default: 100m
 	CPURequest *resource.Quantity `json:"cpuRequest,omitempty"`
-	// CrossSubnet enables Calico's cross-subnet mode when set to true
-	CrossSubnet bool `json:"crossSubnet,omitempty"`
+	// CrossSubnet is deprecated as of kOps 1.22 and has no effect
+	CrossSubnet *bool `json:"crossSubnet,omitempty"`
 	// EncapsulationMode specifies the network packet encapsulation protocol for Calico to use,
 	// employing such encapsulation at the necessary scope per the related CrossSubnet field. In
 	// "ipip" mode, Calico will use IP-in-IP encapsulation as needed. In "vxlan" mode, Calico will
 	// encapsulate packets as needed using the VXLAN scheme.
 	// Options: ipip (default) or vxlan
 	EncapsulationMode string `json:"encapsulationMode,omitempty"`
-	// IPIPMode is the encapsulation mode to use for the default Calico IPv4 pool created at start
-	// up, determining when to use IP-in-IP encapsulation, conveyed to the "calico-node" daemon
-	// container via the CALICO_IPV4POOL_IPIP environment variable.
+	// IPIPMode determines when to use IP-in-IP encapsulation for the default Calico IPv4 pool.
+	// It is conveyed to the "calico-node" daemon container via the CALICO_IPV4POOL_IPIP
+	// environment variable. EncapsulationMode must be set to "ipip".
+	// Options: "CrossSubnet", "Always", or "Never".
+	// Default: "CrossSubnet" if EncapsulationMode is "ipip", "Never" otherwise.
 	IPIPMode string `json:"ipipMode,omitempty"`
 	// IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route
 	// between nodes.  This should be set when the host has multiple interfaces
@@ -183,6 +185,12 @@ type CalicoNetworkingSpec struct {
 	TyphaPrometheusMetricsPort int32 `json:"typhaPrometheusMetricsPort,omitempty"`
 	// TyphaReplicas is the number of replicas of Typha to deploy
 	TyphaReplicas int32 `json:"typhaReplicas,omitempty"`
+	// VXLANMode determines when to use VXLAN encapsulation for the default Calico IPv4 pool.
+	// It is conveyed to the "calico-node" daemon container via the CALICO_IPV4POOL_VXLAN
+	// environment variable. EncapsulationMode must be set to "vxlan".
+	// Options: "CrossSubnet", "Always", or "Never".
+	// Default: "CrossSubnet" if EncapsulationMode is "vxlan", "Never" otherwise.
+	VXLANMode string `json:"vxlanMode,omitempty"`
 	// WireguardEnabled enables WireGuard encryption for all on-the-wire pod-to-pod traffic
 	// (default: false)
 	WireguardEnabled bool `json:"wireguardEnabled,omitempty"`
@@ -203,7 +211,7 @@ type CanalNetworkingSpec struct {
 	// DisableFlannelForwardRules configures Flannel to NOT add the
 	// default ACCEPT traffic rules to the iptables FORWARD chain
 	DisableFlannelForwardRules bool `json:"disableFlannelForwardRules,omitempty"`
-	// DisableTxChecksumOffloading is deprecated as of kops 1.19 and has no effect
+	// DisableTxChecksumOffloading is deprecated as of kOps 1.19 and has no effect.
 	DisableTxChecksumOffloading bool `json:"disableTxChecksumOffloading,omitempty"`
 	// IptablesBackend controls which variant of iptables binary Felix uses
 	// Default: Auto (other options: Legacy, NFT)
@@ -238,7 +246,7 @@ type KuberouterNetworkingSpec struct {
 }
 
 // RomanaNetworkingSpec declares that we want Romana networking
-// Romana is deprecated as of kops 1.18 and removed as of kops 1.19
+// Romana is deprecated as of kOps 1.18 and removed as of kOps 1.19.
 type RomanaNetworkingSpec struct {
 	// DaemonServiceIP is the Kubernetes Service IP for the romana-daemon pod
 	DaemonServiceIP string `json:"daemonServiceIP,omitempty"`

pkg/apis/kops/validation/validation.go

@@ -1085,6 +1085,12 @@ func validateNetworkingCalico(v *kops.CalicoNetworkingSpec, e kops.EtcdClusterSp
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("awsSrcDstCheck"), &v.AWSSrcDstCheck, valid)...)
 	}
 
+	if v.CrossSubnet != nil {
+		if fi.BoolValue(v.CrossSubnet) && v.AWSSrcDstCheck != "Disable" {
+			field.Invalid(fldPath.Child("crossSubnet"), v.CrossSubnet, "crossSubnet is deprecated, use awsSrcDstCheck instead")
+		}
+	}
+
 	if v.BPFExternalServiceMode != "" {
 		valid := []string{"Tunnel", "DSR"}
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("bpfExternalServiceMode"), &v.BPFExternalServiceMode, valid)...)
@@ -1110,14 +1116,24 @@ func validateNetworkingCalico(v *kops.CalicoNetworkingSpec, e kops.EtcdClusterSp
 
 	if v.IPIPMode != "" {
 		child := fldPath.Child("ipipMode")
-		allErrs = append(allErrs, validateCalicoIPPoolEncapsulationMode(v.IPIPMode, child)...)
+		allErrs = append(allErrs, validateCalicoEncapsulationMode(v.IPIPMode, child)...)
 		if v.IPIPMode != "Never" {
 			if v.EncapsulationMode != "" && v.EncapsulationMode != "ipip" {
 				allErrs = append(allErrs, field.Forbidden(child, `IP-in-IP encapsulation requires use of Calico's "ipip" encapsulation mode`))
 			}
 		}
 	}
 
+	if v.VXLANMode != "" {
+		child := fldPath.Child("vxlanMode")
+		allErrs = append(allErrs, validateCalicoEncapsulationMode(v.VXLANMode, child)...)
+		if v.VXLANMode != "Never" {
+			if v.EncapsulationMode != "" && v.EncapsulationMode != "vxlan" {
+				allErrs = append(allErrs, field.Forbidden(child, `VXLAN encapsulation requires use of Calico's "vxlan" encapsulation mode`))
+			}
+		}
+	}
+
 	if v.IPv4AutoDetectionMethod != "" {
 		allErrs = append(allErrs, validateCalicoAutoDetectionMethod(fldPath.Child("ipv4AutoDetectionMethod"), v.IPv4AutoDetectionMethod, ipv4.Version)...)
 	}
@@ -1199,7 +1215,7 @@ func validateCalicoAutoDetectionMethod(fldPath *field.Path, runtime string, vers
 	}
 }
 
-func validateCalicoIPPoolEncapsulationMode(mode string, fldPath *field.Path) field.ErrorList {
+func validateCalicoEncapsulationMode(mode string, fldPath *field.Path) field.ErrorList {
 	valid := []string{"Always", "CrossSubnet", "Never"}
 
 	allErrs := field.ErrorList{}

pkg/model/components/calico.go

@@ -35,27 +35,7 @@ func (b *CalicoOptionsBuilder) BuildOptions(o interface{}) error {
 		return nil
 	}
 
-	rebindIfEmpty := func(s *string, replacement string) bool {
-		if *s != "" {
-			return false
-		}
-		*s = replacement
-		return true
-	}
-
-	activeMode := "Always"
-	if c.CrossSubnet {
-		activeMode = "CrossSubnet"
-	}
-	switch c.EncapsulationMode {
-	case "":
-		c.EncapsulationMode = "ipip"
-		fallthrough
-	case "ipip":
-		rebindIfEmpty(&c.IPIPMode, activeMode)
-	case "vxlan":
-		rebindIfEmpty(&c.IPIPMode, "Never")
-	}
+	c.EncapsulationMode = "ipip"
 
 	return nil
 }