Fail in Agent initialization if GRE tunnel type is used with IPv6 (#3156

)

The 'gre' tunnel type does not work for IPv6 overlays. The correct
tunnel type for OVS would be 'ip6gre'. For dual-stack clusters with both
an IPv4 and an IPv6 overlay, we would need 2 default tunnel ports: one
for IPv4 (with type 'gre') and one for IPv6 (with type 'ip6gre'). This
would add complexity (and require testing) as the code currently assumes
a single default tunnel port. Rather than trying to support IPv6 with
the added complexity that it entails, we choose to fail during Agent
initialization for now if the user-provided tunnel type is 'gre' and the
cluster supports IPv6.

Note that this means that the default manifest for IPsec
(antrea-ipsec.yml) cannot be used in an IPv6 cluster as the tunnel type
defaults to GRE in that case. However, this is not new, and it is better
to fail explicitly rather than have a cluster where the Agent appears to
be running fine but there is no connectivity.

See #3150

Signed-off-by: Antonin Bas <abas@vmware.com>

build/yamls/antrea-aks.yml

@@ -2728,6 +2728,7 @@ data:
     # - vxlan
     # - gre
     # - stt
+    # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
     #tunnelType: geneve
 
     # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.
@@ -2975,7 +2976,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-bkgd5728h8
+  name: antrea-config-665mgk228m
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3046,7 +3047,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-bkgd5728h8
+          value: antrea-config-665mgk228m
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3097,7 +3098,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bkgd5728h8
+          name: antrea-config-665mgk228m
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3333,7 +3334,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-bkgd5728h8
+          name: antrea-config-665mgk228m
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-eks.yml

@@ -2728,6 +2728,7 @@ data:
     # - vxlan
     # - gre
     # - stt
+    # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
     #tunnelType: geneve
 
     # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.
@@ -2975,7 +2976,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-bkgd5728h8
+  name: antrea-config-665mgk228m
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3046,7 +3047,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-bkgd5728h8
+          value: antrea-config-665mgk228m
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3097,7 +3098,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bkgd5728h8
+          name: antrea-config-665mgk228m
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3335,7 +3336,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-bkgd5728h8
+          name: antrea-config-665mgk228m
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-gke.yml

@@ -2728,6 +2728,7 @@ data:
     # - vxlan
     # - gre
     # - stt
+    # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
     #tunnelType: geneve
 
     # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.
@@ -2975,7 +2976,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-c8cgkb72ch
+  name: antrea-config-2cfft84t59
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3046,7 +3047,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-c8cgkb72ch
+          value: antrea-config-2cfft84t59
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3097,7 +3098,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c8cgkb72ch
+          name: antrea-config-2cfft84t59
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3336,7 +3337,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-c8cgkb72ch
+          name: antrea-config-2cfft84t59
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-ipsec.yml

@@ -2728,6 +2728,7 @@ data:
     # - vxlan
     # - gre
     # - stt
+    # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
     tunnelType: gre
 
     # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.
@@ -2980,7 +2981,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-7tbgf2gbc9
+  name: antrea-config-2m674972kf
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3060,7 +3061,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-7tbgf2gbc9
+          value: antrea-config-2m674972kf
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3111,7 +3112,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-7tbgf2gbc9
+          name: antrea-config-2m674972kf
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3382,7 +3383,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-7tbgf2gbc9
+          name: antrea-config-2m674972kf
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea.yml

@@ -2728,6 +2728,7 @@ data:
     # - vxlan
     # - gre
     # - stt
+    # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
     #tunnelType: geneve
 
     # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.
@@ -2980,7 +2981,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-557m769449
+  name: antrea-config-h6d7mmd9hg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3051,7 +3052,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-557m769449
+          value: antrea-config-h6d7mmd9hg
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3102,7 +3103,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-557m769449
+          name: antrea-config-h6d7mmd9hg
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3338,7 +3339,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-557m769449
+          name: antrea-config-h6d7mmd9hg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/base/conf/antrea-agent.conf

@@ -87,6 +87,7 @@ featureGates:
 # - vxlan
 # - gre
 # - stt
+# Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters).
 #tunnelType: geneve
 
 # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode.

docs/design/architecture.md

@@ -321,9 +321,11 @@ to the local Pods on their Nodes.
 
 ### IPsec encryption
 
-Antrea supports encrypting GRE tunnel traffic with IPsec ESP. The IPsec
-implementation leverages [OVS IPsec](http://docs.openvswitch.org/en/latest/tutorials/ipsec)
-and leverages [strongSwan](https://www.strongswan.org) as the IKE daemon.
+Antrea supports encrypting Pod traffic across Linux Nodes with IPsec ESP. The
+IPsec implementation leverages [OVS
+IPsec](http://docs.openvswitch.org/en/latest/tutorials/ipsec) and leverages
+[strongSwan](https://www.strongswan.org) as the IKE daemon. By default GRE
+tunnels are used but other tunnel types are also supported.
 
 To enable IPsec, an extra container -`antrea-ipsec` - must be added to the
 Antrea Agent DaemonSet, which runs the `ovs-monitor-ipsec` and strongSwan

docs/network-requirements.md

@@ -3,15 +3,15 @@
 Antrea has a few network requirements to get started, ensure that your hosts and
 firewalls allow the necessary traffic based on your configuration.
 
-| Configuration                 | Host(s)             | ports/protocols                            |
-| ----------------------------- | ------------------- | ------------------------------------------ |
-| Antrea with VXLAN enabled     | All                 | UDP 4789                                   |
-| Antrea with Geneve enabled    | All                 | UDP 6081                                   |
-| Antrea with STT enabled       | All                 | TCP 7471                                   |
-| Antrea with GRE enabled       | All                 | IP Protocol ID 47                          |
-| Antrea with IPsec ESP enabled | All                 | IP protocol ID 50 and 51, UDP 500 and 4500 |
-| All                           | kube-apiserver host | TCP 443 or 6443\*                          |
-| All                           | All                 | TCP 10349, 10350                           |
+| Configuration                 | Host(s)             | ports/protocols                            | Other |
+| ----------------------------- | ------------------- | ------------------------------------------ | ----- |
+| Antrea with VXLAN enabled     | All                 | UDP 4789                                   |       |
+| Antrea with Geneve enabled    | All                 | UDP 6081                                   |       |
+| Antrea with STT enabled       | All                 | TCP 7471                                   |       |
+| Antrea with GRE enabled       | All                 | IP Protocol ID 47                          | No support for IPv6 clusters |
+| Antrea with IPsec ESP enabled | All                 | IP protocol ID 50 and 51, UDP 500 and 4500 |       |
+| All                           | kube-apiserver host | TCP 443 or 6443\*                          |       |
+| All                           | All                 | TCP 10349, 10350                           |       |
 
 \* _The value passed to kube-apiserver using the --secure-port flag. If you cannot
 locate this, check the targetPort value returned by kubectl get svc kubernetes -o yaml._

docs/traffic-encryption.md

@@ -8,6 +8,10 @@ WireGuard. Traffic encryption is not supported on Windows Nodes yet.
 IPsec encyption works for all tunnel types supported by OVS including Geneve,
 GRE, VXLAN, and STT tunnel.
 
+Note that GRE is not supported for IPv6 clusters (IPv6-only or dual-stack
+clusters). For such clusters, please choose a different tunnel type such as
+Geneve or VXLAN.
+
 ### Prerequisites
 
 IPsec requires a set of Linux kernel modules. Check the required kernel modules
@@ -64,6 +68,10 @@ After updating the PSK value, deploy Antrea with:
 kubectl apply -f antrea-ipsec.yml
 ```
 
+By default, the deployment yaml uses GRE as the tunnel type, which you can
+change by editing the file. You will need to change the tunnel type to another
+one if your cluster supports IPv6.
+
 ## WireGuard
 
 Antrea can leverage [WireGuard](https://www.wireguard.com) to encrypt Pod traffic

pkg/agent/agent.go

@@ -705,6 +705,23 @@ func (i *Initializer) setupDefaultTunnelInterface() error {
 		localIPStr = localIP.String()
 	}
 
+	// The correct OVS tunnel type to use GRE with an IPv6 overlay is
+	// "ip6gre" and not "gre". While it would be possible to support GRE for
+	// an IPv6-only cluster (by simply setting the tunnel type to "ip6gre"),
+	// things would be more complicated for a dual-stack cluster. For such a
+	// cluster, we have both IPv4 and IPv6 tunnels for inter-Node
+	// traffic. We would therefore need to create 2 default tunnel ports:
+	// one with type "gre" and one with type "ip6gre". This would introduce
+	// some complexity as the code currently assumes that we have a single
+	// default tunnel port. So for now, we just reject configurations that
+	// request a GRE tunnel when the Node network supports IPv6.
+	// See https://github.com/antrea-io/antrea/issues/3150
+	if i.networkConfig.TrafficEncapMode.SupportsEncap() &&
+		i.networkConfig.TunnelType == ovsconfig.GRETunnel &&
+		i.nodeConfig.NodeIPv6Addr != nil {
+		return fmt.Errorf("GRE tunnel type is not supported for IPv6 overlay")
+	}
+
 	// Enabling UDP checksum can greatly improve the performance for Geneve and
 	// VXLAN tunnels by triggering GRO on the receiver.
 	shouldEnableCsum := i.networkConfig.TunnelType == ovsconfig.GeneveTunnel || i.networkConfig.TunnelType == ovsconfig.VXLANTunnel
@@ -782,8 +799,7 @@ func (i *Initializer) initNodeLocalConfig() error {
 	}
 	node, err := i.client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 	if err != nil {
-		klog.Errorf("Failed to get node from K8s with name %s: %v", nodeName, err)
-		return err
+		return fmt.Errorf("failed to get Node with name %s from K8s: %w", nodeName, err)
 	}
 
 	// nodeInterface is the interface that has K8s Node IP. transportInterface is the interface that is used for
@@ -850,7 +866,7 @@ func (i *Initializer) initNodeLocalConfig() error {
 	// OVS in noencap case on Windows Nodes. As a mixture of Linux and Windows nodes is possible, Linux Nodes' MAC
 	// addresses should be reported too to make them discoverable for Windows Nodes.
 	if i.networkConfig.TrafficEncapMode.SupportsNoEncap() {
-		klog.Infof("Updating Node MAC annotation")
+		klog.InfoS("Updating Node MAC annotation")
 		if err := i.patchNodeAnnotations(nodeName, types.NodeMACAddressAnnotationKey, transportInterface.HardwareAddr.String()); err != nil {
 			return err
 		}
@@ -875,48 +891,47 @@ func (i *Initializer) initNodeLocalConfig() error {
 		return err
 	}
 	i.nodeConfig.NodeMTU = mtu
-	klog.Infof("Setting Node MTU=%d", mtu)
+	klog.InfoS("Setting Node MTU", "MTU", mtu)
 
 	if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
 		return nil
 	}
 
-	// Parse all PodCIDRs first, so that we could support IPv4/IPv6 dual-stack configurations.
+	// Parse all PodCIDRs first, so that we can support IPv4/IPv6 dual-stack configurations.
 	if node.Spec.PodCIDRs != nil {
 		for _, podCIDR := range node.Spec.PodCIDRs {
 			_, localSubnet, err := net.ParseCIDR(podCIDR)
 			if err != nil {
-				klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err)
+				klog.ErrorS(err, "Failed to parse subnet from Pod CIDR string", "CIDR", podCIDR)
 				return err
 			}
 			if localSubnet.IP.To4() != nil {
 				if i.nodeConfig.PodIPv4CIDR != nil {
-					klog.Warningf("One IPv4 PodCIDR is already configured on this Node, ignore the IPv4 Subnet CIDR %s", localSubnet.String())
+					klog.InfoS("One IPv4 PodCIDR is already configured on this Node, ignoring the IPv4 Subnet CIDR", "subnet", localSubnet)
 				} else {
 					i.nodeConfig.PodIPv4CIDR = localSubnet
-					klog.V(2).Infof("Configure IPv4 Subnet CIDR %s on this Node", localSubnet.String())
+					klog.V(2).InfoS("Configured IPv4 Subnet CIDR on this Node", "subnet", localSubnet)
 				}
 				continue
 			}
 			if i.nodeConfig.PodIPv6CIDR != nil {
-				klog.Warningf("One IPv6 PodCIDR is already configured on this Node, ignore the IPv6 subnet CIDR %s", localSubnet.String())
+				klog.InfoS("One IPv6 PodCIDR is already configured on this Node, ignoring the IPv6 Subnet CIDR", "subnet", localSubnet)
 			} else {
 				i.nodeConfig.PodIPv6CIDR = localSubnet
-				klog.V(2).Infof("Configure IPv6 Subnet CIDR %s on this Node", localSubnet.String())
+				klog.V(2).InfoS("Configured IPv6 Subnet CIDR on this Node", "subnet", localSubnet)
 			}
 		}
 		return nil
 	}
 	// Spec.PodCIDR can be empty due to misconfiguration.
 	if node.Spec.PodCIDR == "" {
-		klog.Errorf("Spec.PodCIDR is empty for Node %s. Please make sure --allocate-node-cidrs is enabled "+
-			"for kube-controller-manager and --cluster-cidr specifies a sufficient CIDR range", nodeName)
-		return fmt.Errorf("CIDR string is empty for node %s", nodeName)
+		klog.ErrorS(nil, "Spec.PodCIDR is empty for Node. Please make sure --allocate-node-cidrs is enabled "+
+			"for kube-controller-manager and --cluster-cidr specifies a sufficient CIDR range", "nodeName", nodeName)
+		return fmt.Errorf("CIDR string is empty for Node %s", nodeName)
 	}
 	_, localSubnet, err := net.ParseCIDR(node.Spec.PodCIDR)
 	if err != nil {
-		klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err)
-		return err
+		return fmt.Errorf("failed to parse subnet from CIDR string %s: %w", node.Spec.PodCIDR, err)
 	}
 	if localSubnet.IP.To4() != nil {
 		i.nodeConfig.PodIPv4CIDR = localSubnet