USHIFT-7382: Add MTU boundary tests for C2CC and C2CC+IPsec

test/assets/c2cc/nettest-pod.yaml

@@ -0,0 +1,19 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: nettest-pod
+spec:
+  terminationGracePeriodSeconds: 0
+  containers:
+  - name: nettest
+    image: registry.access.redhat.com/ubi9/ubi:9.6
+    command: ["sleep", "infinity"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+      runAsNonRoot: true
+      runAsUser: 10001
+      seccompProfile:
+        type: RuntimeDefault

test/assets/network/jumbo-ipv6-network.xml

@@ -0,0 +1,14 @@
+<network>
+  <name>jumbo-ipv6</name>
+  <forward mode="nat">
+    <nat ipv6='yes'>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <mtu size='9000'/>
+  <ip family="ipv6" address="2001:db8:ca7:fe::1" prefix="96">
+    <dhcp>
+      <range start="2001:db8:ca7:fe::1000" end="2001:db8:ca7:fe::2000" />
+    </dhcp>
+  </ip>
+</network>

test/assets/network/jumbo-network.xml

@@ -0,0 +1,10 @@
+<network>
+  <name>jumbo</name>
+  <forward mode='nat'/>
+  <mtu size='9000'/>
+  <ip address='192.168.150.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.150.100' end='192.168.150.254'/>
+    </dhcp>
+  </ip>
+</network>

test/bin/c2cc_common.sh

@@ -188,11 +188,56 @@ ${network_yaml}IEOF
 EOF
 }
 
+# inject_kickstart_mtu sets the NIC MTU via three layered mechanisms to
+# guarantee the interface has jumbo MTU before MicroShift starts:
+#   1. udev rule — sets MTU at device detection time (before NM)
+#   2. NM conf.d — sets default MTU for all ethernet connections
+#   3. systemd service — final safety net, uses ip link set directly
+# Args: host mtu
+inject_kickstart_mtu() {
+    local -r host=$1
+    local -r mtu=$2
+
+    local -r ks_dir="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${host}"
+    cat >> "${ks_dir}/post-microshift.cfg" <<EOF
+# Layer 1: udev rule — earliest point to set MTU, runs before NM
+cat > /etc/udev/rules.d/99-jumbo-mtu.rules <<'UDEVEOF'
+ACTION=="add", SUBSYSTEM=="net", ATTR{type}=="1", ATTR{mtu}="${mtu}"
+UDEVEOF
+
+# Layer 2: NM conf.d — default ethernet MTU for auto-created connections
+cat > /etc/NetworkManager/conf.d/99-jumbo-mtu.conf <<'NMCONF'
+[connection-ethernet-jumbo]
+match-device=type:ethernet
+ethernet.mtu=${mtu}
+NMCONF
+
+# Layer 3: systemd service — runs ip link set before MicroShift
+cat > /etc/systemd/system/set-jumbo-mtu.service <<'SVCEOF'
+[Unit]
+Description=Set jumbo frame MTU on ethernet interfaces
+Before=microshift.service
+After=NetworkManager-wait-online.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c 'for dev in \$(ip -o link show type ether | awk -F: "{print \\\$2}" | tr -d " "); do ip link set dev "\$dev" mtu ${mtu}; done'
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+SVCEOF
+mkdir -p /etc/systemd/system/multi-user.target.wants
+ln -sf /etc/systemd/system/set-jumbo-mtu.service /etc/systemd/system/multi-user.target.wants/set-jumbo-mtu.service
+EOF
+}
+
 c2cc_create_vms() {
     local -r boot_commit_ref="${1}"
     local -r boot_blueprint="${2}"
     local -r network="${3:-default}"
     local -r ip_family="${4:-ipv4}"
+    local -r network_mtu="${5:-}"
 
     # Prepare kickstart for all hosts
     # prepare_kickstart args: vmname template commit_ref fips_enabled ipv6_only
@@ -223,9 +268,74 @@ c2cc_create_vms() {
         "${CLUSTER_C_POD_CIDR}" "${CLUSTER_C_SVC_CIDR}" \
         "${CLUSTER_C_POD_CIDR_DUAL}" "${CLUSTER_C_SVC_CIDR_DUAL}"
 
-    launch_vm "${boot_blueprint}" --vmname host1 --network "${network}"
-    launch_vm "${boot_blueprint}" --vmname host2 --network "${network}"
-    launch_vm "${boot_blueprint}" --vmname host3 --network "${network}"
+    # Set the NIC MTU via NetworkManager in the kickstart so the guest boots
+    # with the correct MTU before MicroShift starts.
+    local mtu_args=()
+    if [ -n "${network_mtu}" ]; then
+        mtu_args=(--network_mtu "${network_mtu}")
+        for host in host1 host2 host3; do
+            inject_kickstart_mtu "${host}" "${network_mtu}"
+        done
+    fi
+
+    launch_vm "${boot_blueprint}" --vmname host1 --network "${network}" "${mtu_args[@]}"
+    launch_vm "${boot_blueprint}" --vmname host2 --network "${network}" "${mtu_args[@]}"
+    launch_vm "${boot_blueprint}" --vmname host3 --network "${network}" "${mtu_args[@]}"
+}
+
+# c2cc_create_jumbo_network creates a libvirt network with MTU 9000 for
+# jumbo frame testing. Idempotent — skips if the network already exists.
+c2cc_create_jumbo_network() {
+    if sudo virsh net-info jumbo &>/dev/null; then
+        return 0
+    fi
+    local -r netconfig="${ROOTDIR}/test/assets/network/jumbo-network.xml"
+    sudo virsh net-define "${netconfig}"
+    sudo virsh net-start jumbo
+    sudo virsh net-autostart jumbo
+}
+
+# c2cc_create_jumbo_ipv6_network creates a single-stack IPv6 libvirt network
+# with MTU 9000 for jumbo frame testing. Idempotent — skips if the network
+# already exists. Kept separate from the shared VM_IPV6_NETWORK ("ipv6") so
+# jumbo MTU doesn't affect other IPv6 scenarios.
+c2cc_create_jumbo_ipv6_network() {
+    if sudo virsh net-info jumbo-ipv6 &>/dev/null; then
+        return 0
+    fi
+    local -r netconfig="${ROOTDIR}/test/assets/network/jumbo-ipv6-network.xml"
+    sudo virsh net-define "${netconfig}"
+    sudo virsh net-start jumbo-ipv6
+    sudo virsh net-autostart jumbo-ipv6
+
+    # Add the bridge's IPv6 subnet to the firewall trusted zone so VMs
+    # can reach the hypervisor's services (kickstart web server, registry).
+    # The standard IPv6 network ("ipv6") gets this via manage_hypervisor_config.sh,
+    # but jumbo-ipv6 is created on-the-fly and needs it explicitly.
+    local bridge
+    bridge=$(sudo virsh net-info jumbo-ipv6 | grep '^Bridge:' | awk '{print $2}')
+    local ip
+    for ip in $(ip addr show "${bridge}" | grep "scope global" | awk '{print $2}'); do
+        sudo firewall-cmd --permanent --zone=trusted --add-source="${ip}"
+    done
+    sudo firewall-cmd --reload
+
+    # Wait for IPv6 DAD (Duplicate Address Detection) to complete and the
+    # web server to become reachable on the new bridge address. Until DAD
+    # finishes, the address is in "tentative" state and the kernel rejects
+    # incoming connections, causing kickstart file fetch to fail.
+    local bridge_ip
+    bridge_ip=$(ip -6 addr show dev "${bridge}" scope global | awk '/inet6/{print $2}' | cut -d/ -f1 | head -1)
+    local -r max_wait=30
+    local waited=0
+    while ! curl -s -o /dev/null --connect-timeout 2 "http://[${bridge_ip}]:${WEB_SERVER_PORT}/"; do
+        if (( waited >= max_wait )); then
+            echo "WARNING: web server not reachable on [${bridge_ip}]:${WEB_SERVER_PORT} after ${max_wait}s"
+            break
+        fi
+        sleep 1
+        (( waited++ ))
+    done
 }
 
 c2cc_remove_vms() {

test/bin/scenario.sh

@@ -804,6 +804,7 @@ EOF
 #           <boot_blueprint> \
 #           [--vmname <name>] \
 #           [--network <name>[,<name>...]] \
+#           [--network_mtu <size>] \
 #           [--vm_vcpus <vcpus>] \
 #           [--vm_memory <memory>] \
 #           [--vm_disksize <disksize>] \
@@ -820,6 +821,14 @@ EOF
 #   [--network <name>[,<name>...]]: A comma-separated list for the networks used
 #                                   when creating the VM. Each network entry will
 #                                   create a NIC and they are repeatable.
+#   [--network_mtu <size>]:         Sets the guest-visible MTU on every NIC via
+#                                   virt-install's mtu.size sub-option. Required
+#                                   in addition to a libvirt network's own <mtu>
+#                                   element — QEMU only negotiates a larger MTU
+#                                   with the guest's virtio-net driver when the
+#                                   domain's own <interface> XML requests it;
+#                                   the network-level MTU alone only affects the
+#                                   host-side bridge and tap devices.
 #   [--no_network]:                 Do not configure any network attachments (and
 #                                   therefore no NICs) for the VM.
 #   [--vm_vcpus <vcpus>]:           Number of vCPUs for the VM.
@@ -830,6 +839,7 @@ launch_vm() {
     # Set default values for the optional arguments
     local vmname="host1"
     local network="default"
+    local network_mtu=""
     local vm_memory=4096
     local vm_vcpus=2
     local vm_disksize=20
@@ -848,7 +858,7 @@ launch_vm() {
     # Parse the optional arguments
     while [ $# -gt 0 ]; do
         case "$1" in
-            --vmname|--vm_vcpus|--vm_memory|--vm_disksize)
+            --vmname|--vm_vcpus|--vm_memory|--vm_disksize|--network_mtu)
                 var="${1/--/}"
                 if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
                     declare "${var}=$2"
@@ -960,6 +970,15 @@ launch_vm() {
         if sudo virsh nwfilter-list | awk '{print $2}' | grep -qx "${n}"; then
             vm_network_args+=",filterref=${n}"
         fi
+
+        # A libvirt network's own <mtu> element only sets the host-side bridge
+        # and tap device MTU. The guest only sees a larger MTU if QEMU also
+        # negotiates it with the guest's virtio-net driver, which requires an
+        # explicit MTU on the domain's own interface.
+        if [ -n "${network_mtu}" ]; then
+            vm_network_args+=",mtu.size=${network_mtu}"
+        fi
+
         vm_network_args+=" "
     done
     if [ -z "${vm_network_args}" ] ; then

test/resources/c2cc.resource

@@ -299,13 +299,14 @@ Verify Corefile Does Not Contain C2CC Server Block
     Should Not Contain    ${stdout}    ${domain}:5353
 
 Deploy Test Workloads
-    [Documentation]    Create namespace and deploy hello-microshift + curl-pod on all clusters.
+    [Documentation]    Create namespace and deploy hello-microshift, curl-pod, and nettest-pod on all clusters.
     VAR    ${assets}=    ${EXECDIR}/assets/c2cc
     FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
         ${ns}=    Create Unique Namespace On Cluster    ${alias}
         Set To Dictionary    ${NAMESPACES}    ${alias}    ${ns}
         Oc On Cluster    ${alias}    oc apply -n ${ns} -f ${assets}/hello-microshift.yaml
         Oc On Cluster    ${alias}    oc apply -n ${ns} -f ${assets}/curl-pod.yaml
+        Oc On Cluster    ${alias}    oc apply -n ${ns} -f ${assets}/nettest-pod.yaml
     END
     Wait For Test Pods
     Wait For Service Endpoints
@@ -315,7 +316,7 @@ Wait For Test Pods
     FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
         Oc On Cluster
         ...    ${alias}
-        ...    oc wait pod/hello-microshift pod/curl-pod -n ${NAMESPACES}[${alias}] --for=condition=Ready --timeout=120s
+        ...    oc wait pod/hello-microshift pod/curl-pod pod/nettest-pod -n ${NAMESPACES}[${alias}] --for=condition=Ready --timeout=120s
     END
 
 Wait For Service Endpoints

test/resources/ipsec.resource

@@ -97,11 +97,13 @@ Add NFTables IPsec Enforcement Rules
     ...    In tunnel mode, decrypted packets have pod/service CIDRs as dst — plaintext
     ...    packets arriving without ESP are caught by this rule.
     [Arguments]    ${alias}    ${local_pod_cidr}
+    ${is_ipv6}=    Evaluate    ':' in '''${local_pod_cidr}'''
+    ${daddr_family}=    Set Variable If    ${is_ipv6}    ip6    ip
     Command On Cluster    ${alias}    nft add table inet c2cc_ipsec_test
     Command On Cluster    ${alias}
     ...    nft 'add chain inet c2cc_ipsec_test enforce { type filter hook input priority -150; policy accept; }'
     Command On Cluster    ${alias}
-    ...    nft add rule inet c2cc_ipsec_test enforce ip daddr ${local_pod_cidr} meta ipsec missing counter drop
+    ...    nft add rule inet c2cc_ipsec_test enforce ${daddr_family} daddr ${local_pod_cidr} meta ipsec missing counter drop
 
 Remove NFTables IPsec Enforcement Rules
     [Documentation]    Remove C2CC IPsec enforcement nftables table and all its rules.
@@ -113,8 +115,13 @@ Remove NFTables IPsec Enforcement Rules
 Curl Should Fail From Cluster
     [Documentation]    Verify curl from curl-pod times out or fails (no "Hello from" in response).
     [Arguments]    ${alias}    ${ip}    ${port}    ${ns}
+    ${is_ipv6}=    Evaluate    ':' in '''${ip}'''
+    ${url}=    Set Variable If
+    ...    ${is_ipv6}
+    ...    http://[${ip}]:${port}/cgi-bin/hello
+    ...    http://${ip}:${port}/cgi-bin/hello
     ${stdout}=    Oc On Cluster    ${alias}
-    ...    oc exec curl-pod -n ${ns} -- curl -sS --max-time 5 http://${ip}:${port}/cgi-bin/hello 2>&1 || true
+    ...    oc exec curl-pod -n ${ns} -- curl -sS --max-time 5 ${url} 2>&1 || true
     ...    allow_fail=${TRUE}
     Should Not Contain    ${stdout}    Hello from
 
@@ -123,6 +130,100 @@ Curl From Host Should Fail
     ...    Expects failure — host-originated traffic is not matched by tunnel-mode
     ...    IPsec selectors scoped to pod/service CIDRs.
     [Arguments]    ${alias}    ${pod_ip}    ${port}
+    ${is_ipv6}=    Evaluate    ':' in '''${pod_ip}'''
+    ${url}=    Set Variable If
+    ...    ${is_ipv6}
+    ...    http://[${pod_ip}]:${port}/cgi-bin/hello
+    ...    http://${pod_ip}:${port}/cgi-bin/hello
     ${stdout}=    Disruptive Command On Cluster    ${alias}
-    ...    curl -sS --max-time 5 http://${pod_ip}:${port}/cgi-bin/hello 2>&1 || true
+    ...    curl -sS --max-time 5 ${url} 2>&1 || true
     Should Not Contain    ${stdout}    Hello from
+
+Ping With DF Bit And Verify
+    [Documentation]    Send a UDP datagram with DF bit set (IP_PMTUDISC_DO / IPV6_DONTFRAG)
+    ...    via Python3. Asserts the kernel accepted the datagram (fits within the
+    ...    ESP-adjusted PMTU). Auto-detects IPv4 vs IPv6 from the destination address.
+    ...    Uses UDP instead of ICMP to avoid requiring NET_RAW capability.
+    ...    UDP overhead (IP+UDP) matches ICMP overhead (IP+ICMP): 28B for IPv4,
+    ...    48B for IPv6 (40B header instead of 20B), so payload sizes are
+    ...    equivalent to ping -s values within each family.
+    [Arguments]    ${alias}    ${dest_ip}    ${payload_size}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc exec nettest-pod -n ${NAMESPACES}[${alias}] -- python3 -c 'import socket,sys;a=sys.argv[1];f=socket.AF_INET6 if ":" in a else socket.AF_INET;s=socket.socket(f,socket.SOCK_DGRAM);s.setsockopt(41,62,1) if f==socket.AF_INET6 else s.setsockopt(socket.IPPROTO_IP,10,2);s.sendto(b"A"*${payload_size},(a,9999));print("OK")' ${dest_ip} 2>&1
+    ...    allow_fail=${TRUE}
+    Should Contain    ${stdout}    OK
+    ...    DF-bit UDP send of ${payload_size}B to ${dest_ip} failed: ${stdout}
+
+Ping With DF Bit Should Fail
+    [Documentation]    Send a UDP datagram with DF bit set. Asserts EMSGSIZE (Message too long).
+    ...    Proves the datagram exceeds the ESP-adjusted PMTU. Auto-detects IPv4 vs
+    ...    IPv6 from the destination address.
+    [Arguments]    ${alias}    ${dest_ip}    ${payload_size}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc exec nettest-pod -n ${NAMESPACES}[${alias}] -- python3 -c 'import socket,sys;a=sys.argv[1];f=socket.AF_INET6 if ":" in a else socket.AF_INET;s=socket.socket(f,socket.SOCK_DGRAM);s.setsockopt(41,62,1) if f==socket.AF_INET6 else s.setsockopt(socket.IPPROTO_IP,10,2);s.sendto(b"A"*${payload_size},(a,9999));print("OK")' ${dest_ip} 2>&1 || true
+    ...    allow_fail=${TRUE}
+    Should Not Contain    ${stdout}    OK
+    ...    DF-bit UDP send of ${payload_size}B should have been rejected but succeeded
+
+Get Pod Interface MTU
+    [Documentation]    Return the MTU of the pod's eth0 interface.
+    [Arguments]    ${alias}    ${pod}    ${ns}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc exec ${pod} -n ${ns} -- cat /sys/class/net/eth0/mtu
+    ${mtu}=    Strip String    ${stdout}
+    RETURN    ${mtu}
+
+Send Large Payload And Verify
+    [Documentation]    Send a large payload via curl POST and verify it succeeds.
+    [Arguments]    ${alias}    ${ip}    ${size}
+    ${is_ipv6}=    Evaluate    ':' in '''${ip}'''
+    ${url}=    Set Variable If
+    ...    ${is_ipv6}
+    ...    http://[${ip}]:8080/cgi-bin/hello
+    ...    http://${ip}:8080/cgi-bin/hello
+    ${stdout}=    Oc On Cluster
+    ...    ${alias}
+    ...    oc exec curl-pod -n ${NAMESPACES}[${alias}] -- sh -c 'dd if=/dev/zero bs=${size} count=1 2>/dev/null | curl -sS --max-time 15 --data-binary @- ${url}'
+    Should Contain    ${stdout}    Hello from
+
+Ping DF Bit Between All Clusters
+    [Documentation]    Send DF-bit UDP payload across all 6 cluster pairs.
+    [Arguments]    ${size}
+    FOR    ${src}    ${dst}    IN
+    ...    cluster-a    cluster-b
+    ...    cluster-a    cluster-c
+    ...    cluster-b    cluster-a
+    ...    cluster-b    cluster-c
+    ...    cluster-c    cluster-a
+    ...    cluster-c    cluster-b
+        ${ip}=    Get Hello Pod IP    ${dst}
+        Ping With DF Bit And Verify    ${src}    ${ip}    ${size}
+    END
+
+Ping DF Bit Should Fail Between All Clusters
+    [Documentation]    Send DF-bit UDP payload expecting rejection on all 6 pairs.
+    [Arguments]    ${size}
+    FOR    ${src}    ${dst}    IN
+    ...    cluster-a    cluster-b
+    ...    cluster-a    cluster-c
+    ...    cluster-b    cluster-a
+    ...    cluster-b    cluster-c
+    ...    cluster-c    cluster-a
+    ...    cluster-c    cluster-b
+        ${ip}=    Get Hello Pod IP    ${dst}
+        Ping With DF Bit Should Fail    ${src}    ${ip}    ${size}
+    END
+
+Large Payload Between All Clusters
+    [Documentation]    Send large TCP payload across all 6 cluster pairs.
+    [Arguments]    ${size}
+    FOR    ${src}    ${dst}    IN
+    ...    cluster-a    cluster-b
+    ...    cluster-a    cluster-c
+    ...    cluster-b    cluster-a
+    ...    cluster-b    cluster-c
+    ...    cluster-c    cluster-a
+    ...    cluster-c    cluster-b
+        ${ip}=    Get Hello Pod IP    ${dst}
+        Send Large Payload And Verify    ${src}    ${ip}    ${size}
+    END