fix(dsr): add family for fwmark

pkg/controllers/proxy/linux_networking.go

@@ -298,8 +298,15 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot
 }
 
 // ipvsAddFWMarkService: creates an IPVS service using FWMARK
-func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, protocol, port uint16,
+func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, family, protocol, port uint16,
 	persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) {
+	var netmaskForFamily uint32
+	switch family {
+	case syscall.AF_INET:
+		netmaskForFamily = ipv4NetMaskBits
+	case syscall.AF_INET6:
+		netmaskForFamily = ipv6NetMaskBits
+	}
 	for _, svc := range svcs {
 		if fwMark == svc.FWMark {
 			if (persistent && (svc.Flags&ipvsPersistentFlagHex) == 0) ||
@@ -340,16 +347,30 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uin
 				klog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc))
 			}
 
+			if svc.AddressFamily != family {
+				svc.AddressFamily = family
+				svc.Netmask = netmaskForFamily
+				err := ln.ipvsUpdateService(svc)
+				if err != nil {
+					return nil, fmt.Errorf("failed to update the address family for service %s due to %v",
+						ipvsServiceString(svc), err)
+				}
+				klog.V(2).Infof("Updated address family for the service: %s", ipvsServiceString(svc))
+			}
+
 			klog.V(2).Infof("ipvs service %s already exists so returning", ipvsServiceString(svc))
 			return svc, nil
 		}
 	}
 
+	// Even though it may seem unintuitive to require a Netmask on an fwmark service, I found that it was necessary in
+	// order to get IPVS IPv6 services to work correctly. After reviewing the code, it the only difference between the
+	// netlink command that we build here and the one that ipvsadm was building was the netmask, after adding it, it
+	// began to work
 	svc := ipvs.Service{
 		FWMark:        fwMark,
-		AddressFamily: syscall.AF_INET,
-		Protocol:      protocol,
-		Port:          port,
+		AddressFamily: family,
+		Netmask:       netmaskForFamily,
 		SchedName:     ipvs.RoundRobin,
 	}
 

pkg/controllers/proxy/network_services_controller.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/cloudnativelabs/kube-router/v2/pkg/healthcheck"
@@ -157,7 +158,7 @@ type ipvsCalls interface {
 	ipvsUpdateDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
 	ipvsGetDestinations(ipvsSvc *ipvs.Service) ([]*ipvs.Destination, error)
 	ipvsDelDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error
-	ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, protocol, port uint16, persistent bool,
+	ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, family, protocol, port uint16, persistent bool,
 		persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error)
 }
 
@@ -1467,7 +1468,14 @@ func ipvsServiceString(s *ipvs.Service) string {
 }
 
 func ipvsDestinationString(d *ipvs.Destination) string {
-	return fmt.Sprintf("%s:%v (Weight: %v)", d.Address, d.Port, d.Weight)
+	var family string
+	switch d.AddressFamily {
+	case syscall.AF_INET:
+		family = "IPv4"
+	case syscall.AF_INET6:
+		family = "IPv6"
+	}
+	return fmt.Sprintf("%s:%v (Family: %s, Weight: %v)", d.Address, d.Port, family, d.Weight)
 }
 
 func ipvsSetPersistence(svc *ipvs.Service, p bool, timeout int32) {

pkg/controllers/proxy/service_endpoints_sync.go

@@ -439,12 +439,15 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svc *serviceI
 	protocol := convertSvcProtoToSysCallProto(svc.protocol)
 	var nodeIP net.IP
 	var family v1.IPFamily
+	var sysFamily uint16
 	if externalIP.To4() != nil {
 		nodeIP = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs)
 		family = v1.IPv4Protocol
+		sysFamily = syscall.AF_INET
 	} else {
 		nodeIP = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs)
 		family = v1.IPv6Protocol
+		sysFamily = syscall.AF_INET6
 	}
 
 	dummyVipInterface, err := nsc.ln.getKubeDummyInterface()
@@ -462,11 +465,11 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svc *serviceI
 		return fmt.Errorf("failed to generate FW mark")
 	}
 
-	ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(ipvsSvcs, fwMark, protocol, uint16(svc.port),
+	ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(ipvsSvcs, fwMark, sysFamily, protocol, uint16(svc.port),
 		svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags)
 	if err != nil {
-		return fmt.Errorf("failed to create IPVS service for External IP: %s due to: %s",
-			externalIP, err.Error())
+		return fmt.Errorf("failed to create IPVS service for FWMark service: %d (external IP: %s) due to: %s",
+			fwMark, externalIP, err.Error())
 	}
 
 	externalIPServiceID := fmt.Sprint(fwMark)