diff --git a/pkg/controllers/proxy/linux_networking.go b/pkg/controllers/proxy/linux_networking.go index 03c12f668..ba45838b1 100644 --- a/pkg/controllers/proxy/linux_networking.go +++ b/pkg/controllers/proxy/linux_networking.go @@ -298,8 +298,15 @@ func (ln *linuxNetworking) ipvsAddService(svcs []*ipvs.Service, vip net.IP, prot } // ipvsAddFWMarkService: creates an IPVS service using FWMARK -func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, protocol, port uint16, +func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, family, protocol, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) { + var netmaskForFamily uint32 + switch family { + case syscall.AF_INET: + netmaskForFamily = ipv4NetMaskBits + case syscall.AF_INET6: + netmaskForFamily = ipv6NetMaskBits + } for _, svc := range svcs { if fwMark == svc.FWMark { if (persistent && (svc.Flags&ipvsPersistentFlagHex) == 0) || @@ -340,16 +347,30 @@ func (ln *linuxNetworking) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uin klog.V(2).Infof("Updated schedule for the service: %s", ipvsServiceString(svc)) } + if svc.AddressFamily != family { + svc.AddressFamily = family + svc.Netmask = netmaskForFamily + err := ln.ipvsUpdateService(svc) + if err != nil { + return nil, fmt.Errorf("failed to update the address family for service %s due to %v", + ipvsServiceString(svc), err) + } + klog.V(2).Infof("Updated address family for the service: %s", ipvsServiceString(svc)) + } + klog.V(2).Infof("ipvs service %s already exists so returning", ipvsServiceString(svc)) return svc, nil } } + // Even though it may seem unintuitive to require a Netmask on an fwmark service, I found that it was necessary in + // order to get IPVS IPv6 services to work correctly. After reviewing the code, it the only difference between the + // netlink command that we build here and the one that ipvsadm was building was the netmask, after adding it, it + // began to work svc := ipvs.Service{ FWMark: fwMark, - AddressFamily: syscall.AF_INET, - Protocol: protocol, - Port: port, + AddressFamily: family, + Netmask: netmaskForFamily, SchedName: ipvs.RoundRobin, } diff --git a/pkg/controllers/proxy/linux_networking_moq.go b/pkg/controllers/proxy/linux_networking_moq.go index 89e7abeab..d0a1f8c4c 100644 --- a/pkg/controllers/proxy/linux_networking_moq.go +++ b/pkg/controllers/proxy/linux_networking_moq.go @@ -33,7 +33,7 @@ var _ LinuxNetworking = &LinuxNetworkingMock{} // ipAddrDelFunc: func(iface netlink.Link, ip string, nodeIP string) error { // panic("mock out the ipAddrDel method") // }, -// ipvsAddFWMarkServiceFunc: func(svcs []*ipvs.Service, fwMark uint32, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) { +// ipvsAddFWMarkServiceFunc: func(svcs []*ipvs.Service, fwMark uint32, family uint16, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) { // panic("mock out the ipvsAddFWMarkService method") // }, // ipvsAddServerFunc: func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error { @@ -98,7 +98,7 @@ type LinuxNetworkingMock struct { ipAddrDelFunc func(iface netlink.Link, ip string, nodeIP string) error // ipvsAddFWMarkServiceFunc mocks the ipvsAddFWMarkService method. - ipvsAddFWMarkServiceFunc func(svcs []*ipvs.Service, fwMark uint32, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) + ipvsAddFWMarkServiceFunc func(svcs []*ipvs.Service, fwMark uint32, family uint16, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) // ipvsAddServerFunc mocks the ipvsAddServer method. ipvsAddServerFunc func(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error @@ -186,6 +186,8 @@ type LinuxNetworkingMock struct { Svcs []*ipvs.Service // FwMark is the fwMark argument value. FwMark uint32 + // Family is the family argument value. + Family uint16 // Protocol is the protocol argument value. Protocol uint16 // Port is the port argument value. @@ -487,13 +489,14 @@ func (mock *LinuxNetworkingMock) ipAddrDelCalls() []struct { } // ipvsAddFWMarkService calls ipvsAddFWMarkServiceFunc. -func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) { +func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, family uint16, protocol uint16, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) { if mock.ipvsAddFWMarkServiceFunc == nil { panic("LinuxNetworkingMock.ipvsAddFWMarkServiceFunc: method is nil but LinuxNetworking.ipvsAddFWMarkService was just called") } callInfo := struct { Svcs []*ipvs.Service FwMark uint32 + Family uint16 Protocol uint16 Port uint16 Persistent bool @@ -503,6 +506,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMa }{ Svcs: svcs, FwMark: fwMark, + Family: family, Protocol: protocol, Port: port, Persistent: persistent, @@ -513,7 +517,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMa mock.lockipvsAddFWMarkService.Lock() mock.calls.ipvsAddFWMarkService = append(mock.calls.ipvsAddFWMarkService, callInfo) mock.lockipvsAddFWMarkService.Unlock() - return mock.ipvsAddFWMarkServiceFunc(svcs, fwMark, protocol, port, persistent, persistentTimeout, scheduler, flags) + return mock.ipvsAddFWMarkServiceFunc(svcs, fwMark, family, protocol, port, persistent, persistentTimeout, scheduler, flags) } // ipvsAddFWMarkServiceCalls gets all the calls that were made to ipvsAddFWMarkService. @@ -523,6 +527,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkService(svcs []*ipvs.Service, fwMa func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct { Svcs []*ipvs.Service FwMark uint32 + Family uint16 Protocol uint16 Port uint16 Persistent bool @@ -533,6 +538,7 @@ func (mock *LinuxNetworkingMock) ipvsAddFWMarkServiceCalls() []struct { var calls []struct { Svcs []*ipvs.Service FwMark uint32 + Family uint16 Protocol uint16 Port uint16 Persistent bool diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index bcd5bcaf9..25b730c84 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" "github.com/cloudnativelabs/kube-router/v2/pkg/healthcheck" @@ -157,7 +158,7 @@ type ipvsCalls interface { ipvsUpdateDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error ipvsGetDestinations(ipvsSvc *ipvs.Service) ([]*ipvs.Destination, error) ipvsDelDestination(ipvsSvc *ipvs.Service, ipvsDst *ipvs.Destination) error - ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, protocol, port uint16, persistent bool, + ipvsAddFWMarkService(svcs []*ipvs.Service, fwMark uint32, family, protocol, port uint16, persistent bool, persistentTimeout int32, scheduler string, flags schedFlags) (*ipvs.Service, error) } @@ -1467,7 +1468,14 @@ func ipvsServiceString(s *ipvs.Service) string { } func ipvsDestinationString(d *ipvs.Destination) string { - return fmt.Sprintf("%s:%v (Weight: %v)", d.Address, d.Port, d.Weight) + var family string + switch d.AddressFamily { + case syscall.AF_INET: + family = "IPv4" + case syscall.AF_INET6: + family = "IPv6" + } + return fmt.Sprintf("%s:%v (Family: %s, Weight: %v)", d.Address, d.Port, family, d.Weight) } func ipvsSetPersistence(svc *ipvs.Service, p bool, timeout int32) { diff --git a/pkg/controllers/proxy/service_endpoints_sync.go b/pkg/controllers/proxy/service_endpoints_sync.go index 14617f3d6..11b4eaff6 100644 --- a/pkg/controllers/proxy/service_endpoints_sync.go +++ b/pkg/controllers/proxy/service_endpoints_sync.go @@ -439,12 +439,15 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svc *serviceI protocol := convertSvcProtoToSysCallProto(svc.protocol) var nodeIP net.IP var family v1.IPFamily + var sysFamily uint16 if externalIP.To4() != nil { nodeIP = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs) family = v1.IPv4Protocol + sysFamily = syscall.AF_INET } else { nodeIP = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs) family = v1.IPv6Protocol + sysFamily = syscall.AF_INET6 } dummyVipInterface, err := nsc.ln.getKubeDummyInterface() @@ -462,11 +465,11 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svc *serviceI return fmt.Errorf("failed to generate FW mark") } - ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(ipvsSvcs, fwMark, protocol, uint16(svc.port), + ipvsExternalIPSvc, err := nsc.ln.ipvsAddFWMarkService(ipvsSvcs, fwMark, sysFamily, protocol, uint16(svc.port), svc.sessionAffinity, svc.sessionAffinityTimeoutSeconds, svc.scheduler, svc.flags) if err != nil { - return fmt.Errorf("failed to create IPVS service for External IP: %s due to: %s", - externalIP, err.Error()) + return fmt.Errorf("failed to create IPVS service for FWMark service: %d (external IP: %s) due to: %s", + fwMark, externalIP, err.Error()) } externalIPServiceID := fmt.Sprint(fwMark)