From 15dbb7d4c5f014f819eab541d299763dd05ac3f6 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 6 Aug 2021 16:28:31 -0700 Subject: [PATCH 1/9] ipv6 dualstack support transparent mode --- ipam/ipv6Ipam.go | 3 + network/endpoint_linux.go | 6 +- network/network_linux.go | 67 +++++++++++++++- network/transparent_endpointclient_linux.go | 86 ++++++++++++++++++--- 4 files changed, 150 insertions(+), 12 deletions(-) diff --git a/ipam/ipv6Ipam.go b/ipam/ipv6Ipam.go index f8613d7594..6e5d2ffdb9 100644 --- a/ipam/ipv6Ipam.go +++ b/ipam/ipv6Ipam.go @@ -152,6 +152,7 @@ func (source *ipv6IpamSource) refresh() error { } if source.isLoaded { + log.Printf("ipv6 source already loaded") return nil } @@ -211,7 +212,9 @@ func (source *ipv6IpamSource) refresh() error { return err } + source.isLoaded = true log.Printf("[ipam] Address space successfully populated from Kubernetes API Server") + return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 61df257f06..bf59c9d9b3 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -305,9 +305,13 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error { ifIndex = interfaceIf.Index } + family := netlink.GetIpAddressFamily(route.Gw) + if route.Gw == nil { + family = netlink.GetIpAddressFamily(route.Dst.IP) + } nlRoute := &netlink.Route{ - Family: netlink.GetIpAddressFamily(route.Gw), + Family: family, Dst: &route.Dst, Gw: route.Gw, LinkIndex: ifIndex, diff --git a/network/network_linux.go b/network/network_linux.go index 384cffca5c..59978becf0 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -67,7 +67,14 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt } handleCommonOptions(extIf.BridgeName, nwInfo) case opModeTransparent: + log.Printf("Transparent mode") handleCommonOptions(extIf.Name, nwInfo) + if nwInfo.IPV6Mode != "" { + if err := nm.handleIpv6Transparent(extIf, nwInfo); err != nil { + log.Errorf("Error configuring ipv6 rules:%+v", err) + return nil, err + } + } default: return nil, errNetworkModeInvalid } @@ -336,6 +343,40 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { return err } +func (nm *networkManager) handleIpv6Transparent(extIf *externalInterface, nwInfo *NetworkInfo) error { + log.Printf("configure ipv6 rules") + + if err := epcommon.EnableIPV6Forwarding(); err != nil { + return err + } + + hostIf, err := net.InterfaceByName(extIf.Name) + if err != nil { + return err + } + + addrs, err := hostIf.Addrs() + if err != nil { + return err + } + + for _, addr := range addrs { + ipAddr, ipNet, err := net.ParseCIDR(addr.String()) + ipNet.IP = ipAddr + if err != nil { + continue + } + + if !ipAddr.IsGlobalUnicast() { + continue + } + + extIf.IPAddresses = append(extIf.IPAddresses, ipNet) + } + + return addIpv6SnatRule(extIf, nwInfo) +} + // ConnectExternalInterface connects the given host interface to a bridge. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var ( @@ -586,15 +627,37 @@ func addIpv6NatGateway(nwInfo *NetworkInfo) error { // snat ipv6 traffic to secondary ipv6 ip before leaving VM func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { - log.Printf("[net] Adding ipv6 snat rule") + var ( + ipv6SnatRuleSet bool + ipv6SubnetPrefix *net.IPNet + ) + + for _, subnet := range nwInfo.Subnets { + if subnet.Family == platform.AfINET6 { + ipv6SubnetPrefix = &subnet.Prefix + break + } + } + + if ipv6SubnetPrefix == nil { + return fmt.Errorf("Couldn't find ipv6 subnet in network info") + } + for _, ipAddr := range extIf.IPAddresses { if ipAddr.IP.To4() == nil { - if err := epcommon.AddSnatRule("", ipAddr.IP); err != nil { + log.Printf("[net] Adding ipv6 snat rule") + matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) + if err := epcommon.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { return err } + ipv6SnatRuleSet = true } } + if !ipv6SnatRuleSet { + return fmt.Errorf("ipv6 snat rule not set. Might be VM ipv6 address missing") + } + return nil } diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 5f7d2b9fd9..7e2abdc474 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -14,6 +14,8 @@ const ( virtualGwIPString = "169.254.1.1/32" defaultGwCidr = "0.0.0.0/0" defaultGw = "0.0.0.0" + virtualv6GwString = "fe80::1234:5678:9abc/128" + defaultv6Cidr = "::/0" ) type TransparentEndpointClient struct { @@ -89,8 +91,16 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) // ip route add dev // This route is needed for incoming packets to pod to route via hostveth for _, ipAddr := range epInfo.IPAddresses { - var routeInfo RouteInfo - ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(128, 128)} + } log.Printf("[net] Adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) @@ -109,17 +119,23 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) } func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { - var routeInfoList []RouteInfo - // ip route del dev // Deleting the route set up for routing the incoming packets to pod for _, ipAddr := range ep.IPAddresses { - var routeInfo RouteInfo - ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(128, 128)} + } + log.Printf("[net] Deleting route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - deleteRoutes(client.hostVethName, routeInfoList) + deleteRoutes(client.hostVethName, []RouteInfo{routeInfo}) } } @@ -185,7 +201,59 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e //arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", virtualGwNet.String(), client.hostVethMac) - return netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, false) + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, false); err != nil { + return err + } + + if err := client.setupIPV6Routes(epInfo); err != nil { + return err + } + + return client.setIPV6NeighEntry(epInfo) + +} + +func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) error { + if epInfo.IPV6Mode != "" { + log.Printf("Setting up ipv6 routes in container") + + //add route for virtualgwip + //ip -6 route add fe80::1234:5678:9abc/128 dev eth0 + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualv6GwString) + gwRoute := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + + //ip -6 route add default via fe80::1234:5678:9abc dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultv6Cidr) + log.Printf("defaultv6ipnet :%+v", defaultIPNet) + defaultRoute := RouteInfo{ + Dst: *defaultIPNet, + Gw: virtualGwIP, + } + + if err := addRoutes(client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { + return err + } + + } + + return nil +} + +func (client *TransparentEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) error { + if epInfo.IPV6Mode != "" { + log.Printf("[net] Add v6 neigh entry for default gw ip") + hostGwIp, _, _ := net.ParseCIDR(virtualv6GwString) + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, + hostGwIp, client.hostVethMac, false); err != nil { + log.Printf("Failed setting neigh entry in container: %v", err) + return err + } + } + + return nil } func (client *TransparentEndpointClient) DeleteEndpoints(ep *endpoint) error { From 34e24781f0c55b5615e57fa1543e0b1509f40aec Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 6 Aug 2021 17:06:57 -0700 Subject: [PATCH 2/9] golint fixes --- network/network_linux.go | 4 ++-- network/transparent_endpointclient_linux.go | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/network/network_linux.go b/network/network_linux.go index 59978becf0..dc83810756 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -640,7 +640,7 @@ func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { } if ipv6SubnetPrefix == nil { - return fmt.Errorf("Couldn't find ipv6 subnet in network info") + return errors.New("Couldn't find ipv6 subnet in network info") } for _, ipAddr := range extIf.IPAddresses { @@ -655,7 +655,7 @@ func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { } if !ipv6SnatRuleSet { - return fmt.Errorf("ipv6 snat rule not set. Might be VM ipv6 address missing") + return errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") } return nil diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 7e2abdc474..fbfb7770a0 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -55,7 +55,6 @@ func setArpProxy(ifName string) error { } func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - if _, err := net.InterfaceByName(client.hostVethName); err == nil { log.Printf("Deleting old host veth %v", client.hostVethName) if err = netlink.DeleteLink(client.hostVethName); err != nil { @@ -164,7 +163,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } - //ip route del 10.240.0.0/12 dev eth0 (removing kernel subnet route added by above call) + // ip route del 10.240.0.0/12 dev eth0 (removing kernel subnet route added by above call) for _, ipAddr := range epInfo.IPAddresses { _, ipnet, _ := net.ParseCIDR(ipAddr.String()) routeInfo := RouteInfo{ @@ -177,8 +176,8 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } } - //add route for virtualgwip - //ip route add 169.254.1.1/32 dev eth0 + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, @@ -188,7 +187,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } - //ip route add default via 169.254.1.1 dev eth0 + // ip route add default via 169.254.1.1 dev eth0 _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ @@ -199,7 +198,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } - //arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac + // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", virtualGwNet.String(), client.hostVethMac) if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, false); err != nil { return err @@ -210,22 +209,21 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } return client.setIPV6NeighEntry(epInfo) - } func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) error { if epInfo.IPV6Mode != "" { log.Printf("Setting up ipv6 routes in container") - //add route for virtualgwip - //ip -6 route add fe80::1234:5678:9abc/128 dev eth0 + // add route for virtualgwip + // ip -6 route add fe80::1234:5678:9abc/128 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualv6GwString) gwRoute := RouteInfo{ Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, } - //ip -6 route add default via fe80::1234:5678:9abc dev eth0 + // ip -6 route add default via fe80::1234:5678:9abc dev eth0 _, defaultIPNet, _ := net.ParseCIDR(defaultv6Cidr) log.Printf("defaultv6ipnet :%+v", defaultIPNet) defaultRoute := RouteInfo{ From 01ae35601289024d7f7c8cce19ffd36ab6522afa Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Aug 2021 12:29:30 -0700 Subject: [PATCH 3/9] fixed linter errors --- network/errors.go | 8 +++++ network/network_linux.go | 18 +++++------ network/transparent_endpointclient_linux.go | 35 ++++++++++++++------- 3 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 network/errors.go diff --git a/network/errors.go b/network/errors.go new file mode 100644 index 0000000000..95ab57d2ca --- /dev/null +++ b/network/errors.go @@ -0,0 +1,8 @@ +package network + +import "errors" + +var ( + errSubnetV6NotFound = errors.New("Couldn't find ipv6 subnet in network info") + errV6SnatRuleNotSet = errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") +) diff --git a/network/network_linux.go b/network/network_linux.go index dc83810756..d4da76b7f7 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -347,17 +347,17 @@ func (nm *networkManager) handleIpv6Transparent(extIf *externalInterface, nwInfo log.Printf("configure ipv6 rules") if err := epcommon.EnableIPV6Forwarding(); err != nil { - return err + return fmt.Errorf("Ipv6 forwarding failed: %w", err) } hostIf, err := net.InterfaceByName(extIf.Name) if err != nil { - return err + return fmt.Errorf("%w", err) } addrs, err := hostIf.Addrs() if err != nil { - return err + return fmt.Errorf("%w", err) } for _, addr := range addrs { @@ -629,18 +629,18 @@ func addIpv6NatGateway(nwInfo *NetworkInfo) error { func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { var ( ipv6SnatRuleSet bool - ipv6SubnetPrefix *net.IPNet + ipv6SubnetPrefix net.IPNet ) for _, subnet := range nwInfo.Subnets { if subnet.Family == platform.AfINET6 { - ipv6SubnetPrefix = &subnet.Prefix + ipv6SubnetPrefix = subnet.Prefix break } } - if ipv6SubnetPrefix == nil { - return errors.New("Couldn't find ipv6 subnet in network info") + if len(ipv6SubnetPrefix.IP) == 0 { + return errSubnetV6NotFound } for _, ipAddr := range extIf.IPAddresses { @@ -648,14 +648,14 @@ func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { log.Printf("[net] Adding ipv6 snat rule") matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) if err := epcommon.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { - return err + return fmt.Errorf("Adding iptable snat rule failed:%w", err) } ipv6SnatRuleSet = true } } if !ipv6SnatRuleSet { - return errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") + return errV6SnatRuleNotSet } return nil diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index fbfb7770a0..3109bb6a08 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -16,6 +16,10 @@ const ( defaultGw = "0.0.0.0" virtualv6GwString = "fe80::1234:5678:9abc/128" defaultv6Cidr = "::/0" + ipv4Bits = 32 + ipv6Bits = 128 + ipv4FullMask = 32 + ipv6FullMask = 128 ) type TransparentEndpointClient struct { @@ -96,9 +100,9 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) ) if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(128, 128)} + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} } log.Printf("[net] Adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet @@ -127,14 +131,16 @@ func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { ) if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(128, 128)} + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} } log.Printf("[net] Deleting route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet - deleteRoutes(client.hostVethName, []RouteInfo{routeInfo}) + if err := deleteRoutes(client.hostVethName, []RouteInfo{routeInfo}); err != nil { + log.Printf("Error removing route from vm:%+v", err) + } } } @@ -199,9 +205,14 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", virtualGwNet.String(), client.hostVethMac) - if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, false); err != nil { - return err + log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", + virtualGwNet.String(), client.hostVethMac) + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, + client.containerVethName, + virtualGwNet.IP, + client.hostVethMac, + false); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) } if err := client.setupIPV6Routes(epInfo); err != nil { @@ -243,11 +254,11 @@ func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) e func (client *TransparentEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) error { if epInfo.IPV6Mode != "" { log.Printf("[net] Add v6 neigh entry for default gw ip") - hostGwIp, _, _ := net.ParseCIDR(virtualv6GwString) + hostGwIP, _, _ := net.ParseCIDR(virtualv6GwString) if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, - hostGwIp, client.hostVethMac, false); err != nil { - log.Printf("Failed setting neigh entry in container: %v", err) - return err + hostGwIP, client.hostVethMac, false); err != nil { + log.Printf("Failed setting neigh entry in container: %+v", err) + return fmt.Errorf("Failed setting neigh entry in container: %w", err) } } From d99ac16267c14f843fbacf916cceaf10e61281a9 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 27 Aug 2021 17:37:22 -0700 Subject: [PATCH 4/9] enable ipv6 setting --- network/bridge_endpointclient_linux.go | 7 ------- network/endpoint_linux.go | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index bc00121a7e..84d2543fbf 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -170,13 +170,6 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if epInfo.IPV6Mode != "" { - // Enable ipv6 setting in container - if err := epcommon.UpdateIPV6Setting(0); err != nil { - return err - } - } - if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index bf59c9d9b3..4a25693061 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" ) const ( @@ -177,6 +178,14 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { }() } + if epInfo.IPV6Mode != "" { + // Enable ipv6 setting in container + log.Printf("Enable ipv6 setting in container.") + if err := epcommon.UpdateIPV6Setting(0); err != nil { + return nil, err + } + } + // If a name for the container interface is specified... if epInfo.IfName != "" { if err = epClient.SetupContainerInterfaces(epInfo); err != nil { From b095c6ad2fe451c675ac7534c74d7de4c0f584d9 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 27 Sep 2021 15:02:36 -0700 Subject: [PATCH 5/9] dualstack transparent changes --- netlink/{ip.go => ip_linux.go} | 0 netlink/{link.go => link_linux.go} | 0 network/endpoint_linux.go | 9 +++-- network/network_linux.go | 44 ++------------------- network/transparent_endpointclient_linux.go | 8 ++-- 5 files changed, 12 insertions(+), 49 deletions(-) rename netlink/{ip.go => ip_linux.go} (100%) rename netlink/{link.go => link_linux.go} (100%) diff --git a/netlink/ip.go b/netlink/ip_linux.go similarity index 100% rename from netlink/ip.go rename to netlink/ip_linux.go diff --git a/netlink/link.go b/netlink/link_linux.go similarity index 100% rename from netlink/link.go rename to netlink/link_linux.go diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 51a5e9790f..a4ba262c0e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" "github.com/Azure/azure-container-networking/ovsctl" ) @@ -182,8 +183,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e if epInfo.IPV6Mode != "" { // Enable ipv6 setting in container log.Printf("Enable ipv6 setting in container.") - if err := epcommon.UpdateIPV6Setting(0); err != nil { - return nil, err + if err = epcommon.UpdateIPV6Setting(0); err != nil { + return nil, fmt.Errorf("Enable ipv6 in container failed:%w", err) } } @@ -315,9 +316,9 @@ func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Ro ifIndex = interfaceIf.Index } - family := netlink.GetIpAddressFamily(route.Gw) + family := netlink.GetIPAddressFamily(route.Gw) if route.Gw == nil { - family = netlink.GetIpAddressFamily(route.Dst.IP) + family = netlink.GetIPAddressFamily(route.Dst.IP) } nlRoute := &netlink.Route{ diff --git a/network/network_linux.go b/network/network_linux.go index 510f2b89fc..53c6971621 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -83,14 +83,10 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt log.Printf("bridge handleCommonOptions failed with error %s", err.Error()) } case opModeTransparent: - log.Printf("Transparent mode") err := nm.handleCommonOptions(extIf.Name, nwInfo) - if err != nil { - log.Printf("transparent handleCommonOptions failed with error %s", err.Error()) - } + log.Printf("Transparent mode") if nwInfo.IPV6Mode != "" { - if err := nm.handleIpv6Transparent(extIf, nwInfo); err != nil { - log.Errorf("Error configuring ipv6 rules:%+v", err) - return nil, err + if err := epcommon.EnableIPV6Forwarding(); err != nil { + return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) } } default: @@ -361,40 +357,6 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { return err } -func (nm *networkManager) handleIpv6Transparent(extIf *externalInterface, nwInfo *NetworkInfo) error { - log.Printf("configure ipv6 rules") - - if err := epcommon.EnableIPV6Forwarding(); err != nil { - return fmt.Errorf("Ipv6 forwarding failed: %w", err) - } - - hostIf, err := net.InterfaceByName(extIf.Name) - if err != nil { - return fmt.Errorf("%w", err) - } - - addrs, err := hostIf.Addrs() - if err != nil { - return fmt.Errorf("%w", err) - } - - for _, addr := range addrs { - ipAddr, ipNet, err := net.ParseCIDR(addr.String()) - ipNet.IP = ipAddr - if err != nil { - continue - } - - if !ipAddr.IsGlobalUnicast() { - continue - } - - extIf.IPAddresses = append(extIf.IPAddresses, ipNet) - } - - return addIpv6SnatRule(extIf, nwInfo) -} - // ConnectExternalInterface connects the given host interface to a bridge. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var ( diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 3461a6efd3..40fc3627fc 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -174,7 +174,7 @@ func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { log.Printf("[net] Deleting route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet - if err := deleteRoutes(client.hostVethName, []RouteInfo{routeInfo}); err != nil { + if err := deleteRoutes(client.netlink, client.hostVethName, []RouteInfo{routeInfo}); err != nil { log.Printf("[net] Failed to delete route on VM for the ip %v: %v", ipNet.String(), err) } } @@ -245,7 +245,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", virtualGwNet.String(), client.hostVethMac) - if err := netlink.AddOrRemoveStaticArp(netlink.ADD, + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, @@ -280,7 +280,7 @@ func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) e Gw: virtualGwIP, } - if err := addRoutes(client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { + if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { return err } @@ -293,7 +293,7 @@ func (client *TransparentEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) if epInfo.IPV6Mode != "" { log.Printf("[net] Add v6 neigh entry for default gw ip") hostGwIP, _, _ := net.ParseCIDR(virtualv6GwString) - if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, hostGwIP, client.hostVethMac, false); err != nil { log.Printf("Failed setting neigh entry in container: %+v", err) return fmt.Errorf("Failed setting neigh entry in container: %w", err) From eb0571104ce336ba86860e821eddcfcd79997fdf Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 6 Oct 2021 16:50:57 -0700 Subject: [PATCH 6/9] abstracted platform execute command --- aitelemetry/telemetrywrapper_test.go | 13 +- cni/network/network.go | 2 +- cnm/network/network.go | 2 +- cnms/service/networkmonitor.go | 2 +- cns/dockerclient/dockerclient.go | 3 +- ebtables/ebtables.go | 7 +- iptables/iptables.go | 3 +- network/bridge_endpointclient_linux.go | 25 +- network/bridge_networkclient_linux.go | 10 +- network/endpoint.go | 9 +- network/endpoint_linux.go | 44 +-- network/manager.go | 8 +- network/network_linux.go | 43 +-- network/network_test.go | 23 ++ network/network_windows.go | 9 +- .../networkutils_linux.go} | 78 ++--- network/ovs_endpoint_infraroute_linux.go | 2 +- network/ovs_endpoint_snatroute_linux.go | 8 +- network/ovs_endpointclient_linux.go | 23 +- network/ovs_networkclient_linux.go | 15 +- network/ovs_networkclient_linux_test.go | 20 +- network/ovsinfravnet/infravnet.go | 11 +- network/ovssnat/ovssnat.go | 37 ++- network/transparent_endpoint_linux_test.go | 281 +++++++++++++++++- network/transparent_endpointclient_linux.go | 34 ++- ovsctl/ovsctl.go | 72 ++--- platform/mockexec.go | 23 ++ platform/osInterface.go | 12 + platform/os_linux.go | 19 +- platform/os_windows.go | 2 +- telemetry/telemetry_windows.go | 6 +- 31 files changed, 631 insertions(+), 215 deletions(-) rename network/{epcommon/endpoint_common.go => networkutils/networkutils_linux.go} (74%) create mode 100644 platform/mockexec.go create mode 100644 platform/osInterface.go diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index 4031f6d9fe..5fda52de70 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -29,12 +29,15 @@ func TestMain(m *testing.M) { fmt.Printf("TestST LogDir configuration succeeded\n") } + p := platform.NewExecClient() if runtime.GOOS == "linux" { - platform.ExecuteCommand("cp metadata_test.json /tmp/azuremetadata.json") + //nolint:errcheck // initial test setup + p.ExecuteCommand("cp metadata_test.json /tmp/azuremetadata.json") } else { metadataFile := filepath.FromSlash(os.Getenv("TEMP")) + "\\azuremetadata.json" cmd := fmt.Sprintf("copy metadata_test.json %s", metadataFile) - platform.ExecuteCommand(cmd) + //nolint:errcheck // initial test setup + p.ExecuteCommand(cmd) } hostu, _ := url.Parse("tcp://" + hostAgentUrl) @@ -54,11 +57,13 @@ func TestMain(m *testing.M) { exitCode := m.Run() if runtime.GOOS == "linux" { - platform.ExecuteCommand("rm /tmp/azuremetadata.json") + //nolint:errcheck // test cleanup + p.ExecuteCommand("rm /tmp/azuremetadata.json") } else { metadataFile := filepath.FromSlash(os.Getenv("TEMP")) + "\\azuremetadata.json" cmd := fmt.Sprintf("del %s", metadataFile) - platform.ExecuteCommand(cmd) + //nolint:errcheck // initial test cleanup + p.ExecuteCommand(cmd) } log.Close() diff --git a/cni/network/network.go b/cni/network/network.go index ef24be504f..7e6807cd45 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -114,7 +114,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index dee59881fa..e811f4d098 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -52,7 +52,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index d01c84e437..6341bc06ee 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -148,7 +148,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 1c43cada6f..b28c108084 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -150,6 +150,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd // DeleteNetwork creates a network using docker network create. func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { + p := platform.NewExecClient() logger.Printf("[Azure CNS] DeleteNetwork") url := dockerClient.connectionURL + inspectNetworkPath + networkName @@ -178,7 +179,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", primaryNic.Subnet) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) if err != nil { logger.Printf("[Azure CNS] Error Removing Outbound SNAT rule %v", err) } diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index 39d4c94337..ba8e215622 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -129,11 +129,11 @@ func GetEbtableRules(tableName, chainName string) ([]string, error) { inChain bool rules []string ) - + p := platform.NewExecClient() command := fmt.Sprintf( "ebtables -t %s -L %s --Lmac2", tableName, chainName) - out, err := platform.ExecuteCommand(command) + out, err := p.ExecuteCommand(command) if err != nil { return nil, err } @@ -226,8 +226,9 @@ func EbTableRuleExists(tableName, chainName, matchSet string) (bool, error) { // runEbCmd runs an EB rule command. func runEbCmd(table, action, chain, rule string) error { + p := platform.NewExecClient() command := fmt.Sprintf("ebtables -t %s %s %s %s", table, action, chain, rule) - _, err := platform.ExecuteCommand(command) + _, err := p.ExecuteCommand(command) return err } diff --git a/iptables/iptables.go b/iptables/iptables.go index dc0a6e8c0c..79d1b28fdf 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -94,6 +94,7 @@ type IPTableEntry struct { func RunCmd(version, params string) error { var cmd string + p := platform.NewExecClient() iptCmd := iptables if version == V6 { iptCmd = ip6tables @@ -105,7 +106,7 @@ func RunCmd(version, params string) error { cmd = fmt.Sprintf("%s -w %d %s", iptCmd, lockTimeout, params) } - if _, err := platform.ExecuteCommand(cmd); err != nil { + if _, err := p.ExecuteCommand(cmd); err != nil { return err } diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index fa0837033c..9b6f871d58 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -2,12 +2,14 @@ package network import ( "fmt" + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" ) const ( @@ -26,6 +28,9 @@ type LinuxBridgeEndpointClient struct { hostIPAddresses []*net.IPNet mode string netlink netlink.NetlinkInterface + plClient platform.ExecClient + netioshim netio.NetIOInterface + nuc networkutils.NetworkUtils } func NewLinuxBridgeEndpointClient( @@ -34,6 +39,7 @@ func NewLinuxBridgeEndpointClient( containerVethName string, mode string, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *LinuxBridgeEndpointClient { client := &LinuxBridgeEndpointClient{ @@ -45,16 +51,17 @@ func NewLinuxBridgeEndpointClient( hostIPAddresses: []*net.IPNet{}, mode: mode, netlink: nl, + plClient: plc, + netioshim: &netio.NetIO{}, } client.hostIPAddresses = append(client.hostIPAddresses, extIf.IPAddresses...) - + client.nuc = networkutils.NewNetworkUtils(nl, plc) return client } func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -164,8 +171,7 @@ func (client *LinuxBridgeEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := client.nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -175,12 +181,11 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } - if err := addRoutes(client.netlink, client.containerVethName, epInfo.Routes); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { return err } @@ -273,7 +278,7 @@ func (client *LinuxBridgeEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) e routes = append(routes, defaultV6Route) log.Printf("[net] Adding ipv6 routes in container %+v", routes) - if err := addRoutes(client.netlink, client.containerVethName, routes); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, routes); err != nil { return nil } } diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index 3ffadbf2e2..b9cc84be2a 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -3,12 +3,13 @@ package network import ( "errors" "fmt" + "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" ) const ( @@ -26,6 +27,7 @@ type LinuxBridgeClient struct { hostInterfaceName string nwInfo NetworkInfo netlink netlink.NetlinkInterface + nuClient networkutils.NetworkUtils } func NewLinuxBridgeClient( @@ -33,12 +35,14 @@ func NewLinuxBridgeClient( hostInterfaceName string, nwInfo NetworkInfo, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *LinuxBridgeClient { client := &LinuxBridgeClient{ bridgeName: bridgeName, nwInfo: nwInfo, hostInterfaceName: hostInterfaceName, netlink: nl, + nuClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -58,7 +62,7 @@ func (client *LinuxBridgeClient) CreateBridge() error { return err } - return epcommon.DisableRAForInterface(client.bridgeName) + return client.nuClient.DisableRAForInterface(client.bridgeName) } func (client *LinuxBridgeClient) DeleteBridge() error { @@ -123,7 +127,7 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { return err } - if err := epcommon.EnableIPV6Forwarding(); err != nil { + if err := client.nuClient.EnableIPV6Forwarding(); err != nil { return err } } diff --git a/network/endpoint.go b/network/endpoint.go index 740f80e611..8e4e52bfcc 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -5,6 +5,7 @@ package network import ( "context" + "github.com/Azure/azure-container-networking/platform" "net" "strings" @@ -98,7 +99,7 @@ type apipaClient interface { } // NewEndpoint creates a new endpoint in the network. -func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint var err error @@ -110,7 +111,7 @@ func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epI }() // Call the platform implementation. - ep, err = nw.newEndpointImpl(cli, nl, epInfo) + ep, err = nw.newEndpointImpl(cli, nl, plc, epInfo) if err != nil { return nil, err } @@ -122,7 +123,7 @@ func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epI } // DeleteEndpoint deletes an existing endpoint from the network. -func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, endpointID string) error { +func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, endpointID string) error { var err error log.Printf("[net] Deleting endpoint %v from network %v.", endpointID, nw.Id) @@ -140,7 +141,7 @@ func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, } // Call the platform implementation. - err = nw.deleteEndpointImpl(cli, nl, ep) + err = nw.deleteEndpointImpl(cli, nl, plc, ep) if err != nil { return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index a4ba262c0e..28e902f44e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -7,12 +7,14 @@ import ( "crypto/sha1" "encoding/hex" "fmt" + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/platform" "net" "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" ) @@ -46,7 +48,7 @@ func ConstructEndpointID(containerID string, _ string, ifName string) (string, s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { var containerIf *net.Interface var ns *Namespace var ep *endpoint @@ -100,13 +102,14 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e vlanid, localIP, nl, - ovsctl.NewOvsctl()) + ovsctl.NewOvsctl(), + plc) } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") - epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } else { log.Printf("Transparent client") - epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl) + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } // Cleanup on failure. @@ -183,7 +186,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e if epInfo.IPV6Mode != "" { // Enable ipv6 setting in container log.Printf("Enable ipv6 setting in container.") - if err = epcommon.UpdateIPV6Setting(0); err != nil { + nuc := networkutils.NewNetworkUtils(nl, plc) + if err = nuc.UpdateIPV6Setting(0); err != nil { return nil, fmt.Errorf("Enable ipv6 in container failed:%w", err) } } @@ -227,7 +231,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, ep *endpoint) error { var epClient EndpointClient // Delete the veth pair by deleting one of the peer interfaces. @@ -235,11 +239,11 @@ func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl()) + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } else if nw.Mode != opModeTransparent { - epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { - epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl) + epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } epClient.DeleteEndpointRules(ep) @@ -252,17 +256,21 @@ func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { } -func addRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []RouteInfo) error { +func addRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, interfaceName string, routes []RouteInfo) error { ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { log.Printf("[net] Adding IP route %+v to link %v.", route, interfaceName) if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) + devIf, _ := netioshim.GetNetworkInterfaceByName(route.DevName) ifIndex = devIf.Index } else { + interfaceIf, err := netioshim.GetNetworkInterfaceByName(interfaceName) + if err != nil { + log.Errorf("Interface not found:%v", err) + return err + } ifIndex = interfaceIf.Index } @@ -293,15 +301,14 @@ func addRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Route return nil } -func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []RouteInfo) error { +func deleteRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, interfaceName string, routes []RouteInfo) error { ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { log.Printf("[net] Deleting IP route %+v from link %v.", route, interfaceName) if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) + devIf, _ := netioshim.GetNetworkInterfaceByName(route.DevName) if devIf == nil { log.Printf("[net] Not deleting route. Interface %v doesn't exist", interfaceName) continue @@ -309,6 +316,7 @@ func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Ro ifIndex = devIf.Index } else { + interfaceIf, _ := netioshim.GetNetworkInterfaceByName(interfaceName) if interfaceIf == nil { log.Printf("[net] Not deleting route. Interface %v doesn't exist", interfaceName) continue @@ -455,12 +463,12 @@ func (nm *networkManager) updateRoutes(existingEp *EndpointInfo, targetEp *Endpo } - err := deleteRoutes(nm.netlink, existingEp.IfName, tobeDeletedRoutes) + err := deleteRoutes(nm.netlink, &netio.NetIO{}, existingEp.IfName, tobeDeletedRoutes) if err != nil { return err } - err = addRoutes(nm.netlink, existingEp.IfName, tobeAddedRoutes) + err = addRoutes(nm.netlink, &netio.NetIO{}, existingEp.IfName, tobeAddedRoutes) if err != nil { return err } diff --git a/network/manager.go b/network/manager.go index c7a954365c..4ae1829924 100644 --- a/network/manager.go +++ b/network/manager.go @@ -58,6 +58,7 @@ type networkManager struct { ExternalInterfaces map[string]*externalInterface store store.KeyValueStore netlink netlink.NetlinkInterface + plClient platform.ExecClient sync.Mutex } @@ -85,10 +86,11 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, + plClient: plc, } return nm, nil @@ -333,7 +335,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } } - _, err = nw.newEndpoint(cli, nm.netlink, epInfo) + _, err = nw.newEndpoint(cli, nm.netlink, nm.plClient, epInfo) if err != nil { return err } @@ -356,7 +358,7 @@ func (nm *networkManager) DeleteEndpoint(cli apipaClient, networkID string, endp return err } - err = nw.deleteEndpoint(cli, nm.netlink, endpointID) + err = nw.deleteEndpoint(cli, nm.netlink, nm.plClient, endpointID) if err != nil { return err } diff --git a/network/network_linux.go b/network/network_linux.go index 53c6971621..0f2ecf8633 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -6,6 +6,7 @@ package network import ( "errors" "fmt" + "github.com/Azure/azure-container-networking/netio" "net" "strconv" "strings" @@ -13,7 +14,7 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" "golang.org/x/sys/unix" @@ -85,7 +86,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt case opModeTransparent: log.Printf("Transparent mode") if nwInfo.IPV6Mode != "" { - if err := epcommon.EnableIPV6Forwarding(); err != nil { + nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + if err := nu.EnableIPV6Forwarding(); err != nil { return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) } } @@ -131,9 +133,9 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, ovsctl.NewOvsctl()) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}, nm.netlink) + networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}, nm.netlink, nm.plClient) } // Disconnect the interface if this was the last network using it. @@ -238,8 +240,9 @@ func isGreaterOrEqaulUbuntuVersion(versionToMatch int) bool { func readDnsInfo(ifName string) (DNSInfo, error) { var dnsInfo DNSInfo + p := platform.NewExecClient() cmd := fmt.Sprintf("systemd-resolve --status %s", ifName) - out, err := platform.ExecuteCommand(cmd) + out, err := p.ExecuteCommand(cmd) if err != nil { return dnsInfo, err } @@ -327,6 +330,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { setDnsList string err error ) + p := platform.NewExecClient() if extIf != nil { for _, server := range extIf.DNSInfo.Servers { @@ -341,7 +345,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { if setDnsList != "" { cmd := fmt.Sprintf("systemd-resolve --interface=%s%s", ifName, setDnsList) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) if err != nil { return err } @@ -349,7 +353,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { if extIf.DNSInfo.Suffix != "" { cmd := fmt.Sprintf("systemd-resolve --interface=%s --set-domain=%s", ifName, extIf.DNSInfo.Suffix) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) } } @@ -387,9 +391,9 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { - networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl()) + networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink, nm.plClient) } // Check if the bridge already exists. @@ -430,8 +434,9 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI isGreaterOrEqualUbuntu17 := isGreaterOrEqaulUbuntuVersion(ubuntuVersion17) isSystemdResolvedActive := false if isGreaterOrEqualUbuntu17 { + p := platform.NewExecClient() // Don't copy dns servers if systemd-resolved isn't available - if _, cmderr := platform.ExecuteCommand("systemctl status systemd-resolved"); cmderr == nil { + if _, cmderr := p.ExecuteCommand("systemctl status systemd-resolved"); cmderr == nil { isSystemdResolvedActive = true log.Printf("[net] Saving dns config from %v", extIf.Name) if err = saveDnsConfig(extIf); err != nil { @@ -502,12 +507,12 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI if nwInfo.IPV6Mode == IPV6Nat { // adds pod cidr gateway ip to bridge - if err = addIpv6NatGateway(nm.netlink, nwInfo); err != nil { + if err = nm.addIpv6NatGateway(nwInfo); err != nil { log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err) return err } - if err = addIpv6SnatRule(extIf, nwInfo); err != nil { + if err = nm.addIpv6SnatRule(extIf, nwInfo); err != nil { log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err) return err } @@ -595,7 +600,7 @@ func (nm *networkManager) addBridgeRoutes(bridgeName string, routes []RouteInfo) } // Add ipv6 nat gateway IP on bridge -func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { +func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { log.Printf("[net] Adding ipv6 nat gateway on azure bridge") for _, subnetInfo := range nwInfo.Subnets { if subnetInfo.Family == platform.AfINET6 { @@ -603,8 +608,8 @@ func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { IP: subnetInfo.Gateway, Mask: subnetInfo.Prefix.Mask, }} - epc := epcommon.NewEPCommon(nl) - err := epc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) + nuc := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + err := nuc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) if err != nil { return newErrorNetworkManager(err.Error()) } @@ -615,7 +620,7 @@ func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { } // snat ipv6 traffic to secondary ipv6 ip before leaving VM -func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { +func (*networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { var ( ipv6SnatRuleSet bool ipv6SubnetPrefix net.IPNet @@ -636,7 +641,7 @@ func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { if ipAddr.IP.To4() == nil { log.Printf("[net] Adding ipv6 snat rule") matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) - if err := epcommon.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { + if err := networkutils.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { return fmt.Errorf("Adding iptable snat rule failed:%w", err) } ipv6SnatRuleSet = true @@ -659,14 +664,14 @@ func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { } // AddStaticRoute adds a static route to the interface. -func AddStaticRoute(nl netlink.NetlinkInterface, ip, interfaceName string) error { +func AddStaticRoute(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, ip, interfaceName string) error { log.Printf("[ovs] Adding %v static route", ip) var routes []RouteInfo _, ipNet, _ := net.ParseCIDR(ip) gwIP := net.ParseIP("0.0.0.0") route := RouteInfo{Dst: *ipNet, Gw: gwIP} routes = append(routes, route) - if err := addRoutes(nl, interfaceName, routes); err != nil { + if err := addRoutes(nl, netioshim, interfaceName, routes); err != nil { if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("addroutes failed with error %v", err) return err diff --git a/network/network_test.go b/network/network_test.go index ec4f691843..6093cef755 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -1,6 +1,7 @@ package network import ( + "github.com/Azure/azure-container-networking/platform" "net" "testing" @@ -158,6 +159,28 @@ var _ = Describe("Test Network", func() { Expect(nw).To(BeNil()) }) }) + + Context("create new network in transparent mode", func() { + It("Should create new network", func() { + nm := &networkManager{ + ExternalInterfaces: map[string]*externalInterface{}, + plClient: platform.NewMockExecClient(false), + } + nm.ExternalInterfaces["eth0"] = &externalInterface{ + Networks: map[string]*network{}, + } + nwInfo := &NetworkInfo{ + Id: "nw", + MasterIfName: "eth0", + Mode: opModeTransparent, + IPV6Mode: IPV6Nat, + } + nw, err := nm.newNetwork(nwInfo) + Expect(err).To(BeNil()) + Expect(nw).NotTo(BeNil()) + Expect(nw.Id).To(Equal(nwInfo.Id)) + }) + }) }) Describe("Test deleteNetwork", func() { diff --git a/network/network_windows.go b/network/network_windows.go index aa6c606684..8544988804 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" - "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" "github.com/google/uuid" @@ -146,7 +145,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern }() // route entry for pod cidr - if err = appIPV6RouteEntry(nwInfo); err != nil { + if err = nm.appIPV6RouteEntry(nwInfo); err != nil { return nil, err } @@ -173,7 +172,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern return nw, nil } -func appIPV6RouteEntry(nwInfo *NetworkInfo) error { +func (nm *networkManager) appIPV6RouteEntry(nwInfo *NetworkInfo) error { var ( err error out string @@ -192,13 +191,13 @@ func appIPV6RouteEntry(nwInfo *NetworkInfo) error { cmd := fmt.Sprintf(routeCmd, "delete", nwInfo.Subnets[1].Prefix.String(), ifName, ipv6DefaultHop) - if out, err = platform.ExecuteCommand(cmd); err != nil { + if out, err = nm.plClient.ExecuteCommand(cmd); err != nil { log.Printf("[net] Deleting ipv6 route failed: %v:%v", out, err) } cmd = fmt.Sprintf(routeCmd, "add", nwInfo.Subnets[1].Prefix.String(), ifName, ipv6DefaultHop) - if out, err = platform.ExecuteCommand(cmd); err != nil { + if out, err = nm.plClient.ExecuteCommand(cmd); err != nil { log.Printf("[net] Adding ipv6 route failed: %v:%v", out, err) } } diff --git a/network/epcommon/endpoint_common.go b/network/networkutils/networkutils_linux.go similarity index 74% rename from network/epcommon/endpoint_common.go rename to network/networkutils/networkutils_linux.go index af9cd403d0..db9e8d6ca7 100644 --- a/network/epcommon/endpoint_common.go +++ b/network/networkutils/networkutils_linux.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package epcommon +package networkutils import ( "errors" @@ -37,23 +37,25 @@ const ( acceptRAV6File = "/proc/sys/net/ipv6/conf/%s/accept_ra" ) -var errorEPCommon = errors.New("ErrorEPCommon Error") +var errorNetworkUtils = errors.New("NetworkUtils Error") -func newErrorEPCommon(errStr string) error { - return fmt.Errorf("%w : %s", errorEPCommon, errStr) +func newErrorNetworkUtils(errStr string) error { + return fmt.Errorf("%w : %s", errorNetworkUtils, errStr) } -type EPCommon struct { - netlink netlink.NetlinkInterface +type NetworkUtils struct { + netlink netlink.NetlinkInterface + plClient platform.ExecClient } -func NewEPCommon(nl netlink.NetlinkInterface) EPCommon { - return EPCommon{ - netlink: nl, +func NewNetworkUtils(nl netlink.NetlinkInterface, plClient platform.ExecClient) NetworkUtils { + return NetworkUtils{ + netlink: nl, + plClient: plClient, } } -func (epc EPCommon) CreateEndpoint(hostVethName string, containerVethName string) error { +func (nu NetworkUtils) CreateEndpoint(hostVethName, containerVethName string) error { log.Printf("[net] Creating veth pair %v %v.", hostVethName, containerVethName) link := netlink.VEthLink{ @@ -64,66 +66,66 @@ func (epc EPCommon) CreateEndpoint(hostVethName string, containerVethName string PeerName: containerVethName, } - err := epc.netlink.AddLink(&link) + err := nu.netlink.AddLink(&link) if err != nil { log.Printf("[net] Failed to create veth pair, err:%v.", err) - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } log.Printf("[net] Setting link %v state up.", hostVethName) - err = epc.netlink.SetLinkState(hostVethName, true) + err = nu.netlink.SetLinkState(hostVethName, true) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } - if err := DisableRAForInterface(hostVethName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.DisableRAForInterface(hostVethName); err != nil { + return newErrorNetworkUtils(err.Error()) } return nil } -func (epc EPCommon) SetupContainerInterface(containerVethName string, targetIfName string) error { +func (nu NetworkUtils) SetupContainerInterface(containerVethName, targetIfName string) error { // Interface needs to be down before renaming. log.Printf("[net] Setting link %v state down.", containerVethName) - if err := epc.netlink.SetLinkState(containerVethName, false); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.netlink.SetLinkState(containerVethName, false); err != nil { + return newErrorNetworkUtils(err.Error()) } // Rename the container interface. log.Printf("[net] Setting link %v name %v.", containerVethName, targetIfName) - if err := epc.netlink.SetLinkName(containerVethName, targetIfName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.netlink.SetLinkName(containerVethName, targetIfName); err != nil { + return newErrorNetworkUtils(err.Error()) } - if err := DisableRAForInterface(targetIfName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.DisableRAForInterface(targetIfName); err != nil { + return newErrorNetworkUtils(err.Error()) } // Bring the interface back up. log.Printf("[net] Setting link %v state up.", targetIfName) - err := epc.netlink.SetLinkState(targetIfName, true) + err := nu.netlink.SetLinkState(targetIfName, true) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } return nil } -func (epc EPCommon) AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { +func (nu NetworkUtils) AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { var err error // Assign IP address to container network interface. for i, ipAddr := range ipAddresses { log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), interfaceName) - err = epc.netlink.AddIPAddress(interfaceName, ipAddr.IP, &ipAddresses[i]) + err = nu.netlink.AddIPAddress(interfaceName, ipAddr.IP, &ipAddresses[i]) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } } return nil } -func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, chainName string, target string) error { +func addOrDeleteFilterRule(bridgeName, action, ipAddress, chainName, target string) error { var err error option := "i" @@ -169,7 +171,7 @@ func AllowIPAddresses(bridgeName string, skipAddresses []string, action string) return nil } -func BlockIPAddresses(bridgeName string, action string) error { +func BlockIPAddresses(bridgeName, action string) error { privateIPAddresses := getPrivateIPSpace() chains := getFilterChains() target := getFilterchainTarget() @@ -194,11 +196,11 @@ func BlockIPAddresses(bridgeName string, action string) error { } // This fucntion enables ip forwarding in VM and allow forwarding packets from the interface -func EnableIPForwarding(ifName string) error { +func (nu NetworkUtils) EnableIPForwarding(ifName string) error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Enable ipforwarding failed with: %v", err) return err @@ -213,9 +215,9 @@ func EnableIPForwarding(ifName string) error { return nil } -func EnableIPV6Forwarding() error { +func (nu NetworkUtils) EnableIPV6Forwarding() error { cmd := fmt.Sprint(enableIPV6ForwardCmd) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Enable ipv6 forwarding failed with: %v", err) return err @@ -225,10 +227,10 @@ func EnableIPV6Forwarding() error { } // This functions enables/disables ipv6 setting based on enable parameter passed. -func UpdateIPV6Setting(disable int) error { +func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { // sysctl -w net.ipv6.conf.all.disable_ipv6=0/1 cmd := fmt.Sprintf(toggleIPV6Cmd, disable) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Update IPV6 Setting failed with: %v", err) } @@ -247,7 +249,7 @@ func AddSnatRule(match string, ip net.IP) error { return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) } -func DisableRAForInterface(ifName string) error { +func (nu NetworkUtils) DisableRAForInterface(ifName string) error { raFilePath := fmt.Sprintf(acceptRAV6File, ifName) exist, err := platform.CheckIfFileExists(raFilePath) if !exist { @@ -256,7 +258,7 @@ func DisableRAForInterface(ifName string) error { } cmd := fmt.Sprintf(disableRACmd, ifName) - out, err := platform.ExecuteCommand(cmd) + out, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Errorf("[net] Diabling ra failed with err: %v out: %v", err, out) } diff --git a/network/ovs_endpoint_infraroute_linux.go b/network/ovs_endpoint_infraroute_linux.go index 91a2c50a06..829a58ee4c 100644 --- a/network/ovs_endpoint_infraroute_linux.go +++ b/network/ovs_endpoint_infraroute_linux.go @@ -12,7 +12,7 @@ func NewInfraVnetClient(client *OVSEndpointClient, epID string) { hostIfName := fmt.Sprintf("%s%s", infraVethInterfacePrefix, epID) contIfName := fmt.Sprintf("%s%s-2", infraVethInterfacePrefix, epID) - client.infraVnetClient = ovsinfravnet.NewInfraVnetClient(hostIfName, contIfName, client.netlink) + client.infraVnetClient = ovsinfravnet.NewInfraVnetClient(hostIfName, contIfName, client.netlink, client.plClient) } } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 8952cd80df..5048bfc54a 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -3,7 +3,7 @@ package network import ( "fmt" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovssnat" ) @@ -20,6 +20,7 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin epInfo.DNS.Servers, client.netlink, client.ovsctlClient, + client.plClient, ) } } @@ -47,11 +48,12 @@ func AddSnatEndpointRules(client *OVSEndpointClient) error { } // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddStaticRoute(client.netlink, ovssnat.ImdsIP, client.bridgeName); err != nil { + if err := AddStaticRoute(client.netlink, client.netioshim, ovssnat.ImdsIP, client.bridgeName); err != nil { return err } - if err := epcommon.EnableIPForwarding(ovssnat.SnatBridgeName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.EnableIPForwarding(ovssnat.SnatBridgeName); err != nil { return err } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 4653fc73f0..b120fcda60 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -4,11 +4,13 @@ package network import ( + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovsinfravnet" "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" @@ -30,7 +32,9 @@ type OVSEndpointClient struct { allowInboundFromNCToHost bool enableSnatForDns bool netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface ovsctlClient ovsctl.OvsInterface + plClient platform.ExecClient } const ( @@ -46,7 +50,8 @@ func NewOVSEndpointClient( vlanid int, localIP string, nl netlink.NetlinkInterface, - ovs ovsctl.OvsInterface) *OVSEndpointClient { + ovs ovsctl.OvsInterface, + plc platform.ExecClient) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, @@ -62,6 +67,8 @@ func NewOVSEndpointClient( enableSnatForDns: epInfo.EnableSnatForDns, netlink: nl, ovsctlClient: ovs, + plClient: plc, + netioshim: &netio.NetIO{}, } NewInfraVnetClient(client, epInfo.Id[:7]) @@ -71,7 +78,7 @@ func NewOVSEndpointClient( } func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -195,8 +202,8 @@ func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo } func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -210,8 +217,8 @@ func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) } func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } @@ -223,7 +230,7 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E return err } - return addRoutes(client.netlink, client.containerVethName, epInfo.Routes) + return addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes) } func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 0703c71f82..9e93d4346f 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -7,11 +7,13 @@ import ( "bytes" "errors" "fmt" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/platform" "os" "strings" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" ) @@ -25,6 +27,9 @@ type OVSNetworkClient struct { bridgeName string hostInterfaceName string ovsctlClient ovsctl.OvsInterface + netlink netlink.NetlinkInterface + plClient platform.ExecClient + nuClient networkutils.NetworkUtils } const ( @@ -67,11 +72,15 @@ func (client *OVSNetworkClient) AddRoutes(nwInfo *NetworkInfo, interfaceName str return nil } -func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface, + netlink netlink.NetlinkInterface, plc platform.ExecClient) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, ovsctlClient: ovsctlClient, + netlink: netlink, + plClient: plc, + nuClient: networkutils.NewNetworkUtils(netlink, plc), } return ovsClient @@ -90,7 +99,7 @@ func (client *OVSNetworkClient) CreateBridge() error { } }() - if err := epcommon.DisableRAForInterface(client.bridgeName); err != nil { + if err := client.nuClient.DisableRAForInterface(client.bridgeName); err != nil { return err } diff --git a/network/ovs_networkclient_linux_test.go b/network/ovs_networkclient_linux_test.go index 146005e80a..912ca4f4d4 100644 --- a/network/ovs_networkclient_linux_test.go +++ b/network/ovs_networkclient_linux_test.go @@ -1,6 +1,8 @@ package network import ( + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/platform" "os" "testing" @@ -18,7 +20,8 @@ func TestMain(m *testing.M) { func TestAddRoutes(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.AddRoutes(nil, ""); err != nil { t.Errorf("Add routes failed") @@ -37,7 +40,8 @@ func TestCreateBridge(t *testing.T) { t.Errorf("Unable to write to file %v: %v", ovsConfigFile, err) } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.CreateBridge(); err != nil { t.Errorf("Error creating OVS bridge: %v", err) } @@ -48,7 +52,8 @@ func TestCreateBridge(t *testing.T) { func TestDeleteBridge(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.DeleteBridge(); err != nil { t.Errorf("Error deleting the OVS bridge: %v", err) } @@ -61,7 +66,8 @@ func TestAddL2Rules(t *testing.T) { MacAddress: []byte("2C:54:91:88:C9:E3"), } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.AddL2Rules(&extIf); err != nil { t.Errorf("Unable to add L2 rules: %v", err) } @@ -74,14 +80,16 @@ func TestDeleteL2Rules(t *testing.T) { MacAddress: []byte("2C:54:91:88:C9:E3"), } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) ovsClient.DeleteL2Rules(&extIf) } func TestSetBridgeMasterToHostInterface(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.SetBridgeMasterToHostInterface(); err != nil { t.Errorf("Unable to set bridge master to host intf: %v", err) } diff --git a/network/ovsinfravnet/infravnet.go b/network/ovsinfravnet/infravnet.go index 5a9df203f0..0934718642 100644 --- a/network/ovsinfravnet/infravnet.go +++ b/network/ovsinfravnet/infravnet.go @@ -10,8 +10,9 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -29,13 +30,15 @@ type OVSInfraVnetClient struct { ContainerInfraVethName string containerInfraMac string netlink netlink.NetlinkInterface + plClient platform.ExecClient } -func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterface) OVSInfraVnetClient { +func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterface, plc platform.ExecClient) OVSInfraVnetClient { infraVnetClient := OVSInfraVnetClient{ hostInfraVethName: hostIfName, ContainerInfraVethName: contIfName, netlink: nl, + plClient: plc, } log.Printf("Initialize new infravnet client %+v", infraVnetClient) @@ -45,7 +48,7 @@ func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterfa func (client *OVSInfraVnetClient) CreateInfraVnetEndpoint(bridgeName string) error { ovs := ovsctl.NewOvsctl() - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostInfraVethName, client.ContainerInfraVethName); err != nil { log.Printf("Creating infraep failed with error %v", err) return err @@ -107,7 +110,7 @@ func (client *OVSInfraVnetClient) MoveInfraEndpointToContainerNS(netnsPath strin } func (client *OVSInfraVnetClient) SetupInfraVnetContainerInterface() error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.ContainerInfraVethName, azureInfraIfName); err != nil { return newErrorOVSInfraVnetClient(err.Error()) } diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index e9df1a7f63..60c531fc67 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -13,7 +13,7 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" ) @@ -45,6 +45,7 @@ type OVSSnatClient struct { SkipAddressesFromBlock []string netlink netlink.NetlinkInterface ovsctlClient ovsctl.OvsInterface + plClient platform.ExecClient } func NewSnatClient(hostIfName string, @@ -55,6 +56,7 @@ func NewSnatClient(hostIfName string, skipAddressesFromBlock []string, nl netlink.NetlinkInterface, ovsctlClient ovsctl.OvsInterface, + plClient platform.ExecClient, ) OVSSnatClient { log.Printf("Initialize new snat client") snatClient := OVSSnatClient{ @@ -65,6 +67,7 @@ func NewSnatClient(hostIfName string, hostPrimaryMac: hostPrimaryMac, netlink: nl, ovsctlClient: ovsctlClient, + plClient: plClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -93,7 +96,7 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return err } - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) // Create veth pair to tie one end to container and other end to linux bridge if err := epc.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { log.Printf("Creating Snat Endpoint failed with error %v", err) @@ -109,7 +112,7 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge func (client *OVSSnatClient) AllowIPAddressesOnSnatBridge() error { - if err := epcommon.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorOVSSnatClient(err.Error()) } @@ -119,7 +122,7 @@ func (client *OVSSnatClient) AllowIPAddressesOnSnatBridge() error { // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge func (client *OVSSnatClient) BlockIPAddressesOnSnatBridge() error { - if err := epcommon.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { + if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorOVSSnatClient(err.Error()) } @@ -143,7 +146,7 @@ func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsI Configure Routes and setup name for container veth **/ func (client *OVSSnatClient) SetupSnatContainerInterface() error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { return newErrorOVSSnatClient(err.Error()) } @@ -404,7 +407,9 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return nil } - if err := epcommon.DisableRAForInterface(SnatBridgeName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + //nolint + if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { return err } @@ -422,11 +427,13 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return err } - if err := epcommon.DisableRAForInterface(azureSnatVeth0); err != nil { + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { return err } - if err := epcommon.DisableRAForInterface(azureSnatVeth1); err != nil { + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { return err } @@ -439,23 +446,23 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(SnatBridgeName, true); err != nil { + if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { + if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { + if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { + if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { return newErrorOVSSnatClient(err.Error()) } @@ -475,7 +482,7 @@ func (client *OVSSnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) er Drop all vlan traffic on linux bridge **/ func (client *OVSSnatClient) addVlanDropRule() error { - out, err := platform.ExecuteCommand(l2PreroutingEntries) + out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) if err != nil { log.Printf("Error while listing ebtable rules %v", err) return err @@ -488,6 +495,6 @@ func (client *OVSSnatClient) addVlanDropRule() error { } log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", vlanDropAddRule) - _, err = platform.ExecuteCommand(vlanDropAddRule) + _, err = client.plClient.ExecuteCommand(vlanDropAddRule) return err } diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index c36548975a..dbd256e872 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -3,14 +3,25 @@ package network import ( + "net" "testing" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" ) -func TestAddEndpoints(t *testing.T) { +const ( + subnetv4Mask = 24 + subnetv6Mask = 64 +) + +func TestTransAddEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + tests := []struct { name string client *TransparentEndpointClient @@ -25,6 +36,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -37,6 +50,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -50,6 +65,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, @@ -62,6 +79,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{}, @@ -75,6 +94,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 3), }, epInfo: &EndpointInfo{}, @@ -88,6 +109,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 4), }, epInfo: &EndpointInfo{}, @@ -109,3 +132,259 @@ func TestAddEndpoints(t *testing.T) { }) } } + +func TestTransAddEndpointsRules(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Add endpoint rules happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Add endpoint rules Dualstack happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPV6Mode: IPV6Nat, + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4FullMask), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + wantErr: false, + }, + { + name: "Add endpoint rules fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(true, "addroute fail"), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "addroute fail", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.AddEndpointRules(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestTransDeleteEndpointsRules(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + ep *endpoint + }{ + { + name: "Delete endpoint rules happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.client.DeleteEndpointRules(tt.ep) + }) + } +} + +func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes dualstack happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes assign ip fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "netlink fail", + }, + { + name: "Configure Interface and routes add routes fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + nuc: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: netio.ErrMockNetIOFail.Error(), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 40fc3627fc..fe8e0107c3 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -8,7 +8,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" ) @@ -41,6 +41,8 @@ type TransparentEndpointClient struct { mode string netlink netlink.NetlinkInterface netioshim netio.NetIOInterface + plClient platform.ExecClient + nuc networkutils.NetworkUtils } func NewTransparentEndpointClient( @@ -49,6 +51,7 @@ func NewTransparentEndpointClient( containerVethName string, mode string, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *TransparentEndpointClient { client := &TransparentEndpointClient{ @@ -60,14 +63,16 @@ func NewTransparentEndpointClient( mode: mode, netlink: nl, netioshim: &netio.NetIO{}, + plClient: plc, + nuc: networkutils.NewNetworkUtils(nl, plc), } return client } -func setArpProxy(ifName string) error { +func (client *TransparentEndpointClient) setArpProxy(ifName string) error { cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName) - _, err := platform.ExecuteCommand(cmd) + _, err := client.plClient.ExecuteCommand(cmd) return err } @@ -85,8 +90,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro return newErrorTransparentEndpointClient(err.Error()) } - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -143,13 +147,13 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) log.Printf("[net] Adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) - if err := addRoutes(client.netlink, client.hostVethName, routeInfoList); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.hostVethName, routeInfoList); err != nil { return newErrorTransparentEndpointClient(err.Error()) } } log.Printf("calling setArpProxy for %v", client.hostVethName) - if err := setArpProxy(client.hostVethName); err != nil { + if err := client.setArpProxy(client.hostVethName); err != nil { log.Printf("setArpProxy failed with: %v", err) return err } @@ -174,7 +178,7 @@ func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { log.Printf("[net] Deleting route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet - if err := deleteRoutes(client.netlink, client.hostVethName, []RouteInfo{routeInfo}); err != nil { + if err := deleteRoutes(client.netlink, client.netioshim, client.hostVethName, []RouteInfo{routeInfo}); err != nil { log.Printf("[net] Failed to delete route on VM for the ip %v: %v", ipNet.String(), err) } } @@ -191,8 +195,7 @@ func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := client.nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -202,8 +205,7 @@ func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -215,7 +217,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Scope: netlink.RT_SCOPE_LINK, Protocol: netlink.RTPROT_KERNEL, } - if err := deleteRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return newErrorTransparentEndpointClient(err.Error()) } } @@ -227,7 +229,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, } - if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -238,7 +240,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Dst: dstIP, Gw: virtualGwIP, } - if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return err } @@ -280,7 +282,7 @@ func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) e Gw: virtualGwIP, } - if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { return err } diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 33f0a64e2f..59f9083def 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -50,17 +50,19 @@ type OvsInterface interface { DeletePortFromOVS(bridgeName string, interfaceName string) error } -type Ovsctl struct{} +type Ovsctl struct { + execcli platform.ExecClient +} func NewOvsctl() Ovsctl { - return Ovsctl{} + return Ovsctl{execcli: platform.NewExecClient()} } -func (Ovsctl) CreateOVSBridge(bridgeName string) error { +func (o Ovsctl) CreateOVSBridge(bridgeName string) error { log.Printf("[ovs] Creating OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) - _, err := platform.ExecuteCommand(ovsCreateCmd) + _, err := o.execcli.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while creating OVS bridge %v", err) return newErrorOvsctl(err.Error()) @@ -69,11 +71,11 @@ func (Ovsctl) CreateOVSBridge(bridgeName string) error { return nil } -func (Ovsctl) DeleteOVSBridge(bridgeName string) error { +func (o Ovsctl) DeleteOVSBridge(bridgeName string) error { log.Printf("[ovs] Deleting OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) - _, err := platform.ExecuteCommand(ovsCreateCmd) + _, err := o.execcli.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while deleting OVS bridge %v", err) return newErrorOvsctl(err.Error()) @@ -82,9 +84,9 @@ func (Ovsctl) DeleteOVSBridge(bridgeName string) error { return nil } -func (Ovsctl) AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error { +func (o Ovsctl) AddPortOnOVSBridge(hostIfName, bridgeName string, vlanID int) error { cmd := fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Error while setting OVS as master to primary interface %v", err) return newErrorOvsctl(err.Error()) @@ -93,9 +95,9 @@ func (Ovsctl) AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID in return nil } -func (Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { +func (o Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", interfaceName) - ofport, err := platform.ExecuteCommand(cmd) + ofport, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Get ofport failed with error %v", err) return "", newErrorOvsctl(err.Error()) @@ -104,9 +106,9 @@ func (Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { return strings.Trim(ofport, "\n"), nil } -func (Ovsctl) AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { +func (o Ovsctl) AddVMIpAcceptRule(bridgeName, primaryIP, mac string) error { cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=%d,actions=normal", bridgeName, primaryIP, mac, high) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -115,10 +117,10 @@ func (Ovsctl) AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) return nil } -func (Ovsctl) AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) error { +func (o Ovsctl) AddArpSnatRule(bridgeName, mac, macHex, ofport string) error { cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=%d,arp,arp_op=1,actions='mod_dl_src:%s, load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, low, mac, macHex, ofport) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -128,7 +130,7 @@ func (Ovsctl) AddArpSnatRule(bridgeName string, mac string, macHex string, ofpor } // IP SNAT Rule - Change src mac to VM Mac for packets coming from container host veth port. -func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port string, mac string, outport string) error { +func (o Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port, mac, outport string) error { var cmd string if outport == "" { outport = "normal" @@ -144,7 +146,7 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin cmd = fmt.Sprintf("%s,strip_vlan,%v", commonPrefix, outport) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -153,7 +155,7 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin // Drop other packets which doesn't satisfy above condition cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=%d,ip,in_port=%s,actions=drop", bridgeName, low, port) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Dropping vlantag packet rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -162,11 +164,11 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin return nil } -func (Ovsctl) AddArpDnatRule(bridgeName string, port string, mac string) error { +func (o Ovsctl) AddArpDnatRule(bridgeName, port, mac string) error { // Add DNAT rule to forward ARP replies to container interfaces. cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, port, mac) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding DNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -175,7 +177,7 @@ func (Ovsctl) AddArpDnatRule(bridgeName string, port string, mac string) error { return nil } -func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { +func (o Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { // If arp fields matches, set arp reply rule for the request macAddrHex := strings.Replace(defaultMacForArpResponse, ":", "", -1) ipAddrInt := common.IpToInt(ip) @@ -186,7 +188,7 @@ func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, bridgeName, high, defaultMacForArpResponse, macAddrHex, ipAddrInt) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -195,14 +197,14 @@ func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { return nil } -func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, mode string) error { +func (o Ovsctl) AddArpReplyRule(bridgeName, port string, ip net.IP, mac string, vlanid int, mode string) error { ipAddrInt := common.IpToInt(ip) macAddrHex := strings.Replace(mac, ":", "", -1) log.Printf("[ovs] Adding ARP reply rule to add vlan %v and forward packet to table 1 for port %v", vlanid, port) cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, bridgeName, port, vlanid) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -215,7 +217,7 @@ func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac str move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, bridgeName, ip.String(), vlanid, high, mac, macAddrHex, ipAddrInt) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -225,7 +227,7 @@ func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac str } // Add MAC DNAT rule based on dst ip and vlanid -func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, containerPort string) error { +func (o Ovsctl) AddMacDnatRule(bridgeName, port string, ip net.IP, mac string, vlanid int, containerPort string) error { var cmd string // This rule changes the destination mac to speciifed mac based on the ip and vlanid. // and forwards the packet to corresponding container hostveth port @@ -236,7 +238,7 @@ func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac stri } else { cmd = fmt.Sprintf("%s,actions=mod_dl_dst:%s,strip_vlan,%s", commonPrefix, mac, containerPort) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding MAC DNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -245,32 +247,32 @@ func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac stri return nil } -func (Ovsctl) DeleteArpReplyRule(bridgeName string, port string, ip net.IP, vlanid int) { +func (o Ovsctl) DeleteArpReplyRule(bridgeName, port string, ip net.IP, vlanid int) { cmd := fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", bridgeName, port) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", bridgeName, ip.String(), vlanid) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } } -func (Ovsctl) DeleteIPSnatRule(bridgeName string, port string) { +func (o Ovsctl) DeleteIPSnatRule(bridgeName, port string) { cmd := fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", bridgeName, port) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("Error while deleting ovs rule %v error %v", cmd, err) } } -func (Ovsctl) DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { +func (o Ovsctl) DeleteMacDnatRule(bridgeName, port string, ip net.IP, vlanid int) { var cmd string if vlanid != 0 { @@ -281,16 +283,16 @@ func (Ovsctl) DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlani bridgeName, ip.String(), port) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) } } -func (Ovsctl) DeletePortFromOVS(bridgeName string, interfaceName string) error { +func (o Ovsctl) DeletePortFromOVS(bridgeName, interfaceName string) error { // Disconnect external interface from its bridge. cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", bridgeName, interfaceName) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Failed to disconnect interface %v from bridge, err:%v.", interfaceName, err) return newErrorOvsctl(err.Error()) diff --git a/platform/mockexec.go b/platform/mockexec.go new file mode 100644 index 0000000000..a1d11a3460 --- /dev/null +++ b/platform/mockexec.go @@ -0,0 +1,23 @@ +package platform + +import "errors" + +type mockExecClient struct { + returnError bool +} + +var ErrMockExec = errors.New("mock exec error") + +func NewMockExecClient(returnErr bool) ExecClient { + return &mockExecClient{ + returnError: returnErr, + } +} + +func (e *mockExecClient) ExecuteCommand(string) (string, error) { + if e.returnError { + return "", ErrMockExec + } + + return "", nil +} diff --git a/platform/osInterface.go b/platform/osInterface.go new file mode 100644 index 0000000000..c0f47ed145 --- /dev/null +++ b/platform/osInterface.go @@ -0,0 +1,12 @@ +package platform + +type execClient struct{} + +//nolint:revive // ExecClient make sense +type ExecClient interface { + ExecuteCommand(command string) (string, error) +} + +func NewExecClient() ExecClient { + return &execClient{} +} diff --git a/platform/os_linux.go b/platform/os_linux.go index 0287d8f6a5..96aa0f208c 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -47,8 +47,9 @@ func GetOSInfo() string { } func GetProcessSupport() error { + p := &execClient{} cmd := fmt.Sprintf("ps -p %v -o comm=", os.Getpid()) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) return err } @@ -72,7 +73,7 @@ func GetLastRebootTime() (time.Time, error) { return rebootTime.UTC(), nil } -func ExecuteCommand(command string) (string, error) { +func (p *execClient) ExecuteCommand(command string) (string, error) { log.Printf("[Azure-Utils] %s", command) var stderr bytes.Buffer @@ -90,9 +91,10 @@ func ExecuteCommand(command string) (string, error) { } func SetOutboundSNAT(subnet string) error { + p := execClient{} cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", subnet) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) if err != nil { log.Printf("SNAT Iptable rule was not set") return err @@ -107,8 +109,9 @@ func ClearNetworkConfiguration() (bool, error) { } func KillProcessByName(processName string) error { + p := &execClient{} cmd := fmt.Sprintf("pkill -f %v", processName) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) return err } @@ -137,9 +140,10 @@ func GetOSDetails() (map[string]string, error) { } func GetProcessNameByID(pidstr string) (string, error) { + p := &execClient{} pidstr = strings.Trim(pidstr, "\n") cmd := fmt.Sprintf("ps -p %s -o comm=", pidstr) - out, err := ExecuteCommand(cmd) + out, err := p.ExecuteCommand(cmd) if err != nil { log.Printf("GetProcessNameByID returned error: %v", err) return "", err @@ -152,10 +156,11 @@ func GetProcessNameByID(pidstr string) (string, error) { } func PrintDependencyPackageDetails() { - out, err := ExecuteCommand("iptables --version") + p := &execClient{} + out, err := p.ExecuteCommand("iptables --version") out = strings.TrimSuffix(out, "\n") log.Printf("[cni-net] iptable version:%s, err:%v", out, err) - out, err = ExecuteCommand("ebtables --version") + out, err = p.ExecuteCommand("ebtables --version") out = strings.TrimSuffix(out, "\n") log.Printf("[cni-net] ebtable version %s, err:%v", out, err) } diff --git a/platform/os_windows.go b/platform/os_windows.go index a5c9ae994d..5285176d10 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -87,7 +87,7 @@ func GetLastRebootTime() (time.Time, error) { return rebootTime.UTC(), nil } -func ExecuteCommand(command string) (string, error) { +func (p *execClient) ExecuteCommand(command string) (string, error) { log.Printf("[Azure-Utils] %s", command) var stderr bytes.Buffer diff --git a/telemetry/telemetry_windows.go b/telemetry/telemetry_windows.go index 974d4bf717..834ad6b8e2 100644 --- a/telemetry/telemetry_windows.go +++ b/telemetry/telemetry_windows.go @@ -4,10 +4,9 @@ package telemetry import ( + "github.com/Azure/azure-container-networking/platform" "runtime" "strings" - - "github.com/Azure/azure-container-networking/platform" ) const ( @@ -38,8 +37,9 @@ func (report *CNIReport) GetSystemDetails() { } func (report *CNIReport) GetOSDetails() { + p := platform.NewExecClient() report.OSDetails = OSInfo{OSType: runtime.GOOS} - out, err := platform.ExecuteCommand(versionCmd) + out, err := p.ExecuteCommand(versionCmd) if err == nil { report.OSDetails.OSVersion = strings.Replace(out, delimiter, "", -1) } From 327fd049b16d1e4497cb8732f1330158356e2df8 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 6 Oct 2021 17:50:57 -0700 Subject: [PATCH 7/9] lint fixes fix compilation issues --- network/bridge_endpointclient_linux.go | 4 ++-- network/bridge_networkclient_linux.go | 8 ++++++-- network/endpoint.go | 2 +- network/endpoint_linux.go | 6 +++--- network/endpoint_windows.go | 4 ++-- network/network_linux.go | 2 +- network/network_test.go | 2 +- network/ovs_endpointclient_linux.go | 4 ++-- network/ovs_networkclient_linux.go | 10 +++++----- network/ovs_networkclient_linux_test.go | 4 ++-- network/transparent_endpointclient_linux.go | 2 +- platform/mockexec.go | 1 + platform/os_windows.go | 3 ++- 13 files changed, 29 insertions(+), 23 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 9b6f871d58..1150e37bfb 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -2,14 +2,14 @@ package network import ( "fmt" - "github.com/Azure/azure-container-networking/netio" - "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" ) const ( diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index b9cc84be2a..25893dc41c 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -3,13 +3,13 @@ package network import ( "errors" "fmt" - "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -62,7 +62,11 @@ func (client *LinuxBridgeClient) CreateBridge() error { return err } - return client.nuClient.DisableRAForInterface(client.bridgeName) + if err := client.nuClient.DisableRAForInterface(client.bridgeName); err != nil { + return fmt.Errorf("CreateBridge:%w", err) + } + + return nil } func (client *LinuxBridgeClient) DeleteBridge() error { diff --git a/network/endpoint.go b/network/endpoint.go index 8e4e52bfcc..f54363e1bb 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -5,13 +5,13 @@ package network import ( "context" - "github.com/Azure/azure-container-networking/platform" "net" "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/policy" + "github.com/Azure/azure-container-networking/platform" ) const ( diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 28e902f44e..312c026e08 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -7,15 +7,15 @@ import ( "crypto/sha1" "encoding/hex" "fmt" - "github.com/Azure/azure-container-networking/netio" - "github.com/Azure/azure-container-networking/platform" "net" "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -269,7 +269,7 @@ func addRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, inte interfaceIf, err := netioshim.GetNetworkInterfaceByName(interfaceName) if err != nil { log.Errorf("Interface not found:%v", err) - return err + return fmt.Errorf("addRoutes failed: %w", err) } ifIndex = interfaceIf.Index } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 09caddde16..ea1d51381e 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -93,7 +93,7 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, _ platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err @@ -415,7 +415,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, _ platform.ExecClient, ep *endpoint) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { return err diff --git a/network/network_linux.go b/network/network_linux.go index 0f2ecf8633..df7355e235 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -6,13 +6,13 @@ package network import ( "errors" "fmt" - "github.com/Azure/azure-container-networking/netio" "net" "strconv" "strings" "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" diff --git a/network/network_test.go b/network/network_test.go index 6093cef755..73d168ee13 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -1,10 +1,10 @@ package network import ( - "github.com/Azure/azure-container-networking/platform" "net" "testing" + "github.com/Azure/azure-container-networking/platform" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index b120fcda60..54054f2572 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -4,16 +4,16 @@ package network import ( - "github.com/Azure/azure-container-networking/netio" - "github.com/Azure/azure-container-networking/platform" "net" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovsinfravnet" "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) type OVSEndpointClient struct { diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 9e93d4346f..bd088b0678 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -7,14 +7,14 @@ import ( "bytes" "errors" "fmt" - "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/platform" "os" "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) var errorOVSNetworkClient = errors.New("OVSNetworkClient Error") @@ -73,14 +73,14 @@ func (client *OVSNetworkClient) AddRoutes(nwInfo *NetworkInfo, interfaceName str } func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface, - netlink netlink.NetlinkInterface, plc platform.ExecClient) *OVSNetworkClient { + nl netlink.NetlinkInterface, plc platform.ExecClient) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, ovsctlClient: ovsctlClient, - netlink: netlink, + netlink: nl, plClient: plc, - nuClient: networkutils.NewNetworkUtils(netlink, plc), + nuClient: networkutils.NewNetworkUtils(nl, plc), } return ovsClient diff --git a/network/ovs_networkclient_linux_test.go b/network/ovs_networkclient_linux_test.go index 912ca4f4d4..924468b474 100644 --- a/network/ovs_networkclient_linux_test.go +++ b/network/ovs_networkclient_linux_test.go @@ -1,12 +1,12 @@ package network import ( - "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/platform" "os" "testing" + "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index fe8e0107c3..56e3a9c302 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -90,7 +90,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro return newErrorTransparentEndpointClient(err.Error()) } - if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err = client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return newErrorTransparentEndpointClient(err.Error()) } diff --git a/platform/mockexec.go b/platform/mockexec.go index a1d11a3460..f768a5c83c 100644 --- a/platform/mockexec.go +++ b/platform/mockexec.go @@ -6,6 +6,7 @@ type mockExecClient struct { returnError bool } +// ErrMockExec - mock exec error var ErrMockExec = errors.New("mock exec error") func NewMockExecClient(returnErr bool) ExecClient { diff --git a/platform/os_windows.go b/platform/os_windows.go index 5285176d10..811bc7fd2b 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -124,8 +124,9 @@ func ClearNetworkConfiguration() (bool, error) { } func KillProcessByName(processName string) { + p := NewExecClient() cmd := fmt.Sprintf("taskkill /IM %v /F", processName) - ExecuteCommand(cmd) + p.ExecuteCommand(cmd) } // ExecutePowershellCommand executes powershell command From c9a85fd77baf0cfa9ba5346cd7782b646591e575 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 11 Oct 2021 15:09:26 -0700 Subject: [PATCH 8/9] addressed comments --- network/transparent_endpoint_linux_test.go | 28 ++++---- network/transparent_endpointclient_linux.go | 73 ++++++++++----------- 2 files changed, 47 insertions(+), 54 deletions(-) diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index dbd256e872..50eaadfce5 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -37,7 +37,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -51,7 +51,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -66,7 +66,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, @@ -80,7 +80,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{}, @@ -95,7 +95,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 3), }, epInfo: &EndpointInfo{}, @@ -110,7 +110,7 @@ func TestTransAddEndpoints(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 4), }, epInfo: &EndpointInfo{}, @@ -152,7 +152,7 @@ func TestTransAddEndpointsRules(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -173,7 +173,7 @@ func TestTransAddEndpointsRules(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -199,7 +199,7 @@ func TestTransAddEndpointsRules(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(true, "addroute fail"), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -246,7 +246,7 @@ func TestTransDeleteEndpointsRules(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ @@ -291,7 +291,7 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -312,7 +312,7 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -337,7 +337,7 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{ @@ -359,7 +359,7 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{ diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 56e3a9c302..caa312aa4a 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -42,7 +42,7 @@ type TransparentEndpointClient struct { netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient - nuc networkutils.NetworkUtils + netUtilsClient networkutils.NetworkUtils } func NewTransparentEndpointClient( @@ -64,7 +64,7 @@ func NewTransparentEndpointClient( netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, - nuc: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -90,7 +90,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro return newErrorTransparentEndpointClient(err.Error()) } - if err = client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err = client.netUtilsClient.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -195,7 +195,7 @@ func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - if err := client.nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -205,7 +205,7 @@ func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if err := client.nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -255,51 +255,44 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return fmt.Errorf("Adding arp in container failed: %w", err) } - if err := client.setupIPV6Routes(epInfo); err != nil { - return err + if epInfo.IPV6Mode != "" { + if err := client.setupIPV6Routes(); err != nil { + return err + } } - return client.setIPV6NeighEntry(epInfo) + return client.setIPV6NeighEntry() } -func (client *TransparentEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) error { - if epInfo.IPV6Mode != "" { - log.Printf("Setting up ipv6 routes in container") - - // add route for virtualgwip - // ip -6 route add fe80::1234:5678:9abc/128 dev eth0 - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualv6GwString) - gwRoute := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - } - - // ip -6 route add default via fe80::1234:5678:9abc dev eth0 - _, defaultIPNet, _ := net.ParseCIDR(defaultv6Cidr) - log.Printf("defaultv6ipnet :%+v", defaultIPNet) - defaultRoute := RouteInfo{ - Dst: *defaultIPNet, - Gw: virtualGwIP, - } +func (client *TransparentEndpointClient) setupIPV6Routes() error { + log.Printf("Setting up ipv6 routes in container") - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}); err != nil { - return err - } + // add route for virtualgwip + // ip -6 route add fe80::1234:5678:9abc/128 dev eth0 + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualv6GwString) + gwRoute := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + // ip -6 route add default via fe80::1234:5678:9abc dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultv6Cidr) + log.Printf("defaultv6ipnet :%+v", defaultIPNet) + defaultRoute := RouteInfo{ + Dst: *defaultIPNet, + Gw: virtualGwIP, } - return nil + return addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}) } -func (client *TransparentEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) error { - if epInfo.IPV6Mode != "" { - log.Printf("[net] Add v6 neigh entry for default gw ip") - hostGwIP, _, _ := net.ParseCIDR(virtualv6GwString) - if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, - hostGwIP, client.hostVethMac, false); err != nil { - log.Printf("Failed setting neigh entry in container: %+v", err) - return fmt.Errorf("Failed setting neigh entry in container: %w", err) - } +func (client *TransparentEndpointClient) setIPV6NeighEntry() error { + log.Printf("[net] Add v6 neigh entry for default gw ip") + hostGwIP, _, _ := net.ParseCIDR(virtualv6GwString) + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, + hostGwIP, client.hostVethMac, false); err != nil { + log.Printf("Failed setting neigh entry in container: %+v", err) + return fmt.Errorf("Failed setting neigh entry in container: %w", err) } return nil From 3b0b629339a1c3bf833a0fc4f58c99617f163dbe Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 14 Oct 2021 14:27:25 -0700 Subject: [PATCH 9/9] fixed a bug --- network/transparent_endpointclient_linux.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index caa312aa4a..17a6d5677e 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -261,7 +261,11 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } } - return client.setIPV6NeighEntry() + if epInfo.IPV6Mode != "" { + return client.setIPV6NeighEntry() + } + + return nil } func (client *TransparentEndpointClient) setupIPV6Routes() error {