refactor: Move CNI bridge/transparent routes to common

acncli/deployment/manager.yaml

@@ -18,17 +18,17 @@ spec:
       hostNetwork: true
       containers:
         - name: azure-cni-installer
-          image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.1.9-alphav1
+          image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63 
           imagePullPolicy: Always
           env:
             - name: AZURE_CNI_OS
               value: linux
             - name: AZURE_CNI_TENANCY
               value: singletenancy
             - name: AZURE_CNI_MODE
-              value: bridge
+              value: transparent
             - name: AZURE_CNI_IPAM
-              value: azure-cns
+              value: azure-vnet-ipam
             - name: AZURE_CNI_EXEMPT
               value: azure-vnet-telemetry,azure-vnet-telemetry.config
           volumeMounts:

cni/network/invoker_cns.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/cnsclient"
+	"github.com/Azure/azure-container-networking/iptables"
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/network"
 	cniTypes "github.com/containernetworking/cni/pkg/types"
@@ -48,7 +49,7 @@ func NewCNSInvoker(podName, namespace string) (*CNSIPAMInvoker, error) {
 }
 
 //Add uses the requestipconfig API in cns, and returns ipv4 and a nil ipv6 as CNS doesn't support IPv6 yet
-func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
+func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
 
 	// Parse Pod arguments.
 	podInfo := cns.KubernetesPodInfo{PodName: invoker.podName, PodNamespace: invoker.podNamespace}
@@ -61,7 +62,7 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
 		return nil, nil, err
 	}
 
-	resultIPv4 := IPv4ResultInfo{
+	info := IPv4ResultInfo{
 		podIPAddress:       response.PodIpInfo.PodIPConfig.IPAddress,
 		ncSubnetPrefix:     response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength,
 		ncPrimaryIP:        response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress,
@@ -71,20 +72,46 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
 		hostGateway:        response.PodIpInfo.HostPrimaryIPInfo.Gateway,
 	}
 
-	ncgw := net.ParseIP(resultIPv4.ncGatewayIPAddress)
+	// set the NC Primary IP in options
+	options[network.SNATIPKey] = info.ncPrimaryIP
+
+	log.Printf("[cni-invoker-cns] Received info %v for pod %v", info, podInfo)
+
+	ncgw := net.ParseIP(info.ncGatewayIPAddress)
 	if ncgw == nil {
-		return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", resultIPv4.ncGatewayIPAddress)
+		return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", info.ncGatewayIPAddress)
 	}
 
-	// set the NC Primary IP in options
-	options[network.SNATIPKey] = resultIPv4.ncPrimaryIP
+	// set result ipconfig from CNS Response Body
+	ip, ncipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
+	if ip == nil {
+		return nil, nil, fmt.Errorf("Unable to parse IP from response: %v with err %v", info.podIPAddress, err)
+	}
 
-	// set host gateway in options
-	options[network.HostGWKey] = resultIPv4.hostGateway
+	// construct ipnet for result
+	resultIPnet := net.IPNet{
+		IP:   ip,
+		Mask: ncipnet.Mask,
+	}
 
-	log.Printf("Received result %+v for pod %v", resultIPv4, podInfo)
+	result := &cniTypesCurr.Result{
+		IPs: []*cniTypesCurr.IPConfig{
+			{
+				Version: "4",
+				Address: resultIPnet,
+				Gateway: ncgw,
+			},
+		},
+		Routes: []*cniTypes.Route{
+			{
+				Dst: network.Ipv4DefaultRouteDstPrefix,
+				GW:  ncgw,
+			},
+		},
+	}
 
-	result, err := getCNIIPv4Result(resultIPv4, subnetPrefix)
+	// set subnet prefix for host vm
+	err = setHostOptions(nwCfg, hostSubnetPrefix, ncipnet, options, info)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -93,54 +120,44 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
 	return result, nil, nil
 }
 
-func getCNIIPv4Result(info IPv4ResultInfo, subnetPrefix *net.IPNet) (*cniTypesCurr.Result, error) {
-
-	gw := net.ParseIP(info.ncGatewayIPAddress)
-	if gw == nil {
-		return nil, fmt.Errorf("Gateway address %v from response is invalid", gw)
+func setHostOptions(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, ncSubnetPrefix *net.IPNet, options map[string]interface{}, info IPv4ResultInfo) error {
+	// get the name of the primary IP address
+	_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
+	if err != nil {
+		return err
 	}
 
+	*hostSubnetPrefix = *hostIPNet
+
+	// get the host ip
 	hostIP := net.ParseIP(info.hostPrimaryIP)
 	if hostIP == nil {
-		return nil, fmt.Errorf("Host IP address %v from response is invalid", hostIP)
+		return fmt.Errorf("Host IP address %v from response is invalid", info.hostPrimaryIP)
 	}
 
-	// set result ipconfig from CNS Response Body
-	ip, ipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
-	if ip == nil {
-		return nil, fmt.Errorf("Unable to parse IP from response: %v", info.podIPAddress)
+	// get host gateway
+	hostGateway := net.ParseIP(info.hostGateway)
+	if hostGateway == nil {
+		return fmt.Errorf("Host Gateway %v from response is invalid", info.hostGateway)
 	}
 
-	// get the name of the primary IP address
-	_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
-	if err != nil {
-		return nil, err
+	// this route is needed when the vm on subnet A needs to send traffic to a pod in subnet B on a different vm
+	options[network.RoutesKey] = []network.RouteInfo{
+		{
+			Dst: *ncSubnetPrefix,
+			Gw:  hostGateway,
+		},
 	}
 
-	// set subnet prefix for host vm
-	*subnetPrefix = *hostIPNet
-
-	// construct ipnet for result
-	resultIPnet := net.IPNet{
-		IP:   ip,
-		Mask: ipnet.Mask,
+	azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
+	snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.ncPrimaryIP)
+	options[network.IPTablesKey] = []iptables.IPTableEntry{
+		iptables.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift),
+		iptables.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift),
+		iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump),
 	}
 
-	return &cniTypesCurr.Result{
-		IPs: []*cniTypesCurr.IPConfig{
-			{
-				Version: "4",
-				Address: resultIPnet,
-				Gateway: gw,
-			},
-		},
-		Routes: []*cniTypes.Route{
-			{
-				Dst: network.Ipv4DefaultRouteDstPrefix,
-				GW:  gw,
-			},
-		},
-	}, nil
+	return nil
 }
 
 // Delete calls into the releaseipconfiguration API in CNS

iptables/iptables.go

@@ -85,8 +85,13 @@ var (
 	DisableIPTableLock bool
 )
 
+type IPTableEntry struct {
+	Version string
+	Params  string
+}
+
 // Run iptables command
-func runCmd(version, params string) error {
+func RunCmd(version, params string) error {
 	var cmd string
 
 	iptCmd := iptables
@@ -110,20 +115,26 @@ func runCmd(version, params string) error {
 // check if iptable chain alreay exists
 func ChainExists(version, tableName, chainName string) bool {
 	params := fmt.Sprintf("-t %s -L %s", tableName, chainName)
-	if err := runCmd(version, params); err != nil {
+	if err := RunCmd(version, params); err != nil {
 		return false
 	}
 
 	return true
 }
+func GetCreateChainCmd(version, tableName, chainName string) IPTableEntry {
+	return IPTableEntry{
+		Version: version,
+		Params:  fmt.Sprintf("-t %s -N %s", tableName, chainName),
+	}
+}
 
 // create new iptable chain under specified table name
 func CreateChain(version, tableName, chainName string) error {
 	var err error
 
 	if !ChainExists(version, tableName, chainName) {
-		params := fmt.Sprintf("-t %s -N %s", tableName, chainName)
-		err = runCmd(version, params)
+		cmd := GetCreateChainCmd(version, tableName, chainName)
+		err = RunCmd(version, cmd.Params)
 	} else {
 		log.Printf("%s Chain exists in table %s", chainName, tableName)
 	}
@@ -134,21 +145,35 @@ func CreateChain(version, tableName, chainName string) error {
 // check if iptable rule alreay exists
 func RuleExists(version, tableName, chainName, match, target string) bool {
 	params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target)
-	if err := runCmd(version, params); err != nil {
+	if err := RunCmd(version, params); err != nil {
 		return false
 	}
 	return true
 }
 
+func GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
+	return IPTableEntry{
+		Version: version,
+		Params:  fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target),
+	}
+}
+
 // Insert iptable rule at beginning of iptable chain
 func InsertIptableRule(version, tableName, chainName, match, target string) error {
 	if RuleExists(version, tableName, chainName, match, target) {
 		log.Printf("Rule already exists")
 		return nil
 	}
 
-	params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target)
-	return runCmd(version, params)
+	cmd := GetInsertIptableRuleCmd(version, tableName, chainName, match, target)
+	return RunCmd(version, cmd.Params)
+}
+
+func GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
+	return IPTableEntry{
+		Version: version,
+		Params:  fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target),
+	}
 }
 
 // Append iptable rule at end of iptable chain
@@ -158,12 +183,12 @@ func AppendIptableRule(version, tableName, chainName, match, target string) erro
 		return nil
 	}
 
-	params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target)
-	return runCmd(version, params)
+	cmd := GetAppendIptableRuleCmd(version, tableName, chainName, match, target)
+	return RunCmd(version, cmd.Params)
 }
 
 // Delete matched iptable rule
 func DeleteIptableRule(version, tableName, chainName, match, target string) error {
 	params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target)
-	return runCmd(version, params)
+	return RunCmd(version, params)
 }

network/bridge_networkclient_linux.go

@@ -1,9 +1,7 @@
 package network
 
 import (
-	"fmt"
 	"net"
-	"strings"
 
 	"github.com/Azure/azure-container-networking/ebtables"
 	"github.com/Azure/azure-container-networking/log"
@@ -48,59 +46,6 @@ func (client *LinuxBridgeClient) CreateBridge() error {
 	return epcommon.DisableRAForInterface(client.bridgeName)
 }
 
-func (client *LinuxBridgeClient) AddRoutes(nwInfo *NetworkInfo, interfaceName string) error {
-	if client.nwInfo.IPAMType == AzureCNS {
-
-		// fetch the host gateway IP from options
-		gwIP := client.nwInfo.Options[HostGWKey]
-		if gwIP == nil {
-			return fmt.Errorf("Host gateway IP in Options not set")
-		}
-
-		gatewayIP := net.ParseIP(gwIP.(string))
-		if gatewayIP == nil {
-			return fmt.Errorf("Invalid host gateway IP: %+v", gwIP)
-		}
-
-		// add host gateway as the default gateway for pod IP's
-		devIf, _ := net.InterfaceByName(interfaceName)
-		ifIndex := devIf.Index
-		family := netlink.GetIpAddressFamily(gatewayIP)
-
-		nlRoute := &netlink.Route{
-			Family:    family,
-			Dst:       &client.nwInfo.PodSubnet.Prefix,
-			Gw:        gatewayIP,
-			LinkIndex: ifIndex,
-		}
-
-		log.Printf("Adding Swift route %+v", nlRoute)
-
-		if err := netlink.AddIpRoute(nlRoute); err != nil {
-			if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
-				return fmt.Errorf("Failed to add route to host interface with error: %v", err)
-			}
-			log.Printf("[cni-cns-net] route already exists: dst %+v, gw %+v, interfaceName %v", nlRoute.Dst, nlRoute.Gw, interfaceName)
-		}
-
-		// Add snat Rules
-		snatIP := client.nwInfo.Options[SNATIPKey]
-		if snatIP == nil {
-			return fmt.Errorf("snatIP in Options not set %v", snatIP)
-		}
-
-		ncPrimaryIP := net.ParseIP(fmt.Sprintf("%v", snatIP))
-		if ncPrimaryIP == nil {
-			return fmt.Errorf("Failed to parse SNAT IP from options %v", client.nwInfo.Options)
-		}
-
-		log.Printf("Adding SNAT rule with snat IP %+v for subnet %s", ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
-		return epcommon.SNATfromSubnetToDNSWithNCPrimaryIP(ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
-
-	}
-	return nil
-}
-
 func (client *LinuxBridgeClient) DeleteBridge() error {
 	// Disconnect external interface from its bridge.
 	err := netlink.SetLinkMaster(client.hostInterfaceName, "")

network/epcommon/endpoint_common.go

@@ -240,39 +240,6 @@ func AddSnatRule(match string, ip net.IP) error {
 	return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target)
 }
 
-// SNATfromSubnetToDNSWithNCPrimaryIP snat's the snattedAddressSpace with the ipForSnat IP
-func SNATfromSubnetToDNSWithNCPrimaryIP(ipForSNAT net.IP, snattedAddressSpace net.IPNet) (err error) {
-
-	// Create SWIFT chain, this checks if the chain already exists
-	// Check if theres a primary IP
-	if ipForSNAT != nil {
-		// Create SWIFT chain, this checks if the chain already exists
-		log.Printf("Creating SWIFT chain...")
-		err := iptables.CreateChain(iptables.V4, iptables.Nat, iptables.Swift)
-		if err != nil {
-			return err
-		}
-
-		log.Printf("Creating SWIFT chain jump from POSTROUTING")
-		// add jump to SWIFT chain from POSTROUTING
-		err = iptables.AppendIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift)
-		if err != nil {
-			return err
-		}
-
-		log.Printf("Adding rule to SNAT subnet %v DNS requests with ip %v", snattedAddressSpace, ipForSNAT)
-		// SNAT requests to Azure DNS
-		azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", snattedAddressSpace.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
-		snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, ipForSNAT)
-		err = iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func DisableRAForInterface(ifName string) error {
 	raFilePath := fmt.Sprintf(acceptRAV6File, ifName)
 	exist, err := common.CheckIfFileExists(raFilePath)