feat: Add SNAT bridge to Native, decouple SNAT bridge

Makefile

@@ -48,6 +48,7 @@ CNM_BUILD_DIR = $(BUILD_DIR)/cnm
 CNI_BUILD_DIR = $(BUILD_DIR)/cni
 ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli
 CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy
+CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan
 CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift
 CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay
 CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal
@@ -77,6 +78,7 @@ CNM_ARCHIVE_NAME = azure-vnet-cnm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 CNI_ARCHIVE_NAME = azure-vnet-cni-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 ACNCLI_ARCHIVE_NAME = acncli-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
+CNI_MULTITENANCY_TRANSPARENT_VLAN_ARCHIVE_NAME = azure-vnet-cni-multitenancy-transparent-vlan-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 CNI_SWIFT_ARCHIVE_NAME = azure-vnet-cni-swift-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 CNI_OVERLAY_ARCHIVE_NAME = azure-vnet-cni-overlay-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
 CNI_BAREMETAL_ARCHIVE_NAME = azure-vnet-cni-baremetal-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
@@ -439,6 +441,15 @@ ifeq ($(GOOS),linux)
 endif
 	cd $(CNI_MULTITENANCY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_MULTITENANCY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
 
+	$(MKDIR) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)
+	cp cni/azure-$(GOOS)-multitenancy-transparent-vlan.conflist $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)/10-azure.conflist
+	cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)
+ifeq ($(GOOS),linux)
+	cp telemetry/azure-vnet-telemetry.config $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)/azure-vnet-telemetry.config
+	cp $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)
+endif
+	cd $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
+
 	$(MKDIR) $(CNI_SWIFT_BUILD_DIR)
 	cp cni/azure-$(GOOS)-swift.conflist $(CNI_SWIFT_BUILD_DIR)/10-azure.conflist
 	cp telemetry/azure-vnet-telemetry.config $(CNI_SWIFT_BUILD_DIR)/azure-vnet-telemetry.config

cni/azure-linux-multitenancy-transparent-vlan.conflist

@@ -0,0 +1,29 @@
+{
+   "cniVersion":"0.3.0",
+   "name":"azure",
+   "plugins":[
+      {
+         "type":"azure-vnet",
+         "mode":"transparent-vlan",
+         "bridge":"azure0",
+         "multiTenancy":true,
+         "infraVnetAddressSpace":"",
+         "podNamespaceForDualNetwork":[],
+         "enableExactMatchForPodName": false,
+         "enableSnatOnHost":true,
+         "ipam":{
+            "type":"azure-cns"
+         },
+         "dns":{
+            "nameservers":[]
+        }
+      },
+      {
+         "type":"portmap",
+         "capabilities":{
+            "portMappings":true
+         },
+         "snat":true
+      }
+   ]
+}

cni/azure-windows-multitenancy-transparent-vlan.conflist

@@ -0,0 +1,52 @@
+{
+    "cniVersion": "0.3.0",
+    "name": "azure",
+    "plugins": [
+        {
+            "type": "azure-vnet",
+            "mode": "transparent-vlan",
+            "bridge": "azure0",
+            "multiTenancy":true,
+            "enableSnatOnHost":true,
+            "enableExactMatchForPodName": true,
+            "capabilities": {
+                "portMappings": true
+            },
+            "ipam": {
+                "type": "azure-cns"
+            },
+            "dns": {
+                "Nameservers": [
+                    "10.0.0.10",
+                    "168.63.129.16"
+                ],
+                "Search": [
+                    "svc.cluster.local"
+                ]
+            },
+            "AdditionalArgs": [
+                {
+                    "Name": "EndpointPolicy",
+                    "Value": {
+                        "Type": "OutBoundNAT",
+                        "ExceptionList": [
+                            "10.240.0.0/16",
+                            "10.0.0.0/8"
+                        ]
+                    }
+                },
+                {
+                    "Name": "EndpointPolicy",
+                    "Value": {
+                        "Type": "ROUTE",
+                        "DestinationPrefix": "10.0.0.0/8",
+                        "NeedEncap": true
+                    }
+                }
+            ],
+             "windowsSettings": {
+                 "hnsTimeoutDurationInSeconds" : 120
+             }
+        }
+    ]
+}

network/endpoint.go

@@ -93,6 +93,7 @@ type RouteInfo struct {
 	DevName  string
 	Scope    int
 	Priority int
+	Table    int
 }
 
 type apipaClient interface {

network/endpoint_linux.go

@@ -13,7 +13,6 @@ import (
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/netio"
 	"github.com/Azure/azure-container-networking/netlink"
-	"github.com/Azure/azure-container-networking/netns"
 	"github.com/Azure/azure-container-networking/network/networkutils"
 	"github.com/Azure/azure-container-networking/ovsctl"
 	"github.com/Azure/azure-container-networking/platform"
@@ -90,25 +89,12 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p
 	}
 
 	if vlanid != 0 {
-		if nw.Mode == opModeNative {
-			log.Printf("Native client")
-			vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid)
-			vnetNSName := fmt.Sprintf("az_ns_%d", vlanid)
-
-			epClient = &NativeEndpointClient{
-				primaryHostIfName: nw.extIf.Name,
-				vlanIfName:        vlanVethName,
-				vnetVethName:      hostIfName,
-				containerVethName: contIfName,
-				vnetNSName:        vnetNSName,
-				nw:                nw,
-				vlanID:            vlanid,
-				netnsClient:       netns.New(),
-				netlink:           nl,
-				netioshim:         &netio.NetIO{},
-				plClient:          plc,
-				netUtilsClient:    networkutils.NewNetworkUtils(nl, plc),
+		if nw.Mode == opModeTransparentVlan {
+			log.Printf("Transparent vlan client")
+			if _, ok := epInfo.Data[SnatBridgeIPKey]; ok {
+				nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string)
 			}
+			epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc)
 		} else {
 			log.Printf("OVS client")
 			if _, ok := epInfo.Data[SnatBridgeIPKey]; ok {
@@ -261,25 +247,10 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.
 	// entering the container netns and hence works both for CNI and CNM.
 	if ep.VlanID != 0 {
 		epInfo := ep.getInfo()
-		if nw.Mode == opModeNative {
-			log.Printf("Native client")
-			vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, ep.VlanID)
-			vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID)
-
-			epClient = &NativeEndpointClient{
-				primaryHostIfName: nw.extIf.Name,
-				vlanIfName:        vlanVethName,
-				vnetVethName:      ep.HostIfName,
-				containerVethName: "",
-				vnetNSName:        vnetNSName,
-				nw:                nw,
-				vlanID:            ep.VlanID,
-				netnsClient:       netns.New(),
-				netlink:           nl,
-				netioshim:         &netio.NetIO{},
-				plClient:          plc,
-				netUtilsClient:    networkutils.NewNetworkUtils(nl, plc),
-			}
+		if nw.Mode == opModeTransparentVlan {
+			log.Printf("Transparent vlan client")
+			epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc)
+
 		} else {
 			epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc)
 		}
@@ -330,6 +301,7 @@ func addRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, inte
 			Priority:  route.Priority,
 			Protocol:  route.Protocol,
 			Scope:     route.Scope,
+			Table:     route.Table,
 		}
 
 		if err := nl.AddIPRoute(nlRoute); err != nil {

network/endpoint_snatroute_linux.go

@@ -0,0 +1,100 @@
+package network
+
+import (
+	"fmt"
+
+	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/netlink"
+	"github.com/Azure/azure-container-networking/network/networkutils"
+	"github.com/Azure/azure-container-networking/network/snat"
+	"github.com/Azure/azure-container-networking/platform"
+	"github.com/pkg/errors"
+)
+
+func GetSnatHostIfName(epInfo *EndpointInfo) string {
+	return fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7])
+}
+
+func GetSnatContIfName(epInfo *EndpointInfo) string {
+	return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7])
+}
+
+func AddSnatEndpoint(snatClient *snat.Client) error {
+	if err := snatClient.CreateSnatEndpoint(); err != nil {
+		return errors.Wrap(err, "failed to add snat endpoint")
+	}
+	return nil
+}
+
+func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error {
+	// Allow specific Private IPs via Snat Bridge
+	if err := snatClient.AllowIPAddressesOnSnatBridge(); err != nil {
+		return errors.Wrap(err, "failed to allow ip addresses on snat bridge")
+	}
+
+	// Block Private IPs via Snat Bridge
+	if err := snatClient.BlockIPAddressesOnSnatBridge(); err != nil {
+		return errors.Wrap(err, "failed to block ip addresses on snat bridge")
+	}
+	nuc := networkutils.NewNetworkUtils(nl, plc)
+	if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil {
+		return errors.Wrap(err, "failed to enable ip forwarding")
+	}
+
+	if hostToNC {
+		if err := snatClient.AllowInboundFromHostToNC(); err != nil {
+			return errors.Wrap(err, "failed to allow inbound from host to nc")
+		}
+	}
+
+	if ncToHost {
+		if err := snatClient.AllowInboundFromNCToHost(); err != nil {
+			return errors.Wrap(err, "failed to allow inbound from nc to host")
+		}
+	}
+	return nil
+}
+
+func MoveSnatEndpointToContainerNS(snatClient *snat.Client, netnsPath string, nsID uintptr) error {
+	if err := snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID); err != nil {
+		return errors.Wrap(err, "failed to move snat endpoint to container ns")
+	}
+	return nil
+}
+
+func SetupSnatContainerInterface(snatClient *snat.Client) error {
+	if err := snatClient.SetupSnatContainerInterface(); err != nil {
+		return errors.Wrap(err, "failed to setup snat container interface")
+	}
+	return nil
+}
+
+func ConfigureSnatContainerInterface(snatClient *snat.Client) error {
+	if err := snatClient.ConfigureSnatContainerInterface(); err != nil {
+		return errors.Wrap(err, "failed to configure snat container interface")
+	}
+	return nil
+}
+
+func DeleteSnatEndpoint(snatClient *snat.Client) error {
+	if err := snatClient.DeleteSnatEndpoint(); err != nil {
+		return errors.Wrap(err, "failed to delete snat endpoint")
+	}
+	return nil
+}
+
+func DeleteSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool) {
+	if hostToNC {
+		err := snatClient.DeleteInboundFromHostToNC()
+		if err != nil {
+			log.Errorf("failed to delete inbound from host to nc rules")
+		}
+	}
+
+	if ncToHost {
+		err := snatClient.DeleteInboundFromNCToHost()
+		if err != nil {
+			log.Errorf("failed to delete inbound from nc to host rules")
+		}
+	}
+}

network/network.go

@@ -15,11 +15,11 @@ import (
 
 const (
 	// Operational modes.
-	opModeBridge      = "bridge"
-	opModeTunnel      = "tunnel"
-	opModeTransparent = "transparent"
-	opModeNative      = "native"
-	opModeDefault     = opModeTunnel
+	opModeBridge          = "bridge"
+	opModeTunnel          = "tunnel"
+	opModeTransparent     = "transparent"
+	opModeTransparentVlan = "transparent-vlan"
+	opModeDefault         = opModeTunnel
 )
 
 const (

network/network_linux.go

@@ -88,8 +88,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
 				return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err)
 			}
 		}
-	case opModeNative:
-		log.Printf("Native mode")
+	case opModeTransparentVlan:
+		log.Printf("Transparent vlan mode")
 		ifName = extIf.Name
 	default:
 		return nil, errNetworkModeInvalid