diff --git a/cni/network/network.go b/cni/network/network.go index 3cb056b03a..eb9c134725 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -131,6 +131,31 @@ func GetEndpointID(args *cniSkel.CmdArgs) string { return infraEpId } +// getPodInfo returns POD info by parsing the CNI args. +func (plugin *netPlugin) getPodInfo(args string) (string, string, error) { + podCfg, err := cni.ParseCniArgs(args) + if err != nil { + log.Printf("Error while parsing CNI Args %v", err) + return "", "", err + } + + k8sNamespace := string(podCfg.K8S_POD_NAMESPACE) + if len(k8sNamespace) == 0 { + errMsg := "Pod Namespace not specified in CNI Args" + log.Printf(errMsg) + return "", "", plugin.Errorf(errMsg) + } + + k8sPodName := string(podCfg.K8S_POD_NAME) + if len(k8sPodName) == 0 { + errMsg := "Pod Name not specified in CNI Args" + log.Printf(errMsg) + return "", "", plugin.Errorf(errMsg) + } + + return k8sPodName, k8sNamespace, nil +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -192,26 +217,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }() // Parse Pod arguments. - podCfg, err := cni.ParseCniArgs(args.Args) + k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args) if err != nil { - log.Printf("Error while parsing CNI Args %v", err) return err } - k8sNamespace := string(podCfg.K8S_POD_NAMESPACE) - if len(k8sNamespace) == 0 { - errMsg := "Pod Namespace not specified in CNI Args" - log.Printf(errMsg) - return plugin.Errorf(errMsg) - } - - k8sPodName := string(podCfg.K8S_POD_NAME) - if len(k8sPodName) == 0 { - errMsg := "Pod Name not specified in CNI Args" - log.Printf(errMsg) - return plugin.Errorf(errMsg) - } - k8sContainerID := args.ContainerID if len(k8sContainerID) == 0 { errMsg := "Container ID not specified in CNI Args" @@ -234,10 +244,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } } - // Initialize values from network config. - networkId := nwCfg.Name - endpointId := GetEndpointID(args) - result, cnsNetworkConfig, subnetPrefix, azIpamResult, err = GetMultiTenancyCNIResult(enableInfraVnet, nwCfg, plugin, k8sPodName, k8sNamespace, args.IfName) if err != nil { log.Printf("GetMultiTenancyCNIResult failed with error %v", err) @@ -252,6 +258,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("Result from multitenancy %+v", result) + // Initialize values from network config. + networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg) + if err != nil { + log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) + return err + } + + endpointId := GetEndpointID(args) + policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) // Check whether the network already exists. @@ -265,13 +280,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) - if err != nil { - log.Printf("handleConsecutiveAdd failed with error %v", err) - return err + resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) + if errConsAdd != nil { + log.Printf("handleConsecutiveAdd failed with error %v", errConsAdd) + result = resultConsAdd + return errConsAdd } - if result != nil { + if resultConsAdd != nil { + result = resultConsAdd return nil } } @@ -335,11 +352,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } log.Printf("[cni-net] nwDNSInfo: %v", nwDNSInfo) + // Update subnet prefix for multi-tenant scenario + updateSubnetPrefix(cnsNetworkConfig, &subnetPrefix) // Create the network. nwInfo := network.NetworkInfo{ - Id: networkId, - Mode: nwCfg.Mode, + Id: networkId, + Mode: nwCfg.Mode, + MasterIfName: masterIfName, Subnets: []network.SubnetInfo{ network.SubnetInfo{ Family: platform.AfINET, @@ -492,8 +512,18 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) + // Parse Pod arguments. + k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args) + if err != nil { + return err + } + // Initialize values from network config. - networkId := nwCfg.Name + networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg) + if err != nil { + log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) + } + endpointId := GetEndpointID(args) // Query the network. @@ -552,8 +582,18 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) + // Parse Pod arguments. + k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args) + if err != nil { + return err + } + // Initialize values from network config. - networkId := nwCfg.Name + networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg) + if err != nil { + log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) + } + endpointId := GetEndpointID(args) // Query the network. diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 5927345256..6a8ccbf88d 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -108,3 +108,10 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { return nil } + +func updateSubnetPrefix(cnsNetworkConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) { +} + +func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) { + return nwCfg.Name, nil +} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index a495894369..ec21be87ca 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net" + "strconv" "strings" "github.com/Azure/azure-container-networking/cni" @@ -70,14 +71,59 @@ func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Resu } func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { + if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { + log.Printf("Setting Network Options") + vlanMap := make(map[string]interface{}) + vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID) + nwInfo.Options[dockerNetworkOption] = vlanMap + } } func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) { + if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { + log.Printf("Setting Endpoint Options") + var cnetAddressMap []string + for _, ipSubnet := range cnsNwConfig.CnetAddressSpace { + cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength))) + } + epInfo.Data[network.CnetAddressSpace] = cnetAddressMap + } } func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } +func updateSubnetPrefix(cnsNwConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) { + if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { + ipconfig := cnsNwConfig.IPConfiguration + ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) + + if ipAddr.To4() != nil { + *subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} + } else { + *subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} + } + + subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) + log.Printf("Updated subnetPrefix: %s", subnetPrefix.String()) + } +} + +func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) { + if nwCfg.MultiTenancy { + _, cnsNetworkConfig, _, err := getContainerNetworkConfiguration(nwCfg, "", podName, podNs, ifName) + if err != nil { + log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", podName, podNs, err) + return "", err + } + + networkName := fmt.Sprintf("%s-vlanid%v", nwCfg.Name, cnsNetworkConfig.MultiTenancyInfo.ID) + return networkName, nil + } + + return nwCfg.Name, nil +} + func setupInfraVnetRoutingForMultitenancy( nwCfg *cni.NetworkConfig, azIpamResult *cniTypesCurr.Result, diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 1d894ca143..f9527944ee 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -44,6 +44,14 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { + var vlanid int + + if epInfo.Data != nil { + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) + } + } + // Get Infrastructure containerID. Handle ADD calls for workload container. var err error infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) @@ -53,7 +61,35 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), - Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies), + } + + // Set outbound NAT policy + outBoundNatPolicy := hcsshim.OutboundNatPolicy{} + outBoundNatPolicy.Policy.Type = hcsshim.OutboundNat + + exceptionList, err := policy.GetOutBoundNatExceptionList(epInfo.Policies) + if err != nil { + log.Printf("[net] Failed to parse outbound NAT policy %v", err) + return nil, err + } + + if exceptionList != nil { + for _, ipAddress := range exceptionList { + outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress) + } + } + + if epInfo.Data[CnetAddressSpace] != nil { + if cnetAddressSpace := epInfo.Data[CnetAddressSpace].([]string); cnetAddressSpace != nil { + for _, ipAddress := range cnetAddressSpace { + outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress) + } + } + } + + if outBoundNatPolicy.Exceptions != nil { + serializedOutboundNatPolicy, _ := json.Marshal(outBoundNatPolicy) + hnsEndpoint.Policies = append(hnsEndpoint.Policies, serializedOutboundNatPolicy) } // HNS currently supports only one IP address per endpoint. @@ -96,13 +132,15 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Create the endpoint object. ep := &endpoint{ - Id: infraEpName, - HnsId: hnsResponse.Id, - SandboxKey: epInfo.ContainerID, - IfName: epInfo.IfName, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{net.ParseIP(hnsResponse.GatewayAddress)}, - DNS: epInfo.DNS, + Id: infraEpName, + HnsId: hnsResponse.Id, + SandboxKey: epInfo.ContainerID, + IfName: epInfo.IfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{net.ParseIP(hnsResponse.GatewayAddress)}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, } for _, route := range epInfo.Routes { diff --git a/network/manager.go b/network/manager.go index cdf03befab..3a6e9e5974 100644 --- a/network/manager.go +++ b/network/manager.go @@ -15,8 +15,9 @@ import ( const ( // Network store key. - storeKey = "Network" - VlanIDKey = "VlanID" + storeKey = "Network" + VlanIDKey = "VlanID" + genericData = "com.docker.network.generic" ) type NetworkClient interface { diff --git a/network/network.go b/network/network.go index 0e68c490b4..cbf177e47a 100644 --- a/network/network.go +++ b/network/network.go @@ -5,6 +5,7 @@ package network import ( "net" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" @@ -46,6 +47,7 @@ type network struct { // NetworkInfo contains read-only information about a container network. type NetworkInfo struct { + MasterIfName string Id string Mode string Subnets []SubnetInfo @@ -121,6 +123,16 @@ func (nm *networkManager) findExternalInterfaceBySubnet(subnet string) *external return nil } +// FindExternalInterfaceByName finds an external interface by name. +func (nm *networkManager) findExternalInterfaceByName(ifName string) *externalInterface { + extIf, exists := nm.ExternalInterfaces[ifName] + if exists && extIf != nil { + return extIf + } + + return nil +} + // NewNetwork creates a new container network. func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) { var nw *network @@ -138,8 +150,14 @@ func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) { nwInfo.Mode = opModeDefault } - // Find the external interface for this subnet. - extIf := nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String()) + // If the master interface name is provided, find the external interface by name + // else use subnet to to find the interface + var extIf *externalInterface + if len(strings.TrimSpace(nwInfo.MasterIfName)) > 0 { + extIf = nm.findExternalInterfaceByName(nwInfo.MasterIfName) + } else { + extIf = nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String()) + } if extIf == nil { err = errSubnetNotFound return nil, err diff --git a/network/network_linux.go b/network/network_linux.go index 4d49a634d2..25217d5b92 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -21,8 +21,6 @@ const ( // Virtual MAC address used by Azure VNET. virtualMacAddress = "12:34:56:78:9a:bc" - genericData = "com.docker.network.generic" - SnatBridgeIPKey = "snatBridgeIP" LocalIPKey = "localIP" diff --git a/network/network_windows.go b/network/network_windows.go index 0a1bc6ca76..b3ecd5e5dd 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -5,6 +5,7 @@ package network import ( "encoding/json" + "strconv" "strings" "time" @@ -15,8 +16,9 @@ import ( const ( // HNS network types. - hnsL2bridge = "l2bridge" - hnsL2tunnel = "l2tunnel" + hnsL2bridge = "l2bridge" + hnsL2tunnel = "l2tunnel" + CnetAddressSpace = "cnetAddressSpace" ) // Windows implementation of route. @@ -24,6 +26,7 @@ type route interface{} // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + var vlanid int networkAdapterName := extIf.Name // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch if strings.HasPrefix(networkAdapterName, "vEthernet") { @@ -37,6 +40,21 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies), } + // Set the VLAN and OutboundNAT policies + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + if opt != nil && opt[VlanIDKey] != nil { + vlanPolicy := hcsshim.VlanPolicy{ + Type: "VLAN", + } + vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) + vlanPolicy.VLAN = uint(vlanID) + + serializedVlanPolicy, _ := json.Marshal(vlanPolicy) + hnsNetwork.Policies = append(hnsNetwork.Policies, serializedVlanPolicy) + + vlanid = (int)(vlanPolicy.VLAN) + } + // Set network mode. switch nwInfo.Mode { case opModeBridge: @@ -74,11 +92,13 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Create the network object. nw := &network{ - Id: nwInfo.Id, - HnsId: hnsResponse.Id, - Mode: nwInfo.Mode, - Endpoints: make(map[string]*endpoint), - extIf: extIf, + Id: nwInfo.Id, + HnsId: hnsResponse.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: vlanid, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, } globals, err := hcsshim.GetHNSGlobals() diff --git a/network/policy/policy.go b/network/policy/policy.go index 3143184a62..355aae6663 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -2,6 +2,13 @@ package policy import ( "encoding/json" + "log" +) + +const ( + NetworkPolicy CNIPolicyType = "NetworkPolicy" + EndpointPolicy CNIPolicyType = "EndpointPolicy" + OutBoundNatPolicy CNIPolicyType = "OutBoundNAT" ) type CNIPolicyType string @@ -21,3 +28,32 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy) []json.RawMe } return jsonPolicies } + +// GetOutBoundNatExceptionList returns exception list for outbound nat policy +func GetOutBoundNatExceptionList(policies []Policy) ([]string, error) { + type KVPair struct { + Type CNIPolicyType `json:"Type"` + ExceptionList json.RawMessage `json:"ExceptionList"` + } + + for _, policy := range policies { + if policy.Type == EndpointPolicy { + var data KVPair + if err := json.Unmarshal(policy.Data, &data); err != nil { + return nil, err + } + + if data.Type == OutBoundNatPolicy { + var exceptionList []string + if err := json.Unmarshal(data.ExceptionList, &exceptionList); err != nil { + return nil, err + } + + return exceptionList, nil + } + } + } + + log.Printf("OutBoundNAT policy not set.") + return nil, nil +} diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go deleted file mode 100644 index 9121d45927..0000000000 --- a/network/policy/policy_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -package policy - -const ( - NetworkPolicy CNIPolicyType = "NetworkPolicy" - EndpointPolicy CNIPolicyType = "EndpointPolicy" - OutBoundNatPolicy CNIPolicyType = "OutBoundNatPolicy" -)