From fb257f8f8c5bcafa7f5c9882923999f897c49643 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Sun, 4 Mar 2018 19:50:10 -0800 Subject: [PATCH 01/51] Temporary commit. Added vlan support --- cni/network/network.go | 6 + cnm/network/network.go | 3 + common/utils.go | 29 ++++ ipam/azure.go | 3 +- network/endpoint_linux.go | 146 +++++++++++++++++---- network/network.go | 1 + network/network_linux.go | 270 +++++++++++++++++++++++++++++++++++++- 7 files changed, 429 insertions(+), 29 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index fb4b258dc7..1cddc7504b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -203,6 +203,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { BridgeName: nwCfg.Bridge, } + nwInfo.Options = make(map[string]interface{}) + nwInfo.Options["vlanid"] = "100" + err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { err = plugin.Errorf("Failed to create network: %v", err) @@ -242,6 +245,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { IfName: args.IfName, } + epInfo.Data = make(map[string]interface{}) + epInfo.Data["vlanid"] = "100" + // Populate addresses. for _, ipconfig := range result.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) diff --git a/cnm/network/network.go b/cnm/network/network.go index 29a0b0aa59..131bb6f0dc 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -231,6 +231,9 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) IPAddresses: []net.IPNet{*ipv4Address}, } + epInfo.Data = make(map[string]interface{}) + epInfo.Data["vlanid"] = 100 + err = plugin.nm.CreateEndpoint(req.NetworkID, &epInfo) if err != nil { plugin.SendErrorResponse(w, err) diff --git a/common/utils.go b/common/utils.go index 33948709a8..6c2a0a61c8 100644 --- a/common/utils.go +++ b/common/utils.go @@ -4,9 +4,13 @@ package common import ( + "bytes" + "encoding/binary" "encoding/xml" + "fmt" "net" "os" + "os/exec" "github.com/Azure/azure-container-networking/log" ) @@ -69,3 +73,28 @@ func CreateDirectory(dirPath string) error { return err } + +func ExecuteShellCommand(command string) (string, error) { + log.Printf("[Azure-Utils] %s", command) + + var stderr bytes.Buffer + var out bytes.Buffer + cmd := exec.Command("sh", "-c", command) + cmd.Stderr = &stderr + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("%s:%s", err.Error(), stderr.String()) + } + + return out.String(), nil +} + +func IpToInt(ip net.IP) uint32 { + if len(ip) == 16 { + return binary.BigEndian.Uint32(ip[12:16]) + } + + return binary.BigEndian.Uint32(ip) +} diff --git a/ipam/azure.go b/ipam/azure.go index 62194e7ecb..4e78854dc1 100644 --- a/ipam/azure.go +++ b/ipam/azure.go @@ -16,7 +16,8 @@ import ( const ( // Host URL to query. - azureQueryUrl = "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1" + //azureQueryUrl = "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1" + azureQueryUrl = "http://localhost:42424/" // Minimum time interval between consecutive queries. azureQueryInterval = 10 * time.Second diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 0019c370cd..7630001637 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -8,7 +8,9 @@ package network import ( "fmt" "net" + "strings" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" @@ -25,6 +27,124 @@ const ( containerInterfacePrefix = "eth" ) +func (nw *network) setupContainerNicAndRules(hostIfName string, contIfName string, containerIf *net.Interface, epInfo *EndpointInfo) error { + // Connect host interface to the bridge. + multitenancy := false + var err error + + if epInfo.Data != nil { + if vlanid, ok := epInfo.Data["vlanid"]; ok { + multitenancy = true + log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) + + cmd := fmt.Sprintf("ovs-vsctl add-port %v %v tag=%v", nw.extIf.BridgeName, hostIfName, vlanid) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding port failed with error %v", err) + return err + } + + log.Printf("[net] Get ovs port for interface %v.", hostIfName) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIfName) + containerPort, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + return err + } + + containerPort = strings.Trim(containerPort, "\n") + mac := nw.extIf.MacAddress.String() + macHex := strings.Replace(mac, ":", "", -1) + log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) + + cmd = fmt.Sprintf(`ovs-ofctl add-flow %v priority=10,arp,in_port=%s,arp_op=1,actions='mod_dl_src:%s, + load:0x%s->NXM_NX_ARP_SHA[],normal'`, nw.extIf.BridgeName, containerPort, mac, macHex) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding SNAT rule failed with error %v", err) + return err + } + + log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) + + cmd = fmt.Sprintf(`ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal`, + nw.extIf.BridgeName, containerPort, mac, macHex) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding SNAT rule failed with error %v", err) + return err + } + + macAddr := containerIf.HardwareAddr.String() + macAddrHex := strings.Replace(macAddr, ":", "", -1) + + log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) + ofport, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + return err + } + + ofport = strings.Trim(ofport, "\n") + + for _, ipAddr := range epInfo.IPAddresses { + // Add ARP reply rule. + ipAddrInt := common.IpToInt(ipAddr.IP) + + log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,actions='load:0x2->NXM_OF_ARP_OP[], + move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], + load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, + nw.extIf.BridgeName, ipAddr.String(), vlanid, macAddr, macAddrHex, ipAddrInt) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding ARP reply rule failed with error %v", err) + return err + } + + // Add MAC address translation rule. + log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) + cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", + nw.extIf.BridgeName, ipAddr.String(), vlanid, ofport, macAddr) + _, err = common.ExecuteShellCommand(cmd) + //err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) + if err != nil { + log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) + return err + } + } + } + } + + if !multitenancy { + log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) + err = netlink.SetLinkMaster(hostIfName, nw.extIf.BridgeName) + if err != nil { + return err + } + + for _, ipAddr := range epInfo.IPAddresses { + // Add ARP reply rule. + log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) + err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(containerIf.HardwareAddr), ebtables.Append) + if err != nil { + return err + } + + // Add MAC address translation rule. + log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) + err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) + if err != nil { + return err + } + } + } + + return nil +} + // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var containerIf *net.Interface @@ -33,7 +153,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var err error if nw.Endpoints[epInfo.Id] != nil { - log.Printf("[net] Endpoint alreday exists.") + log.Printf("[net] Endpoint alreday exists.") err = errEndpointExists return nil, err } @@ -76,39 +196,19 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return nil, err } - // Connect host interface to the bridge. - log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) - err = netlink.SetLinkMaster(hostIfName, nw.extIf.BridgeName) + containerIf, err = net.InterfaceByName(contIfName) if err != nil { return nil, err } + nw.setupContainerNicAndRules(hostIfName, contIfName, containerIf, epInfo) // // Container network interface setup. // // Query container network interface info. - containerIf, err = net.InterfaceByName(contIfName) - if err != nil { - return nil, err - } // Setup rules for IP addresses on the container interface. - for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(containerIf.HardwareAddr), ebtables.Append) - if err != nil { - return nil, err - } - - // Add MAC address translation rule. - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) - if err != nil { - return nil, err - } - } // If a network namespace for the container interface is specified... if epInfo.NetNsPath != "" { diff --git a/network/network.go b/network/network.go index 3340dcf9ce..b33f6f2765 100644 --- a/network/network.go +++ b/network/network.go @@ -35,6 +35,7 @@ type network struct { Id string HnsId string `json:",omitempty"` Mode string + VlanId string Subnets []SubnetInfo Endpoints map[string]*endpoint extIf *externalInterface diff --git a/network/network_linux.go b/network/network_linux.go index 2915d8edf7..deaaede1c5 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -10,6 +10,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" @@ -22,6 +23,8 @@ const ( // Virtual MAC address used by Azure VNET. virtualMacAddress = "12:34:56:78:9a:bc" + + genericData = "com.docker.network.generic" ) // Linux implementation of route. @@ -30,14 +33,29 @@ type route netlink.Route // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Connect the external interface. + multitenancy := false + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + log.Printf("opt %v", opt) + // clusterInfo := opt["nodeList"].(string) + switch nwInfo.Mode { case opModeTunnel: fallthrough case opModeBridge: - err := nm.connectExternalInterface(extIf, nwInfo) - if err != nil { - return nil, err + if opt != nil { + if _, ok := opt["vlanid"].(string); ok { + multitenancy = true + nm.createOVSNetwork(extIf, nwInfo) + } } + + if !multitenancy { + err := nm.connectExternalInterface(extIf, nwInfo) + if err != nil { + return nil, err + } + } + default: return nil, errNetworkModeInvalid } @@ -50,14 +68,25 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt extIf: extIf, } + if multitenancy { + nw.VlanId = opt["vlanid"].(string) + } + return nw, nil } // DeleteNetworkImpl deletes an existing container network. func (nm *networkManager) deleteNetworkImpl(nw *network) error { // Disconnect the interface if this was the last network using it. - if len(nw.extIf.Networks) == 1 { - nm.disconnectExternalInterface(nw.extIf) + + if nw.VlanId != "" { + if len(nw.extIf.Networks) == 1 { + nm.disconnectOVSInterface(nw.extIf) + } + } else { + if len(nw.extIf.Networks) == 1 { + nm.disconnectExternalInterface(nw.extIf) + } } return nil @@ -179,6 +208,58 @@ func (nm *networkManager) addBridgeRules(extIf *externalInterface, hostIf *net.I return nil } +func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Interface, bridgeName string) error { + primary := extIf.IPAddresses[0].IP.String() + primaryInt := common.IpToInt(extIf.IPAddresses[0].IP) + + mac := extIf.MacAddress.String() + macHex := strings.Replace(mac, ":", "", -1) + + cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,priority=20,actions=normal", bridgeName, primary) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding SNAT rule failed with error %v", err) + return err + } + // Add ARP reply rule for host primary IP address. + // ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric + // doesn't respond to ARP requests from the VM for its own primary IP address. + + log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) + + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,arp_tpa=%s,actions='load:0x2->NXM_OF_ARP_OP[], + move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], + load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, bridgeName, primary, mac, macHex, primaryInt) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding ARP reply rule failed with error %v", err) + return err + } + + log.Printf("[net] Get ovs port for interface %v.", hostIf.Name) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIf.Name) + ofport, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + return err + } + + ofport = strings.Trim(ofport, "\n") + + // Add DNAT rule to forward ARP replies to container interfaces. + log.Printf("[net] Adding DNAT rule for ingress ARP traffic on interface %v.", hostIf.Name) + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, + load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, ofport, macHex) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding DNAT rule failed with error %v", err) + return err + } + + return nil +} + // DeleteBridgeRules deletes bridge rules for container traffic. func (nm *networkManager) deleteBridgeRules(extIf *externalInterface) { ebtables.SetVepaMode(extIf.BridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Delete) @@ -187,6 +268,185 @@ func (nm *networkManager) deleteBridgeRules(extIf *externalInterface) { ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) } +func (nm *networkManager) createOVSBridge(bridgeName string) error { + log.Printf("[net] Creating OVS Bridge %v", bridgeName) + + ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) + _, err := common.ExecuteShellCommand(ovsCreateCmd) + if err != nil { + log.Printf("[net] Error while creating OVS bridge %v", err) + return err + } + + return nil +} + +func (nm *networkManager) deleteOVSBridge(bridgeName string) error { + log.Printf("[net] Deleting OVS Bridge %v", bridgeName) + + ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) + _, err := common.ExecuteShellCommand(ovsCreateCmd) + if err != nil { + log.Printf("[net] Error while deleting OVS bridge %v", err) + return err + } + + return nil +} + +func setOVSMaster(hostIfName string, bridgeName string) error { + cmd := fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Error while setting OVS as master to primary interface %v", err) + return err + } + + return nil +} + +func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *NetworkInfo) error { + var err error + + log.Printf("[net] Connecting interface %v.", extIf.Name) + defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() + + // Check whether this interface is already connected. + if extIf.BridgeName != "" { + log.Printf("[net] Interface is already connected to bridge %v.", extIf.BridgeName) + return nil + } + + // Find the external interface. + hostIf, err := net.InterfaceByName(extIf.Name) + if err != nil { + return err + } + + // If a bridge name is not specified, generate one based on the external interface index. + bridgeName := nwInfo.BridgeName + if bridgeName == "" { + bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index) + } + + // Check if the bridge already exists. + bridge, err := net.InterfaceByName(bridgeName) + if err != nil { + // Create the bridge. + log.Printf("[net] Creating bridge %v.", bridgeName) + + err = nm.createOVSBridge(bridgeName) + if err != nil { + return err + } + + // On failure, delete the bridge. + defer func() { + if err != nil { + nm.deleteOVSBridge(bridgeName) + } + }() + + bridge, err = net.InterfaceByName(bridgeName) + if err != nil { + return err + } + } else { + // Use the existing bridge. + log.Printf("[net] Found existing bridge %v.", bridgeName) + } + + // Save host IP configuration. + err = nm.saveIPConfig(hostIf, extIf) + if err != nil { + log.Printf("[net] Failed to save IP configuration for interface %v: %v.", hostIf.Name, err) + } + + // External interface down. + log.Printf("[net] Setting link %v state down.", hostIf.Name) + err = netlink.SetLinkState(hostIf.Name, false) + if err != nil { + return err + } + + // Connect the external interface to the bridge. + log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) + err = setOVSMaster(hostIf.Name, bridgeName) + //err = netlink.SetLinkMaster(hostIf.Name, bridgeName) + if err != nil { + return err + } + + // Add the bridge rules. + err = nm.addOVSRules(extIf, hostIf, bridgeName) + if err != nil { + return err + } + + // External interface up. + log.Printf("[net] Setting link %v state up.", hostIf.Name) + err = netlink.SetLinkState(hostIf.Name, true) + if err != nil { + return err + } + + // Bridge up. + log.Printf("[net] Setting link %v state up.", bridgeName) + err = netlink.SetLinkState(bridgeName, true) + if err != nil { + return err + } + + // Apply IP configuration to the bridge for host traffic. + err = nm.applyIPConfig(extIf, bridge) + if err != nil { + log.Printf("[net] Failed to apply interface IP configuration: %v.", err) + } + + extIf.BridgeName = bridgeName + err = nil + + log.Printf("[net] Connected interface %v to bridge %v.", extIf.Name, extIf.BridgeName) + + return nil +} + +// DisconnectExternalInterface disconnects a host interface from OVS bridge. +func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) error { + log.Printf("[net] Disconnecting interface %v.", extIf.Name) + + // Disconnect external interface from its bridge. + //err := netlink.SetLinkMaster(extIf.Name, "") + cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", extIf.BridgeName, extIf.Name) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Failed to disconnect interface %v from bridge, err:%v.", extIf.Name, err) + } + + // Delete the bridge. + // err = netlink.DeleteLink(extIf.BridgeName) + err = nm.deleteOVSBridge(extIf.BridgeName) + if err != nil { + log.Printf("[net] Failed to delete bridge %v, err:%v.", extIf.BridgeName, err) + } + + extIf.BridgeName = "" + + // Restore IP configuration. + hostIf, _ := net.InterfaceByName(extIf.Name) + err = nm.applyIPConfig(extIf, hostIf) + if err != nil { + log.Printf("[net] Failed to apply IP configuration: %v.", err) + } + + extIf.IPAddresses = nil + extIf.Routes = nil + + log.Printf("[net] Disconnected interface %v.", extIf.Name) + + return nil +} + // ConnectExternalInterface connects the given host interface to a bridge. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var err error From 3cd73008bc1819576dfcf2865ddecbebb87ac6d6 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Sun, 4 Mar 2018 19:55:54 -0800 Subject: [PATCH 02/51] checkpoint --- network/endpoint_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 7630001637..e2ebec03b5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -61,17 +61,17 @@ func (nw *network) setupContainerNicAndRules(hostIfName string, contIfName strin load:0x%s->NXM_NX_ARP_SHA[],normal'`, nw.extIf.BridgeName, containerPort, mac, macHex) _, err = common.ExecuteShellCommand(cmd) if err != nil { - log.Printf("[net] Adding SNAT rule failed with error %v", err) + log.Printf("[net] Adding ARP SNAT rule failed with error %v", err) return err } log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal`, - nw.extIf.BridgeName, containerPort, mac, macHex) + cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", + nw.extIf.BridgeName, containerPort, mac) _, err = common.ExecuteShellCommand(cmd) if err != nil { - log.Printf("[net] Adding SNAT rule failed with error %v", err) + log.Printf("[net] Adding IP SNAT rule failed with error %v", err) return err } From 0c05120ae905b5337604dda4859c8b97ef9326c1 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 7 Mar 2018 14:56:50 -0800 Subject: [PATCH 03/51] fixes --- network/endpoint_linux.go | 8 ++++---- network/network_linux.go | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index e2ebec03b5..1920f92f0c 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -92,12 +92,12 @@ func (nw *network) setupContainerNicAndRules(hostIfName string, contIfName strin // Add ARP reply rule. ipAddrInt := common.IpToInt(ipAddr.IP) - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) + log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,actions='load:0x2->NXM_OF_ARP_OP[], move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, - nw.extIf.BridgeName, ipAddr.String(), vlanid, macAddr, macAddrHex, ipAddrInt) + nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, macAddr, macAddrHex, ipAddrInt) _, err = common.ExecuteShellCommand(cmd) if err != nil { log.Printf("[net] Adding ARP reply rule failed with error %v", err) @@ -105,9 +105,9 @@ func (nw *network) setupContainerNicAndRules(hostIfName string, contIfName strin } // Add MAC address translation rule. - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) + log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", - nw.extIf.BridgeName, ipAddr.String(), vlanid, ofport, macAddr) + nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, ofport, macAddr) _, err = common.ExecuteShellCommand(cmd) //err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) if err != nil { diff --git a/network/network_linux.go b/network/network_linux.go index deaaede1c5..a0bb806376 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -210,12 +210,12 @@ func (nm *networkManager) addBridgeRules(extIf *externalInterface, hostIf *net.I func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Interface, bridgeName string) error { primary := extIf.IPAddresses[0].IP.String() - primaryInt := common.IpToInt(extIf.IPAddresses[0].IP) + //primaryInt := common.IpToInt(extIf.IPAddresses[0].IP) mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) - cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,priority=20,actions=normal", bridgeName, primary) + cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=20,actions=normal", bridgeName, primary, mac) _, err := common.ExecuteShellCommand(cmd) if err != nil { log.Printf("[net] Adding SNAT rule failed with error %v", err) @@ -225,17 +225,17 @@ func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Inte // ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric // doesn't respond to ARP requests from the VM for its own primary IP address. - log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) - - cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,arp_tpa=%s,actions='load:0x2->NXM_OF_ARP_OP[], - move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, - move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], - load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, bridgeName, primary, mac, macHex, primaryInt) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP reply rule failed with error %v", err) - return err - } + // log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) + + // cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,arp_tpa=%s,actions='load:0x2->NXM_OF_ARP_OP[], + // move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + // move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], + // load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, bridgeName, primary, mac, macHex, primaryInt) + // _, err = common.ExecuteShellCommand(cmd) + // if err != nil { + // log.Printf("[net] Adding ARP reply rule failed with error %v", err) + // return err + // } log.Printf("[net] Get ovs port for interface %v.", hostIf.Name) cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIf.Name) From ef6d5397c86c71dfce3a0c91311ac68d63c7010f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 2 Apr 2018 14:25:44 -0700 Subject: [PATCH 04/51] Added code for cleaning up ovs rules on deleting endpoint --- cni/network/network.go | 198 ++++++++++++++++++------ cni/plugin.go | 15 ++ cnm/network/network.go | 1 - network/endpoint.go | 29 ++++ network/endpoint_linux.go | 308 ++++++++++++++++++++++++-------------- network/manager.go | 6 + network/network.go | 2 +- network/network_linux.go | 26 ++-- 8 files changed, 409 insertions(+), 176 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 1cddc7504b..584b64f713 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -4,7 +4,12 @@ package network import ( + "encoding/json" + "io/ioutil" "net" + "net/http" + + "k8s.io/api/core/v1" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/common" @@ -12,14 +17,16 @@ import ( "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/telemetry" - cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) const ( // Plugin name. - name = "azure-vnet" + name = "azure-vnet" + namespacekey = "K8S_POD_NAMESPACE" + podnamekey = "K8S_POD_NAME" ) // NetPlugin represents the CNI network plugin. @@ -116,6 +123,66 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre return "" } +func getPodListFromApiServer(ip string, port string) (*v1.PodList, error) { + url := "http://" + ip + ":" + port + "/api/v1/namespaces/default/pods" + log.Printf("Api server url %v", url) + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + podList := &v1.PodList{} + if err = json.Unmarshal(body, podList); err != nil { + return nil, err + } + + return podList, nil +} +func getNetworkContainerID(podList *v1.PodList, containerID string) string { + log.Printf("Number of pods %v", len(podList.Items)) + for _, pod := range podList.Items { + for _, containerstatus := range pod.Status.ContainerStatuses { + log.Printf("container id %v", containerstatus.ContainerID) + if containerstatus.ContainerID == containerID { + log.Printf("Container ID matched") + log.Printf("Annotations :%v", pod.Annotations) + } + } + } + + return "tid" +} +func convertToCniResult(epInfo *network.EndpointInfo) *cniTypesCurr.Result { + result := &cniTypesCurr.Result{} + + log.Printf("Convert IPAddress") + for index, ipAddr := range epInfo.IPAddresses { + ipconfig := &cniTypesCurr.IPConfig{} + ipconfig.Address = ipAddr + + if ipAddr.IP.To4() != nil { + ipconfig.Version = "4" + } else { + ipconfig.Version = "6" + } + + ipconfig.Gateway = epInfo.Data["Gateways"].([]net.IP)[index] + result.IPs = append(result.IPs, ipconfig) + } + + for _, route := range epInfo.Routes { + localroute := &types.Route{Dst: route.Dst, GW: route.Gw} + result.Routes = append(result.Routes, localroute) + } + + result.DNS.Nameservers = epInfo.DNS.Servers + result.DNS.Domain = epInfo.DNS.Suffix + + return result +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -125,6 +192,7 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var result *cniTypesCurr.Result var err error + var epInfo *network.EndpointInfo log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) @@ -143,6 +211,43 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name endpointId := plugin.GetEndpointID(args) + argsMap := plugin.GetCNIArgs(args.Args) + if argsMap != nil { + log.Printf("Argsmap %v", argsMap) + } + + /* log.Printf("call apiserver..check!!!") + podList, err := getPodListFromApiServer("10.2.240.225", "8080") + if err == nil { + networkContainerID = getNetworkContainerID(podList, args.ContainerID) + } else { + log.Printf("Failed to retrieve pod info from apiserver") + } + */ + + // Initialize endpoint info. + + epInfo, err = network.GetContainerNetworkConfiguration(argsMap[namespacekey].(string), argsMap[podnamekey].(string)) + if err != nil { + log.Printf("SetContainerNetworkConfiguration failed with %v", err) + } + + if epInfo != nil { + epInfo.Id = endpointId + epInfo.ContainerID = args.ContainerID + epInfo.NetNsPath = args.Netns + epInfo.IfName = args.IfName + result = convertToCniResult(epInfo) + log.Printf("Convertocniresult :%v", result) + } else { + epInfo = &network.EndpointInfo{ + Id: endpointId, + ContainerID: args.ContainerID, + NetNsPath: args.Netns, + IfName: args.IfName, + } + epInfo.Data = make(map[string]interface{}) + } // Check whether the network already exists. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) @@ -150,17 +255,19 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Network does not exist. log.Printf("[cni-net] Creating network %v.", networkId) - // Call into IPAM plugin to allocate an address pool for the network. - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate pool: %v", err) - return err + if result == nil { + // Call into IPAM plugin to allocate an address pool for the network. + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + if err != nil { + err = plugin.Errorf("Failed to allocate pool: %v", err) + return err + } } // Derive the subnet prefix from allocated IP address. ipconfig := result.IPs[0] subnetPrefix := ipconfig.Address - subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) + gateway := ipconfig.Gateway // On failure, call into IPAM plugin to release the address and address pool. defer func() { @@ -174,6 +281,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } }() + subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) // Find the master interface. masterIfName := plugin.findMasterInterface(nwCfg, &subnetPrefix) if masterIfName == "" { @@ -197,14 +305,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { network.SubnetInfo{ Family: platform.AfINET, Prefix: subnetPrefix, - Gateway: ipconfig.Gateway, + Gateway: gateway, }, }, BridgeName: nwCfg.Bridge, } nwInfo.Options = make(map[string]interface{}) - nwInfo.Options["vlanid"] = "100" err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { @@ -214,54 +321,47 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String()) } else { - // Network already exists. - subnetPrefix := nwInfo.Subnets[0].Prefix.String() - log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) - - // Call into IPAM plugin to allocate an address for the endpoint. - nwCfg.Ipam.Subnet = subnetPrefix - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate address: %v", err) - return err - } - - ipconfig := result.IPs[0] - - // On failure, call into IPAM plugin to release the address. - defer func() { + if result == nil { + // Network already exists. + subnetPrefix := nwInfo.Subnets[0].Prefix.String() + log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) + + // Call into IPAM plugin to allocate an address for the endpoint. + nwCfg.Ipam.Subnet = subnetPrefix + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + err = plugin.Errorf("Failed to allocate address: %v", err) + return err } - }() - } - // Initialize endpoint info. - epInfo := &network.EndpointInfo{ - Id: endpointId, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, + ipconfig := result.IPs[0] + + // On failure, call into IPAM plugin to release the address. + defer func() { + if err != nil { + nwCfg.Ipam.Address = ipconfig.Address.IP.String() + plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + } + }() + } } - epInfo.Data = make(map[string]interface{}) - epInfo.Data["vlanid"] = "100" + if len(epInfo.IPAddresses) == 0 { + // Populate addresses. + for _, ipconfig := range result.IPs { + epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) + } - // Populate addresses. - for _, ipconfig := range result.IPs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } + // Populate routes. + for _, route := range result.Routes { + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) + } - // Populate routes. - for _, route := range result.Routes { - epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) + // Populate DNS info. + epInfo.DNS.Suffix = result.DNS.Domain + epInfo.DNS.Servers = result.DNS.Nameservers } - // Populate DNS info. - epInfo.DNS.Suffix = result.DNS.Domain - epInfo.DNS.Servers = result.DNS.Nameservers - // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/cni/plugin.go b/cni/plugin.go index 25c7e66df3..c60620255e 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "runtime" + "strings" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -168,6 +169,20 @@ func (plugin *Plugin) GetEndpointID(args *cniSkel.CmdArgs) string { return containerID + "-" + args.IfName } +func (plugin *Plugin) GetCNIArgs(args string) map[string]interface{} { + argsMap := make(map[string]interface{}) + + data := strings.Split(args, ";") + for _, pair := range data { + items := strings.Split(pair, "=") + if len(items) > 1 { + argsMap[items[0]] = items[1] + } + } + + return argsMap +} + // Error creates and logs a structured CNI error. func (plugin *Plugin) Error(err error) *cniTypes.Error { var cniErr *cniTypes.Error diff --git a/cnm/network/network.go b/cnm/network/network.go index 131bb6f0dc..f51c86fe3a 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -232,7 +232,6 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) } epInfo.Data = make(map[string]interface{}) - epInfo.Data["vlanid"] = 100 err = plugin.nm.CreateEndpoint(req.NetworkID, &epInfo) if err != nil { diff --git a/network/endpoint.go b/network/endpoint.go index cb88b87cbd..e2b8419da3 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -19,6 +19,7 @@ type endpoint struct { MacAddress net.HardwareAddr IPAddresses []net.IPNet Gateways []net.IP + VlanID int } // EndpointInfo contains read-only information about an endpoint. @@ -149,3 +150,31 @@ func (ep *endpoint) detach() error { return nil } + +func GetContainerNetworkConfiguration(namespace string, podName string) (*EndpointInfo, error) { + log.Printf("Namespace %v PodName %v", namespace, podName) + epInfo := &EndpointInfo{} + routeInfo := RouteInfo{} + ip, ipAddress, _ := net.ParseCIDR("10.2.0.13/16") + ipnet := net.IPNet{IP: ip, Mask: ipAddress.Mask} + var gateways []net.IP + + gateways = append(gateways, net.ParseIP("10.2.0.1")) + dstIp := [1]string{"0.0.0.0/0"} + dstGw := [1]string{"10.2.0.1"} + + for i := 0; i < len(dstIp); i++ { + _, dstipnet, _ := net.ParseCIDR(dstIp[i]) + routeInfo.Dst = *dstipnet + routeInfo.Gw = net.ParseIP(dstGw[i]) + epInfo.Routes = append(epInfo.Routes, routeInfo) + } + + epInfo.IPAddresses = append(epInfo.IPAddresses, ipnet) + epInfo.DNS.Servers = append(epInfo.DNS.Servers, "168.63.129.16") + epInfo.Data = make(map[string]interface{}) + epInfo.Data["vlanid"] = 100 + epInfo.Data["Gateways"] = gateways + + return epInfo, nil +} diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 1920f92f0c..b0acae129c 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -27,118 +27,124 @@ const ( containerInterfacePrefix = "eth" ) -func (nw *network) setupContainerNicAndRules(hostIfName string, contIfName string, containerIf *net.Interface, epInfo *EndpointInfo) error { - // Connect host interface to the bridge. - multitenancy := false +func (nw *network) setupLinuxBridgeRules(hostIfName string, contIfName string, containerIf *net.Interface, epInfo *EndpointInfo) error { var err error - if epInfo.Data != nil { - if vlanid, ok := epInfo.Data["vlanid"]; ok { - multitenancy = true - log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) + log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) + err = netlink.SetLinkMaster(hostIfName, nw.extIf.BridgeName) + if err != nil { + return err + } - cmd := fmt.Sprintf("ovs-vsctl add-port %v %v tag=%v", nw.extIf.BridgeName, hostIfName, vlanid) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding port failed with error %v", err) - return err - } + for _, ipAddr := range epInfo.IPAddresses { + // Add ARP reply rule. + log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) + err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(containerIf.HardwareAddr), ebtables.Append) + if err != nil { + return err + } - log.Printf("[net] Get ovs port for interface %v.", hostIfName) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIfName) - containerPort, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - return err - } + // Add MAC address translation rule. + log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) + err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) + if err != nil { + return err + } + } - containerPort = strings.Trim(containerPort, "\n") - mac := nw.extIf.MacAddress.String() - macHex := strings.Replace(mac, ":", "", -1) - log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) + return nil +} - cmd = fmt.Sprintf(`ovs-ofctl add-flow %v priority=10,arp,in_port=%s,arp_op=1,actions='mod_dl_src:%s, - load:0x%s->NXM_NX_ARP_SHA[],normal'`, nw.extIf.BridgeName, containerPort, mac, macHex) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP SNAT rule failed with error %v", err) - return err - } +func (nw *network) setupOVSRules(hostIfName string, contIfName string, containerIf *net.Interface, vlanid int, epInfo *EndpointInfo) error { + // Connect host interface to the bridge. + var err error - log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) + log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) + cmd := fmt.Sprintf("ovs-vsctl add-port %v %v tag=%v", nw.extIf.BridgeName, hostIfName, vlanid) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding port failed with error %v", err) + return err + } - cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", - nw.extIf.BridgeName, containerPort, mac) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding IP SNAT rule failed with error %v", err) - return err - } + log.Printf("[net] Get ovs port for interface %v.", hostIfName) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIfName) + containerPort, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + return err + } - macAddr := containerIf.HardwareAddr.String() - macAddrHex := strings.Replace(macAddr, ":", "", -1) + containerPort = strings.Trim(containerPort, "\n") - log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) - ofport, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - return err - } + log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) + ofport, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + return err + } - ofport = strings.Trim(ofport, "\n") - - for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. - ipAddrInt := common.IpToInt(ipAddr.IP) - - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,actions='load:0x2->NXM_OF_ARP_OP[], - move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, - move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], - load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, - nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, macAddr, macAddrHex, ipAddrInt) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP reply rule failed with error %v", err) - return err - } - - // Add MAC address translation rule. - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) - cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", - nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, ofport, macAddr) - _, err = common.ExecuteShellCommand(cmd) - //err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) - if err != nil { - log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) - return err - } - } - } + ofport = strings.Trim(ofport, "\n") + + mac := nw.extIf.MacAddress.String() + macHex := strings.Replace(mac, ":", "", -1) + log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) + + cmd = fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, + load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, nw.extIf.BridgeName, mac, macHex, ofport) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding ARP SNAT rule failed with error %v", err) + return err + } + + log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) + + cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", + nw.extIf.BridgeName, containerPort, mac) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding IP SNAT rule failed with error %v", err) + return err } - if !multitenancy { - log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) - err = netlink.SetLinkMaster(hostIfName, nw.extIf.BridgeName) + macAddr := containerIf.HardwareAddr.String() + macAddrHex := strings.Replace(macAddr, ":", "", -1) + + for _, ipAddr := range epInfo.IPAddresses { + // Add ARP reply rule. + ipAddrInt := common.IpToInt(ipAddr.IP) + + log.Printf("[net] Adding ARP reply rule set vlanid %v for container ifname", vlanid, contIfName) + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, + nw.extIf.BridgeName, containerPort, vlanid) + _, err = common.ExecuteShellCommand(cmd) if err != nil { + log.Printf("[net] Adding ARP reply rule failed with error %v", err) return err } - for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(containerIf.HardwareAddr), ebtables.Append) - if err != nil { - return err - } + log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[], + move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], + load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, + nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, macAddr, macAddrHex, ipAddrInt) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding ARP reply rule failed with error %v", err) + return err + } - // Add MAC address translation rule. - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) - if err != nil { - return err - } + // Add MAC address translation rule. + log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) + cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", + nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, ofport, macAddr) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) + return err } } @@ -151,6 +157,13 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var ns *Namespace var ep *endpoint var err error + var vlanid int = 0 + + if epInfo.Data != nil { + if _, ok := epInfo.Data["vlanid"]; ok { + vlanid = epInfo.Data["vlanid"].(int) + } + } if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -201,7 +214,11 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return nil, err } - nw.setupContainerNicAndRules(hostIfName, contIfName, containerIf, epInfo) + if vlanid != 0 { + nw.setupOVSRules(hostIfName, contIfName, containerIf, vlanid, epInfo) + } else { + nw.setupLinuxBridgeRules(hostIfName, contIfName, containerIf, epInfo) + } // // Container network interface setup. // @@ -303,16 +320,100 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { MacAddress: containerIf.HardwareAddr, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{nw.extIf.IPv4Gateway}, + VlanID: vlanid, } return ep, nil } +func (nw *network) deleteOVSRules(ep *endpoint) { + + log.Printf("[net] Get ovs port for interface %v.", ep.HostIfName) + cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", ep.HostIfName) + containerPort, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + } + + log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) + cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) + ofPort, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Get ofport failed with error %v", err) + } + + containerPort = strings.Trim(containerPort, "\n") + ofPort = strings.Trim(ofPort, "\n") + + // Delete IP SNAT + log.Printf("[net] Deleting IP SNAT for port %v", containerPort) + cmd = fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", + nw.extIf.BridgeName, containerPort) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("Error while deleting ovs rule %v error %v", cmd, err) + } + + log.Printf("[net] Deleting ARP reply rule set vlanid %v for container port", ep.VlanID, containerPort) + cmd = fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", + nw.extIf.BridgeName, containerPort) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Deleting ARP reply rule failed with error %v", err) + } + + log.Printf("[net] Deleting ARP reply rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) + cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", + nw.extIf.BridgeName, ep.IPAddresses[0].IP.String(), ep.VlanID) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Deleting ARP reply rule failed with error %v", err) + } + + // Add MAC address translation rule. + log.Printf("[net] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) + cmd = fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", + nw.extIf.BridgeName, ep.IPAddresses[0].IP.String(), ep.VlanID, ofPort) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) + } + + // Delete port fromk ovs bridge + cmd = fmt.Sprintf("ovs-vsctl del-port %v %v", nw.extIf.BridgeName, ep.HostIfName) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Adding port failed with error %v", err) + } +} + // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. + if ep.VlanID != 0 { + log.Printf("Delete ovs rules and remove port") + nw.deleteOVSRules(ep) + } else { + // Delete rules for IP addresses on the container interface. + for _, ipAddr := range ep.IPAddresses { + // Delete ARP reply rule. + log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) + err := ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(ep.MacAddress), ebtables.Delete) + if err != nil { + log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) + } + + // Delete MAC address translation rule. + log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id) + err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, ep.MacAddress, ebtables.Delete) + if err != nil { + log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err) + } + } + } + log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) err := netlink.DeleteLink(ep.HostIfName) if err != nil { @@ -320,23 +421,6 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { return err } - // Delete rules for IP addresses on the container interface. - for _, ipAddr := range ep.IPAddresses { - // Delete ARP reply rule. - log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(ep.MacAddress), ebtables.Delete) - if err != nil { - log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) - } - - // Delete MAC address translation rule. - log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, ep.MacAddress, ebtables.Delete) - if err != nil { - log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err) - } - } - return nil } diff --git a/network/manager.go b/network/manager.go index 07e1c63132..80cd807b75 100644 --- a/network/manager.go +++ b/network/manager.go @@ -247,6 +247,12 @@ func (nm *networkManager) CreateEndpoint(networkId string, epInfo *EndpointInfo) return err } + if nw.VlanId != 0 { + if epInfo.Data["vlanid"] == nil { + epInfo.Data["vlanid"] = nw.VlanId + } + } + _, err = nw.newEndpoint(epInfo) if err != nil { return err diff --git a/network/network.go b/network/network.go index b33f6f2765..bfd2ccb23f 100644 --- a/network/network.go +++ b/network/network.go @@ -35,7 +35,7 @@ type network struct { Id string HnsId string `json:",omitempty"` Mode string - VlanId string + VlanId int Subnets []SubnetInfo Endpoints map[string]*endpoint extIf *externalInterface diff --git a/network/network_linux.go b/network/network_linux.go index a0bb806376..7643c105aa 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -8,6 +8,7 @@ package network import ( "fmt" "net" + "strconv" "strings" "github.com/Azure/azure-container-networking/common" @@ -33,23 +34,25 @@ type route netlink.Route // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Connect the external interface. - multitenancy := false + var vlanid int opt, _ := nwInfo.Options[genericData].(map[string]interface{}) log.Printf("opt %v", opt) - // clusterInfo := opt["nodeList"].(string) switch nwInfo.Mode { case opModeTunnel: fallthrough case opModeBridge: - if opt != nil { - if _, ok := opt["vlanid"].(string); ok { - multitenancy = true - nm.createOVSNetwork(extIf, nwInfo) + if opt != nil && opt["vlanid"] != nil { + var err error + log.Printf("create ovs bridge") + vlanid, err = strconv.Atoi(opt["vlanid"].(string)) + if err != nil { + log.Printf("Error while converting vlanid from string to integer") } - } - if !multitenancy { + nm.createOVSNetwork(extIf, nwInfo) + } else { + log.Printf("create linux bridge") err := nm.connectExternalInterface(extIf, nwInfo) if err != nil { return nil, err @@ -66,10 +69,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt Mode: nwInfo.Mode, Endpoints: make(map[string]*endpoint), extIf: extIf, - } - - if multitenancy { - nw.VlanId = opt["vlanid"].(string) + VlanId: vlanid, } return nw, nil @@ -79,7 +79,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt func (nm *networkManager) deleteNetworkImpl(nw *network) error { // Disconnect the interface if this was the last network using it. - if nw.VlanId != "" { + if nw.VlanId != 0 { if len(nw.extIf.Networks) == 1 { nm.disconnectOVSInterface(nw.extIf) } From 09122b093ff9b22838a15874ed3a1644a245d960 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 4 Apr 2018 15:30:18 -0700 Subject: [PATCH 05/51] Added code to get network config --- cni/network/network.go | 138 ++++++++++++++++++++++------------- cns/dnccontract.go | 37 ++++++++-- cns/restserver/restserver.go | 115 +++++++++++++++++++++++++++-- cns/service/main.go | 132 ++++++++++++++++++--------------- common/config.go | 3 + network/endpoint.go | 28 ------- 6 files changed, 302 insertions(+), 151 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 584b64f713..b8b27883ed 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -4,14 +4,11 @@ package network import ( - "encoding/json" - "io/ioutil" "net" - "net/http" - - "k8s.io/api/core/v1" + "github.com/Azure/azure-container-networking/client/cnsclient" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -123,7 +120,7 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre return "" } -func getPodListFromApiServer(ip string, port string) (*v1.PodList, error) { +/*func getPodListFromApiServer(ip string, port string) (*v1.PodList, error) { url := "http://" + ip + ":" + port + "/api/v1/namespaces/default/pods" log.Printf("Api server url %v", url) resp, err := http.Get(url) @@ -140,6 +137,7 @@ func getPodListFromApiServer(ip string, port string) (*v1.PodList, error) { return podList, nil } + func getNetworkContainerID(podList *v1.PodList, containerID string) string { log.Printf("Number of pods %v", len(podList.Items)) for _, pod := range podList.Items { @@ -154,35 +152,79 @@ func getNetworkContainerID(podList *v1.PodList, containerID string) string { return "tid" } -func convertToCniResult(epInfo *network.EndpointInfo) *cniTypesCurr.Result { +*/ +func convertToCniResult(networkConfig *cns.GetNetworkConfigResponse) *cniTypesCurr.Result { result := &cniTypesCurr.Result{} log.Printf("Convert IPAddress") - for index, ipAddr := range epInfo.IPAddresses { - ipconfig := &cniTypesCurr.IPConfig{} - ipconfig.Address = ipAddr + ipconfig := networkConfig.IPConfiguration + ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) - if ipAddr.IP.To4() != nil { - ipconfig.Version = "4" - } else { - ipconfig.Version = "6" - } + resultIpconfig := &cniTypesCurr.IPConfig{} - ipconfig.Gateway = epInfo.Data["Gateways"].([]net.IP)[index] - result.IPs = append(result.IPs, ipconfig) + if ipAddr.To4() != nil { + resultIpconfig.Version = "4" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} + } else { + resultIpconfig.Version = "6" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} } - for _, route := range epInfo.Routes { - localroute := &types.Route{Dst: route.Dst, GW: route.Gw} + resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) + result.IPs = append(result.IPs, resultIpconfig) + + result.DNS.Nameservers = ipconfig.DNSServers + + for _, route := range networkConfig.Routes { + _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) + gwIP := net.ParseIP(route.GatewayIPAddress) + localroute := &types.Route{Dst: *routeIPnet, GW: gwIP} result.Routes = append(result.Routes, localroute) } - result.DNS.Nameservers = epInfo.DNS.Servers - result.DNS.Domain = epInfo.DNS.Suffix - return result } +func GetContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, error) { + /* log.Printf("Namespace %v PodName %v", namespace, podName) + epInfo := &EndpointInfo{} + routeInfo := RouteInfo{} + ip, ipAddress, _ := net.ParseCIDR("10.2.0.13/16") + ipnet := net.IPNet{IP: ip, Mask: ipAddress.Mask} + var gateways []net.IP + + gateways = append(gateways, net.ParseIP("10.2.0.1")) + dstIp := [1]string{"0.0.0.0/0"} + dstGw := [1]string{"10.2.0.1"} + + for i := 0; i < len(dstIp); i++ { + _, dstipnet, _ := net.ParseCIDR(dstIp[i]) + routeInfo.Dst = *dstipnet + routeInfo.Gw = net.ParseIP(dstGw[i]) + epInfo.Routes = append(epInfo.Routes, routeInfo) + } + + epInfo.IPAddresses = append(epInfo.IPAddresses, ipnet) + epInfo.DNS.Servers = append(epInfo.DNS.Servers, "168.63.129.16") + epInfo.Data = make(map[string]interface{}) + epInfo.Data["vlanid"] = 100 + epInfo.Data["Gateways"] = gateways*/ + cnsClient, err := cnsclient.NewCnsClient("") + if err != nil { + log.Printf("Initializing CNS client error %v", err) + return nil, 0, err + } + log.Printf("Network config request") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + if err != nil { + log.Printf("GetNetworkConfiguration failed with %v", err) + return nil, 0, err + } + + log.Printf("Network config received from cns %v", networkConfig) + return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, nil +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -193,6 +235,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var result *cniTypesCurr.Result var err error var epInfo *network.EndpointInfo + var vlanid int log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) @@ -227,26 +270,21 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize endpoint info. - epInfo, err = network.GetContainerNetworkConfiguration(argsMap[namespacekey].(string), argsMap[podnamekey].(string)) + result, vlanid, err = GetContainerNetworkConfiguration(argsMap[namespacekey].(string), argsMap[podnamekey].(string)) if err != nil { log.Printf("SetContainerNetworkConfiguration failed with %v", err) } - if epInfo != nil { - epInfo.Id = endpointId - epInfo.ContainerID = args.ContainerID - epInfo.NetNsPath = args.Netns - epInfo.IfName = args.IfName - result = convertToCniResult(epInfo) - log.Printf("Convertocniresult :%v", result) - } else { - epInfo = &network.EndpointInfo{ - Id: endpointId, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, - } - epInfo.Data = make(map[string]interface{}) + epInfo = &network.EndpointInfo{ + Id: endpointId, + ContainerID: args.ContainerID, + NetNsPath: args.Netns, + IfName: args.IfName, + } + epInfo.Data = make(map[string]interface{}) + + if vlanid != 0 { + epInfo.Data["vlanid"] = vlanid } // Check whether the network already exists. @@ -346,22 +384,20 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } } - if len(epInfo.IPAddresses) == 0 { - // Populate addresses. - for _, ipconfig := range result.IPs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } - - // Populate routes. - for _, route := range result.Routes { - epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) - } + // Populate addresses. + for _, ipconfig := range result.IPs { + epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) + } - // Populate DNS info. - epInfo.DNS.Suffix = result.DNS.Domain - epInfo.DNS.Servers = result.DNS.Nameservers + // Populate routes. + for _, route := range result.Routes { + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } + // Populate DNS info. + epInfo.DNS.Suffix = result.DNS.Domain + epInfo.DNS.Servers = result.DNS.Nameservers + // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/cns/dnccontract.go b/cns/dnccontract.go index 54ba756862..5737c333ba 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -1,18 +1,26 @@ package cns +import "encoding/json" + // Container Network Service DNC Contract const ( - CreateOrUpdateNetworkContainer = "/network/createorupdatenetworkcontainer" - DeleteNetworkContainer = "/network/deletenetworkcontainer" - GetNetworkContainerStatus = "/network/getnetworkcontainerstatus" - GetInterfaceForContainer = "/network/getinterfaceforcontainer" + CreateOrUpdateNetworkContainer = "/network/createorupdatenetworkcontainer" + DeleteNetworkContainer = "/network/deletenetworkcontainer" + GetNetworkContainerStatus = "/network/getnetworkcontainerstatus" + GetInterfaceForContainer = "/network/getinterfaceforcontainer" + GetNetworkConfigByOrchestratorInfo = "/network/getnetworkconfigbyorchestratorinfo" ) -// Orchestrator Types +// Network Container Types const ( AzureContainerInstance = "AzureContainerInstance" ) +// Orchestrator Types +const ( + Kubernetes = "Kubernetes" +) + // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { Version string @@ -30,11 +38,11 @@ type CreateNetworkContainerRequest struct { // OrchestratorInfo contains orchestrator type which is used to cast OrchestratorContext. type OrchestratorInfo struct { OrchestratorType string - OrchestratorContext interface{} + OrchestratorContext json.RawMessage } -// AzureContainerInstanceInfo is an OrchestratorContext that holds PodName and PodNamespace. -type AzureContainerInstanceInfo struct { +// KubernetesInfo is an OrchestratorContext that holds PodName and PodNamespace. +type KubernetesInfo struct { PodName string PodNamespace string } @@ -119,3 +127,16 @@ type NetworkInterface struct { Name string IPAddress string } + +// GetInterfaceForContainerResponse specifies the network config for a given Networkcontainerid/NetworkContainerOrchestratorInfo. +type GetNetworkConfigRequest struct { + NetworkcontainerID string + OrchestratorInfo OrchestratorInfo +} + +type GetNetworkConfigResponse struct { + IPConfiguration IPConfiguration + Routes []Route + MultiTenancyInfo MultiTenancyInfo + Response Response +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 23353ad454..237fa5983c 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,6 +4,7 @@ package restserver import ( + "encoding/json" "fmt" "net" "net/http" @@ -51,12 +52,13 @@ type containerstatus struct { // httpRestServiceState contains the state we would like to persist. type httpRestServiceState struct { - Location string - NetworkType string - Initialized bool - ContainerStatus map[string]containerstatus - Networks map[string]*networkInfo - TimeStamp time.Time + Location string + NetworkType string + Initialized bool + ContainerIDByOrchestratorInfo map[string]string + ContainerStatus map[string]containerstatus + Networks map[string]*networkInfo + TimeStamp time.Time } type networkInfo struct { @@ -93,6 +95,7 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { serviceState := &httpRestServiceState{} serviceState.Networks = make(map[string]*networkInfo) + serviceState.ContainerIDByOrchestratorInfo = make(map[string]string) return &httpRestService{ Service: service, @@ -143,6 +146,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.DeleteNetworkContainer, service.deleteNetworkContainer) listener.AddHandler(cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.GetInterfaceForContainer, service.getInterfaceForContainer) + listener.AddHandler(cns.GetNetworkConfigByOrchestratorInfo, service.getNetworkConfigByOrchestratorInfo) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -157,6 +161,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.DeleteNetworkContainer, service.deleteNetworkContainer) listener.AddHandler(cns.V2Prefix+cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.V2Prefix+cns.GetInterfaceForContainer, service.getInterfaceForContainer) + listener.AddHandler(cns.V2Prefix+cns.GetNetworkConfigByOrchestratorInfo, service.getNetworkConfigByOrchestratorInfo) log.Printf("[Azure CNS] Listening.") return nil @@ -849,6 +854,27 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr VMVersion: req.Version, CreateNetworkContainerRequest: req, HostVersion: hostVersion} + + if req.NetworkContainerType == cns.AzureContainerInstance { + orchInfo := req.OrchestratorInfo + switch orchInfo.OrchestratorType { + case cns.Kubernetes: + var azNCInfo cns.AzureContainerInstanceInfo + err = json.Unmarshal(req.OrchestratorInfo.OrchestratorContext, &azNCInfo) + if err != nil { + log.Printf("Unmarshalling AzureContainerInstanceInfo failed with error %v", err) + return + } + + log.Printf("Azure container instance info %v", azNCInfo) + service.state.ContainerIDByOrchestratorInfo[azNCInfo.PodName+azNCInfo.PodNamespace] = req.NetworkContainerid + break + + default: + log.Printf("Invalid orchestrator type %v", orchInfo.OrchestratorType) + } + } + service.saveState() default: @@ -1058,6 +1084,83 @@ func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, log.Response(service.Name, getInterfaceForContainerResponse, err) } +func (service *httpRestService) getNetworkConfigByOrchestratorInfo(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure CNS] getNetworkConfigForContainer") + + var req cns.GetNetworkConfigRequest + var containerID string + returnMessage := "" + returnCode := 0 + + err := service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + log.Printf("Decoding failed") + return + } + + orchInfo := req.OrchestratorInfo + + switch orchInfo.OrchestratorType { + case cns.AzureContainerInstance: + var azNCInfo cns.AzureContainerInstanceInfo + err = json.Unmarshal(req.OrchestratorInfo.OrchestratorContext, &azNCInfo) + if err != nil { + log.Printf("Unmarshalling AzureContainerInstanceInfo failed with error %v", err) + return + } + + log.Printf("azure container instance info %v", azNCInfo) + containerID = service.state.ContainerIDByOrchestratorInfo[azNCInfo.PodName+azNCInfo.PodNamespace] + log.Printf("containerid %v", containerID) + break + default: + log.Printf("Invalid Orchestrator type %v", orchInfo.OrchestratorType) + } + + var ipconfig cns.IPConfiguration + var routes []cns.Route + var route cns.Route + var encapInfo cns.MultiTenancyInfo + + /* containerInfo := service.state.ContainerStatus + containerDetails, ok := containerInfo[containerID] + if ok { + savedReq := containerDetails.CreateNetworkContainerRequest + ipconfig = savedReq.IPConfiguration + routes = savedReq.Routes + encapInfo = savedReq.MultiTenancyInfo + } else { + returnMessage = "[Azure CNS] Never received call to create this container." + returnCode = UnknownContainerID + } + */ + ipconfig.IPSubnet.IPAddress = "10.2.0.13" + ipconfig.IPSubnet.PrefixLength = 24 + ipconfig.GatewayIPAddress = "10.2.0.1" + ipconfig.DNSServers = append(ipconfig.DNSServers, "168.63.129.16") + route.IPAddress = "10.2.0.0/16" + route.GatewayIPAddress = "10.2.0.1" + routes = append(routes, route) + encapInfo.ID = 100 + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + getNetworkConfigResponse := cns.GetNetworkConfigResponse{ + Response: resp, + IPConfiguration: ipconfig, + Routes: routes, + MultiTenancyInfo: encapInfo, + } + + log.Printf("Send networkconfig response") + err = service.Listener.Encode(w, &getNetworkConfigResponse) + + log.Response(service.Name, getNetworkConfigResponse, err) +} + // restoreNetworkState restores Network state that existed before reboot. func (service *httpRestService) restoreNetworkState() error { log.Printf("[Azure CNS] Enter Restoring Network State") diff --git a/cns/service/main.go b/cns/service/main.go index 35c6a85da6..77baa3302c 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -93,6 +93,13 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: "", }, + { + Name: acn.OptStopAzureVnet, + Shorthand: acn.OptStopAzureVnetAlias, + Description: "Start Azure-CNM if flag is true", + Type: "bool", + DefaultValue: true, + }, { Name: acn.OptVersion, Shorthand: acn.OptVersionAlias, @@ -110,6 +117,7 @@ func printVersion() { // Main is the entry point for CNS. func main() { + var stopcnm = false // Initialize and parse command line arguments. acn.ParseArgs(&args, printVersion) @@ -120,6 +128,7 @@ func main() { logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) + stopcnm = acn.GetArg(acn.OptStopAzureVnet).(bool) vers := acn.GetArg(acn.OptVersion).(bool) if vers { @@ -135,40 +144,22 @@ func main() { // Create a channel to receive unhandled errors from CNS. config.ErrChan = make(chan error, 1) - // Create the key value store. var err error - config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") - if err != nil { - fmt.Printf("Failed to create store: %v\n", err) - return - } - - // Create CNS object. - httpRestService, err := restserver.NewHTTPRestService(&config) - if err != nil { - fmt.Printf("Failed to create CNS object, err:%v.\n", err) - return + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if logDirectory != "" { + log.SetLogDirectory(logDirectory) } - var pluginConfig acn.PluginConfig - pluginConfig.Version = version - - // Create a channel to receive unhandled errors from the plugins. - pluginConfig.ErrChan = make(chan error, 1) - - // Create network plugin. - netPlugin, err := network.NewPlugin(&pluginConfig) + err = log.SetTarget(logTarget) if err != nil { - fmt.Printf("Failed to create network plugin, err:%v.\n", err) + fmt.Printf("Failed to configure logging: %v\n", err) return } - // Create IPAM plugin. - ipamPlugin, err := ipam.NewPlugin(&pluginConfig) - if err != nil { - fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) - return - } + // Log platform information. + log.Printf("Running on %v", platform.GetOSInfo()) err = acn.CreateDirectory(platform.CNMRuntimePath) if err != nil { @@ -177,28 +168,20 @@ func main() { } // Create the key value store. - pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") + + config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { fmt.Printf("Failed to create store: %v\n", err) return } - // Create logging provider. - log.SetName(name) - log.SetLevel(logLevel) - if logDirectory != "" { - log.SetLogDirectory(logDirectory) - } - - err = log.SetTarget(logTarget) + // Create CNS object. + httpRestService, err := restserver.NewHTTPRestService(&config) if err != nil { - fmt.Printf("Failed to configure logging: %v\n", err) + fmt.Printf("Failed to create CNS object, err:%v.\n", err) return } - // Log platform information. - log.Printf("Running on %v", platform.GetOSInfo()) - // Set CNS options. httpRestService.SetOption(acn.OptCnsURL, cnsURL) @@ -211,28 +194,59 @@ func main() { } } - // Set plugin options. - netPlugin.SetOption(acn.OptAPIServerURL, url) + log.Printf("stop cnm val %t", stopcnm) + var netPlugin network.NetPlugin + var ipamPlugin ipam.IpamPlugin + + if !stopcnm { + var pluginConfig acn.PluginConfig + pluginConfig.Version = version - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) + // Create a channel to receive unhandled errors from the plugins. + pluginConfig.ErrChan = make(chan error, 1) - if netPlugin != nil { - log.Printf("Start netplugin\n") - err = netPlugin.Start(&pluginConfig) + // Create network plugin. + netPlugin, err = network.NewPlugin(&pluginConfig) if err != nil { - fmt.Printf("Failed to start network plugin, err:%v.\n", err) + fmt.Printf("Failed to create network plugin, err:%v.\n", err) + return + } + + // Create IPAM plugin. + ipamPlugin, err = ipam.NewPlugin(&pluginConfig) + if err != nil { + fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) return } - } - if ipamPlugin != nil { - err = ipamPlugin.Start(&pluginConfig) + // Create the key value store. + pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") if err != nil { - fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + fmt.Printf("Failed to create store: %v\n", err) return } + + // Set plugin options. + netPlugin.SetOption(acn.OptAPIServerURL, url) + if netPlugin != nil { + log.Printf("Start netplugin\n") + err = netPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start network plugin, err:%v.\n", err) + return + } + } + + ipamPlugin.SetOption(acn.OptEnvironment, environment) + ipamPlugin.SetOption(acn.OptAPIServerURL, url) + ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) + if ipamPlugin != nil { + err = ipamPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + return + } + } } // Relay these incoming signals to OS signal channel. @@ -252,11 +266,13 @@ func main() { httpRestService.Stop() } - if netPlugin != nil { - netPlugin.Stop() - } + if !stopcnm { + if netPlugin != nil { + netPlugin.Stop() + } - if ipamPlugin != nil { - ipamPlugin.Stop() + if ipamPlugin != nil { + ipamPlugin.Stop() + } } } diff --git a/common/config.go b/common/config.go index d670c08a5b..84cc690724 100644 --- a/common/config.go +++ b/common/config.go @@ -42,6 +42,9 @@ const ( OptIpamQueryInterval = "ipam-query-interval" OptIpamQueryIntervalAlias = "i" + OptStopAzureVnet = "stop-azure-cnm" + OptStopAzureVnetAlias = "stopcnm" + // Version. OptVersion = "version" OptVersionAlias = "v" diff --git a/network/endpoint.go b/network/endpoint.go index e2b8419da3..e391bd2e7f 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -150,31 +150,3 @@ func (ep *endpoint) detach() error { return nil } - -func GetContainerNetworkConfiguration(namespace string, podName string) (*EndpointInfo, error) { - log.Printf("Namespace %v PodName %v", namespace, podName) - epInfo := &EndpointInfo{} - routeInfo := RouteInfo{} - ip, ipAddress, _ := net.ParseCIDR("10.2.0.13/16") - ipnet := net.IPNet{IP: ip, Mask: ipAddress.Mask} - var gateways []net.IP - - gateways = append(gateways, net.ParseIP("10.2.0.1")) - dstIp := [1]string{"0.0.0.0/0"} - dstGw := [1]string{"10.2.0.1"} - - for i := 0; i < len(dstIp); i++ { - _, dstipnet, _ := net.ParseCIDR(dstIp[i]) - routeInfo.Dst = *dstipnet - routeInfo.Gw = net.ParseIP(dstGw[i]) - epInfo.Routes = append(epInfo.Routes, routeInfo) - } - - epInfo.IPAddresses = append(epInfo.IPAddresses, ipnet) - epInfo.DNS.Servers = append(epInfo.DNS.Servers, "168.63.129.16") - epInfo.Data = make(map[string]interface{}) - epInfo.Data["vlanid"] = 100 - epInfo.Data["Gateways"] = gateways - - return epInfo, nil -} From 9bea2e8f4406d715ae9c558cb14c230d16a01d33 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 4 Apr 2018 15:32:20 -0700 Subject: [PATCH 06/51] Added cnsclient --- client/cnsclient/cnsclient.go | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 client/cnsclient/cnsclient.go diff --git a/client/cnsclient/cnsclient.go b/client/cnsclient/cnsclient.go new file mode 100644 index 0000000000..20a00b4c00 --- /dev/null +++ b/client/cnsclient/cnsclient.go @@ -0,0 +1,85 @@ +package cnsclient + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/log" +) + +// IpamClient specifies a client to connect to Ipam Plugin. +type CNSClient struct { + connectionURL string +} + +const ( + defaultCnsURL = "http://localhost:10090" +) + +// NewCnsClient create a new cns client. +func NewCnsClient(url string) (*CNSClient, error) { + if url == "" { + url = defaultCnsURL + } + return &CNSClient{ + connectionURL: url, + }, nil +} + +// GetNetworkConfiguration Request to get network config. +func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string) (*cns.GetNetworkConfigResponse, error) { + var body bytes.Buffer + httpc := &http.Client{} + + url := cnsClient.connectionURL + cns.GetNetworkConfigByOrchestratorInfo + log.Printf("GetNetworkConfiguration url %v", url) + + azContainerInfo := cns.AzureContainerInstanceInfo{PodName: podName, PodNamespace: podNamespace} + azContainerInfoBytes, err := json.Marshal(azContainerInfo) + if err != nil { + log.Printf("Marshalling azure container instance info failed with %v", err) + return nil, err + } + + ncOrchestratorInfo := cns.OrchestratorInfo{OrchestratorType: cns.AzureContainerInstance, OrchestratorContext: azContainerInfoBytes} + + payload := &cns.GetNetworkConfigRequest{ + OrchestratorInfo: ncOrchestratorInfo, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Printf("encoding json failed with %v", err) + return nil, err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Printf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return nil, err + } + + if res.StatusCode == 200 { + var resp cns.GetNetworkConfigResponse + err := json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) + return nil, err + } + + if resp.Response.ReturnCode != 0 { + log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) + } + + return &resp, nil + } + + var errMsg string + errMsg = fmt.Sprintf(errMsg, "[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) + log.Printf(errMsg) + return nil, fmt.Errorf(errMsg) +} From eecc9ec522eaabe9dbd76905e4679a9c160b0adf Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 4 Apr 2018 18:43:37 -0700 Subject: [PATCH 07/51] Added setorchestrator api --- cns/dnccontract.go | 14 +++++------ cns/restserver/api.go | 1 + cns/restserver/restserver.go | 45 +++++++++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/cns/dnccontract.go b/cns/dnccontract.go index 863255b0cc..c80d378a10 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -4,6 +4,7 @@ import "encoding/json" // Container Network Service DNC Contract const ( + SetOrchestratorType = "/network/setorchestratortype" CreateOrUpdateNetworkContainer = "/network/createorupdatenetworkcontainer" DeleteNetworkContainer = "/network/deletenetworkcontainer" GetNetworkContainerStatus = "/network/getnetworkcontainerstatus" @@ -27,19 +28,13 @@ type CreateNetworkContainerRequest struct { NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. AuthorizationToken string - OrchestratorInfo OrchestratorInfo + OrchestratorContext json.RawMessage IPConfiguration IPConfiguration MultiTenancyInfo MultiTenancyInfo VnetAddressSpace []IPSubnet // To setup SNAT (should include service endpoint vips). Routes []Route } -// OrchestratorInfo contains orchestrator type which is used to cast OrchestratorContext. -type OrchestratorInfo struct { - OrchestratorType string - OrchestratorContext json.RawMessage -} - // KubernetesPodInfo is an OrchestratorContext that holds PodName and PodNamespace. type KubernetesPodInfo struct { PodName string @@ -72,6 +67,11 @@ type Route struct { InterfaceToUse string } +// SetOrchestratorTypeRequest specifies the orchestrator type for the node. +type SetOrchestratorTypeRequest struct { + OrchestratorType string +} + // CreateNetworkContainerResponse specifies response of creating a network container. type CreateNetworkContainerResponse struct { Response Response diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 1b802515db..6db8f6f5f5 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -19,5 +19,6 @@ const ( NetworkContainerNotSpecified = 16 CallToHostFailed = 17 UnknownContainerID = 18 + UnsupportedOrchestratorType = 19 UnexpectedError = 99 ) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 23353ad454..13b14ccbbd 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -51,12 +51,13 @@ type containerstatus struct { // httpRestServiceState contains the state we would like to persist. type httpRestServiceState struct { - Location string - NetworkType string - Initialized bool - ContainerStatus map[string]containerstatus - Networks map[string]*networkInfo - TimeStamp time.Time + Location string + NetworkType string + OrchestratorType string + Initialized bool + ContainerStatus map[string]containerstatus + Networks map[string]*networkInfo + TimeStamp time.Time } type networkInfo struct { @@ -143,6 +144,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.DeleteNetworkContainer, service.deleteNetworkContainer) listener.AddHandler(cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.GetInterfaceForContainer, service.getInterfaceForContainer) + listener.AddHandler(cns.SetOrchestratorType, service.setOrchestratorType) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -157,6 +159,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.DeleteNetworkContainer, service.deleteNetworkContainer) listener.AddHandler(cns.V2Prefix+cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.V2Prefix+cns.GetInterfaceForContainer, service.getInterfaceForContainer) + listener.AddHandler(cns.V2Prefix+cns.SetOrchestratorType, service.setOrchestratorType) log.Printf("[Azure CNS] Listening.") return nil @@ -798,6 +801,36 @@ func (service *httpRestService) restoreState() error { return nil } +func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure CNS] setOrchestratorType") + var req cns.SetOrchestratorTypeRequest + + returnMessage := "" + returnCode := 0 + err := service.Listener.Decode(w, r, &req) + if err != nil { + return + } + + switch req.OrchestratorType { + case cns.Kubernetes: + service.state.OrchestratorType = cns.Kubernetes + service.saveState() + break + default: + returnMessage = fmt.Sprintf("Invalid Orchestrator type %v", req.OrchestratorType) + returnCode = UnsupportedOrchestratorType + } + + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + err = service.Listener.Encode(w, &resp) + log.Response(service.Name, resp, err) +} + func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] createOrUpdateNetworkContainer") From 9a210557a7862b5af2e33aef849ea5ee228a4e2d Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 4 Apr 2018 18:47:13 -0700 Subject: [PATCH 08/51] Fixed styling --- cns/restserver/restserver.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 13b14ccbbd..30c59e48bf 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -803,10 +803,11 @@ func (service *httpRestService) restoreState() error { func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] setOrchestratorType") - var req cns.SetOrchestratorTypeRequest + var req cns.SetOrchestratorTypeRequest returnMessage := "" returnCode := 0 + err := service.Listener.Decode(w, r, &req) if err != nil { return @@ -835,13 +836,11 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr log.Printf("[Azure CNS] createOrUpdateNetworkContainer") var req cns.CreateNetworkContainerRequest - returnMessage := "" returnCode := 0 - err := service.Listener.Decode(w, r, &req) + err := service.Listener.Decode(w, r, &req) log.Request(service.Name, &req, err) - if err != nil { return } @@ -905,13 +904,11 @@ func (service *httpRestService) getNetworkContainer(w http.ResponseWriter, r *ht log.Printf("[Azure CNS] getNetworkContainer") var req cns.GetNetworkContainerRequest - returnMessage := "" returnCode := 0 - err := service.Listener.Decode(w, r, &req) + err := service.Listener.Decode(w, r, &req) log.Request(service.Name, &req, err) - if err != nil { return } @@ -932,13 +929,11 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r log.Printf("[Azure CNS] deleteNetworkContainer") var req cns.DeleteNetworkContainerRequest - returnMessage := "" returnCode := 0 - err := service.Listener.Decode(w, r, &req) + err := service.Listener.Decode(w, r, &req) log.Request(service.Name, &req, err) - if err != nil { return } @@ -987,9 +982,9 @@ func (service *httpRestService) getNetworkContainerStatus(w http.ResponseWriter, var req cns.GetNetworkContainerStatusRequest returnMessage := "" returnCode := 0 + err := service.Listener.Decode(w, r, &req) log.Request(service.Name, &req, err) - if err != nil { return } @@ -1050,9 +1045,9 @@ func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, var req cns.GetInterfaceForContainerRequest returnMessage := "" returnCode := 0 + err := service.Listener.Decode(w, r, &req) log.Request(service.Name, &req, err) - if err != nil { return } From f7ab7ea07287753319ed4576bfc69debec8b33fb Mon Sep 17 00:00:00 2001 From: tamilmani1989 Date: Thu, 5 Apr 2018 13:25:17 -0700 Subject: [PATCH 09/51] Modified networkcontainercreate request to save podinfo (#115) --- cns/restserver/restserver.go | 85 +++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 30c59e48bf..10f5ce6b99 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,6 +4,7 @@ package restserver import ( + "encoding/json" "fmt" "net" "net/http" @@ -51,13 +52,14 @@ type containerstatus struct { // httpRestServiceState contains the state we would like to persist. type httpRestServiceState struct { - Location string - NetworkType string - OrchestratorType string - Initialized bool - ContainerStatus map[string]containerstatus - Networks map[string]*networkInfo - TimeStamp time.Time + Location string + NetworkType string + OrchestratorType string + Initialized bool + ContainerIDByOrchestratorContext map[string]string // OrchestratorContext is key and value is NetworkContainerID. + ContainerStatus map[string]containerstatus // NetworkContainerID is key. + Networks map[string]*networkInfo + TimeStamp time.Time } type networkInfo struct { @@ -832,6 +834,51 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht log.Response(service.Name, resp, err) } +func (service *httpRestService) saveNetworkContainerGoalState(req cns.CreateNetworkContainerRequest) (int, string) { + // we don't want to overwrite what other calls may have written + service.lock.Lock() + defer service.lock.Unlock() + + existing, ok := service.state.ContainerStatus[req.NetworkContainerid] + var hostVersion string + if ok { + hostVersion = existing.HostVersion + } + + if service.state.ContainerStatus == nil { + service.state.ContainerStatus = make(map[string]containerstatus) + } + + service.state.ContainerStatus[req.NetworkContainerid] = + containerstatus{ + ID: req.NetworkContainerid, + VMVersion: req.Version, + CreateNetworkContainerRequest: req, + HostVersion: hostVersion} + + if req.NetworkContainerType == cns.AzureContainerInstance { + switch service.state.OrchestratorType { + case cns.Kubernetes: + var podInfo cns.KubernetesPodInfo + err := json.Unmarshal(req.OrchestratorContext, &podInfo) + if err != nil { + errBuf := fmt.Sprintf("Unmarshalling AzureContainerInstanceInfo failed with error %v", err) + return UnexpectedError, errBuf + } + + log.Printf("Azure container instance info %v", podInfo) + service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] = req.NetworkContainerid + break + + default: + log.Printf("Invalid orchestrator type %v", service.state.OrchestratorType) + } + } + + service.saveState() + return 0, "" +} + func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] createOrUpdateNetworkContainer") @@ -854,39 +901,17 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr case "POST": nc := service.networkContainer err := nc.Create(req) - if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateOrUpdateNetworkContainer failed %v", err.Error()) returnCode = UnexpectedError break } - // we don't want to overwrite what other calls may have written - service.lock.Lock() - defer service.lock.Unlock() - - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] - var hostVersion string - if ok { - hostVersion = existing.HostVersion - } - - if service.state.ContainerStatus == nil { - service.state.ContainerStatus = make(map[string]containerstatus) - } - - service.state.ContainerStatus[req.NetworkContainerid] = - containerstatus{ - ID: req.NetworkContainerid, - VMVersion: req.Version, - CreateNetworkContainerRequest: req, - HostVersion: hostVersion} - service.saveState() + returnCode, returnMessage = service.saveNetworkContainerGoalState(req) default: returnMessage = "[Azure CNS] Error. CreateOrUpdateNetworkContainer did not receive a POST." returnCode = InvalidParameter - } resp := cns.Response{ From 40b87bcf5e013f6f987da04553a9e01ff34b34aa Mon Sep 17 00:00:00 2001 From: tamilmani1989 Date: Thu, 5 Apr 2018 16:06:12 -0700 Subject: [PATCH 10/51] Added GetNetworkContainerByContext Api (#116) Added getnetworkcontainerbyorchestrator context api and relevant test cases --- cns/dnccontract.go | 18 ++-- cns/restserver/restserver.go | 80 +++++++++++++++- cns/restserver/restserver_test.go | 151 ++++++++++++++++++++++++++++-- 3 files changed, 235 insertions(+), 14 deletions(-) diff --git a/cns/dnccontract.go b/cns/dnccontract.go index c80d378a10..c4cde79e36 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -4,11 +4,12 @@ import "encoding/json" // Container Network Service DNC Contract const ( - SetOrchestratorType = "/network/setorchestratortype" - CreateOrUpdateNetworkContainer = "/network/createorupdatenetworkcontainer" - DeleteNetworkContainer = "/network/deletenetworkcontainer" - GetNetworkContainerStatus = "/network/getnetworkcontainerstatus" - GetInterfaceForContainer = "/network/getinterfaceforcontainer" + SetOrchestratorType = "/network/setorchestratortype" + CreateOrUpdateNetworkContainer = "/network/createorupdatenetworkcontainer" + DeleteNetworkContainer = "/network/deletenetworkcontainer" + GetNetworkContainerStatus = "/network/getnetworkcontainerstatus" + GetInterfaceForContainer = "/network/getinterfaceforcontainer" + GetNetworkContainerByOrchestratorContext = "/network/getnetworkcontainerbyorchestratorcontext" ) // NetworkContainer Types @@ -92,11 +93,16 @@ type GetNetworkContainerStatusResponse struct { // GetNetworkContainerRequest specifies the details about the request to retrieve a specifc network container. type GetNetworkContainerRequest struct { + NetworkContainerid string + OrchestratorContext json.RawMessage } // GetNetworkContainerResponse describes the response to retrieve a specifc network container. type GetNetworkContainerResponse struct { - Response Response + IPConfiguration IPConfiguration + Routes []Route + MultiTenancyInfo MultiTenancyInfo + Response Response } // DeleteNetworkContainerRequest specifies the details about the request to delete a specifc network container. diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 10f5ce6b99..c9351117d1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -147,6 +147,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.GetInterfaceForContainer, service.getInterfaceForContainer) listener.AddHandler(cns.SetOrchestratorType, service.setOrchestratorType) + listener.AddHandler(cns.GetNetworkContainerByOrchestratorContext, service.getNetworkContainerByOrchestratorContext) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -162,6 +163,7 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.GetNetworkContainerStatus, service.getNetworkContainerStatus) listener.AddHandler(cns.V2Prefix+cns.GetInterfaceForContainer, service.getInterfaceForContainer) listener.AddHandler(cns.V2Prefix+cns.SetOrchestratorType, service.setOrchestratorType) + listener.AddHandler(cns.V2Prefix+cns.GetNetworkContainerByOrchestratorContext, service.getNetworkContainerByOrchestratorContext) log.Printf("[Azure CNS] Listening.") return nil @@ -867,6 +869,11 @@ func (service *httpRestService) saveNetworkContainerGoalState(req cns.CreateNetw } log.Printf("Azure container instance info %v", podInfo) + + if service.state.ContainerIDByOrchestratorContext == nil { + service.state.ContainerIDByOrchestratorContext = make(map[string]string) + } + service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] = req.NetworkContainerid break @@ -925,7 +932,7 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr log.Response(service.Name, reserveResp, err) } -func (service *httpRestService) getNetworkContainer(w http.ResponseWriter, r *http.Request) { +func (service *httpRestService) getNetworkContainerByID(w http.ResponseWriter, r *http.Request) { log.Printf("[Azure CNS] getNetworkContainer") var req cns.GetNetworkContainerRequest @@ -945,9 +952,67 @@ func (service *httpRestService) getNetworkContainer(w http.ResponseWriter, r *ht reserveResp := &cns.GetNetworkContainerResponse{Response: resp} err = service.Listener.Encode(w, &reserveResp) - log.Response(service.Name, reserveResp, err) +} + +func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { + var containerID string + var getNetworkContainerResponse cns.GetNetworkContainerResponse + + switch service.state.OrchestratorType { + case cns.Kubernetes: + var podInfo cns.KubernetesPodInfo + err := json.Unmarshal(req.OrchestratorContext, &podInfo) + if err != nil { + getNetworkContainerResponse.Response.ReturnCode = UnexpectedError + getNetworkContainerResponse.Response.Message = fmt.Sprintf("Unmarshalling orchestrator context failed with error %v", err) + return getNetworkContainerResponse + } + + log.Printf("azure container instance info %+v", podInfo) + containerID = service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] + log.Printf("containerid %v", containerID) + break + + default: + getNetworkContainerResponse.Response.ReturnCode = UnsupportedOrchestratorType + getNetworkContainerResponse.Response.Message = fmt.Sprintf("Invalid orchestrator type %v", service.state.OrchestratorType) + return getNetworkContainerResponse + } + + containerStatus := service.state.ContainerStatus + containerDetails, ok := containerStatus[containerID] + if !ok { + getNetworkContainerResponse.Response.ReturnCode = UnknownContainerID + getNetworkContainerResponse.Response.Message = "NetworkContainer doesn't exist." + return getNetworkContainerResponse + } + + savedReq := containerDetails.CreateNetworkContainerRequest + getNetworkContainerResponse = cns.GetNetworkContainerResponse{ + IPConfiguration: savedReq.IPConfiguration, + Routes: savedReq.Routes, + MultiTenancyInfo: savedReq.MultiTenancyInfo, + } + return getNetworkContainerResponse +} + +func (service *httpRestService) getNetworkContainerByOrchestratorContext(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure CNS] getNetworkContainer") + + var req cns.GetNetworkContainerRequest + + err := service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + getNetworkContainerResponse := service.getNetworkContainerResponse(req) + + err = service.Listener.Encode(w, &getNetworkContainerResponse) + log.Response(service.Name, getNetworkContainerResponse, err) } func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r *http.Request) { @@ -982,6 +1047,17 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r if service.state.ContainerStatus != nil { delete(service.state.ContainerStatus, req.NetworkContainerid) } + + if service.state.ContainerIDByOrchestratorContext != nil { + for orchestratorContext, networkContainerID := range service.state.ContainerIDByOrchestratorContext { + if networkContainerID == req.NetworkContainerid { + delete(service.state.ContainerIDByOrchestratorContext, orchestratorContext) + break + } + } + } + + service.saveState() service.lock.Unlock() } break diff --git a/cns/restserver/restserver_test.go b/cns/restserver/restserver_test.go index a4affa1a3e..1d9c421a62 100644 --- a/cns/restserver/restserver_test.go +++ b/cns/restserver/restserver_test.go @@ -266,7 +266,36 @@ func TestGetUnhealthyIPAddresses(t *testing.T) { } } -func creatOrUpdateWebAppContainerWithName(t *testing.T, name string, ip string) error { +func setOrchestratorType(t *testing.T, orchestratorType string) error { + var body bytes.Buffer + + info := &cns.SetOrchestratorTypeRequest{OrchestratorType: orchestratorType} + + json.NewEncoder(&body).Encode(info) + + req, err := http.NewRequest(http.MethodPost, cns.SetOrchestratorType, &body) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + var resp cns.Response + err = decodeResponse(w, &resp) + fmt.Printf("Raw response: %+v", w.Body) + if err != nil || resp.ReturnCode != 0 { + t.Errorf("setOrchestratorType failed with response %+v Err:%+v", resp, err) + t.Fatal(err) + } else { + fmt.Printf("setOrchestratorType passed with response %+v Err:%+v", resp, err) + } + + fmt.Printf("setOrchestratorType succeeded with response %+v\n", resp) + return nil +} + +func creatOrUpdateNetworkContainerWithName(t *testing.T, name string, ip string, containerType string) error { var body bytes.Buffer var ipConfig cns.IPConfiguration ipConfig.DNSServers = []string{"8.8.8.8", "8.8.4.4"} @@ -275,11 +304,14 @@ func creatOrUpdateWebAppContainerWithName(t *testing.T, name string, ip string) ipSubnet.IPAddress = ip ipSubnet.PrefixLength = 24 ipConfig.IPSubnet = ipSubnet + podInfo := cns.KubernetesPodInfo{PodName: "testpod", PodNamespace: "testpodnamespace"} + context, _ := json.Marshal(podInfo) info := &cns.CreateNetworkContainerRequest{ Version: "0.1", - NetworkContainerType: "WebApps", + NetworkContainerType: containerType, NetworkContainerid: name, + OrchestratorContext: context, IPConfiguration: ipConfig, PrimaryInterfaceIdentifier: "11.0.0.7", } @@ -335,6 +367,60 @@ func deleteNetworkAdapterWithName(t *testing.T, name string) error { return nil } +func getNetworkCotnainerByContext(t *testing.T, name string) error { + var body bytes.Buffer + var resp cns.GetNetworkContainerResponse + + podInfo := cns.KubernetesPodInfo{PodName: "testpod", PodNamespace: "testpodnamespace"} + podInfoBytes, err := json.Marshal(podInfo) + getReq := &cns.GetNetworkContainerRequest{OrchestratorContext: podInfoBytes} + + json.NewEncoder(&body).Encode(getReq) + req, err := http.NewRequest(http.MethodPost, cns.GetNetworkContainerByOrchestratorContext, &body) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + err = decodeResponse(w, &resp) + if err != nil || resp.Response.ReturnCode != 0 { + t.Errorf("GetNetworkContainerByContext failed with response %+v Err:%+v", resp, err) + t.Fatal(err) + } + + fmt.Printf("**GetNetworkContainerByContext succeded with response %+v, raw:%+v\n", resp, w.Body) + return nil +} + +func getNonExistNetworkCotnainerByContext(t *testing.T, name string) error { + var body bytes.Buffer + var resp cns.GetNetworkContainerResponse + + podInfo := cns.KubernetesPodInfo{PodName: "testpod", PodNamespace: "testpodnamespace"} + podInfoBytes, err := json.Marshal(podInfo) + getReq := &cns.GetNetworkContainerRequest{OrchestratorContext: podInfoBytes} + + json.NewEncoder(&body).Encode(getReq) + req, err := http.NewRequest(http.MethodPost, cns.GetNetworkContainerByOrchestratorContext, &body) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + err = decodeResponse(w, &resp) + if err != nil || resp.Response.ReturnCode != UnknownContainerID { + t.Errorf("GetNetworkContainerByContext unexpected response %+v Err:%+v", resp, err) + t.Fatal(err) + } + + fmt.Printf("**GetNonExistNetworkContainerByContext succeded with response %+v, raw:%+v\n", resp, w.Body) + return nil +} + func getNetworkCotnainerStatus(t *testing.T, name string) error { var body bytes.Buffer var resp cns.GetNetworkContainerStatusResponse @@ -389,17 +475,31 @@ func getInterfaceForContainer(t *testing.T, name string) error { return nil } +func TestSetOrchestratorType(t *testing.T) { + fmt.Println("Test: TestSetOrchestratorType") + + setEnv(t) + + err := setOrchestratorType(t, cns.Kubernetes) + if err != nil { + t.Errorf("setOrchestratorType failed Err:%+v", err) + t.Fatal(err) + } +} + func TestCreateNetworkContainer(t *testing.T) { // requires more than 30 seconds to run fmt.Println("Test: TestCreateNetworkContainer") + setEnv(t) - err := creatOrUpdateWebAppContainerWithName(t, "ethWebApp", "11.0.0.5") + + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps") if err != nil { t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err) t.Fatal(err) } - err = creatOrUpdateWebAppContainerWithName(t, "ethWebApp", "11.0.0.6") + err = creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.6", "WebApps") if err != nil { t.Errorf("Updating interface failed Err:%+v", err) t.Fatal(err) @@ -414,11 +514,48 @@ func TestCreateNetworkContainer(t *testing.T) { } } +func TestGetNetworkContainerByOrchestratorContext(t *testing.T) { + // requires more than 30 seconds to run + fmt.Println("Test: TestGetNetworkContainerByOrchestratorContext") + + setEnv(t) + setOrchestratorType(t, cns.Kubernetes) + + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "AzureContainerInstance") + if err != nil { + t.Errorf("creatOrUpdateNetworkContainerWithName failed Err:%+v", err) + t.Fatal(err) + } + + fmt.Println("Now calling getNetworkCotnainerStatus") + err = getNetworkCotnainerByContext(t, "ethWebApp") + if err != nil { + t.Errorf("TestGetNetworkContainerByOrchestratorContext failed Err:%+v", err) + t.Fatal(err) + } + + fmt.Println("Now calling DeleteNetworkContainer") + + err = deleteNetworkAdapterWithName(t, "ethWebApp") + if err != nil { + t.Errorf("Deleting interface failed Err:%+v", err) + t.Fatal(err) + } + + err = getNonExistNetworkCotnainerByContext(t, "ethWebApp") + if err != nil { + t.Errorf("TestGetNetworkContainerByOrchestratorContext failed Err:%+v", err) + t.Fatal(err) + } +} + func TestGetNetworkContainerStatus(t *testing.T) { // requires more than 30 seconds to run fmt.Println("Test: TestCreateNetworkContainer") + setEnv(t) - err := creatOrUpdateWebAppContainerWithName(t, "ethWebApp", "11.0.0.5") + + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps") if err != nil { t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err) t.Fatal(err) @@ -443,8 +580,10 @@ func TestGetNetworkContainerStatus(t *testing.T) { func TestGetInterfaceForNetworkContainer(t *testing.T) { // requires more than 30 seconds to run fmt.Println("Test: TestCreateNetworkContainer") + setEnv(t) - err := creatOrUpdateWebAppContainerWithName(t, "ethWebApp", "11.0.0.5") + + err := creatOrUpdateNetworkContainerWithName(t, "ethWebApp", "11.0.0.5", "WebApps") if err != nil { t.Errorf("creatOrUpdateWebAppContainerWithName failed Err:%+v", err) t.Fatal(err) From 1a439cbcdae0bcd830c3e15d1f04365f599d690b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Apr 2018 15:08:43 -0700 Subject: [PATCH 11/51] Added testcase to check if state is cleaneup after delete call --- cns/restserver/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 6db8f6f5f5..ed19e524d0 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -20,5 +20,6 @@ const ( CallToHostFailed = 17 UnknownContainerID = 18 UnsupportedOrchestratorType = 19 + NetworkContainerNotExist = 20 UnexpectedError = 99 ) From 0df120d7ea83fe694f128dde2ecab5f1298b922f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Apr 2018 15:17:25 -0700 Subject: [PATCH 12/51] Resued existing errorcode --- cns/restserver/api.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cns/restserver/api.go b/cns/restserver/api.go index ed19e524d0..6db8f6f5f5 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -20,6 +20,5 @@ const ( CallToHostFailed = 17 UnknownContainerID = 18 UnsupportedOrchestratorType = 19 - NetworkContainerNotExist = 20 UnexpectedError = 99 ) From 70e8074794eb895f6f83f845a9da4f46f89d7929 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Apr 2018 17:29:57 -0700 Subject: [PATCH 13/51] Added code in CNI to retrieve network config from CNS based on podname and namespace --- client/cnsclient/cnsclient.go | 83 ++++++++++++++++++++ cni/network/network.go | 144 ++++++++++++++++++++++++++-------- cni/plugin.go | 15 ++++ 3 files changed, 209 insertions(+), 33 deletions(-) create mode 100644 client/cnsclient/cnsclient.go diff --git a/client/cnsclient/cnsclient.go b/client/cnsclient/cnsclient.go new file mode 100644 index 0000000000..842ab62db3 --- /dev/null +++ b/client/cnsclient/cnsclient.go @@ -0,0 +1,83 @@ +package cnsclient + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/log" +) + +// IpamClient specifies a client to connect to Ipam Plugin. +type CNSClient struct { + connectionURL string +} + +const ( + defaultCnsURL = "http://localhost:10090" +) + +// NewCnsClient create a new cns client. +func NewCnsClient(url string) (*CNSClient, error) { + if url == "" { + url = defaultCnsURL + } + return &CNSClient{ + connectionURL: url, + }, nil +} + +// GetNetworkConfiguration Request to get network config. +func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string) (*cns.GetNetworkContainerResponse, error) { + var body bytes.Buffer + httpc := &http.Client{} + + url := cnsClient.connectionURL + cns.GetNetworkContainerByOrchestratorContext + log.Printf("GetNetworkConfiguration url %v", url) + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + podInfoBytes, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Marshalling azure container instance info failed with %v", err) + return nil, err + } + + payload := &cns.GetNetworkContainerRequest{ + OrchestratorContext: podInfoBytes, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Printf("encoding json failed with %v", err) + return nil, err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Printf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return nil, err + } + + if res.StatusCode == 200 { + var resp cns.GetNetworkContainerResponse + err := json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) + return nil, err + } + + if resp.Response.ReturnCode != 0 { + log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) + } + + return &resp, nil + } + + var errMsg string + errMsg = fmt.Sprintf(errMsg, "[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) + log.Printf(errMsg) + return nil, fmt.Errorf(errMsg) +} diff --git a/cni/network/network.go b/cni/network/network.go index fb4b258dc7..492f4676f9 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -6,7 +6,9 @@ package network import ( "net" + "github.com/Azure/azure-container-networking/client/cnsclient" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -14,12 +16,15 @@ import ( "github.com/Azure/azure-container-networking/telemetry" cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) const ( // Plugin name. - name = "azure-vnet" + name = "azure-vnet" + namespacekey = "K8S_POD_NAMESPACE" + podnamekey = "K8S_POD_NAME" ) // NetPlugin represents the CNI network plugin. @@ -116,6 +121,60 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre return "" } +func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { + result := &cniTypesCurr.Result{} + + log.Printf("Convert IPAddress") + ipconfig := networkConfig.IPConfiguration + ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) + + resultIpconfig := &cniTypesCurr.IPConfig{} + + if ipAddr.To4() != nil { + resultIpconfig.Version = "4" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} + } else { + resultIpconfig.Version = "6" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} + } + + resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) + result.IPs = append(result.IPs, resultIpconfig) + + result.DNS.Nameservers = ipconfig.DNSServers + + if networkConfig.Routes == nil && len(networkConfig.Routes) > 0 { + for _, route := range networkConfig.Routes { + _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) + gwIP := net.ParseIP(route.GatewayIPAddress) + result.Routes = append(result.Routes, &types.Route{Dst: *routeIPnet, GW: gwIP}) + } + } else { + gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: resultIpconfig.Address.Mask} + result.Routes = append(result.Routes, &types.Route{Dst: dstIP, GW: gwIP}) + } + + return result +} + +func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, error) { + cnsClient, err := cnsclient.NewCnsClient("") + if err != nil { + log.Printf("Initializing CNS client error %v", err) + return nil, 0, err + } + log.Printf("Network config request") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + if err != nil { + log.Printf("GetNetworkConfiguration failed with %v", err) + return nil, 0, err + } + + log.Printf("Network config received from cns %v", networkConfig) + return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, nil +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -125,6 +184,8 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var result *cniTypesCurr.Result var err error + var epInfo *network.EndpointInfo + var vlanid int log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) @@ -144,19 +205,42 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { networkId := nwCfg.Name endpointId := plugin.GetEndpointID(args) + argsMap := plugin.GetCNIArgs(args.Args) + if argsMap != nil { + log.Printf("Argsmap %v", argsMap) + } + + result, vlanid, err = getContainerNetworkConfiguration(argsMap[namespacekey].(string), argsMap[podnamekey].(string)) + if err != nil { + log.Printf("SetContainerNetworkConfiguration failed with %v", err) + } + + epInfo = &network.EndpointInfo{ + Id: endpointId, + ContainerID: args.ContainerID, + NetNsPath: args.Netns, + IfName: args.IfName, + } + epInfo.Data = make(map[string]interface{}) + + if vlanid != 0 { + epInfo.Data["vlanid"] = vlanid + } + // Check whether the network already exists. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) if err != nil { // Network does not exist. log.Printf("[cni-net] Creating network %v.", networkId) - // Call into IPAM plugin to allocate an address pool for the network. - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate pool: %v", err) - return err + if result == nil { + // Call into IPAM plugin to allocate an address pool for the network. + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + if err != nil { + err = plugin.Errorf("Failed to allocate pool: %v", err) + return err + } } - // Derive the subnet prefix from allocated IP address. ipconfig := result.IPs[0] subnetPrefix := ipconfig.Address @@ -211,35 +295,29 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String()) } else { - // Network already exists. - subnetPrefix := nwInfo.Subnets[0].Prefix.String() - log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) - - // Call into IPAM plugin to allocate an address for the endpoint. - nwCfg.Ipam.Subnet = subnetPrefix - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate address: %v", err) - return err - } - - ipconfig := result.IPs[0] - - // On failure, call into IPAM plugin to release the address. - defer func() { + if result == nil { + // Network already exists. + subnetPrefix := nwInfo.Subnets[0].Prefix.String() + log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) + + // Call into IPAM plugin to allocate an address for the endpoint. + nwCfg.Ipam.Subnet = subnetPrefix + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + err = plugin.Errorf("Failed to allocate address: %v", err) + return err } - }() - } - // Initialize endpoint info. - epInfo := &network.EndpointInfo{ - Id: endpointId, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, + ipconfig := result.IPs[0] + + // On failure, call into IPAM plugin to release the address. + defer func() { + if err != nil { + nwCfg.Ipam.Address = ipconfig.Address.IP.String() + plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + } + }() + } } // Populate addresses. diff --git a/cni/plugin.go b/cni/plugin.go index 25c7e66df3..c60620255e 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "runtime" + "strings" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -168,6 +169,20 @@ func (plugin *Plugin) GetEndpointID(args *cniSkel.CmdArgs) string { return containerID + "-" + args.IfName } +func (plugin *Plugin) GetCNIArgs(args string) map[string]interface{} { + argsMap := make(map[string]interface{}) + + data := strings.Split(args, ";") + for _, pair := range data { + items := strings.Split(pair, "=") + if len(items) > 1 { + argsMap[items[0]] = items[1] + } + } + + return argsMap +} + // Error creates and logs a structured CNI error. func (plugin *Plugin) Error(err error) *cniTypes.Error { var cniErr *cniTypes.Error From 46fc36fd2db28759bb1b661608727c2b09a3b006 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Apr 2018 18:34:28 -0700 Subject: [PATCH 14/51] Added option in cns to not to start cnm by default. --- cns/service/main.go | 132 +++++++++++++++++++++++++------------------- common/config.go | 3 + 2 files changed, 77 insertions(+), 58 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index 35c6a85da6..77baa3302c 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -93,6 +93,13 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: "", }, + { + Name: acn.OptStopAzureVnet, + Shorthand: acn.OptStopAzureVnetAlias, + Description: "Start Azure-CNM if flag is true", + Type: "bool", + DefaultValue: true, + }, { Name: acn.OptVersion, Shorthand: acn.OptVersionAlias, @@ -110,6 +117,7 @@ func printVersion() { // Main is the entry point for CNS. func main() { + var stopcnm = false // Initialize and parse command line arguments. acn.ParseArgs(&args, printVersion) @@ -120,6 +128,7 @@ func main() { logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) + stopcnm = acn.GetArg(acn.OptStopAzureVnet).(bool) vers := acn.GetArg(acn.OptVersion).(bool) if vers { @@ -135,40 +144,22 @@ func main() { // Create a channel to receive unhandled errors from CNS. config.ErrChan = make(chan error, 1) - // Create the key value store. var err error - config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") - if err != nil { - fmt.Printf("Failed to create store: %v\n", err) - return - } - - // Create CNS object. - httpRestService, err := restserver.NewHTTPRestService(&config) - if err != nil { - fmt.Printf("Failed to create CNS object, err:%v.\n", err) - return + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if logDirectory != "" { + log.SetLogDirectory(logDirectory) } - var pluginConfig acn.PluginConfig - pluginConfig.Version = version - - // Create a channel to receive unhandled errors from the plugins. - pluginConfig.ErrChan = make(chan error, 1) - - // Create network plugin. - netPlugin, err := network.NewPlugin(&pluginConfig) + err = log.SetTarget(logTarget) if err != nil { - fmt.Printf("Failed to create network plugin, err:%v.\n", err) + fmt.Printf("Failed to configure logging: %v\n", err) return } - // Create IPAM plugin. - ipamPlugin, err := ipam.NewPlugin(&pluginConfig) - if err != nil { - fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) - return - } + // Log platform information. + log.Printf("Running on %v", platform.GetOSInfo()) err = acn.CreateDirectory(platform.CNMRuntimePath) if err != nil { @@ -177,28 +168,20 @@ func main() { } // Create the key value store. - pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") + + config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { fmt.Printf("Failed to create store: %v\n", err) return } - // Create logging provider. - log.SetName(name) - log.SetLevel(logLevel) - if logDirectory != "" { - log.SetLogDirectory(logDirectory) - } - - err = log.SetTarget(logTarget) + // Create CNS object. + httpRestService, err := restserver.NewHTTPRestService(&config) if err != nil { - fmt.Printf("Failed to configure logging: %v\n", err) + fmt.Printf("Failed to create CNS object, err:%v.\n", err) return } - // Log platform information. - log.Printf("Running on %v", platform.GetOSInfo()) - // Set CNS options. httpRestService.SetOption(acn.OptCnsURL, cnsURL) @@ -211,28 +194,59 @@ func main() { } } - // Set plugin options. - netPlugin.SetOption(acn.OptAPIServerURL, url) + log.Printf("stop cnm val %t", stopcnm) + var netPlugin network.NetPlugin + var ipamPlugin ipam.IpamPlugin + + if !stopcnm { + var pluginConfig acn.PluginConfig + pluginConfig.Version = version - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) + // Create a channel to receive unhandled errors from the plugins. + pluginConfig.ErrChan = make(chan error, 1) - if netPlugin != nil { - log.Printf("Start netplugin\n") - err = netPlugin.Start(&pluginConfig) + // Create network plugin. + netPlugin, err = network.NewPlugin(&pluginConfig) if err != nil { - fmt.Printf("Failed to start network plugin, err:%v.\n", err) + fmt.Printf("Failed to create network plugin, err:%v.\n", err) + return + } + + // Create IPAM plugin. + ipamPlugin, err = ipam.NewPlugin(&pluginConfig) + if err != nil { + fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) return } - } - if ipamPlugin != nil { - err = ipamPlugin.Start(&pluginConfig) + // Create the key value store. + pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") if err != nil { - fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + fmt.Printf("Failed to create store: %v\n", err) return } + + // Set plugin options. + netPlugin.SetOption(acn.OptAPIServerURL, url) + if netPlugin != nil { + log.Printf("Start netplugin\n") + err = netPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start network plugin, err:%v.\n", err) + return + } + } + + ipamPlugin.SetOption(acn.OptEnvironment, environment) + ipamPlugin.SetOption(acn.OptAPIServerURL, url) + ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) + if ipamPlugin != nil { + err = ipamPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + return + } + } } // Relay these incoming signals to OS signal channel. @@ -252,11 +266,13 @@ func main() { httpRestService.Stop() } - if netPlugin != nil { - netPlugin.Stop() - } + if !stopcnm { + if netPlugin != nil { + netPlugin.Stop() + } - if ipamPlugin != nil { - ipamPlugin.Stop() + if ipamPlugin != nil { + ipamPlugin.Stop() + } } } diff --git a/common/config.go b/common/config.go index d670c08a5b..84cc690724 100644 --- a/common/config.go +++ b/common/config.go @@ -42,6 +42,9 @@ const ( OptIpamQueryInterval = "ipam-query-interval" OptIpamQueryIntervalAlias = "i" + OptStopAzureVnet = "stop-azure-cnm" + OptStopAzureVnetAlias = "stopcnm" + // Version. OptVersion = "version" OptVersionAlias = "v" From 37f81147cded55345336dbb586b5a647059ff06a Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Apr 2018 18:53:27 -0700 Subject: [PATCH 15/51] Fixed style and spacing --- client/cnsclient/cnsclient.go | 34 +++++++++++++++++----------------- cni/network/network.go | 7 +++---- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/client/cnsclient/cnsclient.go b/client/cnsclient/cnsclient.go index 842ab62db3..7dcf6c11d9 100644 --- a/client/cnsclient/cnsclient.go +++ b/client/cnsclient/cnsclient.go @@ -32,8 +32,8 @@ func NewCnsClient(url string) (*CNSClient, error) { // GetNetworkConfiguration Request to get network config. func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string) (*cns.GetNetworkContainerResponse, error) { var body bytes.Buffer - httpc := &http.Client{} + httpc := &http.Client{} url := cnsClient.connectionURL + cns.GetNetworkContainerByOrchestratorContext log.Printf("GetNetworkConfiguration url %v", url) @@ -60,24 +60,24 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string return nil, err } - if res.StatusCode == 200 { - var resp cns.GetNetworkContainerResponse - err := json.NewDecoder(res.Body).Decode(&resp) - if err != nil { - log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) - return nil, err - } + if res.StatusCode != 200 { + errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) + log.Printf(errMsg) + return nil, fmt.Errorf(errMsg) + } + + var resp cns.GetNetworkContainerResponse - if resp.Response.ReturnCode != 0 { - log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) - return nil, fmt.Errorf(resp.Response.Message) - } + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) + return nil, err + } - return &resp, nil + if resp.Response.ReturnCode != 0 { + log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) } - var errMsg string - errMsg = fmt.Sprintf(errMsg, "[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) - log.Printf(errMsg) - return nil, fmt.Errorf(errMsg) + return &resp, nil } diff --git a/cni/network/network.go b/cni/network/network.go index 492f4676f9..030371f563 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -123,13 +123,11 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { result := &cniTypesCurr.Result{} + resultIpconfig := &cniTypesCurr.IPConfig{} - log.Printf("Convert IPAddress") ipconfig := networkConfig.IPConfiguration ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) - resultIpconfig := &cniTypesCurr.IPConfig{} - if ipAddr.To4() != nil { resultIpconfig.Version = "4" resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} @@ -164,7 +162,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp log.Printf("Initializing CNS client error %v", err) return nil, 0, err } - log.Printf("Network config request") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) @@ -172,6 +170,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp } log.Printf("Network config received from cns %v", networkConfig) + return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, nil } From 765cac923c6a235e7e72ebf439324d153e31253f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Apr 2018 13:38:19 -0700 Subject: [PATCH 16/51] fixed merge issues --- cni/network/network.go | 5 ----- cns/restserver/restserver.go | 1 - 2 files changed, 6 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 37d61c6bc2..71343585be 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -207,11 +207,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("Argsmap %v", argsMap) } - argsMap := plugin.GetCNIArgs(args.Args) - if argsMap != nil { - log.Printf("Argsmap %v", argsMap) - } - result, vlanid, err = getContainerNetworkConfiguration(argsMap[namespacekey].(string), argsMap[podnamekey].(string)) if err != nil { log.Printf("SetContainerNetworkConfiguration failed with %v", err) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 7217886df5..c9351117d1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -96,7 +96,6 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { serviceState := &httpRestServiceState{} serviceState.Networks = make(map[string]*networkInfo) - serviceState.ContainerIDByOrchestratorInfo = make(map[string]string) return &httpRestService{ Service: service, From 00522aee50875c638551509f5204b2cbb265d02b Mon Sep 17 00:00:00 2001 From: tamilmani1989 Date: Mon, 9 Apr 2018 13:46:50 -0700 Subject: [PATCH 17/51] CNI gets networkconfig from CNS (#117) --- client/cnsclient/cnsclient.go | 84 ++++++++++++++++++++ cni/network/network.go | 143 ++++++++++++++++++++++++++-------- cni/plugin.go | 15 ++++ cns/service/main.go | 118 +++++++++++++++------------- common/config.go | 3 + 5 files changed, 277 insertions(+), 86 deletions(-) create mode 100644 client/cnsclient/cnsclient.go diff --git a/client/cnsclient/cnsclient.go b/client/cnsclient/cnsclient.go new file mode 100644 index 0000000000..55552d159e --- /dev/null +++ b/client/cnsclient/cnsclient.go @@ -0,0 +1,84 @@ +package cnsclient + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/log" +) + +// IpamClient specifies a client to connect to Ipam Plugin. +type CNSClient struct { + connectionURL string +} + +const ( + defaultCnsURL = "http://localhost:10090" +) + +// NewCnsClient create a new cns client. +func NewCnsClient(url string) (*CNSClient, error) { + if url == "" { + url = defaultCnsURL + } + + return &CNSClient{ + connectionURL: url, + }, nil +} + +// GetNetworkConfiguration Request to get network config. +func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string) (*cns.GetNetworkContainerResponse, error) { + var body bytes.Buffer + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.GetNetworkContainerByOrchestratorContext + log.Printf("GetNetworkConfiguration url %v", url) + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + podInfoBytes, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Marshalling azure container instance info failed with %v", err) + return nil, err + } + + payload := &cns.GetNetworkContainerRequest{ + OrchestratorContext: podInfoBytes, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Printf("encoding json failed with %v", err) + return nil, err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Printf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return nil, err + } + + if res.StatusCode != 200 { + errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) + log.Printf(errMsg) + return nil, fmt.Errorf(errMsg) + } + + var resp cns.GetNetworkContainerResponse + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Printf("[Azure CNSClient] Error received while parsing GetNetworkConfiguration response resp:%v err:%v", res.Body, err.Error()) + return nil, err + } + + if resp.Response.ReturnCode != 0 { + log.Printf("[Azure CNSClient] GetNetworkConfiguration received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) + } + + return &resp, nil +} diff --git a/cni/network/network.go b/cni/network/network.go index fb4b258dc7..24b5ed7ccf 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -6,7 +6,9 @@ package network import ( "net" + "github.com/Azure/azure-container-networking/client/cnsclient" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -14,12 +16,15 @@ import ( "github.com/Azure/azure-container-networking/telemetry" cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) const ( // Plugin name. - name = "azure-vnet" + name = "azure-vnet" + namespaceKey = "K8S_POD_NAMESPACE" + podNameKey = "K8S_POD_NAME" ) // NetPlugin represents the CNI network plugin. @@ -116,6 +121,59 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre return "" } +func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { + result := &cniTypesCurr.Result{} + resultIpconfig := &cniTypesCurr.IPConfig{} + + ipconfig := networkConfig.IPConfiguration + ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) + + if ipAddr.To4() != nil { + resultIpconfig.Version = "4" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} + } else { + resultIpconfig.Version = "6" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} + } + + resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) + result.IPs = append(result.IPs, resultIpconfig) + + result.DNS.Nameservers = ipconfig.DNSServers + + if networkConfig.Routes == nil && len(networkConfig.Routes) > 0 { + for _, route := range networkConfig.Routes { + _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) + gwIP := net.ParseIP(route.GatewayIPAddress) + result.Routes = append(result.Routes, &types.Route{Dst: *routeIPnet, GW: gwIP}) + } + } else { + gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: resultIpconfig.Address.Mask} + result.Routes = append(result.Routes, &types.Route{Dst: dstIP, GW: gwIP}) + } + + return result +} + +func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, error) { + cnsClient, err := cnsclient.NewCnsClient("") + if err != nil { + log.Printf("Initializing CNS client error %v", err) + return nil, 0, err + } + + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + if err != nil { + log.Printf("GetNetworkConfiguration failed with %v", err) + return nil, 0, err + } + + log.Printf("Network config received from cns %v", networkConfig) + + return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, nil +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -125,6 +183,8 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var result *cniTypesCurr.Result var err error + var epInfo *network.EndpointInfo + var vlanid int log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) @@ -144,19 +204,42 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { networkId := nwCfg.Name endpointId := plugin.GetEndpointID(args) + argsMap := plugin.GetCNIArgs(args.Args) + if argsMap != nil { + log.Printf("Argsmap %v", argsMap) + } + + result, vlanid, err = getContainerNetworkConfiguration(argsMap[namespaceKey].(string), argsMap[podNameKey].(string)) + if err != nil { + log.Printf("SetContainerNetworkConfiguration failed with %v", err) + } + + epInfo = &network.EndpointInfo{ + Id: endpointId, + ContainerID: args.ContainerID, + NetNsPath: args.Netns, + IfName: args.IfName, + } + epInfo.Data = make(map[string]interface{}) + + if vlanid != 0 { + epInfo.Data["vlanid"] = vlanid + } + // Check whether the network already exists. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) if err != nil { // Network does not exist. log.Printf("[cni-net] Creating network %v.", networkId) - // Call into IPAM plugin to allocate an address pool for the network. - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate pool: %v", err) - return err + if result == nil { + // Call into IPAM plugin to allocate an address pool for the network. + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + if err != nil { + err = plugin.Errorf("Failed to allocate pool: %v", err) + return err + } } - // Derive the subnet prefix from allocated IP address. ipconfig := result.IPs[0] subnetPrefix := ipconfig.Address @@ -211,35 +294,29 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String()) } else { - // Network already exists. - subnetPrefix := nwInfo.Subnets[0].Prefix.String() - log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) - - // Call into IPAM plugin to allocate an address for the endpoint. - nwCfg.Ipam.Subnet = subnetPrefix - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to allocate address: %v", err) - return err - } - - ipconfig := result.IPs[0] - - // On failure, call into IPAM plugin to release the address. - defer func() { + if result == nil { + // Network already exists. + subnetPrefix := nwInfo.Subnets[0].Prefix.String() + log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) + + // Call into IPAM plugin to allocate an address for the endpoint. + nwCfg.Ipam.Subnet = subnetPrefix + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + err = plugin.Errorf("Failed to allocate address: %v", err) + return err } - }() - } - // Initialize endpoint info. - epInfo := &network.EndpointInfo{ - Id: endpointId, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, + ipconfig := result.IPs[0] + + // On failure, call into IPAM plugin to release the address. + defer func() { + if err != nil { + nwCfg.Ipam.Address = ipconfig.Address.IP.String() + plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + } + }() + } } // Populate addresses. diff --git a/cni/plugin.go b/cni/plugin.go index 25c7e66df3..c60620255e 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "runtime" + "strings" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -168,6 +169,20 @@ func (plugin *Plugin) GetEndpointID(args *cniSkel.CmdArgs) string { return containerID + "-" + args.IfName } +func (plugin *Plugin) GetCNIArgs(args string) map[string]interface{} { + argsMap := make(map[string]interface{}) + + data := strings.Split(args, ";") + for _, pair := range data { + items := strings.Split(pair, "=") + if len(items) > 1 { + argsMap[items[0]] = items[1] + } + } + + return argsMap +} + // Error creates and logs a structured CNI error. func (plugin *Plugin) Error(err error) *cniTypes.Error { var cniErr *cniTypes.Error diff --git a/cns/service/main.go b/cns/service/main.go index 35c6a85da6..ede8741b39 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -93,6 +93,13 @@ var args = acn.ArgumentList{ Type: "string", DefaultValue: "", }, + { + Name: acn.OptStopAzureVnet, + Shorthand: acn.OptStopAzureVnetAlias, + Description: "Start Azure-CNM if flag is true", + Type: "bool", + DefaultValue: true, + }, { Name: acn.OptVersion, Shorthand: acn.OptVersionAlias, @@ -110,6 +117,7 @@ func printVersion() { // Main is the entry point for CNS. func main() { + var stopcnm = false // Initialize and parse command line arguments. acn.ParseArgs(&args, printVersion) @@ -120,6 +128,7 @@ func main() { logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) ipamQueryInterval, _ := acn.GetArg(acn.OptIpamQueryInterval).(int) + stopcnm = acn.GetArg(acn.OptStopAzureVnet).(bool) vers := acn.GetArg(acn.OptVersion).(bool) if vers { @@ -135,40 +144,22 @@ func main() { // Create a channel to receive unhandled errors from CNS. config.ErrChan = make(chan error, 1) - // Create the key value store. var err error - config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") - if err != nil { - fmt.Printf("Failed to create store: %v\n", err) - return - } - - // Create CNS object. - httpRestService, err := restserver.NewHTTPRestService(&config) - if err != nil { - fmt.Printf("Failed to create CNS object, err:%v.\n", err) - return + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if logDirectory != "" { + log.SetLogDirectory(logDirectory) } - var pluginConfig acn.PluginConfig - pluginConfig.Version = version - - // Create a channel to receive unhandled errors from the plugins. - pluginConfig.ErrChan = make(chan error, 1) - - // Create network plugin. - netPlugin, err := network.NewPlugin(&pluginConfig) + err = log.SetTarget(logTarget) if err != nil { - fmt.Printf("Failed to create network plugin, err:%v.\n", err) + fmt.Printf("Failed to configure logging: %v\n", err) return } - // Create IPAM plugin. - ipamPlugin, err := ipam.NewPlugin(&pluginConfig) - if err != nil { - fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) - return - } + // Log platform information. + log.Printf("Running on %v", platform.GetOSInfo()) err = acn.CreateDirectory(platform.CNMRuntimePath) if err != nil { @@ -177,28 +168,20 @@ func main() { } // Create the key value store. - pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") + + config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { fmt.Printf("Failed to create store: %v\n", err) return } - // Create logging provider. - log.SetName(name) - log.SetLevel(logLevel) - if logDirectory != "" { - log.SetLogDirectory(logDirectory) - } - - err = log.SetTarget(logTarget) + // Create CNS object. + httpRestService, err := restserver.NewHTTPRestService(&config) if err != nil { - fmt.Printf("Failed to configure logging: %v\n", err) + fmt.Printf("Failed to create CNS object, err:%v.\n", err) return } - // Log platform information. - log.Printf("Running on %v", platform.GetOSInfo()) - // Set CNS options. httpRestService.SetOption(acn.OptCnsURL, cnsURL) @@ -211,23 +194,50 @@ func main() { } } - // Set plugin options. - netPlugin.SetOption(acn.OptAPIServerURL, url) + log.Printf("stop cnm val %t", stopcnm) + var netPlugin network.NetPlugin + var ipamPlugin ipam.IpamPlugin - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) + if !stopcnm { + var pluginConfig acn.PluginConfig + pluginConfig.Version = version - if netPlugin != nil { + // Create a channel to receive unhandled errors from the plugins. + pluginConfig.ErrChan = make(chan error, 1) + + // Create network plugin. + netPlugin, err = network.NewPlugin(&pluginConfig) + if err != nil { + fmt.Printf("Failed to create network plugin, err:%v.\n", err) + return + } + + // Create IPAM plugin. + ipamPlugin, err = ipam.NewPlugin(&pluginConfig) + if err != nil { + fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) + return + } + + // Create the key value store. + pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") + if err != nil { + fmt.Printf("Failed to create store: %v\n", err) + return + } + + // Set plugin options. + netPlugin.SetOption(acn.OptAPIServerURL, url) log.Printf("Start netplugin\n") err = netPlugin.Start(&pluginConfig) if err != nil { fmt.Printf("Failed to start network plugin, err:%v.\n", err) return } - } - if ipamPlugin != nil { + ipamPlugin.SetOption(acn.OptEnvironment, environment) + ipamPlugin.SetOption(acn.OptAPIServerURL, url) + ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) err = ipamPlugin.Start(&pluginConfig) if err != nil { fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) @@ -252,11 +262,13 @@ func main() { httpRestService.Stop() } - if netPlugin != nil { - netPlugin.Stop() - } + if !stopcnm { + if netPlugin != nil { + netPlugin.Stop() + } - if ipamPlugin != nil { - ipamPlugin.Stop() + if ipamPlugin != nil { + ipamPlugin.Stop() + } } } diff --git a/common/config.go b/common/config.go index d670c08a5b..84cc690724 100644 --- a/common/config.go +++ b/common/config.go @@ -42,6 +42,9 @@ const ( OptIpamQueryInterval = "ipam-query-interval" OptIpamQueryIntervalAlias = "i" + OptStopAzureVnet = "stop-azure-cnm" + OptStopAzureVnetAlias = "stopcnm" + // Version. OptVersion = "version" OptVersionAlias = "v" From 3858cec374556fe5a3563c17c92f49c7413fc8a0 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Apr 2018 17:33:03 -0700 Subject: [PATCH 18/51] Fixed issues and tested cnm and cni --- cni/network/network.go | 22 +++++++++++++--------- ipam/azure.go | 4 ++-- network/endpoint_linux.go | 6 ++---- network/manager.go | 1 + network/network_linux.go | 29 ++++++----------------------- 5 files changed, 24 insertions(+), 38 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 57c4f8837a..e061308b13 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -5,6 +5,7 @@ package network import ( "net" + "strconv" "github.com/Azure/azure-container-networking/client/cnsclient" "github.com/Azure/azure-container-networking/cni" @@ -21,9 +22,11 @@ import ( const ( // Plugin name. - name = "azure-vnet" - namespaceKey = "K8S_POD_NAMESPACE" - podNameKey = "K8S_POD_NAME" + name = "azure-vnet" + namespaceKey = "K8S_POD_NAMESPACE" + podNameKey = "K8S_POD_NAME" + vlanIDKey = "vlanid" + dockerNetworkOption = "com.docker.network.generic" ) // NetPlugin represents the CNI network plugin. @@ -162,7 +165,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, err } - networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + networkConfig, err := cnsClient.GetNetworkConfiguration("testpod", "testpodnamespace") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, err @@ -207,11 +210,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("Argsmap %v", argsMap) } - argsMap := plugin.GetCNIArgs(args.Args) - if argsMap != nil { - log.Printf("Argsmap %v", argsMap) - } - result, vlanid, err = getContainerNetworkConfiguration(argsMap[namespaceKey].(string), argsMap[podNameKey].(string)) if err != nil { log.Printf("SetContainerNetworkConfiguration failed with %v", err) @@ -243,6 +241,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } } + // Derive the subnet prefix from allocated IP address. ipconfig := result.IPs[0] subnetPrefix := ipconfig.Address @@ -291,6 +290,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } nwInfo.Options = make(map[string]interface{}) + if vlanid != 0 { + vlanMap := make(map[string]interface{}) + vlanMap[vlanIDKey] = strconv.Itoa(vlanid) + nwInfo.Options[dockerNetworkOption] = vlanMap + } err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { diff --git a/ipam/azure.go b/ipam/azure.go index 4e78854dc1..16c18e7cd6 100644 --- a/ipam/azure.go +++ b/ipam/azure.go @@ -16,8 +16,8 @@ import ( const ( // Host URL to query. - //azureQueryUrl = "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1" - azureQueryUrl = "http://localhost:42424/" + azureQueryUrl = "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1" + //azureQueryUrl = "http://localhost:42424/" // Minimum time interval between consecutive queries. azureQueryInterval = 10 * time.Second diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b0acae129c..092f98aed5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -56,7 +56,6 @@ func (nw *network) setupLinuxBridgeRules(hostIfName string, contIfName string, c } func (nw *network) setupOVSRules(hostIfName string, contIfName string, containerIf *net.Interface, vlanid int, epInfo *EndpointInfo) error { - // Connect host interface to the bridge. var err error log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) @@ -89,8 +88,8 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container mac := nw.extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) - log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) + log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) cmd = fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, nw.extIf.BridgeName, mac, macHex, ofport) _, err = common.ExecuteShellCommand(cmd) @@ -100,7 +99,6 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container } log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) - cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", nw.extIf.BridgeName, containerPort, mac) _, err = common.ExecuteShellCommand(cmd) @@ -113,7 +111,6 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container macAddrHex := strings.Replace(macAddr, ":", "", -1) for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. ipAddrInt := common.IpToInt(ipAddr.IP) log.Printf("[net] Adding ARP reply rule set vlanid %v for container ifname", vlanid, contIfName) @@ -354,6 +351,7 @@ func (nw *network) deleteOVSRules(ep *endpoint) { log.Printf("Error while deleting ovs rule %v error %v", cmd, err) } + // Delete Arp Reply Rules for container log.Printf("[net] Deleting ARP reply rule set vlanid %v for container port", ep.VlanID, containerPort) cmd = fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", nw.extIf.BridgeName, containerPort) diff --git a/network/manager.go b/network/manager.go index 80cd807b75..b2d5cb8b21 100644 --- a/network/manager.go +++ b/network/manager.go @@ -249,6 +249,7 @@ func (nm *networkManager) CreateEndpoint(networkId string, epInfo *EndpointInfo) if nw.VlanId != 0 { if epInfo.Data["vlanid"] == nil { + log.Printf("overriding endpoint vlanid with network vlanid") epInfo.Data["vlanid"] = nw.VlanId } } diff --git a/network/network_linux.go b/network/network_linux.go index 7643c105aa..00e1f65ebe 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -26,6 +26,8 @@ const ( virtualMacAddress = "12:34:56:78:9a:bc" genericData = "com.docker.network.generic" + + vlanIDKey = "vlanid" ) // Linux implementation of route. @@ -36,16 +38,17 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Connect the external interface. var vlanid int opt, _ := nwInfo.Options[genericData].(map[string]interface{}) - log.Printf("opt %v", opt) + log.Printf("opt %v options %v", opt, nwInfo.Options) switch nwInfo.Mode { case opModeTunnel: fallthrough case opModeBridge: - if opt != nil && opt["vlanid"] != nil { + if opt != nil && opt[vlanIDKey] != nil { var err error + log.Printf("create ovs bridge") - vlanid, err = strconv.Atoi(opt["vlanid"].(string)) + vlanid, err = strconv.Atoi(opt[vlanIDKey].(string)) if err != nil { log.Printf("Error while converting vlanid from string to integer") } @@ -210,8 +213,6 @@ func (nm *networkManager) addBridgeRules(extIf *externalInterface, hostIf *net.I func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Interface, bridgeName string) error { primary := extIf.IPAddresses[0].IP.String() - //primaryInt := common.IpToInt(extIf.IPAddresses[0].IP) - mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) @@ -221,21 +222,6 @@ func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Inte log.Printf("[net] Adding SNAT rule failed with error %v", err) return err } - // Add ARP reply rule for host primary IP address. - // ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric - // doesn't respond to ARP requests from the VM for its own primary IP address. - - // log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) - - // cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,arp_tpa=%s,actions='load:0x2->NXM_OF_ARP_OP[], - // move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, - // move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], - // load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],IN_PORT'`, bridgeName, primary, mac, macHex, primaryInt) - // _, err = common.ExecuteShellCommand(cmd) - // if err != nil { - // log.Printf("[net] Adding ARP reply rule failed with error %v", err) - // return err - // } log.Printf("[net] Get ovs port for interface %v.", hostIf.Name) cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIf.Name) @@ -372,7 +358,6 @@ func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *Net // Connect the external interface to the bridge. log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) err = setOVSMaster(hostIf.Name, bridgeName) - //err = netlink.SetLinkMaster(hostIf.Name, bridgeName) if err != nil { return err } @@ -416,7 +401,6 @@ func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) error log.Printf("[net] Disconnecting interface %v.", extIf.Name) // Disconnect external interface from its bridge. - //err := netlink.SetLinkMaster(extIf.Name, "") cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", extIf.BridgeName, extIf.Name) _, err := common.ExecuteShellCommand(cmd) if err != nil { @@ -424,7 +408,6 @@ func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) error } // Delete the bridge. - // err = netlink.DeleteLink(extIf.BridgeName) err = nm.deleteOVSBridge(extIf.BridgeName) if err != nil { log.Printf("[net] Failed to delete bridge %v, err:%v.", extIf.BridgeName, err) From adde6f2f7eb706068c5366e4a10917f4ed610cc5 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Apr 2018 17:37:02 -0700 Subject: [PATCH 19/51] Added comments --- network/endpoint_linux.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 092f98aed5..3c0f6e4128 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,6 +58,7 @@ func (nw *network) setupLinuxBridgeRules(hostIfName string, contIfName string, c func (nw *network) setupOVSRules(hostIfName string, contIfName string, containerIf *net.Interface, vlanid int, epInfo *EndpointInfo) error { var err error + // Set OVS Bridge as master to host veth pair of container interface log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) cmd := fmt.Sprintf("ovs-vsctl add-port %v %v tag=%v", nw.extIf.BridgeName, hostIfName, vlanid) _, err = common.ExecuteShellCommand(cmd) @@ -89,6 +90,7 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container mac := nw.extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) + // Arp SNAT Rule log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) cmd = fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, nw.extIf.BridgeName, mac, macHex, ofport) @@ -98,6 +100,7 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container return err } + // IP SNAT Rule log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", nw.extIf.BridgeName, containerPort, mac) @@ -113,6 +116,8 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container for _, ipAddr := range epInfo.IPAddresses { ipAddrInt := common.IpToInt(ipAddr.IP) + // Add Arp Reply Rules + // Set Vlan id on arp request packet and forward it to table 1 log.Printf("[net] Adding ARP reply rule set vlanid %v for container ifname", vlanid, contIfName) cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, nw.extIf.BridgeName, containerPort, vlanid) @@ -122,6 +127,7 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container return err } + // If arp fields matches, set arp reply rule for the request log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[], move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, @@ -134,7 +140,7 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container return err } - // Add MAC address translation rule. + // Add IP DNAT rule based on dst ip and vlanid log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, ofport, macAddr) From 679dc94a2eac6b4b29feb76d996514d3fad176fd Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Apr 2018 17:42:58 -0700 Subject: [PATCH 20/51] Fixed hardcoded values --- cni/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index e061308b13..57a10181eb 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -165,7 +165,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, err } - networkConfig, err := cnsClient.GetNetworkConfiguration("testpod", "testpodnamespace") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, err From fad692ce38698aacd9c7b6c4c41124fc5a5410b9 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 9 Apr 2018 17:59:45 -0700 Subject: [PATCH 21/51] removed unnecessary checks --- cns/service/main.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index f97ae33b19..665d0353d5 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -253,24 +253,20 @@ func main() { // Set plugin options. netPlugin.SetOption(acn.OptAPIServerURL, url) - if netPlugin != nil { - log.Printf("Start netplugin\n") - err = netPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to start network plugin, err:%v.\n", err) - return - } + log.Printf("Start netplugin\n") + err = netPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start network plugin, err:%v.\n", err) + return } ipamPlugin.SetOption(acn.OptEnvironment, environment) ipamPlugin.SetOption(acn.OptAPIServerURL, url) ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) - if ipamPlugin != nil { - err = ipamPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) - return - } + err = ipamPlugin.Start(&pluginConfig) + if err != nil { + fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + return } } From 82486a8ba0b989bdaa1a2fd189ff5da5500dc9e6 Mon Sep 17 00:00:00 2001 From: tamilmani1989 Date: Thu, 12 Apr 2018 14:48:09 -0700 Subject: [PATCH 22/51] check for network container type (#121) --- cns/dnccontract.go | 1 + cns/restserver/restserver.go | 58 ++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/cns/dnccontract.go b/cns/dnccontract.go index c4cde79e36..21d6e386b9 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -15,6 +15,7 @@ const ( // NetworkContainer Types const ( AzureContainerInstance = "AzureContainerInstance" + WebApps = "WebApps" ) // Orchestrator Types diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index c9351117d1..59b28959cb 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -906,14 +906,14 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr switch r.Method { case "POST": - nc := service.networkContainer - err := nc.Create(req) - if err != nil { - returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateOrUpdateNetworkContainer failed %v", err.Error()) - returnCode = UnexpectedError - break + if req.NetworkContainerType == cns.WebApps { + nc := service.networkContainer + if err := nc.Create(req); err != nil { + returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateOrUpdateNetworkContainer failed %v", err.Error()) + returnCode = UnexpectedError + break + } } - returnCode, returnMessage = service.saveNetworkContainerGoalState(req) default: @@ -1035,31 +1035,39 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r switch r.Method { case "POST": - nc := service.networkContainer - err := nc.Delete(req.NetworkContainerid) + var containerStatus containerstatus + var ok bool - if err != nil { - returnMessage = fmt.Sprintf("[Azure CNS] Error. DeleteNetworkContainer failed %v", err.Error()) - returnCode = UnexpectedError + if containerStatus, ok = service.state.ContainerStatus[req.NetworkContainerid]; !ok { + log.Printf("Not able to retrieve network container details for this container id %v", req.NetworkContainerid) break - } else { - service.lock.Lock() - if service.state.ContainerStatus != nil { - delete(service.state.ContainerStatus, req.NetworkContainerid) + } + + if containerStatus.CreateNetworkContainerRequest.NetworkContainerType == cns.WebApps { + nc := service.networkContainer + if err := nc.Delete(req.NetworkContainerid); err != nil { + returnMessage = fmt.Sprintf("[Azure CNS] Error. DeleteNetworkContainer failed %v", err.Error()) + returnCode = UnexpectedError + break } + } - if service.state.ContainerIDByOrchestratorContext != nil { - for orchestratorContext, networkContainerID := range service.state.ContainerIDByOrchestratorContext { - if networkContainerID == req.NetworkContainerid { - delete(service.state.ContainerIDByOrchestratorContext, orchestratorContext) - break - } + service.lock.Lock() + if service.state.ContainerStatus != nil { + delete(service.state.ContainerStatus, req.NetworkContainerid) + } + + if service.state.ContainerIDByOrchestratorContext != nil { + for orchestratorContext, networkContainerID := range service.state.ContainerIDByOrchestratorContext { + if networkContainerID == req.NetworkContainerid { + delete(service.state.ContainerIDByOrchestratorContext, orchestratorContext) + break } } - - service.saveState() - service.lock.Unlock() } + + service.saveState() + service.lock.Unlock() break default: returnMessage = "[Azure CNS] Error. DeleteNetworkContainer did not receive a POST." From ce6a3571ac631ddd172e1017fb9479e9fe3b6e87 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 12 Apr 2018 18:37:42 -0700 Subject: [PATCH 23/51] Fixed review comments --- cns/service/main.go | 50 +++++++++------------------------------ ipam/azure.go | 1 - network/endpoint_linux.go | 5 ++-- network/network_linux.go | 37 +++++++++-------------------- 4 files changed, 24 insertions(+), 69 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index 665d0353d5..b9c2054446 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -163,7 +163,7 @@ func main() { err = acn.CreateDirectory(platform.CNMRuntimePath) if err != nil { - fmt.Printf("Failed to create File Store directory Error:%v", err.Error()) + log.Printf("Failed to create File Store directory Error:%v", err.Error()) return } @@ -171,14 +171,14 @@ func main() { config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { - fmt.Printf("Failed to create store: %v\n", err) + log.Printf("Failed to create store: %v\n", err) return } // Create CNS object. httpRestService, err := restserver.NewHTTPRestService(&config) if err != nil { - fmt.Printf("Failed to create CNS object, err:%v.\n", err) + log.Printf("Failed to create CNS object, err:%v.\n", err) return } @@ -189,12 +189,11 @@ func main() { if httpRestService != nil { err = httpRestService.Start(&config) if err != nil { - fmt.Printf("Failed to start CNS, err:%v.\n", err) + log.Printf("Failed to start CNS, err:%v.\n", err) return } } - log.Printf("stop cnm val %t", stopcnm) var netPlugin network.NetPlugin var ipamPlugin ipam.IpamPlugin @@ -208,64 +207,37 @@ func main() { // Create network plugin. netPlugin, err = network.NewPlugin(&pluginConfig) if err != nil { - fmt.Printf("Failed to create network plugin, err:%v.\n", err) + log.Printf("Failed to create network plugin, err:%v.\n", err) return } // Create IPAM plugin. ipamPlugin, err = ipam.NewPlugin(&pluginConfig) if err != nil { - fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) + log.Printf("Failed to create IPAM plugin, err:%v.\n", err) return } // Create the key value store. pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") if err != nil { - fmt.Printf("Failed to create store: %v\n", err) + log.Printf("Failed to create store: %v\n", err) return } // Set plugin options. netPlugin.SetOption(acn.OptAPIServerURL, url) log.Printf("Start netplugin\n") - err = netPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to create network plugin, err:%v.\n", err) - return - } - - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) - err = ipamPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) - return - } - - // Create the key value store. - pluginConfig.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + pluginName + ".json") - if err != nil { - fmt.Printf("Failed to create store: %v\n", err) - return - } - - // Set plugin options. - netPlugin.SetOption(acn.OptAPIServerURL, url) - log.Printf("Start netplugin\n") - err = netPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to start network plugin, err:%v.\n", err) + if err := netPlugin.Start(&pluginConfig); err != nil { + log.Printf("Failed to create network plugin, err:%v.\n", err) return } ipamPlugin.SetOption(acn.OptEnvironment, environment) ipamPlugin.SetOption(acn.OptAPIServerURL, url) ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) - err = ipamPlugin.Start(&pluginConfig) - if err != nil { - fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + if err := ipamPlugin.Start(&pluginConfig); err != nil { + log.Printf("Failed to create IPAM plugin, err:%v.\n", err) return } } diff --git a/ipam/azure.go b/ipam/azure.go index 16c18e7cd6..62194e7ecb 100644 --- a/ipam/azure.go +++ b/ipam/azure.go @@ -17,7 +17,6 @@ import ( const ( // Host URL to query. azureQueryUrl = "http://169.254.169.254/machine/plugins?comp=nmagent&type=getinterfaceinfov1" - //azureQueryUrl = "http://localhost:42424/" // Minimum time interval between consecutive queries. azureQueryInterval = 10 * time.Second diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 3c0f6e4128..806921e753 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -86,7 +86,6 @@ func (nw *network) setupOVSRules(hostIfName string, contIfName string, container } ofport = strings.Trim(ofport, "\n") - mac := nw.extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) @@ -380,14 +379,14 @@ func (nw *network) deleteOVSRules(ep *endpoint) { nw.extIf.BridgeName, ep.IPAddresses[0].IP.String(), ep.VlanID, ofPort) _, err = common.ExecuteShellCommand(cmd) if err != nil { - log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) + log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) } // Delete port fromk ovs bridge cmd = fmt.Sprintf("ovs-vsctl del-port %v %v", nw.extIf.BridgeName, ep.HostIfName) _, err = common.ExecuteShellCommand(cmd) if err != nil { - log.Printf("[net] Adding port failed with error %v", err) + log.Printf("[net] Deleting port failed with error %v", err) } } diff --git a/network/network_linux.go b/network/network_linux.go index 00e1f65ebe..cf38267d4c 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -56,8 +56,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt nm.createOVSNetwork(extIf, nwInfo) } else { log.Printf("create linux bridge") - err := nm.connectExternalInterface(extIf, nwInfo) - if err != nil { + if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { return nil, err } } @@ -321,8 +320,7 @@ func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *Net // Create the bridge. log.Printf("[net] Creating bridge %v.", bridgeName) - err = nm.createOVSBridge(bridgeName) - if err != nil { + if err := nm.createOVSBridge(bridgeName); err != nil { return err } @@ -343,61 +341,52 @@ func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *Net } // Save host IP configuration. - err = nm.saveIPConfig(hostIf, extIf) - if err != nil { + if err := nm.saveIPConfig(hostIf, extIf); err != nil { log.Printf("[net] Failed to save IP configuration for interface %v: %v.", hostIf.Name, err) } // External interface down. log.Printf("[net] Setting link %v state down.", hostIf.Name) - err = netlink.SetLinkState(hostIf.Name, false) - if err != nil { + if err := netlink.SetLinkState(hostIf.Name, false); err != nil { return err } // Connect the external interface to the bridge. log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) - err = setOVSMaster(hostIf.Name, bridgeName) - if err != nil { + if err := setOVSMaster(hostIf.Name, bridgeName); err != nil { return err } // Add the bridge rules. - err = nm.addOVSRules(extIf, hostIf, bridgeName) - if err != nil { + if err := nm.addOVSRules(extIf, hostIf, bridgeName); err != nil { return err } // External interface up. log.Printf("[net] Setting link %v state up.", hostIf.Name) - err = netlink.SetLinkState(hostIf.Name, true) - if err != nil { + if err := netlink.SetLinkState(hostIf.Name, true); err != nil { return err } // Bridge up. log.Printf("[net] Setting link %v state up.", bridgeName) - err = netlink.SetLinkState(bridgeName, true) - if err != nil { + if err := netlink.SetLinkState(bridgeName, true); err != nil { return err } // Apply IP configuration to the bridge for host traffic. - err = nm.applyIPConfig(extIf, bridge) - if err != nil { + if err := nm.applyIPConfig(extIf, bridge); err != nil { log.Printf("[net] Failed to apply interface IP configuration: %v.", err) } extIf.BridgeName = bridgeName - err = nil - log.Printf("[net] Connected interface %v to bridge %v.", extIf.Name, extIf.BridgeName) return nil } // DisconnectExternalInterface disconnects a host interface from OVS bridge. -func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) error { +func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) { log.Printf("[net] Disconnecting interface %v.", extIf.Name) // Disconnect external interface from its bridge. @@ -426,8 +415,6 @@ func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) error extIf.Routes = nil log.Printf("[net] Disconnected interface %v.", extIf.Name) - - return nil } // ConnectExternalInterface connects the given host interface to a bridge. @@ -551,7 +538,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } // DisconnectExternalInterface disconnects a host interface from its bridge. -func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) error { +func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) { log.Printf("[net] Disconnecting interface %v.", extIf.Name) // Delete bridge rules set on the external interface. @@ -582,6 +569,4 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) extIf.Routes = nil log.Printf("[net] Disconnected interface %v.", extIf.Name) - - return nil } From c77597ad7199b7e6262ef67d146c316b14652a72 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 25 Apr 2018 19:30:40 -0700 Subject: [PATCH 24/51] Redesigned OVS code and tested --- network/endpoint_linux.go | 233 +------------ network/linuxbridge_endpointclient_linux.go | 98 ++++++ network/linuxbridge_networkclient_linux.go | 109 +++++++ network/manager.go | 14 + network/network_linux.go | 342 +++----------------- network/ovs_endpointclient_linux.go | 110 +++++++ network/ovs_networkclient_linux.go | 70 ++++ ovsrules/ovsrules.go | 202 ++++++++++++ 8 files changed, 654 insertions(+), 524 deletions(-) create mode 100644 network/linuxbridge_endpointclient_linux.go create mode 100644 network/linuxbridge_networkclient_linux.go create mode 100644 network/ovs_endpointclient_linux.go create mode 100644 network/ovs_networkclient_linux.go create mode 100644 ovsrules/ovsrules.go diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 806921e753..51592f39d4 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -8,10 +8,7 @@ package network import ( "fmt" "net" - "strings" - "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" ) @@ -27,138 +24,13 @@ const ( containerInterfacePrefix = "eth" ) -func (nw *network) setupLinuxBridgeRules(hostIfName string, contIfName string, containerIf *net.Interface, epInfo *EndpointInfo) error { - var err error - - log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) - err = netlink.SetLinkMaster(hostIfName, nw.extIf.BridgeName) - if err != nil { - return err - } - - for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(containerIf.HardwareAddr), ebtables.Append) - if err != nil { - return err - } - - // Add MAC address translation rule. - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.String(), contIfName) - err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) - if err != nil { - return err - } - } - - return nil -} - -func (nw *network) setupOVSRules(hostIfName string, contIfName string, containerIf *net.Interface, vlanid int, epInfo *EndpointInfo) error { - var err error - - // Set OVS Bridge as master to host veth pair of container interface - log.Printf("[net] Setting link %v master %v.", hostIfName, nw.extIf.BridgeName) - cmd := fmt.Sprintf("ovs-vsctl add-port %v %v tag=%v", nw.extIf.BridgeName, hostIfName, vlanid) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding port failed with error %v", err) - return err - } - - log.Printf("[net] Get ovs port for interface %v.", hostIfName) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIfName) - containerPort, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - return err - } - - containerPort = strings.Trim(containerPort, "\n") - - log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) - ofport, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - return err - } - - ofport = strings.Trim(ofport, "\n") - mac := nw.extIf.MacAddress.String() - macHex := strings.Replace(mac, ":", "", -1) - - // Arp SNAT Rule - log.Printf("[net] OVS - Adding ARP SNAT rule for egress traffic on %v.", hostIfName) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, - load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, nw.extIf.BridgeName, mac, macHex, ofport) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP SNAT rule failed with error %v", err) - return err - } - - // IP SNAT Rule - log.Printf("[net] OVS - Adding IP SNAT rule for egress traffic on %v.", hostIfName) - cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", - nw.extIf.BridgeName, containerPort, mac) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding IP SNAT rule failed with error %v", err) - return err - } - - macAddr := containerIf.HardwareAddr.String() - macAddrHex := strings.Replace(macAddr, ":", "", -1) - - for _, ipAddr := range epInfo.IPAddresses { - ipAddrInt := common.IpToInt(ipAddr.IP) - - // Add Arp Reply Rules - // Set Vlan id on arp request packet and forward it to table 1 - log.Printf("[net] Adding ARP reply rule set vlanid %v for container ifname", vlanid, contIfName) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, - nw.extIf.BridgeName, containerPort, vlanid) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP reply rule failed with error %v", err) - return err - } - - // If arp fields matches, set arp reply rule for the request - log.Printf("[net] Adding ARP reply rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[], - move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, - move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], - load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, - nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, macAddr, macAddrHex, ipAddrInt) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding ARP reply rule failed with error %v", err) - return err - } - - // Add IP DNAT rule based on dst ip and vlanid - log.Printf("[net] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), contIfName) - cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", - nw.extIf.BridgeName, ipAddr.IP.String(), vlanid, ofport, macAddr) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding MAC DNAT rule failed with error %v", err) - return err - } - } - - return nil -} - // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var containerIf *net.Interface var ns *Namespace var ep *endpoint var err error + var epClient EndpointClient var vlanid int = 0 if epInfo.Data != nil { @@ -217,10 +89,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } if vlanid != 0 { - nw.setupOVSRules(hostIfName, contIfName, containerIf, vlanid, epInfo) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), containerIf.HardwareAddr.String(), vlanid) } else { - nw.setupLinuxBridgeRules(hostIfName, contIfName, containerIf, epInfo) + epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress, containerIf.HardwareAddr, nw.Mode) } + + epClient.AddEndpointRules(epInfo) // // Container network interface setup. // @@ -330,93 +204,23 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { func (nw *network) deleteOVSRules(ep *endpoint) { - log.Printf("[net] Get ovs port for interface %v.", ep.HostIfName) - cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", ep.HostIfName) - containerPort, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - } - - log.Printf("[net] Get ovs port for interface %v.", nw.extIf.Name) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", nw.extIf.Name) - ofPort, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - } - - containerPort = strings.Trim(containerPort, "\n") - ofPort = strings.Trim(ofPort, "\n") - - // Delete IP SNAT - log.Printf("[net] Deleting IP SNAT for port %v", containerPort) - cmd = fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", - nw.extIf.BridgeName, containerPort) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("Error while deleting ovs rule %v error %v", cmd, err) - } - - // Delete Arp Reply Rules for container - log.Printf("[net] Deleting ARP reply rule set vlanid %v for container port", ep.VlanID, containerPort) - cmd = fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", - nw.extIf.BridgeName, containerPort) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Deleting ARP reply rule failed with error %v", err) - } - - log.Printf("[net] Deleting ARP reply rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) - cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", - nw.extIf.BridgeName, ep.IPAddresses[0].IP.String(), ep.VlanID) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Deleting ARP reply rule failed with error %v", err) - } - - // Add MAC address translation rule. - log.Printf("[net] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) - cmd = fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", - nw.extIf.BridgeName, ep.IPAddresses[0].IP.String(), ep.VlanID, ofPort) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) - } - - // Delete port fromk ovs bridge - cmd = fmt.Sprintf("ovs-vsctl del-port %v %v", nw.extIf.BridgeName, ep.HostIfName) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Deleting port failed with error %v", err) - } } // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(ep *endpoint) error { + var epClient EndpointClient + // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { - log.Printf("Delete ovs rules and remove port") - nw.deleteOVSRules(ep) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", ep.VlanID) } else { - // Delete rules for IP addresses on the container interface. - for _, ipAddr := range ep.IPAddresses { - // Delete ARP reply rule. - log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err := ebtables.SetArpReply(ipAddr.IP, nw.getArpReplyAddress(ep.MacAddress), ebtables.Delete) - if err != nil { - log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) - } - - // Delete MAC address translation rule. - log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, ep.MacAddress, ebtables.Delete) - if err != nil { - log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err) - } - } + epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress, nil, nw.Mode) } + epClient.DeleteEndpointRules(ep) + log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) err := netlink.DeleteLink(ep.HostIfName) if err != nil { @@ -427,21 +231,6 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { return nil } -// getArpReplyAddress returns the MAC address to use in ARP replies. -func (nw *network) getArpReplyAddress(epMacAddress net.HardwareAddr) net.HardwareAddr { - var macAddress net.HardwareAddr - - if nw.Mode == opModeTunnel { - // In tunnel mode, resolve all IP addresses to the virtual MAC address for hairpinning. - macAddress, _ = net.ParseMAC(virtualMacAddress) - } else { - // Otherwise, resolve to actual MAC address. - macAddress = epMacAddress - } - - return macAddress -} - // getInfoImpl returns information about the endpoint. func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { } diff --git a/network/linuxbridge_endpointclient_linux.go b/network/linuxbridge_endpointclient_linux.go new file mode 100644 index 0000000000..a11d545af6 --- /dev/null +++ b/network/linuxbridge_endpointclient_linux.go @@ -0,0 +1,98 @@ +package network + +import ( + "log" + "net" + + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/netlink" +) + +type LinuxBridgeEndpointClient struct { + bridgeName string + hostPrimaryIfName string + hostVethName string + hostPrimaryMac net.HardwareAddr + containerMac net.HardwareAddr + mode string +} + +func NewLinuxBridgeEndpointClient( + bridgeName string, + hostPrimaryIfName string, + hostVethName string, + hostPrimaryMac net.HardwareAddr, + containerMac net.HardwareAddr, + mode string, +) *LinuxBridgeEndpointClient { + + client := &LinuxBridgeEndpointClient{ + bridgeName: bridgeName, + hostPrimaryIfName: hostPrimaryIfName, + hostVethName: hostVethName, + hostPrimaryMac: hostPrimaryMac, + containerMac: containerMac, + mode: mode, + } + + return client +} + +func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { + var err error + + log.Printf("[net] Setting link %v master %v.", client.hostVethName, client.bridgeName) + if err := netlink.SetLinkMaster(client.hostVethName, client.bridgeName); err != nil { + return err + } + + for _, ipAddr := range epInfo.IPAddresses { + // Add ARP reply rule. + log.Printf("[net] Adding ARP reply rule for IP address %v", ipAddr.String()) + if err = ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(client.containerMac), ebtables.Append); err != nil { + return err + } + + // Add MAC address translation rule. + log.Printf("[net] Adding MAC DNAT rule for IP address %v", ipAddr.String()) + if err := ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, client.containerMac, ebtables.Append); err != nil { + return err + } + } + + return nil +} + +func (client *LinuxBridgeEndpointClient) DeleteEndpointRules(ep *endpoint) { + // Delete rules for IP addresses on the container interface. + for _, ipAddr := range ep.IPAddresses { + // Delete ARP reply rule. + log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) + err := ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(ep.MacAddress), ebtables.Delete) + if err != nil { + log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) + } + + // Delete MAC address translation rule. + log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id) + err = ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, ep.MacAddress, ebtables.Delete) + if err != nil { + log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err) + } + } +} + +// getArpReplyAddress returns the MAC address to use in ARP replies. +func (client *LinuxBridgeEndpointClient) getArpReplyAddress(epMacAddress net.HardwareAddr) net.HardwareAddr { + var macAddress net.HardwareAddr + + if client.mode == opModeTunnel { + // In tunnel mode, resolve all IP addresses to the virtual MAC address for hairpinning. + macAddress, _ = net.ParseMAC(virtualMacAddress) + } else { + // Otherwise, resolve to actual MAC address. + macAddress = epMacAddress + } + + return macAddress +} diff --git a/network/linuxbridge_networkclient_linux.go b/network/linuxbridge_networkclient_linux.go new file mode 100644 index 0000000000..f146518e95 --- /dev/null +++ b/network/linuxbridge_networkclient_linux.go @@ -0,0 +1,109 @@ +package network + +import ( + "net" + + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" +) + +type LinuxBridgeClient struct { + bridgeName string + hostInterfaceName string + mode string +} + +func NewLinuxBridgeClient(bridgeName string, hostInterfaceName string, mode string) *LinuxBridgeClient { + client := &LinuxBridgeClient{ + bridgeName: bridgeName, + mode: mode, + hostInterfaceName: hostInterfaceName, + } + + return client +} + +func (client *LinuxBridgeClient) CreateBridge() error { + log.Printf("[net] Creating bridge %v.", client.bridgeName) + + link := netlink.BridgeLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_BRIDGE, + Name: client.bridgeName, + }, + } + + return netlink.AddLink(&link) +} + +func (client *LinuxBridgeClient) DeleteBridge() error { + + // Disconnect external interface from its bridge. + err := netlink.SetLinkMaster(client.hostInterfaceName, "") + if err != nil { + log.Printf("[net] Failed to disconnect interface %v from bridge, err:%v.", client.hostInterfaceName, err) + } + + // Delete the bridge. + err = netlink.DeleteLink(client.bridgeName) + if err != nil { + log.Printf("[net] Failed to delete bridge %v, err:%v.", client.bridgeName, err) + } + + return nil +} + +func (client *LinuxBridgeClient) AddBridgeRules(extIf *externalInterface) error { + + hostIf, err := net.InterfaceByName(client.hostInterfaceName) + if err != nil { + return err + } + + // Add SNAT rule to translate container egress traffic. + log.Printf("[net] Adding SNAT rule for egress traffic on %v.", client.hostInterfaceName) + if err := ebtables.SetSnatForInterface(client.hostInterfaceName, hostIf.HardwareAddr, ebtables.Append); err != nil { + return err + } + + // Add ARP reply rule for host primary IP address. + // ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric + // doesn't respond to ARP requests from the VM for its own primary IP address. + primary := extIf.IPAddresses[0].IP + log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) + if err := ebtables.SetArpReply(primary, hostIf.HardwareAddr, ebtables.Append); err != nil { + return err + } + + // Add DNAT rule to forward ARP replies to container interfaces. + log.Printf("[net] Adding DNAT rule for ingress ARP traffic on interface %v.", client.hostInterfaceName) + if err := ebtables.SetDnatForArpReplies(client.hostInterfaceName, ebtables.Append); err != nil { + return err + } + + // Enable VEPA for host policy enforcement if necessary. + if client.mode == opModeTunnel { + log.Printf("[net] Enabling VEPA mode for %v.", client.hostInterfaceName) + if err := ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append); err != nil { + return err + } + } + + return nil +} + +func (client *LinuxBridgeClient) DeleteBridgeRules(extIf *externalInterface) { + ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Delete) + ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) + ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) + ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) +} + +func (client *LinuxBridgeClient) SetBridgeMasterToHostInterface() error { + return netlink.SetLinkMaster(client.hostInterfaceName, client.bridgeName) +} + +func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error { + return netlink.SetLinkHairpin(client.hostInterfaceName, enable) +} diff --git a/network/manager.go b/network/manager.go index b2d5cb8b21..289b37719c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -18,6 +18,20 @@ const ( storeKey = "Network" ) +type NetworkClient interface { + CreateBridge() error + DeleteBridge() error + AddBridgeRules(extIf *externalInterface) error + DeleteBridgeRules(extIf *externalInterface) + SetBridgeMasterToHostInterface() error + SetHairpinOnHostInterface(bool) error +} + +type EndpointClient interface { + AddEndpointRules(epInfo *EndpointInfo) error + DeleteEndpointRules(ep *endpoint) +} + // NetworkManager manages the set of container networking resources. type networkManager struct { Version string diff --git a/network/network_linux.go b/network/network_linux.go index cf38267d4c..db265e2765 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -11,8 +11,6 @@ import ( "strconv" "strings" - "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "golang.org/x/sys/unix" @@ -44,21 +42,13 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt case opModeTunnel: fallthrough case opModeBridge: - if opt != nil && opt[vlanIDKey] != nil { - var err error - - log.Printf("create ovs bridge") - vlanid, err = strconv.Atoi(opt[vlanIDKey].(string)) - if err != nil { - log.Printf("Error while converting vlanid from string to integer") - } + log.Printf("create bridge") + if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { + return nil, err + } - nm.createOVSNetwork(extIf, nwInfo) - } else { - log.Printf("create linux bridge") - if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { - return nil, err - } + if opt != nil && opt[vlanIDKey] != nil { + vlanid, _ = strconv.Atoi(opt[vlanIDKey].(string)) } default: @@ -79,16 +69,17 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // DeleteNetworkImpl deletes an existing container network. func (nm *networkManager) deleteNetworkImpl(nw *network) error { - // Disconnect the interface if this was the last network using it. + var networkClient NetworkClient if nw.VlanId != 0 { - if len(nw.extIf.Networks) == 1 { - nm.disconnectOVSInterface(nw.extIf) - } + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name) } else { - if len(nw.extIf.Networks) == 1 { - nm.disconnectExternalInterface(nw.extIf) - } + networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) + } + + // Disconnect the interface if this was the last network using it. + if len(nw.extIf.Networks) == 1 { + nm.disconnectExternalInterface(nw.extIf, networkClient) } return nil @@ -172,126 +163,10 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net. return nil } -// AddBridgeRules adds bridge frame table rules for container traffic. -func (nm *networkManager) addBridgeRules(extIf *externalInterface, hostIf *net.Interface, bridgeName string, opMode string) error { - // Add SNAT rule to translate container egress traffic. - log.Printf("[net] Adding SNAT rule for egress traffic on %v.", hostIf.Name) - err := ebtables.SetSnatForInterface(hostIf.Name, hostIf.HardwareAddr, ebtables.Append) - if err != nil { - return err - } - - // Add ARP reply rule for host primary IP address. - // ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric - // doesn't respond to ARP requests from the VM for its own primary IP address. - primary := extIf.IPAddresses[0].IP - log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary) - err = ebtables.SetArpReply(primary, hostIf.HardwareAddr, ebtables.Append) - if err != nil { - return err - } - - // Add DNAT rule to forward ARP replies to container interfaces. - log.Printf("[net] Adding DNAT rule for ingress ARP traffic on interface %v.", hostIf.Name) - err = ebtables.SetDnatForArpReplies(hostIf.Name, ebtables.Append) - if err != nil { - return err - } - - // Enable VEPA for host policy enforcement if necessary. - if opMode == opModeTunnel { - log.Printf("[net] Enabling VEPA mode for %v.", hostIf.Name) - err = ebtables.SetVepaMode(bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append) - if err != nil { - return err - } - } - - return nil -} - -func (nm *networkManager) addOVSRules(extIf *externalInterface, hostIf *net.Interface, bridgeName string) error { - primary := extIf.IPAddresses[0].IP.String() - mac := extIf.MacAddress.String() - macHex := strings.Replace(mac, ":", "", -1) - - cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=20,actions=normal", bridgeName, primary, mac) - _, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding SNAT rule failed with error %v", err) - return err - } - - log.Printf("[net] Get ovs port for interface %v.", hostIf.Name) - cmd = fmt.Sprintf("ovs-vsctl get Interface %s ofport", hostIf.Name) - ofport, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Get ofport failed with error %v", err) - return err - } - - ofport = strings.Trim(ofport, "\n") - - // Add DNAT rule to forward ARP replies to container interfaces. - log.Printf("[net] Adding DNAT rule for ingress ARP traffic on interface %v.", hostIf.Name) - cmd = fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, - load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, ofport, macHex) - _, err = common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Adding DNAT rule failed with error %v", err) - return err - } - - return nil -} - -// DeleteBridgeRules deletes bridge rules for container traffic. -func (nm *networkManager) deleteBridgeRules(extIf *externalInterface) { - ebtables.SetVepaMode(extIf.BridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Delete) - ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) - ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) - ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) -} - -func (nm *networkManager) createOVSBridge(bridgeName string) error { - log.Printf("[net] Creating OVS Bridge %v", bridgeName) - - ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) - _, err := common.ExecuteShellCommand(ovsCreateCmd) - if err != nil { - log.Printf("[net] Error while creating OVS bridge %v", err) - return err - } - - return nil -} - -func (nm *networkManager) deleteOVSBridge(bridgeName string) error { - log.Printf("[net] Deleting OVS Bridge %v", bridgeName) - - ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) - _, err := common.ExecuteShellCommand(ovsCreateCmd) - if err != nil { - log.Printf("[net] Error while deleting OVS bridge %v", err) - return err - } - - return nil -} - -func setOVSMaster(hostIfName string, bridgeName string) error { - cmd := fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName) - _, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Error while setting OVS as master to primary interface %v", err) - return err - } - - return nil -} - -func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *NetworkInfo) error { +// ConnectExternalInterface connects the given host interface to a bridge. +func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var err error + var networkClient NetworkClient log.Printf("[net] Connecting interface %v.", extIf.Name) defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() @@ -314,156 +189,27 @@ func (nm *networkManager) createOVSNetwork(extIf *externalInterface, nwInfo *Net bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index) } - // Check if the bridge already exists. - bridge, err := net.InterfaceByName(bridgeName) - if err != nil { - // Create the bridge. - log.Printf("[net] Creating bridge %v.", bridgeName) - - if err := nm.createOVSBridge(bridgeName); err != nil { - return err - } - - // On failure, delete the bridge. - defer func() { - if err != nil { - nm.deleteOVSBridge(bridgeName) - } - }() - - bridge, err = net.InterfaceByName(bridgeName) - if err != nil { - return err - } + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + if opt != nil && opt[vlanIDKey] != nil { + networkClient = NewOVSClient(bridgeName, extIf.Name) } else { - // Use the existing bridge. - log.Printf("[net] Found existing bridge %v.", bridgeName) - } - - // Save host IP configuration. - if err := nm.saveIPConfig(hostIf, extIf); err != nil { - log.Printf("[net] Failed to save IP configuration for interface %v: %v.", hostIf.Name, err) - } - - // External interface down. - log.Printf("[net] Setting link %v state down.", hostIf.Name) - if err := netlink.SetLinkState(hostIf.Name, false); err != nil { - return err - } - - // Connect the external interface to the bridge. - log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) - if err := setOVSMaster(hostIf.Name, bridgeName); err != nil { - return err - } - - // Add the bridge rules. - if err := nm.addOVSRules(extIf, hostIf, bridgeName); err != nil { - return err - } - - // External interface up. - log.Printf("[net] Setting link %v state up.", hostIf.Name) - if err := netlink.SetLinkState(hostIf.Name, true); err != nil { - return err - } - - // Bridge up. - log.Printf("[net] Setting link %v state up.", bridgeName) - if err := netlink.SetLinkState(bridgeName, true); err != nil { - return err - } - - // Apply IP configuration to the bridge for host traffic. - if err := nm.applyIPConfig(extIf, bridge); err != nil { - log.Printf("[net] Failed to apply interface IP configuration: %v.", err) - } - - extIf.BridgeName = bridgeName - log.Printf("[net] Connected interface %v to bridge %v.", extIf.Name, extIf.BridgeName) - - return nil -} - -// DisconnectExternalInterface disconnects a host interface from OVS bridge. -func (nm *networkManager) disconnectOVSInterface(extIf *externalInterface) { - log.Printf("[net] Disconnecting interface %v.", extIf.Name) - - // Disconnect external interface from its bridge. - cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", extIf.BridgeName, extIf.Name) - _, err := common.ExecuteShellCommand(cmd) - if err != nil { - log.Printf("[net] Failed to disconnect interface %v from bridge, err:%v.", extIf.Name, err) - } - - // Delete the bridge. - err = nm.deleteOVSBridge(extIf.BridgeName) - if err != nil { - log.Printf("[net] Failed to delete bridge %v, err:%v.", extIf.BridgeName, err) - } - - extIf.BridgeName = "" - - // Restore IP configuration. - hostIf, _ := net.InterfaceByName(extIf.Name) - err = nm.applyIPConfig(extIf, hostIf) - if err != nil { - log.Printf("[net] Failed to apply IP configuration: %v.", err) - } - - extIf.IPAddresses = nil - extIf.Routes = nil - - log.Printf("[net] Disconnected interface %v.", extIf.Name) -} - -// ConnectExternalInterface connects the given host interface to a bridge. -func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { - var err error - - log.Printf("[net] Connecting interface %v.", extIf.Name) - defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() - - // Check whether this interface is already connected. - if extIf.BridgeName != "" { - log.Printf("[net] Interface is already connected to bridge %v.", extIf.BridgeName) - return nil - } - - // Find the external interface. - hostIf, err := net.InterfaceByName(extIf.Name) - if err != nil { - return err - } - - // If a bridge name is not specified, generate one based on the external interface index. - bridgeName := nwInfo.BridgeName - if bridgeName == "" { - bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } // Check if the bridge already exists. bridge, err := net.InterfaceByName(bridgeName) if err != nil { // Create the bridge. - log.Printf("[net] Creating bridge %v.", bridgeName) - - link := netlink.BridgeLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_BRIDGE, - Name: bridgeName, - }, - } - err = netlink.AddLink(&link) - if err != nil { + if err := networkClient.CreateBridge(); err != nil { + log.Printf("Error while creating bridge %+v", err) return err } // On failure, delete the bridge. defer func() { if err != nil { - netlink.DeleteLink(bridgeName) + networkClient.DeleteBridge() } }() @@ -482,12 +228,6 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI log.Printf("[net] Failed to save IP configuration for interface %v: %v.", hostIf.Name, err) } - // Add the bridge rules. - err = nm.addBridgeRules(extIf, hostIf, bridgeName, nwInfo.Mode) - if err != nil { - return err - } - // External interface down. log.Printf("[net] Setting link %v state down.", hostIf.Name) err = netlink.SetLinkState(hostIf.Name, false) @@ -497,8 +237,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI // Connect the external interface to the bridge. log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) - err = netlink.SetLinkMaster(hostIf.Name, bridgeName) - if err != nil { + if err := networkClient.SetBridgeMasterToHostInterface(); err != nil { return err } @@ -509,10 +248,15 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI return err } + // Add the bridge rules. + err = networkClient.AddBridgeRules(extIf) + if err != nil { + return err + } + // External interface hairpin on. log.Printf("[net] Setting link %v hairpin on.", hostIf.Name) - err = netlink.SetLinkHairpin(hostIf.Name, true) - if err != nil { + if err := networkClient.SetHairpinOnHostInterface(true); err != nil { return err } @@ -538,29 +282,23 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } // DisconnectExternalInterface disconnects a host interface from its bridge. -func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) { +func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface, networkClient NetworkClient) { log.Printf("[net] Disconnecting interface %v.", extIf.Name) + log.Printf("[net] Deleting bridge rules") // Delete bridge rules set on the external interface. - nm.deleteBridgeRules(extIf) - - // Disconnect external interface from its bridge. - err := netlink.SetLinkMaster(extIf.Name, "") - if err != nil { - log.Printf("[net] Failed to disconnect interface %v from bridge, err:%v.", extIf.Name, err) - } + networkClient.DeleteBridgeRules(extIf) - // Delete the bridge. - err = netlink.DeleteLink(extIf.BridgeName) - if err != nil { - log.Printf("[net] Failed to delete bridge %v, err:%v.", extIf.BridgeName, err) - } + log.Printf("[net] Deleting bridge") + // Delete Bridge + networkClient.DeleteBridge() extIf.BridgeName = "" + log.Printf("Restoring ipconfig with primary interface %v", extIf.Name) // Restore IP configuration. hostIf, _ := net.InterfaceByName(extIf.Name) - err = nm.applyIPConfig(extIf, hostIf) + err := nm.applyIPConfig(extIf, hostIf) if err != nil { log.Printf("[net] Failed to apply IP configuration: %v.", err) } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go new file mode 100644 index 0000000000..6bee2c25a7 --- /dev/null +++ b/network/ovs_endpointclient_linux.go @@ -0,0 +1,110 @@ +package network + +import ( + "log" + + "github.com/Azure/azure-container-networking/ovsrules" +) + +type OVSEndpointClient struct { + bridgeName string + hostPrimaryIfName string + hostVethName string + hostPrimaryMac string + containerMac string + vlanID int +} + +func NewOVSEndpointClient( + bridgeName string, + hostPrimaryIfName string, + hostVethName string, + hostPrimaryMac string, + containerMac string, + vlanid int) *OVSEndpointClient { + + client := &OVSEndpointClient{ + bridgeName: bridgeName, + hostPrimaryIfName: hostPrimaryIfName, + hostVethName: hostVethName, + hostPrimaryMac: hostPrimaryMac, + containerMac: containerMac, + vlanID: vlanid, + } + + return client +} + +func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { + log.Printf("[ovs] Setting link %v master %v.", client.hostVethName, client.bridgeName) + if err := ovsrules.AddPortOnOVSBridge(client.hostVethName, client.bridgeName, client.vlanID); err != nil { + return err + } + + log.Printf("[ovs] Get ovs port for interface %v.", client.hostVethName) + containerPort, err := ovsrules.GetOVSPortNumber(client.hostVethName) + if err != nil { + log.Printf("[ovs] Get ofport failed with error %v", err) + return err + } + + log.Printf("[ovs] Get ovs port for interface %v.", client.hostPrimaryIfName) + hostPort, err := ovsrules.GetOVSPortNumber(client.hostPrimaryIfName) + if err != nil { + log.Printf("[ovs] Get ofport failed with error %v", err) + return err + } + + // IP SNAT Rule + log.Printf("[ovs] Adding IP SNAT rule for egress traffic on %v.", containerPort) + if err := ovsrules.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac); err != nil { + return err + } + + for _, ipAddr := range epInfo.IPAddresses { + + // Add Arp Reply Rules + // Set Vlan id on arp request packet and forward it to table 1 + if err := ovsrules.AddArpReplyRule(client.bridgeName, containerPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil { + return err + } + + // Add IP DNAT rule based on dst ip and vlanid + log.Printf("[ovs] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), hostPort) + if err := ovsrules.AddMacDnatRule(client.bridgeName, hostPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil { + return err + } + } + + return nil +} + +func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { + log.Printf("[net] Get ovs port for interface %v.", ep.HostIfName) + containerPort, err := ovsrules.GetOVSPortNumber(client.hostVethName) + if err != nil { + log.Printf("[net] Get portnum failed with error %v", err) + } + + log.Printf("[net] Get ovs port for interface %v.", client.hostPrimaryIfName) + hostPort, err := ovsrules.GetOVSPortNumber(client.hostPrimaryIfName) + if err != nil { + log.Printf("[net] Get portnum failed with error %v", err) + } + + // Delete IP SNAT + log.Printf("[net] Deleting IP SNAT for port %v", containerPort) + ovsrules.DeleteIPSnatRule(client.bridgeName, containerPort) + + // Delete Arp Reply Rules for container + log.Printf("[net] Deleting ARP reply rule for ip %v vlanid %v for container port", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) + ovsrules.DeleteArpReplyRule(client.bridgeName, containerPort, ep.IPAddresses[0].IP, ep.VlanID) + + // Delete MAC address translation rule. + log.Printf("[net] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) + ovsrules.DeleteMacDnatRule(client.bridgeName, hostPort, ep.IPAddresses[0].IP, ep.VlanID) + + // Delete port from ovs bridge + log.Printf("[net] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) + ovsrules.DeletePortFromOVS(client.bridgeName, client.hostVethName) +} diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go new file mode 100644 index 0000000000..7a9099fc09 --- /dev/null +++ b/network/ovs_networkclient_linux.go @@ -0,0 +1,70 @@ +package network + +import ( + "log" + "strings" + + "github.com/Azure/azure-container-networking/ovsrules" +) + +type OVSNetworkClient struct { + bridgeName string + hostInterfaceName string +} + +func NewOVSClient(bridgeName string, hostInterfaceName string) *OVSNetworkClient { + ovsClient := &OVSNetworkClient{ + bridgeName: bridgeName, + hostInterfaceName: hostInterfaceName, + } + + return ovsClient +} + +func (client *OVSNetworkClient) CreateBridge() error { + return ovsrules.CreateOVSBridge(client.bridgeName) +} + +func (client *OVSNetworkClient) DeleteBridge() error { + return ovsrules.DeleteOVSBridge(client.bridgeName) +} + +func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { + primary := extIf.IPAddresses[0].IP.String() + mac := extIf.MacAddress.String() + macHex := strings.Replace(mac, ":", "", -1) + + if err := ovsrules.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { + return err + } + + ofport, err := ovsrules.GetOVSPortNumber(client.hostInterfaceName) + if err != nil { + return err + } + + log.Printf("[ovs] Adding ARP SNAT rule for egress traffic on interface %v", client.hostInterfaceName) + // Arp SNAT Rule + if err := ovsrules.AddArpSnatRule(client.bridgeName, mac, macHex, ofport); err != nil { + return err + } + + log.Printf("[ovs] Adding DNAT rule for ingress ARP traffic on interface %v.", client.hostInterfaceName) + if err := ovsrules.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { + return err + } + + return nil +} + +func (client *OVSNetworkClient) DeleteBridgeRules(extIf *externalInterface) { + ovsrules.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) +} + +func (client *OVSNetworkClient) SetBridgeMasterToHostInterface() error { + return ovsrules.AddPortOnOVSBridge(client.hostInterfaceName, client.bridgeName, 0) +} + +func (client *OVSNetworkClient) SetHairpinOnHostInterface(enable bool) error { + return nil +} diff --git a/ovsrules/ovsrules.go b/ovsrules/ovsrules.go new file mode 100644 index 0000000000..699ab0d644 --- /dev/null +++ b/ovsrules/ovsrules.go @@ -0,0 +1,202 @@ +package ovsrules + +import ( + "fmt" + "log" + "net" + "strings" + + "github.com/Azure/azure-container-networking/common" +) + +func CreateOVSBridge(bridgeName string) error { + log.Printf("[ovs] Creating OVS Bridge %v", bridgeName) + + ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) + _, err := common.ExecuteShellCommand(ovsCreateCmd) + if err != nil { + log.Printf("[ovs] Error while creating OVS bridge %v", err) + return err + } + + return nil +} + +func DeleteOVSBridge(bridgeName string) error { + log.Printf("[ovs] Deleting OVS Bridge %v", bridgeName) + + ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) + _, err := common.ExecuteShellCommand(ovsCreateCmd) + if err != nil { + log.Printf("[ovs] Error while deleting OVS bridge %v", err) + return err + } + + return nil +} + +func AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error { + cmd := "" + + if vlanID == 0 { + cmd = fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName) + } else { + cmd = fmt.Sprintf("ovs-vsctl add-port %s %s tag=%d", bridgeName, hostIfName, vlanID) + } + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Error while setting OVS as master to primary interface %v", err) + return err + } + + return nil +} + +func GetOVSPortNumber(interfaceName string) (string, error) { + log.Printf("[ovs] Get ovs port for interface %v.", interfaceName) + cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", interfaceName) + ofport, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Get ofport failed with error %v", err) + return "", err + } + + ofport = strings.Trim(ofport, "\n") + + return ofport, nil +} + +func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { + cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=20,actions=normal", bridgeName, primaryIP, mac) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding SNAT rule failed with error %v", err) + return err + } + + return nil +} + +func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) error { + cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, + load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, mac, macHex, ofport) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding ARP SNAT rule failed with error %v", err) + return err + } + + return nil +} + +func AddIpSnatRule(bridgeName string, port string, mac string) error { + cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", + bridgeName, port, mac) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err) + return err + } + + return nil +} + +func AddArpDnatRule(bridgeName string, port string, mac string) error { + // Add DNAT rule to forward ARP replies to container interfaces. + cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, + load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, port, mac) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding DNAT rule failed with error %v", err) + return err + } + + return nil +} + +func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error { + ipAddrInt := common.IpToInt(ip) + macAddrHex := strings.Replace(mac, ":", "", -1) + + log.Printf("[ovs] Adding ARP reply rule to add vlan %v and forward packet to table 1 for port %v", vlanid, port) + cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, + bridgeName, port, vlanid) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) + return err + } + + // If arp fields matches, set arp reply rule for the request + log.Printf("[ovs] Adding ARP reply rule for IP address %v and vlanid %v.", ip, vlanid) + cmd = fmt.Sprintf(`ovs-ofctl add-flow %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[], + move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], + load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, + bridgeName, ip.String(), vlanid, mac, macAddrHex, ipAddrInt) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) + return err + } + + return nil +} + +func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error { + cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", + bridgeName, ip.String(), vlanid, port, mac) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding MAC DNAT rule failed with error %v", err) + return err + } + + return nil +} + +func DeleteArpReplyRule(bridgeName string, port string, ip net.IP, vlanid int) { + cmd := fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", + bridgeName, port) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Deleting ARP reply rule failed with error %v", err) + } + + cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", + bridgeName, ip.String(), vlanid) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Deleting ARP reply rule failed with error %v", err) + } +} + +func DeleteIPSnatRule(bridgeName string, port string) { + cmd := fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", + bridgeName, port) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("Error while deleting ovs rule %v error %v", cmd, err) + } +} + +func DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { + cmd := fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", + bridgeName, ip.String(), vlanid, port) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) + } +} + +func DeletePortFromOVS(bridgeName string, interfaceName string) error { + // Disconnect external interface from its bridge. + cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", bridgeName, interfaceName) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Failed to disconnect interface %v from bridge, err:%v.", interfaceName, err) + return err + } + + return nil +} From bb62d01c6fe9eb367e254dedf2c31d1fbc52f97b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 3 May 2018 12:18:55 -0700 Subject: [PATCH 25/51] Added fake arp reply for all arp requests --- network/linuxbridge_endpointclient_linux.go | 2 +- network/ovs_endpointclient_linux.go | 5 ++--- network/ovs_networkclient_linux.go | 14 ++++++------- ovsrules/ovsrules.go | 23 ++++++++++++++++++++- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/network/linuxbridge_endpointclient_linux.go b/network/linuxbridge_endpointclient_linux.go index a11d545af6..5aaeb8ecb6 100644 --- a/network/linuxbridge_endpointclient_linux.go +++ b/network/linuxbridge_endpointclient_linux.go @@ -1,10 +1,10 @@ package network import ( - "log" "net" "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" ) diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 6bee2c25a7..525362f795 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -1,8 +1,7 @@ package network import ( - "log" - + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/ovsrules" ) @@ -65,7 +64,7 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // Add Arp Reply Rules // Set Vlan id on arp request packet and forward it to table 1 - if err := ovsrules.AddArpReplyRule(client.bridgeName, containerPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil { + if err := ovsrules.AddFakeArpReply(client.bridgeName, ipAddr.IP); err != nil { return err } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 7a9099fc09..a54ed7d1f4 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -1,9 +1,9 @@ package network import ( - "log" "strings" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/ovsrules" ) @@ -43,12 +43,12 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { return err } - log.Printf("[ovs] Adding ARP SNAT rule for egress traffic on interface %v", client.hostInterfaceName) - // Arp SNAT Rule - if err := ovsrules.AddArpSnatRule(client.bridgeName, mac, macHex, ofport); err != nil { - return err - } - + /* log.Printf("[ovs] Adding ARP SNAT rule for egress traffic on interface %v", client.hostInterfaceName) + // Arp SNAT Rule + if err := ovsrules.AddArpSnatRule(client.bridgeName, mac, macHex, ofport); err != nil { + return err + } + */ log.Printf("[ovs] Adding DNAT rule for ingress ARP traffic on interface %v.", client.hostInterfaceName) if err := ovsrules.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { return err diff --git a/ovsrules/ovsrules.go b/ovsrules/ovsrules.go index 699ab0d644..9745cd1232 100644 --- a/ovsrules/ovsrules.go +++ b/ovsrules/ovsrules.go @@ -114,7 +114,28 @@ func AddArpDnatRule(bridgeName string, port string, mac string) error { return nil } -func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error { +func AddFakeArpReply(bridgeName string, ip net.IP) error { + // If arp fields matches, set arp reply rule for the request + mac := "12:34:56:78:9a:bc" + macAddrHex := strings.Replace(mac, ":", "", -1) + ipAddrInt := common.IpToInt(ip) + + log.Printf("[ovs] Adding ARP reply rule for IP address %v ", ip.String()) + cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,priority=20,actions='load:0x2->NXM_OF_ARP_OP[], + move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, + move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], + load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, + bridgeName, mac, macAddrHex, ipAddrInt) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) + return err + } + + return nil +} + +func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, mode string) error { ipAddrInt := common.IpToInt(ip) macAddrHex := strings.Replace(mac, ":", "", -1) From b633dd6b4a74eb7e67563a2cf84f4fed438f3060 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 8 May 2018 10:21:21 -0700 Subject: [PATCH 26/51] removed unused functions --- network/endpoint_linux.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 51592f39d4..4024e57f87 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -202,10 +202,6 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return ep, nil } -func (nw *network) deleteOVSRules(ep *endpoint) { - -} - // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(ep *endpoint) error { var epClient EndpointClient @@ -230,7 +226,3 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { return nil } - -// getInfoImpl returns information about the endpoint. -func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { -} From 7cdab6b28077d9c35ada760f3c17445347ee7701 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 9 May 2018 16:35:42 -0700 Subject: [PATCH 27/51] Addressed review comments --- cni/network/network.go | 29 +++++++++++++------ cns/dnccontract.go | 9 +++--- cns/restserver/restserver.go | 7 +++-- common/utils.go | 20 +++++++++++++ ...inux.go => bridge_endpointclient_linux.go} | 0 ...linux.go => bridge_networkclient_linux.go} | 0 network/endpoint_linux.go | 12 ++++---- network/ovs_endpointclient_linux.go | 26 ++++++++--------- network/ovs_networkclient_linux.go | 28 +++++++++--------- ovsrules/ovsrules.go => ovsctl/ovsctl.go | 4 +-- 10 files changed, 83 insertions(+), 52 deletions(-) rename network/{linuxbridge_endpointclient_linux.go => bridge_endpointclient_linux.go} (100%) rename network/{linuxbridge_networkclient_linux.go => bridge_networkclient_linux.go} (100%) rename ovsrules/ovsrules.go => ovsctl/ovsctl.go (99%) diff --git a/cni/network/network.go b/cni/network/network.go index 57a10181eb..9e519c9f95 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -4,6 +4,7 @@ package network import ( + "fmt" "net" "strconv" @@ -158,22 +159,29 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType return result } -func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, error) { +func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, net.IPNet, error) { cnsClient, err := cnsclient.NewCnsClient("") if err != nil { log.Printf("Initializing CNS client error %v", err) - return nil, 0, err + return nil, 0, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + networkConfig, err := cnsClient.GetNetworkConfiguration("testpod", "testpodnamespace") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) - return nil, 0, err + return nil, 0, net.IPNet{}, err } log.Printf("Network config received from cns %v", networkConfig) - return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, nil + subnetPrefix := common.GetIpNet(networkConfig.PrimaryInterfaceIdentifier) + if subnetPrefix == nil { + errBuf := fmt.Sprintf("Interface not found for this ip %v", networkConfig.PrimaryInterfaceIdentifier) + log.Printf(errBuf) + return nil, 0, net.IPNet{}, fmt.Errorf(errBuf) + } + + return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, *subnetPrefix, nil } // @@ -210,11 +218,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("Argsmap %v", argsMap) } - result, vlanid, err = getContainerNetworkConfiguration(argsMap[namespaceKey].(string), argsMap[podNameKey].(string)) + result, vlanid, subnetPrefix, err := getContainerNetworkConfiguration(argsMap[namespaceKey].(string), argsMap[podNameKey].(string)) if err != nil { - log.Printf("SetContainerNetworkConfiguration failed with %v", err) + log.Printf("getContainerNetworkConfiguration failed with %v", err) } + log.Printf("subnetprefix :%v", subnetPrefix.IP.String()) + epInfo = &network.EndpointInfo{ Id: endpointId, ContainerID: args.ContainerID, @@ -240,11 +250,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { err = plugin.Errorf("Failed to allocate pool: %v", err) return err } + + // Derive the subnet prefix from allocated IP address. + subnetPrefix = result.IPs[0].Address } - // Derive the subnet prefix from allocated IP address. ipconfig := result.IPs[0] - subnetPrefix := ipconfig.Address gateway := ipconfig.Gateway // On failure, call into IPAM plugin to release the address and address pool. diff --git a/cns/dnccontract.go b/cns/dnccontract.go index 21d6e386b9..8141601615 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -100,10 +100,11 @@ type GetNetworkContainerRequest struct { // GetNetworkContainerResponse describes the response to retrieve a specifc network container. type GetNetworkContainerResponse struct { - IPConfiguration IPConfiguration - Routes []Route - MultiTenancyInfo MultiTenancyInfo - Response Response + IPConfiguration IPConfiguration + Routes []Route + MultiTenancyInfo MultiTenancyInfo + PrimaryInterfaceIdentifier string + Response Response } // DeleteNetworkContainerRequest specifies the details about the request to delete a specifc network container. diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 59b28959cb..bf126381b7 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -990,9 +990,10 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo savedReq := containerDetails.CreateNetworkContainerRequest getNetworkContainerResponse = cns.GetNetworkContainerResponse{ - IPConfiguration: savedReq.IPConfiguration, - Routes: savedReq.Routes, - MultiTenancyInfo: savedReq.MultiTenancyInfo, + IPConfiguration: savedReq.IPConfiguration, + Routes: savedReq.Routes, + MultiTenancyInfo: savedReq.MultiTenancyInfo, + PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, } return getNetworkContainerResponse diff --git a/common/utils.go b/common/utils.go index e208ffcc95..cc396ce385 100644 --- a/common/utils.go +++ b/common/utils.go @@ -103,3 +103,23 @@ func IpToInt(ip net.IP) uint32 { return binary.BigEndian.Uint32(ip) } + +func GetIpNet(ipAddr string) *net.IPNet { + addrs, err := net.InterfaceAddrs() + if err != nil { + log.Printf("InterfaceAddrs failed with %v: " + err.Error() + "\n") + return nil + } + + for _, a := range addrs { + if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + if ipnet.IP.String() == ipAddr { + return ipnet + } + } + } + } + + return nil +} diff --git a/network/linuxbridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go similarity index 100% rename from network/linuxbridge_endpointclient_linux.go rename to network/bridge_endpointclient_linux.go diff --git a/network/linuxbridge_networkclient_linux.go b/network/bridge_networkclient_linux.go similarity index 100% rename from network/linuxbridge_networkclient_linux.go rename to network/bridge_networkclient_linux.go diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4024e57f87..57dfb98785 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -94,14 +94,8 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress, containerIf.HardwareAddr, nw.Mode) } - epClient.AddEndpointRules(epInfo) - // - // Container network interface setup. - // - - // Query container network interface info. - // Setup rules for IP addresses on the container interface. + epClient.AddEndpointRules(epInfo) // If a network namespace for the container interface is specified... if epInfo.NetNsPath != "" { @@ -226,3 +220,7 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { return nil } + +// getInfoImpl returns information about the endpoint. +func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { +} diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 525362f795..fc357f5b93 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -2,7 +2,7 @@ package network import ( "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/ovsrules" + "github.com/Azure/azure-container-networking/ovsctl" ) type OVSEndpointClient struct { @@ -36,19 +36,19 @@ func NewOVSEndpointClient( func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { log.Printf("[ovs] Setting link %v master %v.", client.hostVethName, client.bridgeName) - if err := ovsrules.AddPortOnOVSBridge(client.hostVethName, client.bridgeName, client.vlanID); err != nil { + if err := ovsctl.AddPortOnOVSBridge(client.hostVethName, client.bridgeName, client.vlanID); err != nil { return err } log.Printf("[ovs] Get ovs port for interface %v.", client.hostVethName) - containerPort, err := ovsrules.GetOVSPortNumber(client.hostVethName) + containerPort, err := ovsctl.GetOVSPortNumber(client.hostVethName) if err != nil { log.Printf("[ovs] Get ofport failed with error %v", err) return err } log.Printf("[ovs] Get ovs port for interface %v.", client.hostPrimaryIfName) - hostPort, err := ovsrules.GetOVSPortNumber(client.hostPrimaryIfName) + hostPort, err := ovsctl.GetOVSPortNumber(client.hostPrimaryIfName) if err != nil { log.Printf("[ovs] Get ofport failed with error %v", err) return err @@ -56,7 +56,7 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // IP SNAT Rule log.Printf("[ovs] Adding IP SNAT rule for egress traffic on %v.", containerPort) - if err := ovsrules.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac); err != nil { + if err := ovsctl.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac); err != nil { return err } @@ -64,13 +64,13 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // Add Arp Reply Rules // Set Vlan id on arp request packet and forward it to table 1 - if err := ovsrules.AddFakeArpReply(client.bridgeName, ipAddr.IP); err != nil { + if err := ovsctl.AddFakeArpReply(client.bridgeName, ipAddr.IP); err != nil { return err } // Add IP DNAT rule based on dst ip and vlanid log.Printf("[ovs] Adding MAC DNAT rule for IP address %v on %v.", ipAddr.IP.String(), hostPort) - if err := ovsrules.AddMacDnatRule(client.bridgeName, hostPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil { + if err := ovsctl.AddMacDnatRule(client.bridgeName, hostPort, ipAddr.IP, client.containerMac, client.vlanID); err != nil { return err } } @@ -80,30 +80,30 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { log.Printf("[net] Get ovs port for interface %v.", ep.HostIfName) - containerPort, err := ovsrules.GetOVSPortNumber(client.hostVethName) + containerPort, err := ovsctl.GetOVSPortNumber(client.hostVethName) if err != nil { log.Printf("[net] Get portnum failed with error %v", err) } log.Printf("[net] Get ovs port for interface %v.", client.hostPrimaryIfName) - hostPort, err := ovsrules.GetOVSPortNumber(client.hostPrimaryIfName) + hostPort, err := ovsctl.GetOVSPortNumber(client.hostPrimaryIfName) if err != nil { log.Printf("[net] Get portnum failed with error %v", err) } // Delete IP SNAT log.Printf("[net] Deleting IP SNAT for port %v", containerPort) - ovsrules.DeleteIPSnatRule(client.bridgeName, containerPort) + ovsctl.DeleteIPSnatRule(client.bridgeName, containerPort) // Delete Arp Reply Rules for container log.Printf("[net] Deleting ARP reply rule for ip %v vlanid %v for container port", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) - ovsrules.DeleteArpReplyRule(client.bridgeName, containerPort, ep.IPAddresses[0].IP, ep.VlanID) + ovsctl.DeleteArpReplyRule(client.bridgeName, containerPort, ep.IPAddresses[0].IP, ep.VlanID) // Delete MAC address translation rule. log.Printf("[net] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) - ovsrules.DeleteMacDnatRule(client.bridgeName, hostPort, ep.IPAddresses[0].IP, ep.VlanID) + ovsctl.DeleteMacDnatRule(client.bridgeName, hostPort, ep.IPAddresses[0].IP, ep.VlanID) // Delete port from ovs bridge log.Printf("[net] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) - ovsrules.DeletePortFromOVS(client.bridgeName, client.hostVethName) + ovsctl.DeletePortFromOVS(client.bridgeName, client.hostVethName) } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index a54ed7d1f4..a91449cbaf 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -4,7 +4,7 @@ import ( "strings" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/ovsrules" + "github.com/Azure/azure-container-networking/ovsctl" ) type OVSNetworkClient struct { @@ -22,11 +22,11 @@ func NewOVSClient(bridgeName string, hostInterfaceName string) *OVSNetworkClient } func (client *OVSNetworkClient) CreateBridge() error { - return ovsrules.CreateOVSBridge(client.bridgeName) + return ovsctl.CreateOVSBridge(client.bridgeName) } func (client *OVSNetworkClient) DeleteBridge() error { - return ovsrules.DeleteOVSBridge(client.bridgeName) + return ovsctl.DeleteOVSBridge(client.bridgeName) } func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { @@ -34,23 +34,23 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) - if err := ovsrules.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { + if err := ovsctl.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { return err } - ofport, err := ovsrules.GetOVSPortNumber(client.hostInterfaceName) + ofport, err := ovsctl.GetOVSPortNumber(client.hostInterfaceName) if err != nil { return err } - /* log.Printf("[ovs] Adding ARP SNAT rule for egress traffic on interface %v", client.hostInterfaceName) - // Arp SNAT Rule - if err := ovsrules.AddArpSnatRule(client.bridgeName, mac, macHex, ofport); err != nil { - return err - } - */ + // Arp SNAT Rule + log.Printf("[ovs] Adding ARP SNAT rule for egress traffic on interface %v", client.hostInterfaceName) + if err := ovsctl.AddArpSnatRule(client.bridgeName, mac, macHex, ofport); err != nil { + return err + } + log.Printf("[ovs] Adding DNAT rule for ingress ARP traffic on interface %v.", client.hostInterfaceName) - if err := ovsrules.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { + if err := ovsctl.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { return err } @@ -58,11 +58,11 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { } func (client *OVSNetworkClient) DeleteBridgeRules(extIf *externalInterface) { - ovsrules.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) + ovsctl.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) } func (client *OVSNetworkClient) SetBridgeMasterToHostInterface() error { - return ovsrules.AddPortOnOVSBridge(client.hostInterfaceName, client.bridgeName, 0) + return ovsctl.AddPortOnOVSBridge(client.hostInterfaceName, client.bridgeName, 0) } func (client *OVSNetworkClient) SetHairpinOnHostInterface(enable bool) error { diff --git a/ovsrules/ovsrules.go b/ovsctl/ovsctl.go similarity index 99% rename from ovsrules/ovsrules.go rename to ovsctl/ovsctl.go index 9745cd1232..48d202918f 100644 --- a/ovsrules/ovsrules.go +++ b/ovsctl/ovsctl.go @@ -1,12 +1,12 @@ -package ovsrules +package ovsctl import ( "fmt" - "log" "net" "strings" "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" ) func CreateOVSBridge(bridgeName string) error { From ad2f472f26e58fb465116cefa01643106efb238c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 9 May 2018 17:35:12 -0700 Subject: [PATCH 28/51] removed hardcoded values --- cni/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 9e519c9f95..e5a5430995 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -166,7 +166,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration("testpod", "testpodnamespace") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, net.IPNet{}, err From ecdcd97f029b74d8dacebadf414daf013fb6b832 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 25 May 2018 18:49:38 -0700 Subject: [PATCH 29/51] removed ipaccept rule --- network/ovs_networkclient_linux.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index a91449cbaf..abbab23eb3 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -30,13 +30,13 @@ func (client *OVSNetworkClient) DeleteBridge() error { } func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { - primary := extIf.IPAddresses[0].IP.String() + //primary := extIf.IPAddresses[0].IP.String() mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) - if err := ovsctl.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { + /*if err := ovsctl.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { return err - } + }*/ ofport, err := ovsctl.GetOVSPortNumber(client.hostInterfaceName) if err != nil { From 74a8f874cb8706fa7541fdb2387d5b8990c20567 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 29 May 2018 19:07:05 -0700 Subject: [PATCH 30/51] Added stdout support --- cni/network/network.go | 17 ++++++++++++++++- cnm/plugin/main.go | 2 ++ cns/service/main.go | 4 ++++ common/config.go | 3 +++ log/logger.go | 2 ++ log/logger_linux.go | 17 +++++++++++++++++ 6 files changed, 44 insertions(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 435f7dc9c1..35794de490 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -95,6 +95,7 @@ func (plugin *netPlugin) Stop() { plugin.nm.Uninitialize() plugin.Uninitialize() log.Printf("[cni-net] Plugin stopped.") + log.Close() } // FindMasterInterface returns the name of the master interface. @@ -191,6 +192,17 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, *subnetPrefix, nil } +func getPodNameWithoutSuffix(podName string) string { + nameSplit := strings.Split(podName, "-") + if len(nameSplit) > 2 { + nameSplit = nameSplit[:2] + } else { + return podName + } + + return strings.Join(nameSplit, "-") +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -258,11 +270,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) + podNameWithoutSuffix := getPodNameWithoutSuffix(k8sPodName) + log.Printf("Podname without suffix %v", podNameWithoutSuffix) + // Initialize values from network config. networkId := nwCfg.Name endpointId := GetEndpointID(args) - result, vlanid, subnetPrefix, err := getContainerNetworkConfiguration(k8sNamespace, k8sPodName) + result, vlanid, subnetPrefix, err := getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) if err != nil { log.Printf("getContainerNetworkConfiguration failed with %v", err) } diff --git a/cnm/plugin/main.go b/cnm/plugin/main.go index 5b479ac569..0aebc6aa2f 100644 --- a/cnm/plugin/main.go +++ b/cnm/plugin/main.go @@ -205,4 +205,6 @@ func main() { if ipamPlugin != nil { ipamPlugin.Stop() } + + log.Close() } diff --git a/cns/service/main.go b/cns/service/main.go index b9c2054446..49674c6150 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -70,6 +70,8 @@ var args = acn.ArgumentList{ acn.OptLogTargetSyslog: log.TargetSyslog, acn.OptLogTargetStderr: log.TargetStderr, acn.OptLogTargetFile: log.TargetLogfile, + acn.OptLogStdout: log.TargetStdout, + acn.OptLogMultiWrite: log.TargetMultiWrite, }, }, { @@ -268,4 +270,6 @@ func main() { ipamPlugin.Stop() } } + + log.Close() } diff --git a/common/config.go b/common/config.go index 84cc690724..c986cc990f 100644 --- a/common/config.go +++ b/common/config.go @@ -29,6 +29,8 @@ const ( OptLogTargetSyslog = "syslog" OptLogTargetStderr = "stderr" OptLogTargetFile = "logfile" + OptLogStdout = "stdout" + OptLogMultiWrite = "stdoutfile" // Logging location OptLogLocation = "log-location" @@ -42,6 +44,7 @@ const ( OptIpamQueryInterval = "ipam-query-interval" OptIpamQueryIntervalAlias = "i" + // Don't Start CNM OptStopAzureVnet = "stop-azure-cnm" OptStopAzureVnetAlias = "stopcnm" diff --git a/log/logger.go b/log/logger.go index 2841fafc61..1defb5d981 100644 --- a/log/logger.go +++ b/log/logger.go @@ -27,6 +27,8 @@ const ( TargetStderr = iota TargetSyslog TargetLogfile + TargetStdout + TargetMultiWrite ) const ( diff --git a/log/logger_linux.go b/log/logger_linux.go index e1c1372e1e..c599fda53b 100644 --- a/log/logger_linux.go +++ b/log/logger_linux.go @@ -5,6 +5,7 @@ package log import ( "fmt" + "io" "log" "log/syslog" "os" @@ -15,12 +16,28 @@ func (logger *Logger) SetTarget(target int) error { var err error switch target { + case TargetStdout: + logger.out = os.Stdout + break case TargetStderr: logger.out = os.Stderr + break case TargetSyslog: logger.out, err = syslog.New(log.LstdFlags, logger.name) + break case TargetLogfile: logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + break + case TargetMultiWrite: + logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + mw := io.MultiWriter(os.Stdout, logger.out) + if err == nil { + logger.l.SetOutput(mw) + logger.target = target + return nil + } + + break default: err = fmt.Errorf("Invalid log target %d", target) } From 6720293a785d82663e269950f4db664f3d7e8d4b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 29 May 2018 19:19:45 -0700 Subject: [PATCH 31/51] Added lock before saving state in setorchestrator api --- cns/restserver/restserver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index bf126381b7..97c91bfbff 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -820,7 +820,9 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht switch req.OrchestratorType { case cns.Kubernetes: service.state.OrchestratorType = cns.Kubernetes + service.lock.Lock() service.saveState() + service.lock.Unlock() break default: returnMessage = fmt.Sprintf("Invalid Orchestrator type %v", req.OrchestratorType) From 9d74bc15cd1bbeffea38a571ba8bd949a0c2c78d Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 29 May 2018 19:20:35 -0700 Subject: [PATCH 32/51] moved lock before setting state --- cns/restserver/restserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 97c91bfbff..9df6c1fedd 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -819,8 +819,8 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht switch req.OrchestratorType { case cns.Kubernetes: - service.state.OrchestratorType = cns.Kubernetes service.lock.Lock() + service.state.OrchestratorType = cns.Kubernetes service.saveState() service.lock.Unlock() break From 323c49f07ec350db886c38ee08cd307ca988c6cc Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 30 May 2018 14:01:16 -0700 Subject: [PATCH 33/51] Fixed review comments and added rule to drop packet if already tagged --- client/cnsclient/cnsclient.go | 1 + cni/network/network.go | 5 ++--- cns/restserver/restserver.go | 5 +++-- common/utils.go | 2 +- ovsctl/ovsctl.go | 19 +++++++++++++++---- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/client/cnsclient/cnsclient.go b/client/cnsclient/cnsclient.go index 55552d159e..d1a646a1b0 100644 --- a/client/cnsclient/cnsclient.go +++ b/client/cnsclient/cnsclient.go @@ -61,6 +61,7 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string return nil, err } + defer res.Body.Close() if res.StatusCode != 200 { errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) log.Printf(errMsg) diff --git a/cni/network/network.go b/cni/network/network.go index 35794de490..a3436cb469 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -149,10 +149,9 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) result.IPs = append(result.IPs, resultIpconfig) - result.DNS.Nameservers = ipconfig.DNSServers - if networkConfig.Routes == nil && len(networkConfig.Routes) > 0 { + if networkConfig.Routes != nil && len(networkConfig.Routes) > 0 { for _, route := range networkConfig.Routes { _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) gwIP := net.ParseIP(route.GatewayIPAddress) @@ -174,7 +173,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, net.IPNet{}, err diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9df6c1fedd..9899e95559 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1041,6 +1041,9 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r var containerStatus containerstatus var ok bool + service.lock.Lock() + defer service.lock.Unlock() + if containerStatus, ok = service.state.ContainerStatus[req.NetworkContainerid]; !ok { log.Printf("Not able to retrieve network container details for this container id %v", req.NetworkContainerid) break @@ -1055,7 +1058,6 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r } } - service.lock.Lock() if service.state.ContainerStatus != nil { delete(service.state.ContainerStatus, req.NetworkContainerid) } @@ -1070,7 +1072,6 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r } service.saveState() - service.lock.Unlock() break default: returnMessage = "[Azure CNS] Error. DeleteNetworkContainer did not receive a POST." diff --git a/common/utils.go b/common/utils.go index cc396ce385..12f0db1d81 100644 --- a/common/utils.go +++ b/common/utils.go @@ -107,7 +107,7 @@ func IpToInt(ip net.IP) uint32 { func GetIpNet(ipAddr string) *net.IPNet { addrs, err := net.InterfaceAddrs() if err != nil { - log.Printf("InterfaceAddrs failed with %v: " + err.Error() + "\n") + log.Printf("InterfaceAddrs failed with %+v", err) return nil } diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 48d202918f..b3324018ee 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -9,6 +9,10 @@ import ( "github.com/Azure/azure-container-networking/log" ) +const ( + macAddress = "12:34:56:78:9a:bc" +) + func CreateOVSBridge(bridgeName string) error { log.Printf("[ovs] Creating OVS Bridge %v", bridgeName) @@ -90,7 +94,7 @@ func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) } func AddIpSnatRule(bridgeName string, port string, mac string) error { - cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=mod_dl_src:%s,normal", + cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=20,ip,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s,strip_vlan,normal", bridgeName, port, mac) _, err := common.ExecuteShellCommand(cmd) if err != nil { @@ -98,6 +102,14 @@ func AddIpSnatRule(bridgeName string, port string, mac string) error { return err } + cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=drop", + bridgeName, port) + _, err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[ovs] Dropping vlantag packet rule failed with error %v", err) + return err + } + return nil } @@ -116,8 +128,7 @@ func AddArpDnatRule(bridgeName string, port string, mac string) error { func AddFakeArpReply(bridgeName string, ip net.IP) error { // If arp fields matches, set arp reply rule for the request - mac := "12:34:56:78:9a:bc" - macAddrHex := strings.Replace(mac, ":", "", -1) + macAddrHex := strings.Replace(macAddress, ":", "", -1) ipAddrInt := common.IpToInt(ip) log.Printf("[ovs] Adding ARP reply rule for IP address %v ", ip.String()) @@ -125,7 +136,7 @@ func AddFakeArpReply(bridgeName string, ip net.IP) error { move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, - bridgeName, mac, macAddrHex, ipAddrInt) + bridgeName, macAddress, macAddrHex, ipAddrInt) _, err := common.ExecuteShellCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) From 82c0df4a240e7ea02efb38820f799ee5f31f7ca0 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 8 Jun 2018 12:37:56 -0700 Subject: [PATCH 34/51] fixed merge conflict bugs --- cni/network/network.go | 4 ++-- network/endpoint.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 1e01fdd96e..6178590746 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -158,12 +158,12 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType for _, route := range networkConfig.Routes { _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) gwIP := net.ParseIP(route.GatewayIPAddress) - result.Routes = append(result.Routes, &types.Route{Dst: *routeIPnet, GW: gwIP}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: *routeIPnet, GW: gwIP}) } } else { gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: resultIpconfig.Address.Mask} - result.Routes = append(result.Routes, &types.Route{Dst: dstIP, GW: gwIP}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } return result diff --git a/network/endpoint.go b/network/endpoint.go index 502eef4f7a..ba66e8b9ba 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -23,6 +23,7 @@ type endpoint struct { Gateways []net.IP DNS DNSInfo Routes []RouteInfo + VlanID int } // EndpointInfo contains read-only information about an endpoint. From e7f0136692f4028ae2b642a8ad0c22209d9fb86a Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 8 Jun 2018 13:57:58 -0700 Subject: [PATCH 35/51] jaeryn changes in cns --- cns/dnccontract.go | 10 ++++++++-- cns/restserver/restserver.go | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cns/dnccontract.go b/cns/dnccontract.go index 8141601615..dd32031d79 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -23,6 +23,12 @@ const ( Kubernetes = "Kubernetes" ) +// Encap Types +const ( + Vlan = "Vlan" + Vxlan = "Vxlan" +) + // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { Version string @@ -33,7 +39,7 @@ type CreateNetworkContainerRequest struct { OrchestratorContext json.RawMessage IPConfiguration IPConfiguration MultiTenancyInfo MultiTenancyInfo - VnetAddressSpace []IPSubnet // To setup SNAT (should include service endpoint vips). + CnetAddressSpace []IPSubnet // To setup SNAT (should include service endpoint vips). Routes []Route } @@ -125,7 +131,7 @@ type GetInterfaceForContainerRequest struct { // GetInterfaceForContainerResponse specifies the interface for a given container ID. type GetInterfaceForContainerResponse struct { NetworkInterface NetworkInterface - VnetAddressSpace []IPSubnet + CnetAddressSpace []IPSubnet Response Response } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9899e95559..80ddc99278 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1169,12 +1169,12 @@ func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, containerDetails, ok := containerInfo[req.NetworkContainerID] var interfaceName string var ipaddress string - var vnetSpace []cns.IPSubnet + var cnetSpace []cns.IPSubnet if ok { savedReq := containerDetails.CreateNetworkContainerRequest interfaceName = savedReq.NetworkContainerid - vnetSpace = savedReq.VnetAddressSpace + cnetSpace = savedReq.CnetAddressSpace ipaddress = savedReq.IPConfiguration.IPSubnet.IPAddress // it has to exist } else { returnMessage = "[Azure CNS] Never received call to create this container." @@ -1191,7 +1191,7 @@ func (service *httpRestService) getInterfaceForContainer(w http.ResponseWriter, getInterfaceForContainerResponse := cns.GetInterfaceForContainerResponse{ Response: resp, NetworkInterface: cns.NetworkInterface{Name: interfaceName, IPAddress: ipaddress}, - VnetAddressSpace: vnetSpace, + CnetAddressSpace: cnetSpace, } err = service.Listener.Encode(w, &getInterfaceForContainerResponse) From 08fdbdc050b5587f575ef9738db2e938aca31dd7 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Jun 2018 16:09:08 -0700 Subject: [PATCH 36/51] Missing changes from devaci branch. --- cns/restserver/restserver.go | 6 ++++++ cns/service/main.go | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 80ddc99278..a20daca0fe 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -824,6 +824,12 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht service.saveState() service.lock.Unlock() break + case cns.WebApps: + service.lock.Lock() + service.state.OrchestratorType = cns.WebApps + service.saveState() + service.lock.Unlock() + break default: returnMessage = fmt.Sprintf("Invalid Orchestrator type %v", req.OrchestratorType) returnCode = UnsupportedOrchestratorType diff --git a/cns/service/main.go b/cns/service/main.go index 161c0ee764..cc8e47cbfc 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -207,10 +207,6 @@ func main() { var netPlugin network.NetPlugin var ipamPlugin ipam.IpamPlugin - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryUrl, ipamQueryUrl) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) if !stopcnm { var pluginConfig acn.PluginConfig pluginConfig.Version = version @@ -249,6 +245,7 @@ func main() { ipamPlugin.SetOption(acn.OptEnvironment, environment) ipamPlugin.SetOption(acn.OptAPIServerURL, url) + ipamPlugin.SetOption(acn.OptIpamQueryUrl, ipamQueryUrl) ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) if err := ipamPlugin.Start(&pluginConfig); err != nil { log.Printf("Failed to create IPAM plugin, err:%v.\n", err) From b6682bd3bb289a6ce05282279e389109e3bf13bd Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 12 Jun 2018 14:49:29 -0700 Subject: [PATCH 37/51] fixed route issue --- cni/azure-linux.conflist | 1 + cni/netconfig.go | 19 ++++++++++--------- cni/network/network.go | 32 +++++++++++++++++++------------- cns/service/main.go | 5 +---- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index 2e07c28d79..a2a4881ef9 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -6,6 +6,7 @@ "type":"azure-vnet", "mode":"bridge", "bridge":"azure0", + "multiTenancy":true, "ipam":{ "type":"azure-vnet-ipam" } diff --git a/cni/netconfig.go b/cni/netconfig.go index 33252f27a5..c1628a4d2f 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -24,15 +24,16 @@ type KVPair struct { // NetworkConfig represents Azure CNI plugin network configuration. type NetworkConfig struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Type string `json:"type"` - Mode string `json:"mode"` - Master string `json:"master"` - Bridge string `json:"bridge,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - LogTarget string `json:"logTarget,omitempty"` - Ipam struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Type string `json:"type"` + Mode string `json:"mode"` + Master string `json:"master"` + Bridge string `json:"bridge,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + LogTarget string `json:"logTarget,omitempty"` + MultiTenancy bool `json:"multiTenancy,omitempty"` + Ipam struct { Type string `json:"type"` Environment string `json:"environment,omitempty"` AddrSpace string `json:"addressSpace,omitempty"` diff --git a/cni/network/network.go b/cni/network/network.go index 6178590746..6b240d5174 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -160,12 +160,14 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType gwIP := net.ParseIP(route.GatewayIPAddress) result.Routes = append(result.Routes, &cniTypes.Route{Dst: *routeIPnet, GW: gwIP}) } - } else { - gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) - dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: resultIpconfig.Address.Mask} - result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } + // route for default gw + gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) + _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) + return result } @@ -213,12 +215,13 @@ func getPodNameWithoutSuffix(podName string) string { // Add handles CNI add commands. func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var ( - result *cniTypesCurr.Result - err error - nwCfg *cni.NetworkConfig - epInfo *network.EndpointInfo - iface *cniTypesCurr.Interface - vlanid int + result *cniTypesCurr.Result + err error + nwCfg *cni.NetworkConfig + epInfo *network.EndpointInfo + iface *cniTypesCurr.Interface + subnetPrefix net.IPNet + vlanid int ) log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", @@ -279,9 +282,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { networkId := nwCfg.Name endpointId := GetEndpointID(args) - result, vlanid, subnetPrefix, err := getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) - if err != nil { - log.Printf("getContainerNetworkConfiguration failed with %v", err) + if nwCfg.MultiTenancy { + result, vlanid, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) + if err != nil { + log.Printf("[CNI Multitenancy] GetContainerNetworkConfiguration failed with %v", err) + return err + } } log.Printf("subnetprefix :%v", subnetPrefix.IP.String()) diff --git a/cns/service/main.go b/cns/service/main.go index 161c0ee764..cc8e47cbfc 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -207,10 +207,6 @@ func main() { var netPlugin network.NetPlugin var ipamPlugin ipam.IpamPlugin - ipamPlugin.SetOption(acn.OptEnvironment, environment) - ipamPlugin.SetOption(acn.OptAPIServerURL, url) - ipamPlugin.SetOption(acn.OptIpamQueryUrl, ipamQueryUrl) - ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) if !stopcnm { var pluginConfig acn.PluginConfig pluginConfig.Version = version @@ -249,6 +245,7 @@ func main() { ipamPlugin.SetOption(acn.OptEnvironment, environment) ipamPlugin.SetOption(acn.OptAPIServerURL, url) + ipamPlugin.SetOption(acn.OptIpamQueryUrl, ipamQueryUrl) ipamPlugin.SetOption(acn.OptIpamQueryInterval, ipamQueryInterval) if err := ipamPlugin.Start(&pluginConfig); err != nil { log.Printf("Failed to create IPAM plugin, err:%v.\n", err) From 385d9c7f9caf2e42897d45a5f55d9d105853478d Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 12 Jun 2018 15:54:32 -0700 Subject: [PATCH 38/51] updated ovs config to clean ovs setup on bootup --- cni/network/network.go | 48 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 6b240d5174..dc34cb5603 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -4,8 +4,10 @@ package network import ( + "bytes" "fmt" "net" + "os" "strconv" "strings" @@ -29,6 +31,7 @@ const ( podNameKey = "K8S_POD_NAME" vlanIDKey = "vlanid" dockerNetworkOption = "com.docker.network.generic" + ovsConfigFile = "/etc/default/openvswitch-switch" // Supported IP version. Currently support only IPv4 ipVersion = "4" @@ -178,7 +181,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, net.IPNet{}, err @@ -207,6 +210,38 @@ func getPodNameWithoutSuffix(podName string) string { return strings.Join(nameSplit, "-") } +func updateOVSConfig(option string) error { + f, err := os.OpenFile(ovsConfigFile, os.O_APPEND|os.O_RDWR, 0666) + if err != nil { + log.Printf("Error while opening ovs config %v", err) + return err + } + + defer f.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(f) + contents := buf.String() + + conSplit := strings.Split(contents, "\n") + + for _, existingOption := range conSplit { + if option == existingOption { + log.Printf("Not updating ovs config. Found option already written") + return nil + } + } + + log.Printf("writing ovsconfig option %v", option) + + if _, err = f.WriteString(option); err != nil { + log.Printf("Error while writing ovs config %v", err) + return err + } + + return nil +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -229,6 +264,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { // Add Interfaces to result. + if result == nil { + result = &cniTypesCurr.Result{} + } + iface = &cniTypesCurr.Interface{ Name: args.IfName, } @@ -318,8 +357,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { if nwInfoErr != nil { // Network does not exist. + log.Printf("[cni-net] Creating network %v.", networkId) + if nwCfg.MultiTenancy { + if err := updateOVSConfig("OVS_CTL_OPTS='--delete-bridges'"); err != nil { + return err + } + } + if result == nil { // Call into IPAM plugin to allocate an address pool for the network. result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) From ed58e92da69a83a3dbee66892dcd9fd1e97f430f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 12 Jun 2018 16:55:31 -0700 Subject: [PATCH 39/51] fixed the hardcoded part --- cni/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index dc34cb5603..124aad197f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -181,7 +181,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, 0, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, 0, net.IPNet{}, err From c76a859e066bad74efb6b2767ffaa06f3adc9811 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 13 Jun 2018 11:19:47 -0700 Subject: [PATCH 40/51] Added multiwrite option for windows logger --- log/logger_linux.go | 110 +++++++++++++++++++++--------------------- log/logger_windows.go | 81 +++++++++++++++++-------------- 2 files changed, 100 insertions(+), 91 deletions(-) diff --git a/log/logger_linux.go b/log/logger_linux.go index d8148a99da..5c2e34f130 100644 --- a/log/logger_linux.go +++ b/log/logger_linux.go @@ -1,56 +1,54 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package log - -import ( - "fmt" - "io" - "log" - "log/syslog" - "os" -) - -const ( - // LogPath is the path where log files are stored. - LogPath = "/var/log/" -) - -// SetTarget sets the log target. -func (logger *Logger) SetTarget(target int) error { - var err error - - switch target { - case TargetStdout: - logger.out = os.Stdout - break - case TargetStderr: - logger.out = os.Stderr - break - case TargetSyslog: - logger.out, err = syslog.New(log.LstdFlags, logger.name) - break - case TargetLogfile: - logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) - break - case TargetMultiWrite: - logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) - mw := io.MultiWriter(os.Stdout, logger.out) - if err == nil { - logger.l.SetOutput(mw) - logger.target = target - return nil - } - - break - default: - err = fmt.Errorf("Invalid log target %d", target) - } - - if err == nil { - logger.l.SetOutput(logger.out) - logger.target = target - } - - return err -} +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package log + +import ( + "fmt" + "io" + "log" + "log/syslog" + "os" +) + +const ( + // LogPath is the path where log files are stored. + LogPath = "/var/log/" +) + +// SetTarget sets the log target. +func (logger *Logger) SetTarget(target int) error { + var err error + + switch target { + case TargetStdout: + logger.out = os.Stdout + + case TargetStderr: + logger.out = os.Stderr + + case TargetSyslog: + logger.out, err = syslog.New(log.LstdFlags, logger.name) + + case TargetLogfile: + logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + + case TargetMultiWrite: + logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + if err == nil { + logger.l.SetOutput(io.MultiWriter(os.Stdout, logger.out)) + logger.target = target + return nil + } + + default: + err = fmt.Errorf("Invalid log target %d", target) + } + + if err == nil { + logger.l.SetOutput(logger.out) + logger.target = target + } + + return err +} diff --git a/log/logger_windows.go b/log/logger_windows.go index 815e93faa5..f504ee20b3 100644 --- a/log/logger_windows.go +++ b/log/logger_windows.go @@ -1,35 +1,46 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package log - -import ( - "fmt" - "os" -) - -const ( - // LogPath is the path where log files are stored. - LogPath = "" -) - -// SetTarget sets the log target. -func (logger *Logger) SetTarget(target int) error { - var err error - - switch target { - case TargetStderr: - logger.out = os.Stderr - case TargetLogfile: - logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) - default: - err = fmt.Errorf("Invalid log target %d", target) - } - - if err == nil { - logger.l.SetOutput(logger.out) - logger.target = target - } - - return err -} +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package log + +import ( + "fmt" + "io" + "os" +) + +const ( + // LogPath is the path where log files are stored. + LogPath = "" +) + +// SetTarget sets the log target. +func (logger *Logger) SetTarget(target int) error { + var err error + + switch target { + case TargetStderr: + logger.out = os.Stderr + + case TargetLogfile: + logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + + case TargetMultiWrite: + logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) + if err == nil { + logger.l.SetOutput(io.MultiWriter(os.Stdout, logger.out)) + logger.target = target + return nil + } + + default: + err = fmt.Errorf("Invalid log target %d", target) + } + + if err == nil { + logger.l.SetOutput(logger.out) + logger.target = target + } + + return err +} From 8313b04392541a2deb52f268b0137eead7453851 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 19 Jun 2018 22:15:08 -0700 Subject: [PATCH 41/51] fixed getting podname without suffix --- cni/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/network.go b/cni/network/network.go index 124aad197f..a3af7a6ce2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -202,7 +202,7 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp func getPodNameWithoutSuffix(podName string) string { nameSplit := strings.Split(podName, "-") if len(nameSplit) > 2 { - nameSplit = nameSplit[:2] + nameSplit = nameSplit[:len(nameSplit)-2] } else { return podName } From 1151913796f0a193d4225f9804828c3f4a0738a4 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 19 Jun 2018 22:25:37 -0700 Subject: [PATCH 42/51] added logs --- cni/network/network.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cni/network/network.go b/cni/network/network.go index a3af7a6ce2..52e3feab02 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -201,12 +201,14 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp func getPodNameWithoutSuffix(podName string) string { nameSplit := strings.Split(podName, "-") + log.Printf("namesplit %v", nameSplit) if len(nameSplit) > 2 { nameSplit = nameSplit[:len(nameSplit)-2] } else { return podName } + log.Printf("Final namesplit %v", nameSplit) return strings.Join(nameSplit, "-") } From e2ad9fd811c4503ecf41aa93f57ad14be4c6bd53 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 21 Jun 2018 12:37:49 -0700 Subject: [PATCH 43/51] Added snat support for outbound traffic --- cni/netconfig.go | 21 ++-- cni/network/network.go | 62 +++++++--- network/api.go | 3 +- network/bridge_endpointclient_linux.go | 61 +++++++++- network/endpoint.go | 55 ++++----- network/endpoint_linux.go | 100 +++------------- network/manager.go | 5 + network/network.go | 30 ++--- network/network_linux.go | 16 +-- network/ovs_endpointclient_linux.go | 154 +++++++++++++++++++++++-- network/ovs_networkclient_linux.go | 112 +++++++++++++++++- ovsctl/ovsctl.go | 1 - 12 files changed, 440 insertions(+), 180 deletions(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index c1628a4d2f..7ce5057bcd 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -24,16 +24,17 @@ type KVPair struct { // NetworkConfig represents Azure CNI plugin network configuration. type NetworkConfig struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Type string `json:"type"` - Mode string `json:"mode"` - Master string `json:"master"` - Bridge string `json:"bridge,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - LogTarget string `json:"logTarget,omitempty"` - MultiTenancy bool `json:"multiTenancy,omitempty"` - Ipam struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Type string `json:"type"` + Mode string `json:"mode"` + Master string `json:"master"` + Bridge string `json:"bridge,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + LogTarget string `json:"logTarget,omitempty"` + MultiTenancy bool `json:"multiTenancy,omitempty"` + EnableSnatOnHost bool `json:"enableSnatOnHost,omitempty"` + Ipam struct { Type string `json:"type"` Environment string `json:"environment,omitempty"` AddrSpace string `json:"addressSpace,omitempty"` diff --git a/cni/network/network.go b/cni/network/network.go index 52e3feab02..6c5251886d 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -32,6 +32,7 @@ const ( vlanIDKey = "vlanid" dockerNetworkOption = "com.docker.network.generic" ovsConfigFile = "/etc/default/openvswitch-switch" + ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" // Supported IP version. Currently support only IPv4 ipVersion = "4" @@ -165,26 +166,21 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType } } - // route for default gw - gwIP := net.ParseIP(networkConfig.IPConfiguration.GatewayIPAddress) - _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") - dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) - return result } -func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, int, net.IPNet, error) { +func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { cnsClient, err := cnsclient.NewCnsClient("") if err != nil { log.Printf("Initializing CNS client error %v", err) - return nil, 0, net.IPNet{}, err + return nil, nil, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + // networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) - return nil, 0, net.IPNet{}, err + return nil, nil, net.IPNet{}, err } log.Printf("Network config received from cns %v", networkConfig) @@ -193,10 +189,10 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp if subnetPrefix == nil { errBuf := fmt.Sprintf("Interface not found for this ip %v", networkConfig.PrimaryInterfaceIdentifier) log.Printf(errBuf) - return nil, 0, net.IPNet{}, fmt.Errorf(errBuf) + return nil, nil, net.IPNet{}, fmt.Errorf(errBuf) } - return convertToCniResult(networkConfig), networkConfig.MultiTenancyInfo.ID, *subnetPrefix, nil + return convertToCniResult(networkConfig), networkConfig, *subnetPrefix, nil } func getPodNameWithoutSuffix(podName string) string { @@ -259,6 +255,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { iface *cniTypesCurr.Interface subnetPrefix net.IPNet vlanid int + cnsNetworkConfig *cns.GetNetworkContainerResponse ) log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", @@ -273,7 +270,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { iface = &cniTypesCurr.Interface{ Name: args.IfName, } + + defaultIface := &cniTypesCurr.Interface{ + Name: "eth1", + } + result.Interfaces = append(result.Interfaces, iface) + result.Interfaces = append(result.Interfaces, defaultIface) // Convert result to the requested CNI version. res, err := result.GetAsVersion(nwCfg.CNIVersion) @@ -324,11 +327,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { endpointId := GetEndpointID(args) if nwCfg.MultiTenancy { - result, vlanid, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) + result, cnsNetworkConfig, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) if err != nil { log.Printf("[CNI Multitenancy] GetContainerNetworkConfiguration failed with %v", err) return err } + + vlanid = cnsNetworkConfig.MultiTenancyInfo.ID } log.Printf("subnetprefix :%v", subnetPrefix.IP.String()) @@ -363,7 +368,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Creating network %v.", networkId) if nwCfg.MultiTenancy { - if err := updateOVSConfig("OVS_CTL_OPTS='--delete-bridges'"); err != nil { + if err := updateOVSConfig(ovsOpt); err != nil { return err } } @@ -422,7 +427,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Gateway: gateway, }, }, - BridgeName: nwCfg.Bridge, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, DNS: network.DNSInfo{ Servers: nwCfg.DNS.Nameservers, Suffix: k8sNamespace + "." + strings.Join(nwCfg.DNS.Search, ","), @@ -471,10 +477,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } epInfo = &network.EndpointInfo{ - Id: endpointId, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, + Id: endpointId, + ContainerID: args.ContainerID, + NetNsPath: args.Netns, + IfName: args.IfName, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, } epInfo.Data = make(map[string]interface{}) @@ -515,6 +522,23 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } + if nwCfg.MultiTenancy { + // route for default gw + var gwIP net.IP + _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + + if nwCfg.EnableSnatOnHost { + gwIP = net.ParseIP("192.168.0.1") + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: "eth1"}) + } else { + gwIP = net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) + } + + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) + } + // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/network/api.go b/network/api.go index 9ce77c4dde..e8f6d8353e 100644 --- a/network/api.go +++ b/network/api.go @@ -18,5 +18,6 @@ var ( errEndpointInUse = fmt.Errorf("Endpoint is already joined to a sandbox") errEndpointNotInUse = fmt.Errorf("Endpoint is not joined to a sandbox") - OptVethName = "vethname" + OptVethName = "vethname" + OptEnableSnatOnHost = "enableSnatOnHost" ) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 5aaeb8ecb6..b3cbc34fac 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -12,6 +12,7 @@ type LinuxBridgeEndpointClient struct { bridgeName string hostPrimaryIfName string hostVethName string + containerVethName string hostPrimaryMac net.HardwareAddr containerMac net.HardwareAddr mode string @@ -21,8 +22,8 @@ func NewLinuxBridgeEndpointClient( bridgeName string, hostPrimaryIfName string, hostVethName string, + containerVethName string, hostPrimaryMac net.HardwareAddr, - containerMac net.HardwareAddr, mode string, ) *LinuxBridgeEndpointClient { @@ -30,14 +31,28 @@ func NewLinuxBridgeEndpointClient( bridgeName: bridgeName, hostPrimaryIfName: hostPrimaryIfName, hostVethName: hostVethName, + containerVethName: containerVethName, hostPrimaryMac: hostPrimaryMac, - containerMac: containerMac, mode: mode, } return client } +func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + if err := createEndpoint(client.hostVethName, client.containerVethName); err != nil { + return err + } + + containerIf, err := net.InterfaceByName(client.containerVethName) + if err != nil { + return err + } + + client.containerMac = containerIf.HardwareAddr + return nil +} + func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { var err error @@ -96,3 +111,45 @@ func (client *LinuxBridgeEndpointClient) getArpReplyAddress(epMacAddress net.Har return macAddress } + +func (client *LinuxBridgeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + // Move the container interface to container's network namespace. + log.Printf("[net] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath) + if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return err + } + + return nil +} + +func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := setupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return err + } + + client.containerVethName = epInfo.IfName + return nil +} + +func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := assignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return err + } + + if err := addRoutes(client.containerVethName, epInfo.Routes); err != nil { + return err + } + + return nil +} + +func (client *LinuxBridgeEndpointClient) DeleteEndpoints(ep *endpoint) error { + log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) + err := netlink.DeleteLink(ep.HostIfName) + if err != nil { + log.Printf("[net] Failed to delete veth pair %v: %v.", ep.HostIfName, err) + return err + } + + return nil +} diff --git a/network/endpoint.go b/network/endpoint.go index ba66e8b9ba..5bc020c262 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -13,40 +13,43 @@ import ( // Endpoint represents a container network interface. type endpoint struct { - Id string - HnsId string `json:",omitempty"` - SandboxKey string - IfName string - HostIfName string - MacAddress net.HardwareAddr - IPAddresses []net.IPNet - Gateways []net.IP - DNS DNSInfo - Routes []RouteInfo - VlanID int + Id string + HnsId string `json:",omitempty"` + SandboxKey string + IfName string + HostIfName string + MacAddress net.HardwareAddr + IPAddresses []net.IPNet + Gateways []net.IP + DNS DNSInfo + Routes []RouteInfo + VlanID int + EnableSnatOnHost bool } // EndpointInfo contains read-only information about an endpoint. type EndpointInfo struct { - Id string - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - Data map[string]interface{} + Id string + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + Data map[string]interface{} } // RouteInfo contains information about an IP route. type RouteInfo struct { - Dst net.IPNet - Gw net.IP + Dst net.IPNet + Gw net.IP + DevName string } // ConstructEndpointID constructs endpoint name from netNsPath. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index c4fcd9768c..bfb867932f 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -68,19 +68,13 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, epInfo.Id[:7]) } - log.Printf("[net] Creating veth pair %v %v.", hostIfName, contIfName) - - link := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: contIfName, - }, - PeerName: hostIfName, + if vlanid != 0 { + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), contIfName, vlanid, epInfo.EnableSnatOnHost) + } else { + epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, contIfName, nw.extIf.MacAddress, nw.Mode) } - err = netlink.AddLink(&link) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) + if err := epClient.AddEndpoints(epInfo); err != nil { return nil, err } @@ -91,28 +85,11 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } }() - // - // Host network interface setup. - // - - // Host interface up. - log.Printf("[net] Setting link %v state up.", hostIfName) - err = netlink.SetLinkState(hostIfName, true) - if err != nil { - return nil, err - } - containerIf, err = net.InterfaceByName(contIfName) if err != nil { return nil, err } - if vlanid != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), containerIf.HardwareAddr.String(), vlanid) - } else { - epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress, containerIf.HardwareAddr, nw.Mode) - } - // Setup rules for IP addresses on the container interface. epClient.AddEndpointRules(epInfo) @@ -126,17 +103,13 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } defer ns.Close() - // Move the container interface to container's network namespace. - log.Printf("[net] Setting link %v netns %v.", contIfName, epInfo.NetNsPath) - err = netlink.SetLinkNetNs(contIfName, ns.GetFd()) - if err != nil { + if err := epClient.MoveEndpointsToContainerNS(epInfo, ns.GetFd()); err != nil { return nil, err } // Enter the container network namespace. log.Printf("[net] Entering netns %v.", epInfo.NetNsPath) - err = ns.Enter() - if err != nil { + if err := ns.Enter(); err != nil { return nil, err } @@ -152,53 +125,13 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // If a name for the container interface is specified... if epInfo.IfName != "" { - // Interface needs to be down before renaming. - log.Printf("[net] Setting link %v state down.", contIfName) - err = netlink.SetLinkState(contIfName, false) - if err != nil { - return nil, err - } - - // Rename the container interface. - log.Printf("[net] Setting link %v name %v.", contIfName, epInfo.IfName) - err = netlink.SetLinkName(contIfName, epInfo.IfName) - if err != nil { - return nil, err - } - contIfName = epInfo.IfName - - // Bring the interface back up. - log.Printf("[net] Setting link %v state up.", contIfName) - err = netlink.SetLinkState(contIfName, true) - if err != nil { - return nil, err - } - } - - // Assign IP address to container network interface. - for _, ipAddr := range epInfo.IPAddresses { - log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), contIfName) - err = netlink.AddIpAddress(contIfName, ipAddr.IP, &ipAddr) - if err != nil { + if err := epClient.SetupContainerInterfaces(epInfo); err != nil { return nil, err } } - // Add IP routes to container network interface. - for _, route := range epInfo.Routes { - log.Printf("[net] Adding IP route %+v to link %v.", route, contIfName) - - nlRoute := &netlink.Route{ - Family: netlink.GetIpAddressFamily(route.Gw), - Dst: &route.Dst, - Gw: route.Gw, - LinkIndex: containerIf.Index, - } - - err = netlink.AddIpRoute(nlRoute) - if err != nil { - return nil, err - } + if err := epClient.ConfigureContainerInterfacesAndRoutes(epInfo); err != nil { + return nil, err } // Create the endpoint object. @@ -211,6 +144,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { Gateways: []net.IP{nw.extIf.IPv4Gateway}, DNS: epInfo.DNS, VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, } for _, route := range epInfo.Routes { @@ -228,19 +162,13 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", ep.VlanID) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", ep.VlanID, ep.EnableSnatOnHost) } else { - epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress, nil, nw.Mode) + epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, "", nw.extIf.MacAddress, nw.Mode) } epClient.DeleteEndpointRules(ep) - - log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) - err := netlink.DeleteLink(ep.HostIfName) - if err != nil { - log.Printf("[net] Failed to delete veth pair %v: %v.", ep.HostIfName, err) - return err - } + epClient.DeleteEndpoints(ep) return nil } diff --git a/network/manager.go b/network/manager.go index a2ffc0dfe3..6b635f1155 100644 --- a/network/manager.go +++ b/network/manager.go @@ -28,8 +28,13 @@ type NetworkClient interface { } type EndpointClient interface { + AddEndpoints(epInfo *EndpointInfo) error AddEndpointRules(epInfo *EndpointInfo) error DeleteEndpointRules(ep *endpoint) + MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error + SetupContainerInterfaces(epInfo *EndpointInfo) error + ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error + DeleteEndpoints(ep *endpoint) error } // NetworkManager manages the set of container networking resources. diff --git a/network/network.go b/network/network.go index 1008ba8d72..f2def56489 100644 --- a/network/network.go +++ b/network/network.go @@ -33,24 +33,26 @@ type externalInterface struct { // A container network is a set of endpoints allowed to communicate with each other. type network struct { - Id string - HnsId string `json:",omitempty"` - Mode string - VlanId int - Subnets []SubnetInfo - Endpoints map[string]*endpoint - extIf *externalInterface + Id string + HnsId string `json:",omitempty"` + Mode string + VlanId int + Subnets []SubnetInfo + Endpoints map[string]*endpoint + extIf *externalInterface + EnableSnatOnHost bool } // NetworkInfo contains read-only information about a container network. type NetworkInfo struct { - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - Options map[string]interface{} + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + Options map[string]interface{} } // SubnetInfo contains subnet information for a container network. diff --git a/network/network_linux.go b/network/network_linux.go index db265e2765..e1783ede6e 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -57,11 +57,12 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Create the network object. nw := &network{ - Id: nwInfo.Id, - Mode: nwInfo.Mode, - Endpoints: make(map[string]*endpoint), - extIf: extIf, - VlanId: vlanid, + Id: nwInfo.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: vlanid, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, } return nw, nil @@ -72,7 +73,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, nw.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) } @@ -167,6 +168,7 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var err error var networkClient NetworkClient + // var disableMultiTenancyInternet bool log.Printf("[net] Connecting interface %v.", extIf.Name) defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() @@ -191,7 +193,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[vlanIDKey] != nil { - networkClient = NewOVSClient(bridgeName, extIf.Name) + networkClient = NewOVSClient(bridgeName, extIf.Name, nwInfo.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index fc357f5b93..7371d2f423 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -1,7 +1,12 @@ package network import ( + "fmt" + "net" + + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" ) @@ -10,30 +15,81 @@ type OVSEndpointClient struct { hostPrimaryIfName string hostVethName string hostPrimaryMac string + containerVethName string containerMac string + intVethName string vlanID int + enableSnatOnHost bool } +const ( + intVethInterfacePrefix = commonInterfacePrefix + "vint" + azureInternetIfName = "eth1" +) + func NewOVSEndpointClient( bridgeName string, hostPrimaryIfName string, hostVethName string, hostPrimaryMac string, - containerMac string, - vlanid int) *OVSEndpointClient { + containerVethName string, + vlanid int, + enableSnatOnHost bool) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: bridgeName, hostPrimaryIfName: hostPrimaryIfName, hostVethName: hostVethName, hostPrimaryMac: hostPrimaryMac, - containerMac: containerMac, + containerVethName: containerVethName, vlanID: vlanid, + enableSnatOnHost: enableSnatOnHost, } return client } +func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + if err := createEndpoint(client.hostVethName, client.containerVethName); err != nil { + return err + } + + containerIf, err := net.InterfaceByName(client.containerVethName) + if err != nil { + return err + } + + client.containerMac = containerIf.HardwareAddr.String() + + if client.enableSnatOnHost { + hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, epInfo.Id[:7]) + contIfName := fmt.Sprintf("%s%s-2", intVethInterfacePrefix, epInfo.Id[:7]) + + if err := createEndpoint(hostIfName, contIfName); err != nil { + return err + } + + if err := netlink.SetLinkMaster(hostIfName, internetBridgeName); err != nil { + return err + } + + internetIf, err := net.InterfaceByName(contIfName) + if err != nil { + return err + } + + intVethMac := internetIf.HardwareAddr.String() + arpCmd := fmt.Sprintf("arp -s 192.168.0.2 %v", intVethMac) + if _, err := common.ExecuteShellCommand(arpCmd); err != nil { + log.Printf("Setting static arp failed for internet interface %v", err) + } + + client.intVethName = contIfName + } + + return nil +} + func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { log.Printf("[ovs] Setting link %v master %v.", client.hostVethName, client.bridgeName) if err := ovsctl.AddPortOnOVSBridge(client.hostVethName, client.bridgeName, client.vlanID); err != nil { @@ -79,31 +135,107 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { } func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { - log.Printf("[net] Get ovs port for interface %v.", ep.HostIfName) + log.Printf("[ovs] Get ovs port for interface %v.", ep.HostIfName) containerPort, err := ovsctl.GetOVSPortNumber(client.hostVethName) if err != nil { - log.Printf("[net] Get portnum failed with error %v", err) + log.Printf("[ovs] Get portnum failed with error %v", err) } - log.Printf("[net] Get ovs port for interface %v.", client.hostPrimaryIfName) + log.Printf("[ovs] Get ovs port for interface %v.", client.hostPrimaryIfName) hostPort, err := ovsctl.GetOVSPortNumber(client.hostPrimaryIfName) if err != nil { - log.Printf("[net] Get portnum failed with error %v", err) + log.Printf("[ovs] Get portnum failed with error %v", err) } // Delete IP SNAT - log.Printf("[net] Deleting IP SNAT for port %v", containerPort) + log.Printf("[ovs] Deleting IP SNAT for port %v", containerPort) ovsctl.DeleteIPSnatRule(client.bridgeName, containerPort) // Delete Arp Reply Rules for container - log.Printf("[net] Deleting ARP reply rule for ip %v vlanid %v for container port", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) + log.Printf("[ovs] Deleting ARP reply rule for ip %v vlanid %v for container port", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) ovsctl.DeleteArpReplyRule(client.bridgeName, containerPort, ep.IPAddresses[0].IP, ep.VlanID) // Delete MAC address translation rule. - log.Printf("[net] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) + log.Printf("[ovs] Deleting MAC DNAT rule for IP address %v and vlan %v.", ep.IPAddresses[0].IP.String(), ep.VlanID) ovsctl.DeleteMacDnatRule(client.bridgeName, hostPort, ep.IPAddresses[0].IP, ep.VlanID) // Delete port from ovs bridge - log.Printf("[net] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) + log.Printf("[ovs] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) ovsctl.DeletePortFromOVS(client.bridgeName, client.hostVethName) } + +func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + // Move the container interface to container's network namespace. + log.Printf("[ovs] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath) + if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return err + } + + if client.enableSnatOnHost { + log.Printf("[ovs] Setting link %v netns %v.", client.intVethName, epInfo.NetNsPath) + if err := netlink.SetLinkNetNs(client.intVethName, nsID); err != nil { + return err + } + } + + return nil +} + +func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + + if err := setupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return err + } + + client.containerVethName = epInfo.IfName + + if client.enableSnatOnHost { + if err := setupContainerInterface(client.intVethName, azureInternetIfName); err != nil { + return err + } + client.intVethName = azureInternetIfName + } + + return nil +} + +func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := assignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return err + } + + if client.enableSnatOnHost { + log.Printf("[ovs] Adding IP address 192.168.0.2/16 to link %v.", client.intVethName) + ip, intIpAddr, _ := net.ParseCIDR("192.168.0.2/16") + if err := netlink.AddIpAddress(client.intVethName, ip, intIpAddr); err != nil { + return err + } + } + + if err := addRoutes(client.containerVethName, epInfo.Routes); err != nil { + return err + } + + return nil +} + +func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { + log.Printf("[ovs] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName) + err := netlink.DeleteLink(ep.HostIfName) + if err != nil { + log.Printf("[ovs] Failed to delete veth pair %v: %v.", ep.HostIfName, err) + return err + } + + if client.enableSnatOnHost { + hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, ep.Id[:7]) + log.Printf("[ovs] Deleting internet veth pair %v.", hostIfName) + err = netlink.DeleteLink(hostIfName) + if err != nil { + log.Printf("[ovs] Failed to delete veth pair %v: %v.", hostIfName, err) + return err + } + } + + return nil +} diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index abbab23eb3..e399dfb79d 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -1,32 +1,138 @@ package network import ( + "net" "strings" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" ) type OVSNetworkClient struct { bridgeName string hostInterfaceName string + enableSnatOnHost bool } -func NewOVSClient(bridgeName string, hostInterfaceName string) *OVSNetworkClient { +const ( + azureInternetVeth0 = "azintveth0" + azureInternetVeth1 = "azintveth1" + internetBridgeName = "azintbr" +) + +func NewOVSClient(bridgeName string, hostInterfaceName string, enableSnatOnHost bool) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, + enableSnatOnHost: enableSnatOnHost, } return ovsClient } func (client *OVSNetworkClient) CreateBridge() error { - return ovsctl.CreateOVSBridge(client.bridgeName) + if err := ovsctl.CreateOVSBridge(client.bridgeName); err != nil { + return err + } + + if err := CreateInternetBridge(); err != nil { + log.Printf("[net] Creating internet bridge failed with erro %v", err) + //return err + } + + link := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureInternetVeth0, + }, + PeerName: azureInternetVeth1, + } + + err := netlink.AddLink(&link) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + if client.enableSnatOnHost { + ip, addr, _ := net.ParseCIDR("192.168.0.1/16") + err = netlink.AddIpAddress(internetBridgeName, ip, addr) + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("[net] Failed to add IP address %v: %v.", addr, err) + return err + } + + if err := netlink.SetLinkState(internetBridgeName, true); err != nil { + return err + } + + if err := netlink.SetLinkState(azureInternetVeth0, true); err != nil { + return err + } + + if err := netlink.SetLinkMaster(azureInternetVeth0, internetBridgeName); err != nil { + return err + } + + if err := netlink.SetLinkState(azureInternetVeth1, true); err != nil { + return err + } + + if err := ovsctl.AddPortOnOVSBridge(azureInternetVeth1, client.bridgeName, 0); err != nil { + return err + } + + _, err := common.ExecuteShellCommand("iptables -t nat -A POSTROUTING -j MASQUERADE") + return err + + _, err := common.ExecuteShellCommand("ebtables -t nat -A PREROUTING -p 802_1Q -j DROP") + return err + } + + return nil } func (client *OVSNetworkClient) DeleteBridge() error { - return ovsctl.DeleteOVSBridge(client.bridgeName) + if err := ovsctl.DeleteOVSBridge(client.bridgeName); err != nil { + log.Printf("Deleting ovs bridge failed with error %v", err) + return err + } + + if client.enableSnatOnHost { + if err := DeleteInternetBridge(); err != nil { + log.Printf("Deleting internet bridge failed with error %v", err) + return err + } + + return netlink.DeleteLink(azureInternetVeth0) + } + + return nil +} + +func CreateInternetBridge() error { + log.Printf("[net] Creating Internet bridge %v.", internetBridgeName) + + link := netlink.BridgeLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_BRIDGE, + Name: internetBridgeName, + }, + } + + return netlink.AddLink(&link) +} + +func DeleteInternetBridge() error { + // Delete the bridge. + err := netlink.DeleteLink(internetBridgeName) + if err != nil { + log.Printf("[net] Failed to delete bridge %v, err:%v.", internetBridgeName, err) + } + + return nil } func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index b3324018ee..4a516cdca2 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -57,7 +57,6 @@ func AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error } func GetOVSPortNumber(interfaceName string) (string, error) { - log.Printf("[ovs] Get ovs port for interface %v.", interfaceName) cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", interfaceName) ofport, err := common.ExecuteShellCommand(cmd) if err != nil { From eaec5d9ab50f52e881eba89bceb986a1752befc5 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 21 Jun 2018 18:00:41 -0700 Subject: [PATCH 44/51] Added cnetspace routes --- cni/network/network.go | 13 ++++++++++--- cns/dnccontract.go | 1 + cns/restserver/restserver.go | 1 + network/ovs_networkclient_linux.go | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 6c5251886d..ca62bfa55a 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -166,6 +166,13 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType } } + for _, ipRouteSubnet := range networkConfig.CnetAddressSpace { + log.Printf("Adding cnetAddressspace routes %v %v", ipRouteSubnet.IPAddress, ipRouteSubnet.PrefixLength) + routeIPnet := net.IPNet{IP: net.ParseIP(ipRouteSubnet.IPAddress), Mask:net.CIDRMask(int(ipRouteSubnet.PrefixLength), 32)} + gwIP := net.ParseIP(ipconfig.GatewayIPAddress) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: routeIPnet, GW: gwIP}) + } + return result } @@ -176,14 +183,14 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, nil, net.IPNet{}, err } - // networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) - networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") + networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + // networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, nil, net.IPNet{}, err } - log.Printf("Network config received from cns %v", networkConfig) + log.Printf("Network config received from cns %+v", networkConfig) subnetPrefix := common.GetIpNet(networkConfig.PrimaryInterfaceIdentifier) if subnetPrefix == nil { diff --git a/cns/dnccontract.go b/cns/dnccontract.go index dd32031d79..0e955662a1 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -108,6 +108,7 @@ type GetNetworkContainerRequest struct { type GetNetworkContainerResponse struct { IPConfiguration IPConfiguration Routes []Route + CnetAddressSpace []IPSubnet MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string Response Response diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index a20daca0fe..143773e638 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1000,6 +1000,7 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo getNetworkContainerResponse = cns.GetNetworkContainerResponse{ IPConfiguration: savedReq.IPConfiguration, Routes: savedReq.Routes, + CnetAddressSpace: savedReq.CnetAddressSpace, MultiTenancyInfo: savedReq.MultiTenancyInfo, PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index e399dfb79d..9ee0d70cb0 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -87,7 +87,7 @@ func (client *OVSNetworkClient) CreateBridge() error { _, err := common.ExecuteShellCommand("iptables -t nat -A POSTROUTING -j MASQUERADE") return err - _, err := common.ExecuteShellCommand("ebtables -t nat -A PREROUTING -p 802_1Q -j DROP") + _, err = common.ExecuteShellCommand("ebtables -t nat -A PREROUTING -p 802_1Q -j DROP") return err } From fa0fd464d925efbc679d45c7b43f39247494c06c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 25 Jun 2018 16:41:09 -0700 Subject: [PATCH 45/51] snat changes --- cni/network/network.go | 2 +- network/endpoint_common_linux.go | 99 +++++++++++++++++++++++++++++ network/ovs_endpointclient_linux.go | 15 ++++- network/ovs_networkclient_linux.go | 23 +++++-- 4 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 network/endpoint_common_linux.go diff --git a/cni/network/network.go b/cni/network/network.go index ca62bfa55a..1d929b75c2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -536,7 +536,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} if nwCfg.EnableSnatOnHost { - gwIP = net.ParseIP("192.168.0.1") + gwIP = net.ParseIP("169.254.0.1") epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: "eth1"}) } else { gwIP = net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) diff --git a/network/endpoint_common_linux.go b/network/endpoint_common_linux.go new file mode 100644 index 0000000000..5466833f40 --- /dev/null +++ b/network/endpoint_common_linux.go @@ -0,0 +1,99 @@ +package network + +import ( + "net" + "strings" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" +) + +func createEndpoint(hostVethName string, containerVethName string) error { + log.Printf("[net] Creating veth pair %v %v.", hostVethName, containerVethName) + + link := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: hostVethName, + }, + PeerName: containerVethName, + } + + err := netlink.AddLink(&link) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + log.Printf("[net] Setting link %v state up.", hostVethName) + err = netlink.SetLinkState(hostVethName, true) + if err != nil { + return err + } + + return nil +} + +func setupContainerInterface(containerVethName string, targetIfName string) error { + // Interface needs to be down before renaming. + log.Printf("[net] Setting link %v state down.", containerVethName) + if err := netlink.SetLinkState(containerVethName, false); err != nil { + return err + } + + // Rename the container interface. + log.Printf("[net] Setting link %v name %v.", containerVethName, targetIfName) + if err := netlink.SetLinkName(containerVethName, targetIfName); err != nil { + return err + } + + // Bring the interface back up. + log.Printf("[net] Setting link %v state up.", targetIfName) + return netlink.SetLinkState(targetIfName, true) +} + +func assignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { + // Assign IP address to container network interface. + for _, ipAddr := range ipAddresses { + log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), interfaceName) + err := netlink.AddIpAddress(interfaceName, ipAddr.IP, &ipAddr) + if err != nil { + return err + } + } + + return nil +} + +func addRoutes(interfaceName string, routes []RouteInfo) error { + ifIndex := 0 + containerIf, _ := net.InterfaceByName(interfaceName) + + for _, route := range routes { + log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) + + if route.DevName != "" { + devIf, _ := net.InterfaceByName(route.DevName) + ifIndex = devIf.Index + } else { + ifIndex = containerIf.Index + } + + nlRoute := &netlink.Route{ + Family: netlink.GetIpAddressFamily(route.Gw), + Dst: &route.Dst, + Gw: route.Gw, + LinkIndex: ifIndex, + } + + if err := netlink.AddIpRoute(nlRoute); err != nil { + if !strings.Contains(strings.ToLower(err.Error()), "file exists") { + return err + } else { + log.Printf("route already exists") + } + } + } + + return nil +} diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 7371d2f423..1b599f319e 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -79,7 +79,8 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } intVethMac := internetIf.HardwareAddr.String() - arpCmd := fmt.Sprintf("arp -s 192.168.0.2 %v", intVethMac) + arpCmd := fmt.Sprintf("arp -s 169.254.0.2 %v", intVethMac) + log.Printf("Adding arp entry for internet veth %v", arpCmd) if _, err := common.ExecuteShellCommand(arpCmd); err != nil { log.Printf("Setting static arp failed for internet interface %v", err) } @@ -205,8 +206,8 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E } if client.enableSnatOnHost { - log.Printf("[ovs] Adding IP address 192.168.0.2/16 to link %v.", client.intVethName) - ip, intIpAddr, _ := net.ParseCIDR("192.168.0.2/16") + log.Printf("[ovs] Adding IP address 169.254.0.2/16 to link %v.", client.intVethName) + ip, intIpAddr, _ := net.ParseCIDR("169.254.0.2/16") if err := netlink.AddIpAddress(client.intVethName, ip, intIpAddr); err != nil { return err } @@ -228,6 +229,14 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { } if client.enableSnatOnHost { + arpCmd := fmt.Sprintf("arp -d 169.254.0.2") + + log.Printf("Adding arp entry for internet veth %v", arpCmd) + if _, err := common.ExecuteShellCommand(arpCmd); err != nil { + log.Printf("Setting static arp failed for internet interface %v", err) + } + + hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, ep.Id[:7]) log.Printf("[ovs] Deleting internet veth pair %v.", hostIfName) err = netlink.DeleteLink(hostIfName) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 9ee0d70cb0..09124a6864 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -57,7 +57,7 @@ func (client *OVSNetworkClient) CreateBridge() error { } if client.enableSnatOnHost { - ip, addr, _ := net.ParseCIDR("192.168.0.1/16") + ip, addr, _ := net.ParseCIDR("169.254.0.1/16") err = netlink.AddIpAddress(internetBridgeName, ip, addr) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("[net] Failed to add IP address %v: %v.", addr, err) @@ -84,11 +84,26 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - _, err := common.ExecuteShellCommand("iptables -t nat -A POSTROUTING -j MASQUERADE") - return err + /*var routes []RouteInfo + route = RouteInfo{Dst:net.ParseCIDR("169.254.169.254/32")} + routes = append(routes, route) + if err := addRoutes(client.bridgeName, routes); err != nil { + log.Printf("addroutes failed with error %v", err) + return err + }*/ + + cmd := "iptables -t nat -A POSTROUTING -j MASQUERADE" + log.Printf("Adding iptable snat rule %v", cmd) + _, err := common.ExecuteShellCommand(cmd) + if err != nil { + return err + } - _, err = common.ExecuteShellCommand("ebtables -t nat -A PREROUTING -p 802_1Q -j DROP") + cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" + log.Printf("Adding ebtable rule to drop vlan traffic on internet bridge %v", cmd) + _, err = common.ExecuteShellCommand(cmd) return err + } return nil From 5d190d914fb016d8ff55f4ffc6c9e11c171a787c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 26 Jun 2018 19:36:11 -0700 Subject: [PATCH 46/51] snat changes integrated with cns --- cni/network/network.go | 44 +++++++++++++---------------- cni/network/network_linux.go | 23 +++++++++++++++ cni/network/network_windows.go | 6 ++++ cns/dnccontract.go | 2 ++ cns/restserver/restserver.go | 1 + network/endpoint_common_linux.go | 34 ++++++++++++++++++++-- network/endpoint_linux.go | 9 ++++-- network/network_linux.go | 35 ++++++++++++++--------- network/ovs_endpointclient_linux.go | 30 +++++--------------- network/ovs_networkclient_linux.go | 32 ++++++++++++++------- 10 files changed, 142 insertions(+), 74 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 1d929b75c2..5dbfaed5dc 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -8,7 +8,6 @@ import ( "fmt" "net" "os" - "strconv" "strings" "github.com/Azure/azure-container-networking/client/cnsclient" @@ -29,10 +28,10 @@ const ( name = "azure-vnet" namespaceKey = "K8S_POD_NAMESPACE" podNameKey = "K8S_POD_NAME" - vlanIDKey = "vlanid" dockerNetworkOption = "com.docker.network.generic" ovsConfigFile = "/etc/default/openvswitch-switch" ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" + snatInterface = "eth1" // Supported IP version. Currently support only IPv4 ipVersion = "4" @@ -183,8 +182,8 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, nil, net.IPNet{}, err } - networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) - // networkConfig, err := cnsClient.GetNetworkConfiguration("TestPod", "TestPodNamespace") + // networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) + networkConfig, err := cnsClient.GetNetworkConfiguration("nginx-deployment2", "default") if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, nil, net.IPNet{}, err @@ -278,12 +277,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Name: args.IfName, } - defaultIface := &cniTypesCurr.Interface{ - Name: "eth1", + snatIface := &cniTypesCurr.Interface{ + Name: snatInterface, } result.Interfaces = append(result.Interfaces, iface) - result.Interfaces = append(result.Interfaces, defaultIface) + result.Interfaces = append(result.Interfaces, snatIface) // Convert result to the requested CNI version. res, err := result.GetAsVersion(nwCfg.CNIVersion) @@ -333,6 +332,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { networkId := nwCfg.Name endpointId := GetEndpointID(args) + // If multitenancy is enabled, get ip and vlan from cns if nwCfg.MultiTenancy { result, cnsNetworkConfig, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) if err != nil { @@ -380,7 +380,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } } - if result == nil { + if !nwCfg.MultiTenancy { // Call into IPAM plugin to allocate an address pool for the network. result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { @@ -444,11 +444,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } nwInfo.Options = make(map[string]interface{}) - if vlanid != 0 { - vlanMap := make(map[string]interface{}) - vlanMap[vlanIDKey] = strconv.Itoa(vlanid) - nwInfo.Options[dockerNetworkOption] = vlanMap - } + log.Printf("set network options") + setNetworkOptions(vlanid, &nwInfo) err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { @@ -458,7 +455,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnetPrefix.String()) } else { - if result == nil { + if !nwCfg.MultiTenancy { // Network already exists. subnetPrefix := nwInfo.Subnets[0].Prefix.String() log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) @@ -494,6 +491,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { if vlanid != 0 { epInfo.Data["vlanid"] = vlanid + epInfo.Data["LocalIP"] = cnsNetworkConfig.LocalIP } epInfo.Data[network.OptVethName] = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) @@ -529,21 +527,19 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } + // Adding default gateway if nwCfg.MultiTenancy { - // route for default gw - var gwIP net.IP - _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") - dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - + // if snat enabled, add 169.254.0.1 as default gateway if nwCfg.EnableSnatOnHost { - gwIP = net.ParseIP("169.254.0.1") - epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: "eth1"}) + log.Printf("add default route for multitenancy.snat on host enabled") + addDefaultRoute(epInfo, result) } else { - gwIP = net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) + _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } - - result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } // Create the endpoint. diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index aa8110ec52..cee6c14a8d 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,12 +1,35 @@ package network import ( + "net" + "strconv" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/network" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" + cniTypes "github.com/containernetworking/cni/pkg/types" ) +const ( + internetBridgeIP = "169.254.0.1" +) // handleConsecutiveAdd is a dummy function for Linux platform. func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } + +func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { + _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + gwIP := net.ParseIP(internetBridgeIP) + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: snatInterface}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) +} + +func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { + if vlanid != 0 { + vlanMap := make(map[string]interface{}) + vlanMap[network.VlanIDKey] = strconv.Itoa(vlanid) + vlanMap[network.InternetBridgeIPKey] = internetBridgeIP + nwInfo.Options[dockerNetworkOption] = vlanMap + } +} \ No newline at end of file diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index d98be65894..499676889c 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -56,3 +56,9 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ return nil, nil } + +func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { +} + +func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +} \ No newline at end of file diff --git a/cns/dnccontract.go b/cns/dnccontract.go index 0e955662a1..be02055fb0 100644 --- a/cns/dnccontract.go +++ b/cns/dnccontract.go @@ -36,6 +36,7 @@ type CreateNetworkContainerRequest struct { NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. AuthorizationToken string + LocalIP string OrchestratorContext json.RawMessage IPConfiguration IPConfiguration MultiTenancyInfo MultiTenancyInfo @@ -111,6 +112,7 @@ type GetNetworkContainerResponse struct { CnetAddressSpace []IPSubnet MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string + LocalIP string Response Response } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 143773e638..62647e5527 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1003,6 +1003,7 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo CnetAddressSpace: savedReq.CnetAddressSpace, MultiTenancyInfo: savedReq.MultiTenancyInfo, PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, + LocalIP: savedReq.LocalIP, } return getNetworkContainerResponse diff --git a/network/endpoint_common_linux.go b/network/endpoint_common_linux.go index 5466833f40..eabb14c4cf 100644 --- a/network/endpoint_common_linux.go +++ b/network/endpoint_common_linux.go @@ -67,7 +67,7 @@ func assignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { func addRoutes(interfaceName string, routes []RouteInfo) error { ifIndex := 0 - containerIf, _ := net.InterfaceByName(interfaceName) + interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) @@ -76,7 +76,7 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { devIf, _ := net.InterfaceByName(route.DevName) ifIndex = devIf.Index } else { - ifIndex = containerIf.Index + ifIndex = interfaceIf.Index } nlRoute := &netlink.Route{ @@ -97,3 +97,33 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { return nil } + + +func deleteRoutes(interfaceName string, routes []RouteInfo) error { + ifIndex := 0 + interfaceIf, _ := net.InterfaceByName(interfaceName) + + for _, route := range routes { + log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) + + if route.DevName != "" { + devIf, _ := net.InterfaceByName(route.DevName) + ifIndex = devIf.Index + } else { + ifIndex = interfaceIf.Index + } + + nlRoute := &netlink.Route{ + Family: netlink.GetIpAddressFamily(route.Gw), + Dst: &route.Dst, + Gw: route.Gw, + LinkIndex: ifIndex, + } + + if err := netlink.DeleteIpRoute(nlRoute); err != nil { + return err + } + } + + return nil +} diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index bfb867932f..901f88280d 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -42,11 +42,16 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var contIfName string var epClient EndpointClient var vlanid int = 0 + var localIP string if epInfo.Data != nil { if _, ok := epInfo.Data["vlanid"]; ok { vlanid = epInfo.Data["vlanid"].(int) } + + if _, ok := epInfo.Data["LocalIP"]; ok { + localIP = epInfo.Data["LocalIP"].(string) + } } if nw.Endpoints[epInfo.Id] != nil { @@ -69,7 +74,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } if vlanid != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), contIfName, vlanid, epInfo.EnableSnatOnHost) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), contIfName, localIP, vlanid, epInfo.EnableSnatOnHost) } else { epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, contIfName, nw.extIf.MacAddress, nw.Mode) } @@ -162,7 +167,7 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", ep.VlanID, ep.EnableSnatOnHost) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", "", ep.VlanID, ep.EnableSnatOnHost) } else { epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, "", nw.extIf.MacAddress, nw.Mode) } diff --git a/network/network_linux.go b/network/network_linux.go index e1783ede6e..d1de654bbe 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -25,7 +25,10 @@ const ( genericData = "com.docker.network.generic" - vlanIDKey = "vlanid" + VlanIDKey = "vlanid" + + InternetBridgeIPKey = "internetBridgeIP" + ) // Linux implementation of route. @@ -47,8 +50,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, err } - if opt != nil && opt[vlanIDKey] != nil { - vlanid, _ = strconv.Atoi(opt[vlanIDKey].(string)) + if opt != nil && opt[VlanIDKey] != nil { + vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string)) } default: @@ -73,7 +76,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, nw.EnableSnatOnHost) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nw.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) } @@ -192,8 +195,14 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } opt, _ := nwInfo.Options[genericData].(map[string]interface{}) - if opt != nil && opt[vlanIDKey] != nil { - networkClient = NewOVSClient(bridgeName, extIf.Name, nwInfo.EnableSnatOnHost) + if opt != nil && opt[VlanIDKey] != nil { + internetBridgeIP := "" + + if opt != nil && opt[InternetBridgeIPKey] != nil { + internetBridgeIP, _ = opt[InternetBridgeIPKey].(string) + } + + networkClient = NewOVSClient(bridgeName, extIf.Name, internetBridgeIP, nwInfo.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } @@ -250,6 +259,13 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI return err } + // Bridge up. + log.Printf("[net] Setting link %v state up.", bridgeName) + err = netlink.SetLinkState(bridgeName, true) + if err != nil { + return err + } + // Add the bridge rules. err = networkClient.AddBridgeRules(extIf) if err != nil { @@ -262,13 +278,6 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI return err } - // Bridge up. - log.Printf("[net] Setting link %v state up.", bridgeName) - err = netlink.SetLinkState(bridgeName, true) - if err != nil { - return err - } - // Apply IP configuration to the bridge for host traffic. err = nm.applyIPConfig(extIf, bridge) if err != nil { diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 1b599f319e..f1415d931b 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -4,7 +4,6 @@ import ( "fmt" "net" - "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" @@ -18,6 +17,7 @@ type OVSEndpointClient struct { containerVethName string containerMac string intVethName string + localIP string vlanID int enableSnatOnHost bool } @@ -25,6 +25,7 @@ type OVSEndpointClient struct { const ( intVethInterfacePrefix = commonInterfacePrefix + "vint" azureInternetIfName = "eth1" + localIPPrefix = "/16" ) func NewOVSEndpointClient( @@ -33,6 +34,7 @@ func NewOVSEndpointClient( hostVethName string, hostPrimaryMac string, containerVethName string, + localIP string, vlanid int, enableSnatOnHost bool) *OVSEndpointClient { @@ -42,6 +44,7 @@ func NewOVSEndpointClient( hostVethName: hostVethName, hostPrimaryMac: hostPrimaryMac, containerVethName: containerVethName, + localIP: localIP, vlanID: vlanid, enableSnatOnHost: enableSnatOnHost, } @@ -73,18 +76,6 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } - internetIf, err := net.InterfaceByName(contIfName) - if err != nil { - return err - } - - intVethMac := internetIf.HardwareAddr.String() - arpCmd := fmt.Sprintf("arp -s 169.254.0.2 %v", intVethMac) - log.Printf("Adding arp entry for internet veth %v", arpCmd) - if _, err := common.ExecuteShellCommand(arpCmd); err != nil { - log.Printf("Setting static arp failed for internet interface %v", err) - } - client.intVethName = contIfName } @@ -206,8 +197,9 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E } if client.enableSnatOnHost { - log.Printf("[ovs] Adding IP address 169.254.0.2/16 to link %v.", client.intVethName) - ip, intIpAddr, _ := net.ParseCIDR("169.254.0.2/16") + log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.intVethName) + localIPSubnetString := client.localIP + localIPPrefix + ip, intIpAddr, _ := net.ParseCIDR(localIPSubnetString) if err := netlink.AddIpAddress(client.intVethName, ip, intIpAddr); err != nil { return err } @@ -229,14 +221,6 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { } if client.enableSnatOnHost { - arpCmd := fmt.Sprintf("arp -d 169.254.0.2") - - log.Printf("Adding arp entry for internet veth %v", arpCmd) - if _, err := common.ExecuteShellCommand(arpCmd); err != nil { - log.Printf("Setting static arp failed for internet interface %v", err) - } - - hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, ep.Id[:7]) log.Printf("[ovs] Deleting internet veth pair %v.", hostIfName) err = netlink.DeleteLink(hostIfName) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 09124a6864..11db26c9d1 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -13,6 +13,7 @@ import ( type OVSNetworkClient struct { bridgeName string hostInterfaceName string + internetBridgeIP string enableSnatOnHost bool } @@ -22,10 +23,11 @@ const ( internetBridgeName = "azintbr" ) -func NewOVSClient(bridgeName string, hostInterfaceName string, enableSnatOnHost bool) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName, internetBridgeIP string, enableSnatOnHost bool) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, + internetBridgeIP: internetBridgeIP, enableSnatOnHost: enableSnatOnHost, } @@ -57,7 +59,10 @@ func (client *OVSNetworkClient) CreateBridge() error { } if client.enableSnatOnHost { - ip, addr, _ := net.ParseCIDR("169.254.0.1/16") + internetBridgeIPSubnetString := client.internetBridgeIP + localIPPrefix + ip, addr, _ := net.ParseCIDR(internetBridgeIPSubnetString) + + log.Printf("Assigning %v on internet bridge", internetBridgeIPSubnetString) err = netlink.AddIpAddress(internetBridgeName, ip, addr) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("[net] Failed to add IP address %v: %v.", addr, err) @@ -84,14 +89,6 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - /*var routes []RouteInfo - route = RouteInfo{Dst:net.ParseCIDR("169.254.169.254/32")} - routes = append(routes, route) - if err := addRoutes(client.bridgeName, routes); err != nil { - log.Printf("addroutes failed with error %v", err) - return err - }*/ - cmd := "iptables -t nat -A POSTROUTING -j MASQUERADE" log.Printf("Adding iptable snat rule %v", cmd) _, err := common.ExecuteShellCommand(cmd) @@ -174,6 +171,21 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { if err := ovsctl.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { return err } + + if client.enableSnatOnHost { + log.Printf("[ovs] Adding 169.254.169.254 static route") + var routes []RouteInfo + _, ipNet, _ := net.ParseCIDR("169.254.169.254/32") + gwIP := net.ParseIP("0.0.0.0") + route := RouteInfo{Dst: *ipNet, Gw: gwIP} + routes = append(routes, route) + if err := addRoutes(client.bridgeName, routes); err != nil { + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("addroutes failed with error %v", err) + return err + } + } + } return nil } From 265c28515a531c82994becde6a1193960bebada1 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 27 Jun 2018 13:52:42 -0700 Subject: [PATCH 47/51] moved linux specific code to appropriate files --- cni/network/network.go | 43 +---------------------------- cni/network/network_linux.go | 5 ++++ cni/network/network_windows.go | 5 +++- network/endpoint_linux.go | 8 +++--- network/manager.go | 5 ++-- network/network_linux.go | 5 ++-- network/ovs_networkclient_linux.go | 44 +++++++++++++++++++++++++++++- 7 files changed, 62 insertions(+), 53 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 5dbfaed5dc..b8f4fad5e8 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -4,10 +4,8 @@ package network import ( - "bytes" "fmt" "net" - "os" "strings" "github.com/Azure/azure-container-networking/client/cnsclient" @@ -29,8 +27,6 @@ const ( namespaceKey = "K8S_POD_NAMESPACE" podNameKey = "K8S_POD_NAME" dockerNetworkOption = "com.docker.network.generic" - ovsConfigFile = "/etc/default/openvswitch-switch" - ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" snatInterface = "eth1" // Supported IP version. Currently support only IPv4 @@ -214,37 +210,6 @@ func getPodNameWithoutSuffix(podName string) string { return strings.Join(nameSplit, "-") } -func updateOVSConfig(option string) error { - f, err := os.OpenFile(ovsConfigFile, os.O_APPEND|os.O_RDWR, 0666) - if err != nil { - log.Printf("Error while opening ovs config %v", err) - return err - } - - defer f.Close() - - buf := new(bytes.Buffer) - buf.ReadFrom(f) - contents := buf.String() - - conSplit := strings.Split(contents, "\n") - - for _, existingOption := range conSplit { - if option == existingOption { - log.Printf("Not updating ovs config. Found option already written") - return nil - } - } - - log.Printf("writing ovsconfig option %v", option) - - if _, err = f.WriteString(option); err != nil { - log.Printf("Error while writing ovs config %v", err) - return err - } - - return nil -} // // CNI implementation @@ -374,11 +339,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Creating network %v.", networkId) - if nwCfg.MultiTenancy { - if err := updateOVSConfig(ovsOpt); err != nil { - return err - } - } if !nwCfg.MultiTenancy { // Call into IPAM plugin to allocate an address pool for the network. @@ -490,8 +450,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Data = make(map[string]interface{}) if vlanid != 0 { - epInfo.Data["vlanid"] = vlanid - epInfo.Data["LocalIP"] = cnsNetworkConfig.LocalIP + setEndpointOptions(vlanid, cnsNetworkConfig.LocalIP, epInfo) } epInfo.Data[network.OptVethName] = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index cee6c14a8d..b717e05eea 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -32,4 +32,9 @@ func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { vlanMap[network.InternetBridgeIPKey] = internetBridgeIP nwInfo.Options[dockerNetworkOption] = vlanMap } +} + +func setEndpointOptions(vlanid int, localIP string, epInfo *network.EndpointInfo) { + epInfo.Data[network.VlanIDKey] = vlanid + epInfo.Data[network.LocalIPKey] = localIP } \ No newline at end of file diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 499676889c..e9e7b0479a 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -57,8 +57,11 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ return nil, nil } +func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +} + func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { } -func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +func setEndpointOptions(vlanid int, localIP string, epInfo *EndpointInfo) { } \ No newline at end of file diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 901f88280d..96aa292a4f 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -45,12 +45,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var localIP string if epInfo.Data != nil { - if _, ok := epInfo.Data["vlanid"]; ok { - vlanid = epInfo.Data["vlanid"].(int) + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) } - if _, ok := epInfo.Data["LocalIP"]; ok { - localIP = epInfo.Data["LocalIP"].(string) + if _, ok := epInfo.Data[LocalIPKey]; ok { + localIP = epInfo.Data[LocalIPKey].(string) } } diff --git a/network/manager.go b/network/manager.go index 6b635f1155..606194b4fe 100644 --- a/network/manager.go +++ b/network/manager.go @@ -16,6 +16,7 @@ import ( const ( // Network store key. storeKey = "Network" + VlanIDKey = "vlanid" ) type NetworkClient interface { @@ -265,9 +266,9 @@ func (nm *networkManager) CreateEndpoint(networkId string, epInfo *EndpointInfo) } if nw.VlanId != 0 { - if epInfo.Data["vlanid"] == nil { + if epInfo.Data[VlanIDKey] == nil { log.Printf("overriding endpoint vlanid with network vlanid") - epInfo.Data["vlanid"] = nw.VlanId + epInfo.Data[VlanIDKey] = nw.VlanId } } diff --git a/network/network_linux.go b/network/network_linux.go index d1de654bbe..8b236adf7d 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -24,11 +24,10 @@ const ( virtualMacAddress = "12:34:56:78:9a:bc" genericData = "com.docker.network.generic" - - VlanIDKey = "vlanid" - + InternetBridgeIPKey = "internetBridgeIP" + LocalIPKey = "localIP" ) // Linux implementation of route. diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 11db26c9d1..ebec89df5b 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -1,6 +1,8 @@ package network import ( + "os" + "bytes" "net" "strings" @@ -21,8 +23,44 @@ const ( azureInternetVeth0 = "azintveth0" azureInternetVeth1 = "azintveth1" internetBridgeName = "azintbr" + imdsIP = "169.254.169.254/32" + ovsConfigFile = "/etc/default/openvswitch-switch" + ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" ) +func updateOVSConfig(option string) error { + f, err := os.OpenFile(ovsConfigFile, os.O_APPEND|os.O_RDWR, 0666) + if err != nil { + log.Printf("Error while opening ovs config %v", err) + return err + } + + defer f.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(f) + contents := buf.String() + + conSplit := strings.Split(contents, "\n") + + for _, existingOption := range conSplit { + if option == existingOption { + log.Printf("Not updating ovs config. Found option already written") + return nil + } + } + + log.Printf("writing ovsconfig option %v", option) + + if _, err = f.WriteString(option); err != nil { + log.Printf("Error while writing ovs config %v", err) + return err + } + + return nil +} + + func NewOVSClient(bridgeName, hostInterfaceName, internetBridgeIP string, enableSnatOnHost bool) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, @@ -39,6 +77,10 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } + if err := updateOVSConfig(ovsOpt); err != nil { + return err + } + if err := CreateInternetBridge(); err != nil { log.Printf("[net] Creating internet bridge failed with erro %v", err) //return err @@ -175,7 +217,7 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { if client.enableSnatOnHost { log.Printf("[ovs] Adding 169.254.169.254 static route") var routes []RouteInfo - _, ipNet, _ := net.ParseCIDR("169.254.169.254/32") + _, ipNet, _ := net.ParseCIDR(imdsIP) gwIP := net.ParseIP("0.0.0.0") route := RouteInfo{Dst: *ipNet, Gw: gwIP} routes = append(routes, route) From 11aeb205f41a855b0d68a4d85d2896fb36c924c8 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 27 Jun 2018 18:32:16 -0700 Subject: [PATCH 48/51] addressed review comments and fixed issues --- cni/azure-linux-multitenancy.conflist | 23 +++++ cni/azure-linux.conflist | 1 - cni/network/network.go | 35 ++++---- cni/network/network_linux.go | 24 ++--- cni/network/network_windows.go | 7 +- ...ontract.go => NetworkContainerContract.go} | 4 +- {client => cns}/cnsclient/cnsclient.go | 14 +-- cns/common/utils.go | 29 ------ cns/dockerclient/dockerclient.go | 6 +- cns/restserver/restserver.go | 18 ++-- cns/service/main.go | 4 +- common/utils.go | 20 ----- network/api.go | 1 - network/ovs_endpointclient_linux.go | 4 +- network/ovs_networkclient_linux.go | 88 +++++++++++++------ ovsctl/ovsctl.go | 37 ++++---- platform/os_linux.go | 31 +++++-- platform/os_windows.go | 8 ++ 18 files changed, 192 insertions(+), 162 deletions(-) create mode 100644 cni/azure-linux-multitenancy.conflist rename cns/{dnccontract.go => NetworkContainerContract.go} (98%) rename {client => cns}/cnsclient/cnsclient.go (79%) delete mode 100644 cns/common/utils.go diff --git a/cni/azure-linux-multitenancy.conflist b/cni/azure-linux-multitenancy.conflist new file mode 100644 index 0000000000..c3218f3b71 --- /dev/null +++ b/cni/azure-linux-multitenancy.conflist @@ -0,0 +1,23 @@ +{ + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"bridge", + "bridge":"azure0", + "multiTenancy":true, + "enableSnatOnHost":true, + "ipam":{ + "type":"azure-vnet-ipam" + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + }, + "snat":true + } + ] +} \ No newline at end of file diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index a2a4881ef9..2e07c28d79 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -6,7 +6,6 @@ "type":"azure-vnet", "mode":"bridge", "bridge":"azure0", - "multiTenancy":true, "ipam":{ "type":"azure-vnet-ipam" } diff --git a/cni/network/network.go b/cni/network/network.go index b8f4fad5e8..11915a41f2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -7,10 +7,10 @@ import ( "fmt" "net" "strings" - - "github.com/Azure/azure-container-networking/client/cnsclient" + "encoding/json" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -24,8 +24,6 @@ import ( const ( // Plugin name. name = "azure-vnet" - namespaceKey = "K8S_POD_NAMESPACE" - podNameKey = "K8S_POD_NAME" dockerNetworkOption = "com.docker.network.generic" snatInterface = "eth1" @@ -178,8 +176,14 @@ func getContainerNetworkConfiguration(namespace string, podName string) (*cniTyp return nil, nil, net.IPNet{}, err } - // networkConfig, err := cnsClient.GetNetworkConfiguration(podName, namespace) - networkConfig, err := cnsClient.GetNetworkConfiguration("nginx-deployment2", "default") + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: namespace} + orchestratorContext, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Marshalling azure container instance info failed with %v", err) + return nil, nil, net.IPNet{}, err + } + + networkConfig, err := cnsClient.GetNetworkConfiguration(orchestratorContext) if err != nil { log.Printf("GetNetworkConfiguration failed with %v", err) return nil, nil, net.IPNet{}, err @@ -225,7 +229,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo *network.EndpointInfo iface *cniTypesCurr.Interface subnetPrefix net.IPNet - vlanid int cnsNetworkConfig *cns.GetNetworkContainerResponse ) @@ -290,8 +293,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) - podNameWithoutSuffix := getPodNameWithoutSuffix(k8sPodName) - log.Printf("Podname without suffix %v", podNameWithoutSuffix) // Initialize values from network config. networkId := nwCfg.Name @@ -299,16 +300,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // If multitenancy is enabled, get ip and vlan from cns if nwCfg.MultiTenancy { + podNameWithoutSuffix := getPodNameWithoutSuffix(k8sPodName) + log.Printf("Podname without suffix %v", podNameWithoutSuffix) + result, cnsNetworkConfig, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) if err != nil { log.Printf("[CNI Multitenancy] GetContainerNetworkConfiguration failed with %v", err) return err } - - vlanid = cnsNetworkConfig.MultiTenancyInfo.ID } - log.Printf("subnetprefix :%v", subnetPrefix.IP.String()) + log.Printf("PrimaryInterfaceIdentifier :%v", subnetPrefix.IP.String()) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) @@ -325,6 +327,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { if epInfo != nil { result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) if err != nil { + log.Printf("handleConsecutiveAdd failed with error %v", err) return err } @@ -405,7 +408,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { nwInfo.Options = make(map[string]interface{}) log.Printf("set network options") - setNetworkOptions(vlanid, &nwInfo) + setNetworkOptions(cnsNetworkConfig, &nwInfo) err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { @@ -449,9 +452,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } epInfo.Data = make(map[string]interface{}) - if vlanid != 0 { - setEndpointOptions(vlanid, cnsNetworkConfig.LocalIP, epInfo) - } + setEndpointOptions(cnsNetworkConfig, epInfo) epInfo.Data[network.OptVethName] = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) @@ -491,7 +492,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // if snat enabled, add 169.254.0.1 as default gateway if nwCfg.EnableSnatOnHost { log.Printf("add default route for multitenancy.snat on host enabled") - addDefaultRoute(epInfo, result) + addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) } else { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index b717e05eea..eb6e1b342a 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,6 +1,7 @@ package network import ( + "github.com/Azure/azure-container-networking/cns" "net" "strconv" "github.com/Azure/azure-container-networking/cni" @@ -9,32 +10,31 @@ import ( cniTypes "github.com/containernetworking/cni/pkg/types" ) -const ( - internetBridgeIP = "169.254.0.1" -) // handleConsecutiveAdd is a dummy function for Linux platform. func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } -func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - gwIP := net.ParseIP(internetBridgeIP) + gwIP := net.ParseIP(gwIPString) epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: snatInterface}) result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } -func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { - if vlanid != 0 { +func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { + if cnsNwConfig.MultiTenancyInfo.ID != 0 { vlanMap := make(map[string]interface{}) - vlanMap[network.VlanIDKey] = strconv.Itoa(vlanid) - vlanMap[network.InternetBridgeIPKey] = internetBridgeIP + vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID) + vlanMap[network.InternetBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) nwInfo.Options[dockerNetworkOption] = vlanMap } } -func setEndpointOptions(vlanid int, localIP string, epInfo *network.EndpointInfo) { - epInfo.Data[network.VlanIDKey] = vlanid - epInfo.Data[network.LocalIPKey] = localIP +func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { + if cnsNwConfig.MultiTenancyInfo.ID != 0 { + epInfo.Data[network.VlanIDKey] = cnsNwConfig.MultiTenancyInfo.ID + epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + } } \ No newline at end of file diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index e9e7b0479a..276c7789f2 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -4,6 +4,7 @@ import ( "net" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Microsoft/hcsshim" @@ -57,11 +58,11 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ return nil, nil } -func addDefaultRoute(epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { } -func setNetworkOptions(vlanid int, nwInfo *network.NetworkInfo) { +func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { } -func setEndpointOptions(vlanid int, localIP string, epInfo *EndpointInfo) { +func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { } \ No newline at end of file diff --git a/cns/dnccontract.go b/cns/NetworkContainerContract.go similarity index 98% rename from cns/dnccontract.go rename to cns/NetworkContainerContract.go index be02055fb0..345a2ef940 100644 --- a/cns/dnccontract.go +++ b/cns/NetworkContainerContract.go @@ -36,7 +36,7 @@ type CreateNetworkContainerRequest struct { NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. AuthorizationToken string - LocalIP string + LocalIPConfiguration IPConfiguration OrchestratorContext json.RawMessage IPConfiguration IPConfiguration MultiTenancyInfo MultiTenancyInfo @@ -112,7 +112,7 @@ type GetNetworkContainerResponse struct { CnetAddressSpace []IPSubnet MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string - LocalIP string + LocalIPConfiguration IPConfiguration Response Response } diff --git a/client/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go similarity index 79% rename from client/cnsclient/cnsclient.go rename to cns/cnsclient/cnsclient.go index d1a646a1b0..0aa244958b 100644 --- a/client/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -31,25 +31,18 @@ func NewCnsClient(url string) (*CNSClient, error) { } // GetNetworkConfiguration Request to get network config. -func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string) (*cns.GetNetworkContainerResponse, error) { +func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) (*cns.GetNetworkContainerResponse, error) { var body bytes.Buffer httpc := &http.Client{} url := cnsClient.connectionURL + cns.GetNetworkContainerByOrchestratorContext log.Printf("GetNetworkConfiguration url %v", url) - podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} - podInfoBytes, err := json.Marshal(podInfo) - if err != nil { - log.Printf("Marshalling azure container instance info failed with %v", err) - return nil, err - } - payload := &cns.GetNetworkContainerRequest{ - OrchestratorContext: podInfoBytes, + OrchestratorContext: orchestratorContext, } - err = json.NewEncoder(&body).Encode(payload) + err := json.NewEncoder(&body).Encode(payload) if err != nil { log.Printf("encoding json failed with %v", err) return nil, err @@ -62,6 +55,7 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(podName, podNamespace string } defer res.Body.Close() + if res.StatusCode != 200 { errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) log.Printf(errMsg) diff --git a/cns/common/utils.go b/cns/common/utils.go deleted file mode 100644 index b807929e51..0000000000 --- a/cns/common/utils.go +++ /dev/null @@ -1,29 +0,0 @@ -package common - -import ( - "fmt" - "os/exec" - - "github.com/Azure/azure-container-networking/log" -) - -func ExecuteShellCommand(command string) error { - log.Printf("[Azure-CNS] %s", command) - cmd := exec.Command("sh", "-c", command) - err := cmd.Start() - if err != nil { - return err - } - return cmd.Wait() -} - -func SetOutboundSNAT(subnet string) error { - cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", - subnet) - err := ExecuteShellCommand(cmd) - if err != nil { - log.Printf("SNAT Iptable rule was not set") - return err - } - return nil -} diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 86384d848c..eba45b5780 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -9,7 +9,7 @@ import ( "fmt" "net/http" - "github.com/Azure/azure-container-networking/cns/common" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/log" ) @@ -140,7 +140,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd } if enableSnat { - err = common.SetOutboundSNAT(nicInfo.Subnet) + err = platform.SetOutboundSNAT(nicInfo.Subnet) if err != nil { log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err) } @@ -179,7 +179,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", primaryNic.Subnet) - err = common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) if err != nil { log.Printf("[Azure CNS] Error Removing Outbound SNAT rule %v", err) } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 62647e5527..8f1812b37b 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -12,7 +12,6 @@ import ( "time" "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/ipamclient" @@ -20,6 +19,7 @@ import ( "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" + "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/store" ) @@ -817,24 +817,24 @@ func (service *httpRestService) setOrchestratorType(w http.ResponseWriter, r *ht return } + service.lock.Lock() + switch req.OrchestratorType { case cns.Kubernetes: - service.lock.Lock() service.state.OrchestratorType = cns.Kubernetes service.saveState() - service.lock.Unlock() break case cns.WebApps: - service.lock.Lock() service.state.OrchestratorType = cns.WebApps service.saveState() - service.lock.Unlock() break default: returnMessage = fmt.Sprintf("Invalid Orchestrator type %v", req.OrchestratorType) returnCode = UnsupportedOrchestratorType } + service.lock.Unlock() + resp := cns.Response{ ReturnCode: returnCode, Message: returnMessage, @@ -941,7 +941,7 @@ func (service *httpRestService) createOrUpdateNetworkContainer(w http.ResponseWr } func (service *httpRestService) getNetworkContainerByID(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure CNS] getNetworkContainer") + log.Printf("[Azure CNS] getNetworkContainerByID") var req cns.GetNetworkContainerRequest returnMessage := "" @@ -1003,14 +1003,14 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo CnetAddressSpace: savedReq.CnetAddressSpace, MultiTenancyInfo: savedReq.MultiTenancyInfo, PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, - LocalIP: savedReq.LocalIP, + LocalIPConfiguration: savedReq.LocalIPConfiguration, } return getNetworkContainerResponse } func (service *httpRestService) getNetworkContainerByOrchestratorContext(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure CNS] getNetworkContainer") + log.Printf("[Azure CNS] getNetworkContainerByOrchestratorContext") var req cns.GetNetworkContainerRequest @@ -1242,7 +1242,7 @@ func (service *httpRestService) restoreNetworkState() error { } if enableSnat { - err := common.SetOutboundSNAT(nwInfo.NicInfo.Subnet) + err := platform.SetOutboundSNAT(nwInfo.NicInfo.Subnet) if err != nil { log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err) return err diff --git a/cns/service/main.go b/cns/service/main.go index cc8e47cbfc..9a39e619c2 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -105,9 +105,9 @@ var args = acn.ArgumentList{ { Name: acn.OptStopAzureVnet, Shorthand: acn.OptStopAzureVnetAlias, - Description: "Start Azure-CNM if flag is true", + Description: "Stop Azure-CNM if flag is true", Type: "bool", - DefaultValue: true, + DefaultValue: false, }, { Name: acn.OptVersion, diff --git a/common/utils.go b/common/utils.go index 12f0db1d81..c754fc5c70 100644 --- a/common/utils.go +++ b/common/utils.go @@ -4,13 +4,10 @@ package common import ( - "bytes" "encoding/binary" "encoding/xml" - "fmt" "net" "os" - "os/exec" "github.com/Azure/azure-container-networking/log" ) @@ -79,23 +76,6 @@ func CreateDirectory(dirPath string) error { return err } -func ExecuteShellCommand(command string) (string, error) { - log.Printf("[Azure-Utils] %s", command) - - var stderr bytes.Buffer - var out bytes.Buffer - cmd := exec.Command("sh", "-c", command) - cmd.Stderr = &stderr - cmd.Stdout = &out - - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("%s:%s", err.Error(), stderr.String()) - } - - return out.String(), nil -} - func IpToInt(ip net.IP) uint32 { if len(ip) == 16 { return binary.BigEndian.Uint32(ip[12:16]) diff --git a/network/api.go b/network/api.go index e8f6d8353e..e18c8644ec 100644 --- a/network/api.go +++ b/network/api.go @@ -19,5 +19,4 @@ var ( errEndpointNotInUse = fmt.Errorf("Endpoint is not joined to a sandbox") OptVethName = "vethname" - OptEnableSnatOnHost = "enableSnatOnHost" ) diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index f1415d931b..96aeacb6b4 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -25,7 +25,6 @@ type OVSEndpointClient struct { const ( intVethInterfacePrefix = commonInterfacePrefix + "vint" azureInternetIfName = "eth1" - localIPPrefix = "/16" ) func NewOVSEndpointClient( @@ -198,8 +197,7 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E if client.enableSnatOnHost { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.intVethName) - localIPSubnetString := client.localIP + localIPPrefix - ip, intIpAddr, _ := net.ParseCIDR(localIPSubnetString) + ip, intIpAddr, _ := net.ParseCIDR(client.localIP ) if err := netlink.AddIpAddress(client.intVethName, ip, intIpAddr); err != nil { return err } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index ebec89df5b..6a3710f1e6 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -1,12 +1,13 @@ package network import ( + "fmt" "os" "bytes" "net" "strings" - "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" @@ -81,30 +82,29 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - if err := CreateInternetBridge(); err != nil { - log.Printf("[net] Creating internet bridge failed with erro %v", err) - //return err - } - - link := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: azureInternetVeth0, - }, - PeerName: azureInternetVeth1, - } - - err := netlink.AddLink(&link) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err - } - if client.enableSnatOnHost { - internetBridgeIPSubnetString := client.internetBridgeIP + localIPPrefix - ip, addr, _ := net.ParseCIDR(internetBridgeIPSubnetString) + if err := CreateInternetBridge(); err != nil { + log.Printf("[net] Creating internet bridge failed with erro %v", err) + return err + } + + link := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureInternetVeth0, + }, + PeerName: azureInternetVeth1, + } + + err := netlink.AddLink(&link) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + ip, addr, _ := net.ParseCIDR(client.internetBridgeIP) - log.Printf("Assigning %v on internet bridge", internetBridgeIPSubnetString) + log.Printf("Assigning %v on internet bridge", client.internetBridgeIP) err = netlink.AddIpAddress(internetBridgeName, ip, addr) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("[net] Failed to add IP address %v: %v.", addr, err) @@ -131,16 +131,17 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - cmd := "iptables -t nat -A POSTROUTING -j MASQUERADE" + + cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", addr.String()) log.Printf("Adding iptable snat rule %v", cmd) - _, err := common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) if err != nil { return err } cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" log.Printf("Adding ebtable rule to drop vlan traffic on internet bridge %v", cmd) - _, err = common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) return err } @@ -148,6 +149,32 @@ func (client *OVSNetworkClient) CreateBridge() error { return nil } +func deleteMasQueradeRule(interfaceName string) error { + internetIf, _ := net.InterfaceByName(interfaceName) + + addrs, _ := internetIf.Addrs() + for _, addr := range addrs { + ipAddr, ipNet, err := net.ParseCIDR(addr.String()) + if err != nil { + log.Printf("error %v", err) + continue + } + + if ipAddr.To4() != nil { + cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + log.Printf("Deleting iptable snat rule %v", cmd) + _, err = platform.ExecuteCommand(cmd) + if err != nil { + return err + } + + return nil + } + } + + return nil +} + func (client *OVSNetworkClient) DeleteBridge() error { if err := ovsctl.DeleteOVSBridge(client.bridgeName); err != nil { log.Printf("Deleting ovs bridge failed with error %v", err) @@ -155,6 +182,15 @@ func (client *OVSNetworkClient) DeleteBridge() error { } if client.enableSnatOnHost { + + deleteMasQueradeRule(internetBridgeName) + + cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" + _, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) + } + if err := DeleteInternetBridge(); err != nil { log.Printf("Deleting internet bridge failed with error %v", err) return err diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 4a516cdca2..6382b4dec6 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/log" ) @@ -17,7 +18,7 @@ func CreateOVSBridge(bridgeName string) error { log.Printf("[ovs] Creating OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) - _, err := common.ExecuteShellCommand(ovsCreateCmd) + _, err := platform.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while creating OVS bridge %v", err) return err @@ -30,7 +31,7 @@ func DeleteOVSBridge(bridgeName string) error { log.Printf("[ovs] Deleting OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) - _, err := common.ExecuteShellCommand(ovsCreateCmd) + _, err := platform.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while deleting OVS bridge %v", err) return err @@ -47,7 +48,7 @@ func AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error } else { cmd = fmt.Sprintf("ovs-vsctl add-port %s %s tag=%d", bridgeName, hostIfName, vlanID) } - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Error while setting OVS as master to primary interface %v", err) return err @@ -58,7 +59,7 @@ func AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error func GetOVSPortNumber(interfaceName string) (string, error) { cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", interfaceName) - ofport, err := common.ExecuteShellCommand(cmd) + ofport, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Get ofport failed with error %v", err) return "", err @@ -71,7 +72,7 @@ func GetOVSPortNumber(interfaceName string) (string, error) { func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=20,actions=normal", bridgeName, primaryIP, mac) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding SNAT rule failed with error %v", err) return err @@ -83,7 +84,7 @@ func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) error { cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=10,arp,arp_op=1,actions='mod_dl_src:%s, load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, mac, macHex, ofport) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP SNAT rule failed with error %v", err) return err @@ -95,7 +96,7 @@ func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) func AddIpSnatRule(bridgeName string, port string, mac string) error { cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=20,ip,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s,strip_vlan,normal", bridgeName, port, mac) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err) return err @@ -103,7 +104,7 @@ func AddIpSnatRule(bridgeName string, port string, mac string) error { cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=10,ip,in_port=%s,actions=drop", bridgeName, port) - _, err = common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Dropping vlantag packet rule failed with error %v", err) return err @@ -116,7 +117,7 @@ func AddArpDnatRule(bridgeName string, port string, mac string) error { // Add DNAT rule to forward ARP replies to container interfaces. cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, port, mac) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding DNAT rule failed with error %v", err) return err @@ -136,7 +137,7 @@ func AddFakeArpReply(bridgeName string, ip net.IP) error { move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, bridgeName, macAddress, macAddrHex, ipAddrInt) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return err @@ -152,7 +153,7 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan log.Printf("[ovs] Adding ARP reply rule to add vlan %v and forward packet to table 1 for port %v", vlanid, port) cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, bridgeName, port, vlanid) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return err @@ -165,7 +166,7 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, bridgeName, ip.String(), vlanid, mac, macAddrHex, ipAddrInt) - _, err = common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return err @@ -177,7 +178,7 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error { cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", bridgeName, ip.String(), vlanid, port, mac) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding MAC DNAT rule failed with error %v", err) return err @@ -189,14 +190,14 @@ func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlani func DeleteArpReplyRule(bridgeName string, port string, ip net.IP, vlanid int) { cmd := fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", bridgeName, port) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", bridgeName, ip.String(), vlanid) - _, err = common.ExecuteShellCommand(cmd) + _, err = platform.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } @@ -205,7 +206,7 @@ func DeleteArpReplyRule(bridgeName string, port string, ip net.IP, vlanid int) { func DeleteIPSnatRule(bridgeName string, port string) { cmd := fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", bridgeName, port) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("Error while deleting ovs rule %v error %v", cmd, err) } @@ -214,7 +215,7 @@ func DeleteIPSnatRule(bridgeName string, port string) { func DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { cmd := fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", bridgeName, ip.String(), vlanid, port) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) } @@ -223,7 +224,7 @@ func DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { func DeletePortFromOVS(bridgeName string, interfaceName string) error { // Disconnect external interface from its bridge. cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", bridgeName, interfaceName) - _, err := common.ExecuteShellCommand(cmd) + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Failed to disconnect interface %v from bridge, err:%v.", interfaceName, err) return err diff --git a/platform/os_linux.go b/platform/os_linux.go index fa79a022e8..768748dee4 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -4,10 +4,12 @@ package platform import ( + "bytes" + "fmt" "io/ioutil" - "log" "os/exec" "time" + "github.com/Azure/azure-container-networking/log" ) const ( @@ -49,13 +51,30 @@ func GetLastRebootTime() (time.Time, error) { return rebootTime.UTC(), nil } -// ExecuteShellCommand executes a shell command. -func ExecuteShellCommand(command string) error { - //log.Debugf("[shell] %s", command) +func ExecuteCommand(command string) (string, error) { + log.Printf("[Azure-Utils] %s", command) + + var stderr bytes.Buffer + var out bytes.Buffer cmd := exec.Command("sh", "-c", command) - err := cmd.Start() + cmd.Stderr = &stderr + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("%s:%s", err.Error(), stderr.String()) + } + + return out.String(), nil +} + +func SetOutboundSNAT(subnet string) error { + cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", + subnet) + _, err := ExecuteCommand(cmd) if err != nil { + log.Printf("SNAT Iptable rule was not set") return err } - return cmd.Wait() + return nil } diff --git a/platform/os_windows.go b/platform/os_windows.go index 5d4a467c6c..d8974ba9fd 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -26,3 +26,11 @@ func GetLastRebootTime() (time.Time, error) { var rebootTime time.Time return rebootTime, nil } + +func ExecuteCommand(command string) (string, error) { + return "", nil +} + +func SetOutboundSNAT(subnet string) error { + return nil +} \ No newline at end of file From bab14a5167c5047982576974e93682613cc1f033 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 29 Jun 2018 16:52:11 -0700 Subject: [PATCH 49/51] handled failure case and create internet bridge if enablesnatonhost is set to true without reboot --- cni/network/network_linux.go | 2 + network/endpoint_linux.go | 63 +++++++--- network/ovs_endpointclient_linux.go | 23 ++++ network/ovs_networkclient_linux.go | 185 ++++++++++++++++++---------- 4 files changed, 195 insertions(+), 78 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index eb6e1b342a..3eaecc940f 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -36,5 +36,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne if cnsNwConfig.MultiTenancyInfo.ID != 0 { epInfo.Data[network.VlanIDKey] = cnsNwConfig.MultiTenancyInfo.ID epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + epInfo.Data[network.InternetBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + } } \ No newline at end of file diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 96aa292a4f..7d5c442648 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -12,7 +12,6 @@ import ( "net" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/netlink" ) const ( @@ -43,6 +42,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var epClient EndpointClient var vlanid int = 0 var localIP string + var internetBridgeIP string if epInfo.Data != nil { if _, ok := epInfo.Data[VlanIDKey]; ok { @@ -52,6 +52,10 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { if _, ok := epInfo.Data[LocalIPKey]; ok { localIP = epInfo.Data[LocalIPKey].(string) } + + if _, ok := epInfo.Data[InternetBridgeIPKey]; ok { + internetBridgeIP = epInfo.Data[InternetBridgeIPKey].(string) + } } if nw.Endpoints[epInfo.Id] != nil { @@ -74,21 +78,47 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } if vlanid != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, nw.extIf.MacAddress.String(), contIfName, localIP, vlanid, epInfo.EnableSnatOnHost) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, + nw.extIf.Name, + hostIfName, + nw.extIf.MacAddress.String(), + contIfName, + internetBridgeIP, + localIP, + vlanid, + epInfo.EnableSnatOnHost) } else { epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, contIfName, nw.extIf.MacAddress, nw.Mode) } - if err := epClient.AddEndpoints(epInfo); err != nil { - return nil, err - } - - // On failure, delete the veth pair. + // On failure, cleanup the things. defer func() { - if err != nil { - netlink.DeleteLink(contIfName) + if err != nil { + log.Printf("CNI error. Delete Endpoint %v and rules that are created.", contIfName) + endpt := &endpoint { + Id: epInfo.Id, + IfName: contIfName, + HostIfName: hostIfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{nw.extIf.IPv4Gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + } + + if containerIf != nil { + endpt.MacAddress = containerIf.HardwareAddr + epClient.DeleteEndpointRules(endpt) + } + + epClient.DeleteEndpoints(endpt) } }() + + + if err = epClient.AddEndpoints(epInfo); err != nil { + return nil, err + } containerIf, err = net.InterfaceByName(contIfName) if err != nil { @@ -96,7 +126,9 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } // Setup rules for IP addresses on the container interface. - epClient.AddEndpointRules(epInfo) + if err = epClient.AddEndpointRules(epInfo); err != nil { + return nil, err + } // If a network namespace for the container interface is specified... if epInfo.NetNsPath != "" { @@ -114,15 +146,14 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Enter the container network namespace. log.Printf("[net] Entering netns %v.", epInfo.NetNsPath) - if err := ns.Enter(); err != nil { + if err = ns.Enter(); err != nil { return nil, err } // Return to host network namespace. defer func() { log.Printf("[net] Exiting netns %v.", epInfo.NetNsPath) - err = ns.Exit() - if err != nil { + if err := ns.Exit(); err != nil { log.Printf("[net] Failed to exit netns, err:%v.", err) } }() @@ -130,12 +161,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // If a name for the container interface is specified... if epInfo.IfName != "" { - if err := epClient.SetupContainerInterfaces(epInfo); err != nil { + if err = epClient.SetupContainerInterfaces(epInfo); err != nil { return nil, err } } - if err := epClient.ConfigureContainerInterfacesAndRoutes(epInfo); err != nil { + if err = epClient.ConfigureContainerInterfacesAndRoutes(epInfo); err != nil { return nil, err } @@ -167,7 +198,7 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", "", ep.VlanID, ep.EnableSnatOnHost) + epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", "", "", ep.VlanID, ep.EnableSnatOnHost) } else { epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, "", nw.extIf.MacAddress, nw.Mode) } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 96aeacb6b4..6bd3251563 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -17,6 +17,7 @@ type OVSEndpointClient struct { containerVethName string containerMac string intVethName string + internetBridgeIP string localIP string vlanID int enableSnatOnHost bool @@ -33,6 +34,7 @@ func NewOVSEndpointClient( hostVethName string, hostPrimaryMac string, containerVethName string, + internetBridgeIP string, localIP string, vlanid int, enableSnatOnHost bool) *OVSEndpointClient { @@ -43,6 +45,7 @@ func NewOVSEndpointClient( hostVethName: hostVethName, hostPrimaryMac: hostPrimaryMac, containerVethName: containerVethName, + internetBridgeIP: internetBridgeIP, localIP: localIP, vlanID: vlanid, enableSnatOnHost: enableSnatOnHost, @@ -64,6 +67,26 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { client.containerMac = containerIf.HardwareAddr.String() if client.enableSnatOnHost { + if err := createInternetBridge(client.internetBridgeIP, client.bridgeName); err != nil { + log.Printf("creating internet bridge failed with error %v", err) + return err + } + + if err := addMasQueradeRule(client.internetBridgeIP); err != nil { + log.Printf("Adding snat rule failed with error %v", err) + return err + } + + if err := addVlanDropRule(); err != nil { + log.Printf("Adding vlan drop rule failed with error %v", err) + return err + } + + if err := addStaticRoute(imdsIP, client.bridgeName); err != nil { + log.Printf("Adding imds static route failed with error %v", err) + return err + } + hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, epInfo.Id[:7]) contIfName := fmt.Sprintf("%s%s-2", intVethInterfacePrefix, epInfo.Id[:7]) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 6a3710f1e6..ffb0d5cb2b 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -83,70 +83,60 @@ func (client *OVSNetworkClient) CreateBridge() error { } if client.enableSnatOnHost { - if err := CreateInternetBridge(); err != nil { + if err := createInternetBridge(client.internetBridgeIP, client.bridgeName); err != nil { log.Printf("[net] Creating internet bridge failed with erro %v", err) return err } - - link := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: azureInternetVeth0, - }, - PeerName: azureInternetVeth1, - } - - err := netlink.AddLink(&link) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) + + if err := addMasQueradeRule(client.internetBridgeIP); err != nil { return err } - - ip, addr, _ := net.ParseCIDR(client.internetBridgeIP) - log.Printf("Assigning %v on internet bridge", client.internetBridgeIP) - err = netlink.AddIpAddress(internetBridgeName, ip, addr) - if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { - log.Printf("[net] Failed to add IP address %v: %v.", addr, err) - return err - } + return addVlanDropRule() + } - if err := netlink.SetLinkState(internetBridgeName, true); err != nil { - return err - } + return nil +} - if err := netlink.SetLinkState(azureInternetVeth0, true); err != nil { - return err - } +func addVlanDropRule() error { + cmd := "ebtables -t nat -L PREROUTING" + out, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("Error while listing ebtable rules %v", err) + return err + } - if err := netlink.SetLinkMaster(azureInternetVeth0, internetBridgeName); err != nil { - return err - } + out = strings.TrimSpace(out) - if err := netlink.SetLinkState(azureInternetVeth1, true); err != nil { - return err - } + if strings.Contains(out, "-p 802_1Q -j DROP") { + log.Printf("vlan drop rule already exists") + return nil + } - if err := ovsctl.AddPortOnOVSBridge(azureInternetVeth1, client.bridgeName, 0); err != nil { - return err - } + cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" + log.Printf("Adding ebtable rule to drop vlan traffic on internet bridge %v", cmd) + _, err = platform.ExecuteCommand(cmd) + return err +} - - cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", addr.String()) - log.Printf("Adding iptable snat rule %v", cmd) - _, err = platform.ExecuteCommand(cmd) - if err != nil { - return err - } +func addMasQueradeRule(internetBridgeIPWithPrefix string) error { + _, ipNet, _ := net.ParseCIDR(internetBridgeIPWithPrefix) - cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" - log.Printf("Adding ebtable rule to drop vlan traffic on internet bridge %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err + cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("iptable snat rule already exists") + return nil + } + cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + log.Printf("Adding iptable snat rule %v", cmd) + _, err = platform.ExecuteCommand(cmd) + if err != nil { + return err } - return nil + return nil } func deleteMasQueradeRule(interfaceName string) error { @@ -191,6 +181,10 @@ func (client *OVSNetworkClient) DeleteBridge() error { log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) } + if err := ovsctl.DeletePortFromOVS(client.bridgeName, azureInternetVeth1); err != nil { + return err + } + if err := DeleteInternetBridge(); err != nil { log.Printf("Deleting internet bridge failed with error %v", err) return err @@ -202,7 +196,15 @@ func (client *OVSNetworkClient) DeleteBridge() error { return nil } -func CreateInternetBridge() error { +func createInternetBridge(internetBridgeIP string, mainInterface string) error { + + _, err := net.InterfaceByName(internetBridgeName) + if err == nil { + log.Printf("Internet Bridge already exists") + return nil + } + + log.Printf("[net] Creating Internet bridge %v.", internetBridgeName) link := netlink.BridgeLink{ @@ -212,7 +214,77 @@ func CreateInternetBridge() error { }, } - return netlink.AddLink(&link) + if err := netlink.AddLink(&link); err != nil { + return err + } + + _, err = net.InterfaceByName(azureInternetVeth0) + if err == nil { + log.Printf("Azure internet veth already exists") + return nil + } + + vethLink := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureInternetVeth0, + }, + PeerName: azureInternetVeth1, + } + + err = netlink.AddLink(&vethLink) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + ip, addr, _ := net.ParseCIDR(internetBridgeIP) + + log.Printf("Assigning %v on internet bridge", internetBridgeIP) + err = netlink.AddIpAddress(internetBridgeName, ip, addr) + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("[net] Failed to add IP address %v: %v.", addr, err) + return err + } + + if err := netlink.SetLinkState(internetBridgeName, true); err != nil { + return err + } + + if err := netlink.SetLinkState(azureInternetVeth0, true); err != nil { + return err + } + + if err := netlink.SetLinkMaster(azureInternetVeth0, internetBridgeName); err != nil { + return err + } + + if err := netlink.SetLinkState(azureInternetVeth1, true); err != nil { + return err + } + + if err := ovsctl.AddPortOnOVSBridge(azureInternetVeth1, mainInterface, 0); err != nil { + return err + } + + return nil +} + +func addStaticRoute(ip string, interfaceName string) error { + log.Printf("[ovs] Adding %v static route", ip) + var routes []RouteInfo + _, ipNet, _ := net.ParseCIDR(imdsIP) + gwIP := net.ParseIP("0.0.0.0") + route := RouteInfo{Dst: *ipNet, Gw: gwIP} + routes = append(routes, route) + if err := addRoutes(interfaceName, routes); err != nil { + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("addroutes failed with error %v", err) + return err + } + } + + return nil } func DeleteInternetBridge() error { @@ -251,18 +323,7 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { } if client.enableSnatOnHost { - log.Printf("[ovs] Adding 169.254.169.254 static route") - var routes []RouteInfo - _, ipNet, _ := net.ParseCIDR(imdsIP) - gwIP := net.ParseIP("0.0.0.0") - route := RouteInfo{Dst: *ipNet, Gw: gwIP} - routes = append(routes, route) - if err := addRoutes(client.bridgeName, routes); err != nil { - if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { - log.Printf("addroutes failed with error %v", err) - return err - } - } + addStaticRoute(imdsIP, client.bridgeName) } return nil From 99770aee771d43548dc03f307e420d7a087bc1ca Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 3 Jul 2018 17:23:34 -0700 Subject: [PATCH 50/51] addressed review comments and tested for regression --- cni/network/mutlitenancy.go | 126 ++++++++++++++++++++++ cni/network/network.go | 139 +++---------------------- cni/network/network_linux.go | 30 ++++-- cni/network/network_windows.go | 7 +- cns/cnsclient/cnsclient.go | 2 +- cns/restserver/restserver.go | 4 +- cns/service/main.go | 1 - network/bridge_endpointclient_linux.go | 10 +- network/bridge_networkclient_linux.go | 4 +- network/endpoint.go | 15 +-- network/endpoint_common_linux.go | 1 - network/endpoint_linux.go | 79 ++++++-------- network/manager.go | 11 +- network/network_linux.go | 28 +++-- network/network_windows.go | 3 + network/ovs_endpointclient_linux.go | 71 +++++++------ network/ovs_networkclient_linux.go | 128 +++++++++++------------ ovsctl/ovsctl.go | 12 +-- platform/os_linux.go | 1 + platform/os_windows.go | 2 +- 20 files changed, 349 insertions(+), 325 deletions(-) create mode 100644 cni/network/mutlitenancy.go diff --git a/cni/network/mutlitenancy.go b/cni/network/mutlitenancy.go new file mode 100644 index 0000000000..baa422628a --- /dev/null +++ b/cni/network/mutlitenancy.go @@ -0,0 +1,126 @@ +package network + +import ( + "encoding/json" + "fmt" + "net" + "strings" + + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/cnsclient" + "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + cniTypes "github.com/containernetworking/cni/pkg/types" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +func addDefaultGateway(nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { + // Adding default gateway + if nwCfg.MultiTenancy { + // if snat enabled, add 169.254.0.1 as default gateway + if nwCfg.EnableSnatOnHost { + log.Printf("add default route for multitenancy.snat on host enabled") + addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) + } else { + _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) + } + } +} + +func GetContainerNetworkConfiguration(multiTenancy bool, address string, podName string, podNamespace string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { + if multiTenancy { + podNameWithoutSuffix := getPodNameWithoutSuffix(podName) + log.Printf("Podname without suffix %v", podNameWithoutSuffix) + return getContainerNetworkConfiguration(address, podNamespace, podNameWithoutSuffix) + } + + return nil, nil, net.IPNet{}, nil +} + +func getContainerNetworkConfiguration(address string, namespace string, podName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { + cnsClient, err := cnsclient.NewCnsClient(address) + if err != nil { + log.Printf("Initializing CNS client error %v", err) + return nil, nil, net.IPNet{}, err + } + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: namespace} + orchestratorContext, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Marshalling KubernetesPodInfo failed with %v", err) + return nil, nil, net.IPNet{}, err + } + + networkConfig, err := cnsClient.GetNetworkConfiguration(orchestratorContext) + if err != nil { + log.Printf("GetNetworkConfiguration failed with %v", err) + return nil, nil, net.IPNet{}, err + } + + log.Printf("Network config received from cns %+v", networkConfig) + + subnetPrefix := common.GetIpNet(networkConfig.PrimaryInterfaceIdentifier) + if subnetPrefix == nil { + errBuf := fmt.Sprintf("Interface not found for this ip %v", networkConfig.PrimaryInterfaceIdentifier) + log.Printf(errBuf) + return nil, nil, net.IPNet{}, fmt.Errorf(errBuf) + } + + return convertToCniResult(networkConfig), networkConfig, *subnetPrefix, nil +} + +func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { + result := &cniTypesCurr.Result{} + resultIpconfig := &cniTypesCurr.IPConfig{} + + ipconfig := networkConfig.IPConfiguration + ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) + + if ipAddr.To4() != nil { + resultIpconfig.Version = "4" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} + } else { + resultIpconfig.Version = "6" + resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} + } + + resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) + result.IPs = append(result.IPs, resultIpconfig) + result.DNS.Nameservers = ipconfig.DNSServers + + if networkConfig.Routes != nil && len(networkConfig.Routes) > 0 { + for _, route := range networkConfig.Routes { + _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) + gwIP := net.ParseIP(route.GatewayIPAddress) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: *routeIPnet, GW: gwIP}) + } + } + + for _, ipRouteSubnet := range networkConfig.CnetAddressSpace { + log.Printf("Adding cnetAddressspace routes %v %v", ipRouteSubnet.IPAddress, ipRouteSubnet.PrefixLength) + routeIPnet := net.IPNet{IP: net.ParseIP(ipRouteSubnet.IPAddress), Mask: net.CIDRMask(int(ipRouteSubnet.PrefixLength), 32)} + gwIP := net.ParseIP(ipconfig.GatewayIPAddress) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: routeIPnet, GW: gwIP}) + } + + return result +} + +func getPodNameWithoutSuffix(podName string) string { + nameSplit := strings.Split(podName, "-") + log.Printf("namesplit %v", nameSplit) + if len(nameSplit) > 2 { + nameSplit = nameSplit[:len(nameSplit)-2] + } else { + return podName + } + + log.Printf("Pod name after splitting based on - : %v", nameSplit) + return strings.Join(nameSplit, "-") +} diff --git a/cni/network/network.go b/cni/network/network.go index 11915a41f2..b709fe8f48 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -7,10 +7,9 @@ import ( "fmt" "net" "strings" - "encoding/json" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -25,7 +24,6 @@ const ( // Plugin name. name = "azure-vnet" dockerNetworkOption = "com.docker.network.generic" - snatInterface = "eth1" // Supported IP version. Currently support only IPv4 ipVersion = "4" @@ -132,89 +130,6 @@ func GetEndpointID(args *cniSkel.CmdArgs) string { return infraEpId } -func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { - result := &cniTypesCurr.Result{} - resultIpconfig := &cniTypesCurr.IPConfig{} - - ipconfig := networkConfig.IPConfiguration - ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress) - - if ipAddr.To4() != nil { - resultIpconfig.Version = "4" - resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)} - } else { - resultIpconfig.Version = "6" - resultIpconfig.Address = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)} - } - - resultIpconfig.Gateway = net.ParseIP(ipconfig.GatewayIPAddress) - result.IPs = append(result.IPs, resultIpconfig) - result.DNS.Nameservers = ipconfig.DNSServers - - if networkConfig.Routes != nil && len(networkConfig.Routes) > 0 { - for _, route := range networkConfig.Routes { - _, routeIPnet, _ := net.ParseCIDR(route.IPAddress) - gwIP := net.ParseIP(route.GatewayIPAddress) - result.Routes = append(result.Routes, &cniTypes.Route{Dst: *routeIPnet, GW: gwIP}) - } - } - - for _, ipRouteSubnet := range networkConfig.CnetAddressSpace { - log.Printf("Adding cnetAddressspace routes %v %v", ipRouteSubnet.IPAddress, ipRouteSubnet.PrefixLength) - routeIPnet := net.IPNet{IP: net.ParseIP(ipRouteSubnet.IPAddress), Mask:net.CIDRMask(int(ipRouteSubnet.PrefixLength), 32)} - gwIP := net.ParseIP(ipconfig.GatewayIPAddress) - result.Routes = append(result.Routes, &cniTypes.Route{Dst: routeIPnet, GW: gwIP}) - } - - return result -} - -func getContainerNetworkConfiguration(namespace string, podName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { - cnsClient, err := cnsclient.NewCnsClient("") - if err != nil { - log.Printf("Initializing CNS client error %v", err) - return nil, nil, net.IPNet{}, err - } - - podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: namespace} - orchestratorContext, err := json.Marshal(podInfo) - if err != nil { - log.Printf("Marshalling azure container instance info failed with %v", err) - return nil, nil, net.IPNet{}, err - } - - networkConfig, err := cnsClient.GetNetworkConfiguration(orchestratorContext) - if err != nil { - log.Printf("GetNetworkConfiguration failed with %v", err) - return nil, nil, net.IPNet{}, err - } - - log.Printf("Network config received from cns %+v", networkConfig) - - subnetPrefix := common.GetIpNet(networkConfig.PrimaryInterfaceIdentifier) - if subnetPrefix == nil { - errBuf := fmt.Sprintf("Interface not found for this ip %v", networkConfig.PrimaryInterfaceIdentifier) - log.Printf(errBuf) - return nil, nil, net.IPNet{}, fmt.Errorf(errBuf) - } - - return convertToCniResult(networkConfig), networkConfig, *subnetPrefix, nil -} - -func getPodNameWithoutSuffix(podName string) string { - nameSplit := strings.Split(podName, "-") - log.Printf("namesplit %v", nameSplit) - if len(nameSplit) > 2 { - nameSplit = nameSplit[:len(nameSplit)-2] - } else { - return podName - } - - log.Printf("Final namesplit %v", nameSplit) - return strings.Join(nameSplit, "-") -} - - // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -223,12 +138,12 @@ func getPodNameWithoutSuffix(podName string) string { // Add handles CNI add commands. func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var ( - result *cniTypesCurr.Result - err error - nwCfg *cni.NetworkConfig - epInfo *network.EndpointInfo - iface *cniTypesCurr.Interface - subnetPrefix net.IPNet + result *cniTypesCurr.Result + err error + nwCfg *cni.NetworkConfig + epInfo *network.EndpointInfo + iface *cniTypesCurr.Interface + subnetPrefix net.IPNet cnsNetworkConfig *cns.GetNetworkContainerResponse ) @@ -245,12 +160,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Name: args.IfName, } - snatIface := &cniTypesCurr.Interface{ - Name: snatInterface, - } - result.Interfaces = append(result.Interfaces, iface) - result.Interfaces = append(result.Interfaces, snatIface) + + addSnatInterface(nwCfg, result) // Convert result to the requested CNI version. res, err := result.GetAsVersion(nwCfg.CNIVersion) @@ -293,21 +205,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) - // Initialize values from network config. networkId := nwCfg.Name endpointId := GetEndpointID(args) - // If multitenancy is enabled, get ip and vlan from cns - if nwCfg.MultiTenancy { - podNameWithoutSuffix := getPodNameWithoutSuffix(k8sPodName) - log.Printf("Podname without suffix %v", podNameWithoutSuffix) - - result, cnsNetworkConfig, subnetPrefix, err = getContainerNetworkConfiguration(k8sNamespace, podNameWithoutSuffix) - if err != nil { - log.Printf("[CNI Multitenancy] GetContainerNetworkConfiguration failed with %v", err) - return err - } + result, cnsNetworkConfig, subnetPrefix, err = GetContainerNetworkConfiguration(nwCfg.MultiTenancy, "", k8sPodName, k8sNamespace) + if err != nil { + log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", k8sPodName, k8sNamespace, err) + return err } log.Printf("PrimaryInterfaceIdentifier :%v", subnetPrefix.IP.String()) @@ -342,7 +247,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Creating network %v.", networkId) - if !nwCfg.MultiTenancy { // Call into IPAM plugin to allocate an address pool for the network. result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) @@ -407,7 +311,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } nwInfo.Options = make(map[string]interface{}) - log.Printf("set network options") + log.Printf("Set Network Options") setNetworkOptions(cnsNetworkConfig, &nwInfo) err = plugin.nm.CreateNetwork(&nwInfo) @@ -487,20 +391,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - // Adding default gateway - if nwCfg.MultiTenancy { - // if snat enabled, add 169.254.0.1 as default gateway - if nwCfg.EnableSnatOnHost { - log.Printf("add default route for multitenancy.snat on host enabled") - addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) - } else { - _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") - dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) - epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) - result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) - } - } + addDefaultGateway(nwCfg, cnsNetworkConfig, epInfo, result) // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 3eaecc940f..055cba016a 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,13 +1,18 @@ package network import ( - "github.com/Azure/azure-container-networking/cns" "net" "strconv" + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network" - cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" cniTypes "github.com/containernetworking/cni/pkg/types" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +const ( + snatInterface = "eth1" ) // handleConsecutiveAdd is a dummy function for Linux platform. @@ -17,26 +22,35 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") - dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} + dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} gwIP := net.ParseIP(gwIPString) epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP, DevName: snatInterface}) result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { - if cnsNwConfig.MultiTenancyInfo.ID != 0 { + if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { vlanMap := make(map[string]interface{}) vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID) - vlanMap[network.InternetBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + vlanMap[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) nwInfo.Options[dockerNetworkOption] = vlanMap } } func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { - if cnsNwConfig.MultiTenancyInfo.ID != 0 { + if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { epInfo.Data[network.VlanIDKey] = cnsNwConfig.MultiTenancyInfo.ID epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) - epInfo.Data[network.InternetBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + } +} +func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { + if nwCfg != nil && nwCfg.MultiTenancy { + snatIface := &cniTypesCurr.Interface{ + Name: snatInterface, + } + + result.Interfaces = append(result.Interfaces, snatIface) } -} \ No newline at end of file +} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 276c7789f2..318fd67534 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -61,8 +61,11 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { } -func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { +func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { } func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { -} \ No newline at end of file +} + +func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { +} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 0aa244958b..18a3a19b7a 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -56,7 +56,7 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) defer res.Body.Close() - if res.StatusCode != 200 { + if res.StatusCode != http.StatusOK { errMsg := fmt.Sprintf("[Azure CNSClient] GetNetworkConfiguration invalid http status code: %v", res.StatusCode) log.Printf(errMsg) return nil, fmt.Errorf(errMsg) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 8f1812b37b..dbf13a26b0 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/ipamclient" @@ -19,7 +20,6 @@ import ( "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" - "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/store" ) @@ -1000,7 +1000,7 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo getNetworkContainerResponse = cns.GetNetworkContainerResponse{ IPConfiguration: savedReq.IPConfiguration, Routes: savedReq.Routes, - CnetAddressSpace: savedReq.CnetAddressSpace, + CnetAddressSpace: savedReq.CnetAddressSpace, MultiTenancyInfo: savedReq.MultiTenancyInfo, PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, LocalIPConfiguration: savedReq.LocalIPConfiguration, diff --git a/cns/service/main.go b/cns/service/main.go index 9a39e619c2..3200f1fc6d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -178,7 +178,6 @@ func main() { } // Create the key value store. - config.Store, err = store.NewJsonFileStore(platform.CNMRuntimePath + name + ".json") if err != nil { log.Printf("Failed to create store: %v\n", err) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index b3cbc34fac..9f1d95e363 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -19,20 +19,18 @@ type LinuxBridgeEndpointClient struct { } func NewLinuxBridgeEndpointClient( - bridgeName string, - hostPrimaryIfName string, + extIf *externalInterface, hostVethName string, containerVethName string, - hostPrimaryMac net.HardwareAddr, mode string, ) *LinuxBridgeEndpointClient { client := &LinuxBridgeEndpointClient{ - bridgeName: bridgeName, - hostPrimaryIfName: hostPrimaryIfName, + bridgeName: extIf.BridgeName, + hostPrimaryIfName: extIf.Name, hostVethName: hostVethName, containerVethName: containerVethName, - hostPrimaryMac: hostPrimaryMac, + hostPrimaryMac: extIf.MacAddress, mode: mode, } diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index f146518e95..f14060dd54 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -54,7 +54,7 @@ func (client *LinuxBridgeClient) DeleteBridge() error { return nil } -func (client *LinuxBridgeClient) AddBridgeRules(extIf *externalInterface) error { +func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { hostIf, err := net.InterfaceByName(client.hostInterfaceName) if err != nil { @@ -93,7 +93,7 @@ func (client *LinuxBridgeClient) AddBridgeRules(extIf *externalInterface) error return nil } -func (client *LinuxBridgeClient) DeleteBridgeRules(extIf *externalInterface) { +func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) { ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Delete) ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) diff --git a/network/endpoint.go b/network/endpoint.go index 5bc020c262..ee0e945573 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -151,13 +151,14 @@ func (nw *network) getEndpoint(endpointId string) (*endpoint, error) { // GetInfo returns information about the endpoint. func (ep *endpoint) getInfo() *EndpointInfo { info := &EndpointInfo{ - Id: ep.Id, - IPAddresses: ep.IPAddresses, - Data: make(map[string]interface{}), - MacAddress: ep.MacAddress, - SandboxKey: ep.SandboxKey, - IfIndex: 0, // Azure CNI supports only one interface - DNS: ep.DNS, + Id: ep.Id, + IPAddresses: ep.IPAddresses, + Data: make(map[string]interface{}), + MacAddress: ep.MacAddress, + SandboxKey: ep.SandboxKey, + IfIndex: 0, // Azure CNI supports only one interface + DNS: ep.DNS, + EnableSnatOnHost: ep.EnableSnatOnHost, } for _, route := range ep.Routes { diff --git a/network/endpoint_common_linux.go b/network/endpoint_common_linux.go index eabb14c4cf..d96934d0c5 100644 --- a/network/endpoint_common_linux.go +++ b/network/endpoint_common_linux.go @@ -98,7 +98,6 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { return nil } - func deleteRoutes(interfaceName string, routes []RouteInfo) error { ifIndex := 0 interfaceIf, _ := net.InterfaceByName(interfaceName) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 7d5c442648..123fc84f6b 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -41,22 +41,6 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var contIfName string var epClient EndpointClient var vlanid int = 0 - var localIP string - var internetBridgeIP string - - if epInfo.Data != nil { - if _, ok := epInfo.Data[VlanIDKey]; ok { - vlanid = epInfo.Data[VlanIDKey].(int) - } - - if _, ok := epInfo.Data[LocalIPKey]; ok { - localIP = epInfo.Data[LocalIPKey].(string) - } - - if _, ok := epInfo.Data[InternetBridgeIPKey]; ok { - internetBridgeIP = epInfo.Data[InternetBridgeIPKey].(string) - } - } if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -64,6 +48,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return nil, err } + if epInfo.Data != nil { + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) + } + } + if _, ok := epInfo.Data[OptVethName]; ok { log.Printf("Generate veth name based on the key provided") key := epInfo.Data[OptVethName].(string) @@ -78,31 +68,28 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { } if vlanid != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, - nw.extIf.Name, - hostIfName, - nw.extIf.MacAddress.String(), - contIfName, - internetBridgeIP, - localIP, - vlanid, - epInfo.EnableSnatOnHost) + epClient = NewOVSEndpointClient( + nw.extIf, + epInfo, + hostIfName, + contIfName, + vlanid) } else { - epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, hostIfName, contIfName, nw.extIf.MacAddress, nw.Mode) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) } // On failure, cleanup the things. defer func() { - if err != nil { + if err != nil { log.Printf("CNI error. Delete Endpoint %v and rules that are created.", contIfName) - endpt := &endpoint { - Id: epInfo.Id, - IfName: contIfName, - HostIfName: hostIfName, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{nw.extIf.IPv4Gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, + endpt := &endpoint{ + Id: epInfo.Id, + IfName: contIfName, + HostIfName: hostIfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{nw.extIf.IPv4Gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, } @@ -114,7 +101,6 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { epClient.DeleteEndpoints(endpt) } }() - if err = epClient.AddEndpoints(epInfo); err != nil { return nil, err @@ -172,14 +158,14 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Create the endpoint object. ep = &endpoint{ - Id: epInfo.Id, - IfName: contIfName, - HostIfName: hostIfName, - MacAddress: containerIf.HardwareAddr, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{nw.extIf.IPv4Gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, + Id: epInfo.Id, + IfName: contIfName, + HostIfName: hostIfName, + MacAddress: containerIf.HardwareAddr, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{nw.extIf.IPv4Gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, } @@ -198,9 +184,10 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { - epClient = NewOVSEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, nw.extIf.MacAddress.String(), "", "", "", ep.VlanID, ep.EnableSnatOnHost) + epInfo := ep.getInfo() + epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID) } else { - epClient = NewLinuxBridgeEndpointClient(nw.extIf.BridgeName, nw.extIf.Name, ep.HostIfName, "", nw.extIf.MacAddress, nw.Mode) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) } epClient.DeleteEndpointRules(ep) diff --git a/network/manager.go b/network/manager.go index 606194b4fe..80e2519c9d 100644 --- a/network/manager.go +++ b/network/manager.go @@ -15,15 +15,15 @@ import ( const ( // Network store key. - storeKey = "Network" - VlanIDKey = "vlanid" + storeKey = "Network" + VlanIDKey = "VlanID" ) type NetworkClient interface { CreateBridge() error DeleteBridge() error - AddBridgeRules(extIf *externalInterface) error - DeleteBridgeRules(extIf *externalInterface) + AddL2Rules(extIf *externalInterface) error + DeleteL2Rules(extIf *externalInterface) SetBridgeMasterToHostInterface() error SetHairpinOnHostInterface(bool) error } @@ -246,8 +246,11 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) Id: networkId, Subnets: nw.Subnets, Mode: nw.Mode, + Options: make(map[string]interface{}), } + getNetworkInfoImpl(nwInfo, nw) + if nw.extIf != nil { nwInfo.BridgeName = nw.extIf.BridgeName } diff --git a/network/network_linux.go b/network/network_linux.go index 8b236adf7d..3af08f8aa7 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -24,9 +24,9 @@ const ( virtualMacAddress = "12:34:56:78:9a:bc" genericData = "com.docker.network.generic" - - InternetBridgeIPKey = "internetBridgeIP" - + + SnatBridgeIPKey = "snatBridgeIP" + LocalIPKey = "localIP" ) @@ -170,8 +170,6 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { var err error var networkClient NetworkClient - // var disableMultiTenancyInternet bool - log.Printf("[net] Connecting interface %v.", extIf.Name) defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() @@ -195,13 +193,13 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { - internetBridgeIP := "" + snatBridgeIP := "" - if opt != nil && opt[InternetBridgeIPKey] != nil { - internetBridgeIP, _ = opt[InternetBridgeIPKey].(string) + if opt != nil && opt[SnatBridgeIPKey] != nil { + snatBridgeIP, _ = opt[SnatBridgeIPKey].(string) } - networkClient = NewOVSClient(bridgeName, extIf.Name, internetBridgeIP, nwInfo.EnableSnatOnHost) + networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } @@ -266,7 +264,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI } // Add the bridge rules. - err = networkClient.AddBridgeRules(extIf) + err = networkClient.AddL2Rules(extIf) if err != nil { return err } @@ -297,7 +295,7 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface, log.Printf("[net] Deleting bridge rules") // Delete bridge rules set on the external interface. - networkClient.DeleteBridgeRules(extIf) + networkClient.DeleteL2Rules(extIf) log.Printf("[net] Deleting bridge") // Delete Bridge @@ -318,3 +316,11 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface, log.Printf("[net] Disconnected interface %v.", extIf.Name) } + +func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { + if nw.VlanId != 0 { + vlanMap := make(map[string]interface{}) + vlanMap[VlanIDKey] = strconv.Itoa(nw.VlanId) + nwInfo.Options[genericData] = vlanMap + } +} diff --git a/network/network_windows.go b/network/network_windows.go index 29f23cf84d..19f3bf1895 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -90,3 +90,6 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { return err } + +func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { +} diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 6bd3251563..93ef994155 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -16,39 +16,42 @@ type OVSEndpointClient struct { hostPrimaryMac string containerVethName string containerMac string - intVethName string - internetBridgeIP string - localIP string + snatVethName string + snatBridgeIP string + localIP string vlanID int enableSnatOnHost bool } const ( - intVethInterfacePrefix = commonInterfacePrefix + "vint" - azureInternetIfName = "eth1" + snatVethInterfacePrefix = commonInterfacePrefix + "vint" + azureSnatIfName = "eth1" ) func NewOVSEndpointClient( - bridgeName string, - hostPrimaryIfName string, + extIf *externalInterface, + epInfo *EndpointInfo, hostVethName string, - hostPrimaryMac string, containerVethName string, - internetBridgeIP string, - localIP string, vlanid int, - enableSnatOnHost bool) *OVSEndpointClient { +) *OVSEndpointClient { client := &OVSEndpointClient{ - bridgeName: bridgeName, - hostPrimaryIfName: hostPrimaryIfName, + bridgeName: extIf.BridgeName, + hostPrimaryIfName: extIf.Name, hostVethName: hostVethName, - hostPrimaryMac: hostPrimaryMac, + hostPrimaryMac: extIf.MacAddress.String(), containerVethName: containerVethName, - internetBridgeIP: internetBridgeIP, - localIP: localIP, vlanID: vlanid, - enableSnatOnHost: enableSnatOnHost, + enableSnatOnHost: epInfo.EnableSnatOnHost, + } + + if _, ok := epInfo.Data[LocalIPKey]; ok { + client.localIP = epInfo.Data[LocalIPKey].(string) + } + + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + client.snatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } return client @@ -67,16 +70,16 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { client.containerMac = containerIf.HardwareAddr.String() if client.enableSnatOnHost { - if err := createInternetBridge(client.internetBridgeIP, client.bridgeName); err != nil { - log.Printf("creating internet bridge failed with error %v", err) + if err := createSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { + log.Printf("creating snat bridge failed with error %v", err) return err } - if err := addMasQueradeRule(client.internetBridgeIP); err != nil { + if err := addMasQueradeRule(client.snatBridgeIP); err != nil { log.Printf("Adding snat rule failed with error %v", err) return err } - + if err := addVlanDropRule(); err != nil { log.Printf("Adding vlan drop rule failed with error %v", err) return err @@ -87,18 +90,18 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } - hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, epInfo.Id[:7]) - contIfName := fmt.Sprintf("%s%s-2", intVethInterfacePrefix, epInfo.Id[:7]) + hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) + contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) if err := createEndpoint(hostIfName, contIfName); err != nil { return err } - if err := netlink.SetLinkMaster(hostIfName, internetBridgeName); err != nil { + if err := netlink.SetLinkMaster(hostIfName, snatBridgeName); err != nil { return err } - client.intVethName = contIfName + client.snatVethName = contIfName } return nil @@ -186,8 +189,8 @@ func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo } if client.enableSnatOnHost { - log.Printf("[ovs] Setting link %v netns %v.", client.intVethName, epInfo.NetNsPath) - if err := netlink.SetLinkNetNs(client.intVethName, nsID); err != nil { + log.Printf("[ovs] Setting link %v netns %v.", client.snatVethName, epInfo.NetNsPath) + if err := netlink.SetLinkNetNs(client.snatVethName, nsID); err != nil { return err } } @@ -204,10 +207,10 @@ func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) client.containerVethName = epInfo.IfName if client.enableSnatOnHost { - if err := setupContainerInterface(client.intVethName, azureInternetIfName); err != nil { + if err := setupContainerInterface(client.snatVethName, azureSnatIfName); err != nil { return err } - client.intVethName = azureInternetIfName + client.snatVethName = azureSnatIfName } return nil @@ -219,9 +222,9 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E } if client.enableSnatOnHost { - log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.intVethName) - ip, intIpAddr, _ := net.ParseCIDR(client.localIP ) - if err := netlink.AddIpAddress(client.intVethName, ip, intIpAddr); err != nil { + log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.snatVethName) + ip, intIpAddr, _ := net.ParseCIDR(client.localIP) + if err := netlink.AddIpAddress(client.snatVethName, ip, intIpAddr); err != nil { return err } } @@ -242,8 +245,8 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { } if client.enableSnatOnHost { - hostIfName := fmt.Sprintf("%s%s", intVethInterfacePrefix, ep.Id[:7]) - log.Printf("[ovs] Deleting internet veth pair %v.", hostIfName) + hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, ep.Id[:7]) + log.Printf("[ovs] Deleting snat veth pair %v.", hostIfName) err = netlink.DeleteLink(hostIfName) if err != nil { log.Printf("[ovs] Failed to delete veth pair %v: %v.", hostIfName, err) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index ffb0d5cb2b..ef1580e2c5 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -1,32 +1,32 @@ package network import ( - "fmt" - "os" "bytes" + "fmt" "net" + "os" "strings" - "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) type OVSNetworkClient struct { bridgeName string hostInterfaceName string - internetBridgeIP string + snatBridgeIP string enableSnatOnHost bool } const ( - azureInternetVeth0 = "azintveth0" - azureInternetVeth1 = "azintveth1" - internetBridgeName = "azintbr" - imdsIP = "169.254.169.254/32" - ovsConfigFile = "/etc/default/openvswitch-switch" - ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" + azureSnatVeth0 = "azSnatveth0" + azureSnatVeth1 = "azSnatveth1" + snatBridgeName = "azSnatbr" + imdsIP = "169.254.169.254/32" + ovsConfigFile = "/etc/default/openvswitch-switch" + ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" ) func updateOVSConfig(option string) error { @@ -61,12 +61,11 @@ func updateOVSConfig(option string) error { return nil } - -func NewOVSClient(bridgeName, hostInterfaceName, internetBridgeIP string, enableSnatOnHost bool) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, enableSnatOnHost bool) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, - internetBridgeIP: internetBridgeIP, + snatBridgeIP: snatBridgeIP, enableSnatOnHost: enableSnatOnHost, } @@ -83,12 +82,12 @@ func (client *OVSNetworkClient) CreateBridge() error { } if client.enableSnatOnHost { - if err := createInternetBridge(client.internetBridgeIP, client.bridgeName); err != nil { - log.Printf("[net] Creating internet bridge failed with erro %v", err) + if err := createSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { + log.Printf("[net] Creating snat bridge failed with erro %v", err) return err } - - if err := addMasQueradeRule(client.internetBridgeIP); err != nil { + + if err := addMasQueradeRule(client.snatBridgeIP); err != nil { return err } @@ -114,55 +113,55 @@ func addVlanDropRule() error { } cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" - log.Printf("Adding ebtable rule to drop vlan traffic on internet bridge %v", cmd) + log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", cmd) _, err = platform.ExecuteCommand(cmd) return err } -func addMasQueradeRule(internetBridgeIPWithPrefix string) error { - _, ipNet, _ := net.ParseCIDR(internetBridgeIPWithPrefix) +func addMasQueradeRule(snatBridgeIPWithPrefix string) error { + _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) _, err := platform.ExecuteCommand(cmd) if err == nil { log.Printf("iptable snat rule already exists") return nil } - cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) log.Printf("Adding iptable snat rule %v", cmd) _, err = platform.ExecuteCommand(cmd) if err != nil { return err } - return nil + return nil } func deleteMasQueradeRule(interfaceName string) error { - internetIf, _ := net.InterfaceByName(interfaceName) + snatIf, _ := net.InterfaceByName(interfaceName) - addrs, _ := internetIf.Addrs() + addrs, _ := snatIf.Addrs() for _, addr := range addrs { ipAddr, ipNet, err := net.ParseCIDR(addr.String()) if err != nil { log.Printf("error %v", err) continue } - + if ipAddr.To4() != nil { - cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) log.Printf("Deleting iptable snat rule %v", cmd) _, err = platform.ExecuteCommand(cmd) if err != nil { return err - } + } return nil } } - - return nil + + return nil } func (client *OVSNetworkClient) DeleteBridge() error { @@ -172,45 +171,42 @@ func (client *OVSNetworkClient) DeleteBridge() error { } if client.enableSnatOnHost { - - deleteMasQueradeRule(internetBridgeName) + deleteMasQueradeRule(snatBridgeName) cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" _, err := platform.ExecuteCommand(cmd) if err != nil { - log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) + log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) } - if err := ovsctl.DeletePortFromOVS(client.bridgeName, azureInternetVeth1); err != nil { + if err := ovsctl.DeletePortFromOVS(client.bridgeName, azureSnatVeth1); err != nil { return err } - - if err := DeleteInternetBridge(); err != nil { - log.Printf("Deleting internet bridge failed with error %v", err) + + if err := DeleteSnatBridge(); err != nil { + log.Printf("Deleting snat bridge failed with error %v", err) return err } - return netlink.DeleteLink(azureInternetVeth0) + return netlink.DeleteLink(azureSnatVeth0) } return nil } -func createInternetBridge(internetBridgeIP string, mainInterface string) error { - - _, err := net.InterfaceByName(internetBridgeName) +func createSnatBridge(snatBridgeIP string, mainInterface string) error { + _, err := net.InterfaceByName(snatBridgeName) if err == nil { - log.Printf("Internet Bridge already exists") + log.Printf("Snat Bridge already exists") return nil } - - log.Printf("[net] Creating Internet bridge %v.", internetBridgeName) + log.Printf("[net] Creating Snat bridge %v.", snatBridgeName) link := netlink.BridgeLink{ LinkInfo: netlink.LinkInfo{ Type: netlink.LINK_TYPE_BRIDGE, - Name: internetBridgeName, + Name: snatBridgeName, }, } @@ -218,18 +214,18 @@ func createInternetBridge(internetBridgeIP string, mainInterface string) error { return err } - _, err = net.InterfaceByName(azureInternetVeth0) + _, err = net.InterfaceByName(azureSnatVeth0) if err == nil { - log.Printf("Azure internet veth already exists") + log.Printf("Azure snat veth already exists") return nil } vethLink := netlink.VEthLink{ LinkInfo: netlink.LinkInfo{ Type: netlink.LINK_TYPE_VETH, - Name: azureInternetVeth0, + Name: azureSnatVeth0, }, - PeerName: azureInternetVeth1, + PeerName: azureSnatVeth1, } err = netlink.AddLink(&vethLink) @@ -238,32 +234,32 @@ func createInternetBridge(internetBridgeIP string, mainInterface string) error { return err } - ip, addr, _ := net.ParseCIDR(internetBridgeIP) + log.Printf("Assigning %v on snat bridge", snatBridgeIP) - log.Printf("Assigning %v on internet bridge", internetBridgeIP) - err = netlink.AddIpAddress(internetBridgeName, ip, addr) + ip, addr, _ := net.ParseCIDR(snatBridgeIP) + err = netlink.AddIpAddress(snatBridgeName, ip, addr) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("[net] Failed to add IP address %v: %v.", addr, err) return err } - if err := netlink.SetLinkState(internetBridgeName, true); err != nil { + if err := netlink.SetLinkState(snatBridgeName, true); err != nil { return err } - if err := netlink.SetLinkState(azureInternetVeth0, true); err != nil { + if err := netlink.SetLinkState(azureSnatVeth0, true); err != nil { return err } - if err := netlink.SetLinkMaster(azureInternetVeth0, internetBridgeName); err != nil { + if err := netlink.SetLinkMaster(azureSnatVeth0, snatBridgeName); err != nil { return err } - if err := netlink.SetLinkState(azureInternetVeth1, true); err != nil { + if err := netlink.SetLinkState(azureSnatVeth1, true); err != nil { return err } - if err := ovsctl.AddPortOnOVSBridge(azureInternetVeth1, mainInterface, 0); err != nil { + if err := ovsctl.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { return err } @@ -287,25 +283,21 @@ func addStaticRoute(ip string, interfaceName string) error { return nil } -func DeleteInternetBridge() error { +func DeleteSnatBridge() error { // Delete the bridge. - err := netlink.DeleteLink(internetBridgeName) + err := netlink.DeleteLink(snatBridgeName) if err != nil { - log.Printf("[net] Failed to delete bridge %v, err:%v.", internetBridgeName, err) + log.Printf("[net] Failed to delete bridge %v, err:%v.", snatBridgeName, err) } - return nil + return err } -func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { +func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { //primary := extIf.IPAddresses[0].IP.String() mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) - /*if err := ovsctl.AddVMIpAcceptRule(client.bridgeName, primary, mac); err != nil { - return err - }*/ - ofport, err := ovsctl.GetOVSPortNumber(client.hostInterfaceName) if err != nil { return err @@ -321,7 +313,7 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { if err := ovsctl.AddArpDnatRule(client.bridgeName, ofport, macHex); err != nil { return err } - + if client.enableSnatOnHost { addStaticRoute(imdsIP, client.bridgeName) } @@ -329,7 +321,7 @@ func (client *OVSNetworkClient) AddBridgeRules(extIf *externalInterface) error { return nil } -func (client *OVSNetworkClient) DeleteBridgeRules(extIf *externalInterface) { +func (client *OVSNetworkClient) DeleteL2Rules(extIf *externalInterface) { ovsctl.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) } diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 6382b4dec6..7e777121a3 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -6,12 +6,12 @@ import ( "strings" "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/platform" ) const ( - macAddress = "12:34:56:78:9a:bc" + defaultMacForArpResponse = "12:34:56:78:9a:bc" ) func CreateOVSBridge(bridgeName string) error { @@ -65,9 +65,7 @@ func GetOVSPortNumber(interfaceName string) (string, error) { return "", err } - ofport = strings.Trim(ofport, "\n") - - return ofport, nil + return strings.Trim(ofport, "\n"), nil } func AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { @@ -128,7 +126,7 @@ func AddArpDnatRule(bridgeName string, port string, mac string) error { func AddFakeArpReply(bridgeName string, ip net.IP) error { // If arp fields matches, set arp reply rule for the request - macAddrHex := strings.Replace(macAddress, ":", "", -1) + macAddrHex := strings.Replace(defaultMacForArpResponse, ":", "", -1) ipAddrInt := common.IpToInt(ip) log.Printf("[ovs] Adding ARP reply rule for IP address %v ", ip.String()) @@ -136,7 +134,7 @@ func AddFakeArpReply(bridgeName string, ip net.IP) error { move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:%s, move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, - bridgeName, macAddress, macAddrHex, ipAddrInt) + bridgeName, defaultMacForArpResponse, macAddrHex, ipAddrInt) _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) diff --git a/platform/os_linux.go b/platform/os_linux.go index 768748dee4..d07a12e62a 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os/exec" "time" + "github.com/Azure/azure-container-networking/log" ) diff --git a/platform/os_windows.go b/platform/os_windows.go index d8974ba9fd..706dc95b03 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -33,4 +33,4 @@ func ExecuteCommand(command string) (string, error) { func SetOutboundSNAT(subnet string) error { return nil -} \ No newline at end of file +} From d1f405e9a4288d5fff5a888a0bed2b97f240573b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 5 Jul 2018 12:39:35 -0700 Subject: [PATCH 51/51] fixed review comments --- cni/network/mutlitenancy.go | 4 ++-- cni/network/network.go | 8 +++----- cni/network/network_linux.go | 7 ++++++- cni/network/network_windows.go | 2 +- cns/NetworkContainerContract.go | 4 ++-- cns/cnsclient/cnsclient.go | 2 +- cns/restserver/restserver.go | 10 +++++----- cns/service/main.go | 2 +- common/utils.go | 2 +- log/logger.go | 2 +- log/logger_linux.go | 2 +- log/logger_windows.go | 2 +- network/api.go | 2 -- network/bridge_networkclient_linux.go | 2 -- network/endpoint_linux.go | 2 +- network/network_linux.go | 4 +++- network/ovs_endpointclient_linux.go | 3 +-- network/ovs_networkclient_linux.go | 26 +++++++++----------------- 18 files changed, 39 insertions(+), 47 deletions(-) diff --git a/cni/network/mutlitenancy.go b/cni/network/mutlitenancy.go index baa422628a..e3d26a2e08 100644 --- a/cni/network/mutlitenancy.go +++ b/cni/network/mutlitenancy.go @@ -16,7 +16,7 @@ import ( cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) -func addDefaultGateway(nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +func SetupRoutingForMultitenancy(nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { // Adding default gateway if nwCfg.MultiTenancy { // if snat enabled, add 169.254.0.1 as default gateway @@ -65,7 +65,7 @@ func getContainerNetworkConfiguration(address string, namespace string, podName log.Printf("Network config received from cns %+v", networkConfig) - subnetPrefix := common.GetIpNet(networkConfig.PrimaryInterfaceIdentifier) + subnetPrefix := common.GetInterfaceSubnetWithSpecificIp(networkConfig.PrimaryInterfaceIdentifier) if subnetPrefix == nil { errBuf := fmt.Sprintf("Interface not found for this ip %v", networkConfig.PrimaryInterfaceIdentifier) log.Printf(errBuf) diff --git a/cni/network/network.go b/cni/network/network.go index b709fe8f48..63be90e394 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -311,7 +311,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } nwInfo.Options = make(map[string]interface{}) - log.Printf("Set Network Options") setNetworkOptions(cnsNetworkConfig, &nwInfo) err = plugin.nm.CreateNetwork(&nwInfo) @@ -356,9 +355,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } epInfo.Data = make(map[string]interface{}) - setEndpointOptions(cnsNetworkConfig, epInfo) - - epInfo.Data[network.OptVethName] = fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) + vethName := fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) + setEndpointOptions(cnsNetworkConfig, epInfo, vethName) var dns network.DNSInfo if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) { @@ -391,7 +389,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - addDefaultGateway(nwCfg, cnsNetworkConfig, epInfo, result) + SetupRoutingForMultitenancy(nwCfg, cnsNetworkConfig, epInfo, result) // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 055cba016a..e1b9fb5af4 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -6,6 +6,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" @@ -30,6 +31,7 @@ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cn func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { + log.Printf("Setting Network Options") vlanMap := make(map[string]interface{}) vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID) vlanMap[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) @@ -37,12 +39,15 @@ func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *net } } -func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { +func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { + log.Printf("Setting Endpoint Options") epInfo.Data[network.VlanIDKey] = cnsNwConfig.MultiTenancyInfo.ID epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) } + + epInfo.Data[network.OptVethName] = vethName } func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 318fd67534..ed2634ade1 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -64,7 +64,7 @@ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cn func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { } -func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo) { +func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) { } func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 345a2ef940..117012b9e4 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -36,7 +36,7 @@ type CreateNetworkContainerRequest struct { NetworkContainerid string // Mandatory input. PrimaryInterfaceIdentifier string // Primary CA. AuthorizationToken string - LocalIPConfiguration IPConfiguration + LocalIPConfiguration IPConfiguration OrchestratorContext json.RawMessage IPConfiguration IPConfiguration MultiTenancyInfo MultiTenancyInfo @@ -109,7 +109,7 @@ type GetNetworkContainerRequest struct { type GetNetworkContainerResponse struct { IPConfiguration IPConfiguration Routes []Route - CnetAddressSpace []IPSubnet + CnetAddressSpace []IPSubnet MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string LocalIPConfiguration IPConfiguration diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 18a3a19b7a..2a24c50134 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -10,7 +10,7 @@ import ( "github.com/Azure/azure-container-networking/log" ) -// IpamClient specifies a client to connect to Ipam Plugin. +// CNSClient specifies a client to connect to Ipam Plugin. type CNSClient struct { connectionURL string } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index dbf13a26b0..542e5dfe99 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -876,7 +876,7 @@ func (service *httpRestService) saveNetworkContainerGoalState(req cns.CreateNetw return UnexpectedError, errBuf } - log.Printf("Azure container instance info %v", podInfo) + log.Printf("Pod info %v", podInfo) if service.state.ContainerIDByOrchestratorContext == nil { service.state.ContainerIDByOrchestratorContext = make(map[string]string) @@ -977,7 +977,7 @@ func (service *httpRestService) getNetworkContainerResponse(req cns.GetNetworkCo return getNetworkContainerResponse } - log.Printf("azure container instance info %+v", podInfo) + log.Printf("pod info %+v", podInfo) containerID = service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] log.Printf("containerid %v", containerID) break @@ -1049,9 +1049,6 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r var containerStatus containerstatus var ok bool - service.lock.Lock() - defer service.lock.Unlock() - if containerStatus, ok = service.state.ContainerStatus[req.NetworkContainerid]; !ok { log.Printf("Not able to retrieve network container details for this container id %v", req.NetworkContainerid) break @@ -1066,6 +1063,9 @@ func (service *httpRestService) deleteNetworkContainer(w http.ResponseWriter, r } } + service.lock.Lock() + defer service.lock.Unlock() + if service.state.ContainerStatus != nil { delete(service.state.ContainerStatus, req.NetworkContainerid) } diff --git a/cns/service/main.go b/cns/service/main.go index 3200f1fc6d..4a4ee0a98d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -71,7 +71,7 @@ var args = acn.ArgumentList{ acn.OptLogTargetStderr: log.TargetStderr, acn.OptLogTargetFile: log.TargetLogfile, acn.OptLogStdout: log.TargetStdout, - acn.OptLogMultiWrite: log.TargetMultiWrite, + acn.OptLogMultiWrite: log.TargetStdOutAndLogFile, }, }, { diff --git a/common/utils.go b/common/utils.go index c754fc5c70..ba6dcf7d51 100644 --- a/common/utils.go +++ b/common/utils.go @@ -84,7 +84,7 @@ func IpToInt(ip net.IP) uint32 { return binary.BigEndian.Uint32(ip) } -func GetIpNet(ipAddr string) *net.IPNet { +func GetInterfaceSubnetWithSpecificIp(ipAddr string) *net.IPNet { addrs, err := net.InterfaceAddrs() if err != nil { log.Printf("InterfaceAddrs failed with %+v", err) diff --git a/log/logger.go b/log/logger.go index a1c9302a20..e5176d6688 100644 --- a/log/logger.go +++ b/log/logger.go @@ -27,7 +27,7 @@ const ( TargetSyslog TargetLogfile TargetStdout - TargetMultiWrite + TargetStdOutAndLogFile ) const ( diff --git a/log/logger_linux.go b/log/logger_linux.go index 5c2e34f130..fcd4d6829d 100644 --- a/log/logger_linux.go +++ b/log/logger_linux.go @@ -33,7 +33,7 @@ func (logger *Logger) SetTarget(target int) error { case TargetLogfile: logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) - case TargetMultiWrite: + case TargetStdOutAndLogFile: logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) if err == nil { logger.l.SetOutput(io.MultiWriter(os.Stdout, logger.out)) diff --git a/log/logger_windows.go b/log/logger_windows.go index f504ee20b3..3e5a5bc6ec 100644 --- a/log/logger_windows.go +++ b/log/logger_windows.go @@ -25,7 +25,7 @@ func (logger *Logger) SetTarget(target int) error { case TargetLogfile: logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) - case TargetMultiWrite: + case TargetStdOutAndLogFile: logger.out, err = os.OpenFile(logger.getLogFileName(), os.O_CREATE|os.O_APPEND|os.O_RDWR, logFilePerm) if err == nil { logger.l.SetOutput(io.MultiWriter(os.Stdout, logger.out)) diff --git a/network/api.go b/network/api.go index e18c8644ec..4b6b93ba81 100644 --- a/network/api.go +++ b/network/api.go @@ -17,6 +17,4 @@ var ( errEndpointNotFound = fmt.Errorf("Endpoint not found") errEndpointInUse = fmt.Errorf("Endpoint is already joined to a sandbox") errEndpointNotInUse = fmt.Errorf("Endpoint is not joined to a sandbox") - - OptVethName = "vethname" ) diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index f14060dd54..24ebb0b79d 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -38,7 +38,6 @@ func (client *LinuxBridgeClient) CreateBridge() error { } func (client *LinuxBridgeClient) DeleteBridge() error { - // Disconnect external interface from its bridge. err := netlink.SetLinkMaster(client.hostInterfaceName, "") if err != nil { @@ -55,7 +54,6 @@ func (client *LinuxBridgeClient) DeleteBridge() error { } func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { - hostIf, err := net.InterfaceByName(client.hostInterfaceName) if err != nil { return err diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 123fc84f6b..690fab492e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -78,7 +78,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) } - // On failure, cleanup the things. + // Cleanup on failure. defer func() { if err != nil { log.Printf("CNI error. Delete Endpoint %v and rules that are created.", contIfName) diff --git a/network/network_linux.go b/network/network_linux.go index 3af08f8aa7..65a46bb3f0 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -28,6 +28,8 @@ const ( SnatBridgeIPKey = "snatBridgeIP" LocalIPKey = "localIP" + + OptVethName = "vethname" ) // Linux implementation of route. @@ -38,7 +40,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Connect the external interface. var vlanid int opt, _ := nwInfo.Options[genericData].(map[string]interface{}) - log.Printf("opt %v options %v", opt, nwInfo.Options) + log.Printf("opt %+v options %+v", opt, nwInfo.Options) switch nwInfo.Mode { case opModeTunnel: diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 93ef994155..848ab31c25 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -75,7 +75,7 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } - if err := addMasQueradeRule(client.snatBridgeIP); err != nil { + if err := addMasqueradeRule(client.snatBridgeIP); err != nil { log.Printf("Adding snat rule failed with error %v", err) return err } @@ -134,7 +134,6 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { } for _, ipAddr := range epInfo.IPAddresses { - // Add Arp Reply Rules // Set Vlan id on arp request packet and forward it to table 1 if err := ovsctl.AddFakeArpReply(client.bridgeName, ipAddr.IP); err != nil { diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index ef1580e2c5..2e8afd4520 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -43,7 +43,6 @@ func updateOVSConfig(option string) error { contents := buf.String() conSplit := strings.Split(contents, "\n") - for _, existingOption := range conSplit { if option == existingOption { log.Printf("Not updating ovs config. Found option already written") @@ -87,7 +86,7 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - if err := addMasQueradeRule(client.snatBridgeIP); err != nil { + if err := addMasqueradeRule(client.snatBridgeIP); err != nil { return err } @@ -106,7 +105,6 @@ func addVlanDropRule() error { } out = strings.TrimSpace(out) - if strings.Contains(out, "-p 802_1Q -j DROP") { log.Printf("vlan drop rule already exists") return nil @@ -118,9 +116,8 @@ func addVlanDropRule() error { return err } -func addMasQueradeRule(snatBridgeIPWithPrefix string) error { +func addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) _, err := platform.ExecuteCommand(cmd) if err == nil { @@ -131,16 +128,15 @@ func addMasQueradeRule(snatBridgeIPWithPrefix string) error { cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) log.Printf("Adding iptable snat rule %v", cmd) _, err = platform.ExecuteCommand(cmd) + return err +} + +func deleteMasqueradeRule(interfaceName string) error { + snatIf, err := net.InterfaceByName(interfaceName) if err != nil { return err } - return nil -} - -func deleteMasQueradeRule(interfaceName string) error { - snatIf, _ := net.InterfaceByName(interfaceName) - addrs, _ := snatIf.Addrs() for _, addr := range addrs { ipAddr, ipNet, err := net.ParseCIDR(addr.String()) @@ -153,11 +149,7 @@ func deleteMasQueradeRule(interfaceName string) error { cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) log.Printf("Deleting iptable snat rule %v", cmd) _, err = platform.ExecuteCommand(cmd) - if err != nil { - return err - } - - return nil + return err } } @@ -171,7 +163,7 @@ func (client *OVSNetworkClient) DeleteBridge() error { } if client.enableSnatOnHost { - deleteMasQueradeRule(snatBridgeName) + deleteMasqueradeRule(snatBridgeName) cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" _, err := platform.ExecuteCommand(cmd)