From 2c88f49041624ed4a09111e923e167267a51f7c3 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 26 Mar 2019 13:28:24 -0700 Subject: [PATCH 01/11] allow inbound connection to container from host if the feature is enabled --- cni/network/network.go | 9 +++--- network/manager.go | 11 +++++--- network/network.go | 38 ++++++++++++++------------ network/network_linux.go | 25 +++++++++++------ network/ovs_networkclient_linux.go | 44 ++++++++++++++++++++---------- network/ovssnat/ovssnat_linux.go | 36 ++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 49 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 61e620effb..f7c88ac003 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -389,10 +389,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Gateway: gateway, }, }, - BridgeName: nwCfg.Bridge, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - DNS: nwDNSInfo, - Policies: policies, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, + AllowInboundFromHostToNC: true, + DNS: nwDNSInfo, + Policies: policies, } nwInfo.Options = make(map[string]interface{}) diff --git a/network/manager.go b/network/manager.go index 7ac8aaab43..75c182efc7 100644 --- a/network/manager.go +++ b/network/manager.go @@ -282,10 +282,13 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) } nwInfo := &NetworkInfo{ - Id: networkId, - Subnets: nw.Subnets, - Mode: nw.Mode, - Options: make(map[string]interface{}), + Id: networkId, + Subnets: nw.Subnets, + Mode: nw.Mode, + EnableSnatOnHost: nw.EnableSnatOnHost, + DNS: nw.DNS, + AllowInboundFromHostToNC: nw.AllowInboundFromHostToNC, + Options: make(map[string]interface{}), } getNetworkInfoImpl(nwInfo, nw) diff --git a/network/network.go b/network/network.go index e0a11fe567..c12096d262 100644 --- a/network/network.go +++ b/network/network.go @@ -35,28 +35,30 @@ type externalInterface struct { // A container network is a set of endpoints allowed to communicate with each other. type network struct { - Id string - HnsId string `json:",omitempty"` - Mode string - VlanId int - Subnets []SubnetInfo - Endpoints map[string]*endpoint - extIf *externalInterface - DNS DNSInfo - EnableSnatOnHost bool + Id string + HnsId string `json:",omitempty"` + Mode string + VlanId int + Subnets []SubnetInfo + Endpoints map[string]*endpoint + extIf *externalInterface + DNS DNSInfo + EnableSnatOnHost bool + AllowInboundFromHostToNC bool } // NetworkInfo contains read-only information about a container network. type NetworkInfo struct { - MasterIfName string - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - EnableSnatOnHost bool - Options map[string]interface{} + MasterIfName string + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + AllowInboundFromHostToNC bool + Options map[string]interface{} } // SubnetInfo contains subnet information for a container network. diff --git a/network/network_linux.go b/network/network_linux.go index 6da8aaa704..c5f8b7bd67 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -60,13 +60,14 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Create the network object. nw := &network{ - Id: nwInfo.Id, - Mode: nwInfo.Mode, - Endpoints: make(map[string]*endpoint), - extIf: extIf, - VlanId: vlanid, - DNS: nwInfo.DNS, - EnableSnatOnHost: nwInfo.EnableSnatOnHost, + Id: nwInfo.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: vlanid, + DNS: nwInfo.DNS, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, + AllowInboundFromHostToNC: nwInfo.AllowInboundFromHostToNC, } return nw, nil @@ -76,8 +77,14 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient + nwInfo, err := nm.GetNetworkInfo(nw.Id) + if err != nil { + log.Printf("Error while getting network info: %v", err) + return nil + } + if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nw.DNS.Servers, nw.EnableSnatOnHost) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nwInfo) } else { networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) } @@ -201,7 +208,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI snatBridgeIP, _ = opt[SnatBridgeIPKey].(string) } - networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo.DNS.Servers, nwInfo.EnableSnatOnHost) + networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 590abef73f..6d2e6e5b98 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -12,11 +12,12 @@ import ( ) type OVSNetworkClient struct { - bridgeName string - hostInterfaceName string - snatBridgeIP string - skipAddressesFromBlock []string - enableSnatOnHost bool + bridgeName string + hostInterfaceName string + snatBridgeIP string + skipAddressesFromBlock []string + enableSnatOnHost bool + allowInboundFromHostToNC bool } const ( @@ -55,14 +56,16 @@ func updateOVSConfig(option string) error { return nil } -func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, skipAddressesFromBlock []string, enableSnatOnHost bool) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, nwInfo *NetworkInfo) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ - bridgeName: bridgeName, - hostInterfaceName: hostInterfaceName, - snatBridgeIP: snatBridgeIP, - skipAddressesFromBlock: skipAddressesFromBlock, - enableSnatOnHost: enableSnatOnHost, + bridgeName: bridgeName, + hostInterfaceName: hostInterfaceName, + snatBridgeIP: snatBridgeIP, + skipAddressesFromBlock: nwInfo.DNS.Servers, + enableSnatOnHost: nwInfo.EnableSnatOnHost, + allowInboundFromHostToNC: nwInfo.AllowInboundFromHostToNC, } + log.Printf("allowInboundFromHostToNC %v", nwInfo.AllowInboundFromHostToNC) return ovsClient } @@ -76,7 +79,7 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { if err := ovssnat.CreateSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { log.Printf("[net] Creating snat bridge failed with erro %v", err) return err @@ -86,7 +89,14 @@ func (client *OVSNetworkClient) CreateBridge() error { return err } - return ovssnat.AddVlanDropRule() + if err := ovssnat.AddVlanDropRule(); err != nil { + return err + } + } + + log.Printf("allowInboundFromHostToNC %v", client.allowInboundFromHostToNC) + if client.allowInboundFromHostToNC { + return ovssnat.AllowInboundFromContainerHost(client.snatBridgeIP) } return nil @@ -98,7 +108,7 @@ func (client *OVSNetworkClient) DeleteBridge() error { return err } - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { ovssnat.DeleteMasqueradeRule() if err := ovssnat.DeleteSnatBridge(client.bridgeName); err != nil { @@ -107,6 +117,12 @@ func (client *OVSNetworkClient) DeleteBridge() error { } } + if client.allowInboundFromHostToNC { + err := ovssnat.DeleteInboundFromContainerHost(client.snatBridgeIP) + log.Printf("Deleting allowInboundFromHostToNC rule failed with: %v", err) + return nil + } + return nil } diff --git a/network/ovssnat/ovssnat_linux.go b/network/ovssnat/ovssnat_linux.go index a6e9ba7d42..61ea3f2344 100644 --- a/network/ovssnat/ovssnat_linux.go +++ b/network/ovssnat/ovssnat_linux.go @@ -265,3 +265,39 @@ func AddVlanDropRule() error { _, err = platform.ExecuteCommand(cmd) return err } + +func AllowInboundFromContainerHost(snatBridgeIPWithPrefix string) error { + ip, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) + cmd := fmt.Sprintf("iptables -t filter -I OUTPUT 1 -s %s -d %s -j ACCEPT", ip.String(), ipNet.String()) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) + return err + } + + cmd = fmt.Sprintf("iptables -t filter -I INPUT 1 -i %s -m state --state ESTABLISHED,RELATED -j ACCEPT", SnatBridgeName) + _, err = platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) + } + + return err +} + +func DeleteInboundFromContainerHost(snatBridgeIPWithPrefix string) error { + ip, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) + cmd := fmt.Sprintf("iptables -t filter -D OUTPUT -s %s -d %s -j ACCEPT", ip.String(), ipNet.String()) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) + return err + } + + cmd = fmt.Sprintf("iptables -t filter -D INPUT -i %s -m state --state ESTABLISHED,RELATED -j ACCEPT", SnatBridgeName) + _, err = platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) + } + + return err +} From 591b379040fb3a364bddaad2b83b3d9ce3b3b478 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 27 Mar 2019 18:23:38 -0700 Subject: [PATCH 02/11] Allowinbound from host to NC if flag is enabled --- cni/network/network_linux.go | 2 + cns/NetworkContainerContract.go | 1 + iptables/iptables.go | 99 +++++++++++++++ network/endpoint.go | 116 +++++++++--------- network/endpoint_linux.go | 70 ++++++----- ...int_common_linux.go => endpoint_common.go} | 2 + network/epcommon/endpoint_common_windows.go | 1 - network/network.go | 39 +++--- network/network_linux.go | 32 ++--- network/ovs_endpoint_snatroute_linux.go | 47 ++++--- network/ovs_endpointclient_linux.go | 48 ++++---- network/ovs_networkclient_linux.go | 80 ++---------- .../{infravnet_linux.go => infravnet.go} | 0 network/ovsinfravnet/infravnet_windows.go | 1 - .../ovssnat/{ovssnat_linux.go => ovssnat.go} | 110 +++++++++++------ network/ovssnat/ovssnat_windows.go | 1 - 16 files changed, 375 insertions(+), 274 deletions(-) create mode 100644 iptables/iptables.go rename network/epcommon/{endpoint_common_linux.go => endpoint_common.go} (99%) delete mode 100644 network/epcommon/endpoint_common_windows.go rename network/ovsinfravnet/{infravnet_linux.go => infravnet.go} (100%) delete mode 100644 network/ovsinfravnet/infravnet_windows.go rename network/ovssnat/{ovssnat_linux.go => ovssnat.go} (70%) delete mode 100644 network/ovssnat/ovssnat_windows.go diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 6a8ccbf88d..714ab0f990 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -54,6 +54,8 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne epInfo.Data[network.VlanIDKey] = cnsNwConfig.MultiTenancyInfo.ID epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) + epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication + } epInfo.Data[network.OptVethName] = vethName diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 263e19bf2d..7510c17e66 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -134,6 +134,7 @@ type GetNetworkContainerResponse struct { MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string LocalIPConfiguration IPConfiguration + AllowHostToNCCommunication bool Response Response } diff --git a/iptables/iptables.go b/iptables/iptables.go new file mode 100644 index 0000000000..f0d3b762fa --- /dev/null +++ b/iptables/iptables.go @@ -0,0 +1,99 @@ +package iptables + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/log" + + "github.com/Azure/azure-container-networking/platform" +) + +// cni iptable chains +const ( + CNIInputChain = "AZURECNIINPUT" + CNIOutputChain = "AZURECNIOUTPUT" +) + +// standard iptable chains +const ( + Input = "INPUT" + Output = "OUTPUT" +) + +// Standard Table names +const ( + Filter = "filter" +) + +// target +const ( + Accept = "ACCEPT" + Drop = "Drop" +) + +// actions +const ( + Insert = "I" + Append = "A" + Delete = "D" +) + +func ChainExists(tableName, chainName string) bool { + cmd := fmt.Sprintf("iptables -t %s -L %s", tableName, chainName) + if _, err := platform.ExecuteCommand(cmd); err != nil { + return false + } + + return true +} + +func CreateCNIChain(tableName, chainName string) error { + var err error + + if !ChainExists(tableName, chainName) { + cmd := fmt.Sprintf("iptables -t %s -N %s", tableName, chainName) + _, err = platform.ExecuteCommand(cmd) + } else { + log.Printf("%s Chain exists in table %s", chainName, tableName) + } + + return err +} + +func RuleExists(tableName, chainName, match, target string) bool { + cmd := fmt.Sprintf("iptables -t %s -C %s %s -j %s", tableName, chainName, match, target) + _, err := platform.ExecuteCommand(cmd) + if err != nil { + return false + } + + return true +} + +func InsertIptableRule(tableName, chainName, match, target string) error { + if RuleExists(tableName, chainName, match, target) { + log.Printf("Rule already exists") + return nil + } + + cmd := fmt.Sprintf("iptables -t %s -I %s 1 %s -j %s", tableName, chainName, match, target) + _, err := platform.ExecuteCommand(cmd) + return err +} + +func AppendIptableRule(tableName, chainName, match, target string) error { + if RuleExists(tableName, chainName, match, target) { + log.Printf("Rule already exists") + return nil + } + + cmd := fmt.Sprintf("iptables -t %s -A %s %s -j %s", tableName, chainName, match, target) + _, err := platform.ExecuteCommand(cmd) + return err +} + +func DeleteIptableRule(tableName, chainName, match, target string) error { + cmd := fmt.Sprintf("iptables -t %s -D %s %s -j %s", tableName, chainName, match, target) + _, err := platform.ExecuteCommand(cmd) + return err +} diff --git a/network/endpoint.go b/network/endpoint.go index b7c44e584d..bafff5df21 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -17,50 +17,53 @@ const ( // Endpoint represents a container network interface. type endpoint struct { - Id string - HnsId string `json:",omitempty"` - SandboxKey string - IfName string - HostIfName string - MacAddress net.HardwareAddr - InfraVnetIP net.IPNet - IPAddresses []net.IPNet - Gateways []net.IP - DNS DNSInfo - Routes []RouteInfo - VlanID int - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultitenancy bool - NetworkNameSpace string `json:",omitempty"` - ContainerID string - PODName string `json:",omitempty"` - PODNameSpace string `json:",omitempty"` - InfraVnetAddressSpace string `json:",omitempty"` + Id string + HnsId string `json:",omitempty"` + SandboxKey string + IfName string + HostIfName string + MacAddress net.HardwareAddr + InfraVnetIP net.IPNet + LocalIP string + IPAddresses []net.IPNet + Gateways []net.IP + DNS DNSInfo + Routes []RouteInfo + VlanID int + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultitenancy bool + AllowInboundFromHostToNC bool + NetworkNameSpace string `json:",omitempty"` + ContainerID string + PODName string `json:",omitempty"` + PODNameSpace string `json:",omitempty"` + InfraVnetAddressSpace string `json:",omitempty"` } // EndpointInfo contains read-only information about an endpoint. type EndpointInfo struct { - Id string - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - InfraVnetIP net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultiTenancy bool - PODName string - PODNameSpace string - Data map[string]interface{} - InfraVnetAddressSpace string + Id string + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + InfraVnetIP net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultiTenancy bool + AllowInboundFromHostToNC bool + PODName string + PODNameSpace string + Data map[string]interface{} + InfraVnetAddressSpace string } // RouteInfo contains information about an IP route. @@ -182,22 +185,23 @@ func podNameMatches(source string, actualValue string, doExactMatch bool) bool { // GetInfo returns information about the endpoint. func (ep *endpoint) getInfo() *EndpointInfo { info := &EndpointInfo{ - Id: ep.Id, - IPAddresses: ep.IPAddresses, - InfraVnetIP: ep.InfraVnetIP, - Data: make(map[string]interface{}), - MacAddress: ep.MacAddress, - SandboxKey: ep.SandboxKey, - IfIndex: 0, // Azure CNI supports only one interface - DNS: ep.DNS, - EnableSnatOnHost: ep.EnableSnatOnHost, - EnableInfraVnet: ep.EnableInfraVnet, - EnableMultiTenancy: ep.EnableMultitenancy, - IfName: ep.IfName, - ContainerID: ep.ContainerID, - NetNsPath: ep.NetworkNameSpace, - PODName: ep.PODName, - PODNameSpace: ep.PODNameSpace, + Id: ep.Id, + IPAddresses: ep.IPAddresses, + InfraVnetIP: ep.InfraVnetIP, + Data: make(map[string]interface{}), + MacAddress: ep.MacAddress, + SandboxKey: ep.SandboxKey, + IfIndex: 0, // Azure CNI supports only one interface + DNS: ep.DNS, + EnableSnatOnHost: ep.EnableSnatOnHost, + EnableInfraVnet: ep.EnableInfraVnet, + EnableMultiTenancy: ep.EnableMultitenancy, + AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, + IfName: ep.IfName, + ContainerID: ep.ContainerID, + NetNsPath: ep.NetworkNameSpace, + PODName: ep.PODName, + PODNameSpace: ep.PODNameSpace, } for _, route := range ep.Routes { diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 0c39d0de15..e97e21598b 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -52,6 +52,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { var err error var hostIfName string var contIfName string + var localIP string var epClient EndpointClient var vlanid int = 0 @@ -65,6 +66,10 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { if _, ok := epInfo.Data[VlanIDKey]; ok { vlanid = epInfo.Data[VlanIDKey].(int) } + + if _, ok := epInfo.Data[LocalIPKey]; ok { + localIP = epInfo.Data[LocalIPKey].(string) + } } if _, ok := epInfo.Data[OptVethName]; ok { @@ -82,12 +87,17 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { if vlanid != 0 { log.Printf("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + epClient = NewOVSEndpointClient( - nw.extIf, + nw, epInfo, hostIfName, contIfName, - vlanid) + vlanid, + localIP) } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode) @@ -101,15 +111,17 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { if err != nil { log.Printf("CNI error. Delete Endpoint %v and rules that are created.", contIfName) endpt := &endpoint{ - Id: epInfo.Id, - IfName: contIfName, - HostIfName: hostIfName, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{nw.extIf.IPv4Gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - EnableMultitenancy: epInfo.EnableMultiTenancy, + Id: epInfo.Id, + IfName: contIfName, + HostIfName: hostIfName, + LocalIP: localIP, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{nw.extIf.IPv4Gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + EnableMultitenancy: epInfo.EnableMultiTenancy, + AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, } if containerIf != nil { @@ -177,22 +189,24 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Create the endpoint object. ep = &endpoint{ - Id: epInfo.Id, - IfName: epInfo.IfName, - HostIfName: hostIfName, - MacAddress: containerIf.HardwareAddr, - InfraVnetIP: epInfo.InfraVnetIP, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{nw.extIf.IPv4Gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - EnableInfraVnet: epInfo.EnableInfraVnet, - EnableMultitenancy: epInfo.EnableMultiTenancy, - NetworkNameSpace: epInfo.NetNsPath, - ContainerID: epInfo.ContainerID, - PODName: epInfo.PODName, - PODNameSpace: epInfo.PODNameSpace, + Id: epInfo.Id, + IfName: epInfo.IfName, + HostIfName: hostIfName, + MacAddress: containerIf.HardwareAddr, + InfraVnetIP: epInfo.InfraVnetIP, + LocalIP: localIP, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{nw.extIf.IPv4Gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + EnableInfraVnet: epInfo.EnableInfraVnet, + EnableMultitenancy: epInfo.EnableMultiTenancy, + AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + NetworkNameSpace: epInfo.NetNsPath, + ContainerID: epInfo.ContainerID, + PODName: epInfo.PODName, + PODNameSpace: epInfo.PODNameSpace, } for _, route := range epInfo.Routes { @@ -211,7 +225,7 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID) + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP) } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode) } else { diff --git a/network/epcommon/endpoint_common_linux.go b/network/epcommon/endpoint_common.go similarity index 99% rename from network/epcommon/endpoint_common_linux.go rename to network/epcommon/endpoint_common.go index fba220070d..c166bc919c 100644 --- a/network/epcommon/endpoint_common_linux.go +++ b/network/epcommon/endpoint_common.go @@ -1,3 +1,5 @@ +// +build linux + package epcommon import ( diff --git a/network/epcommon/endpoint_common_windows.go b/network/epcommon/endpoint_common_windows.go deleted file mode 100644 index 0a1eaf507b..0000000000 --- a/network/epcommon/endpoint_common_windows.go +++ /dev/null @@ -1 +0,0 @@ -package epcommon diff --git a/network/network.go b/network/network.go index c12096d262..b39865e8de 100644 --- a/network/network.go +++ b/network/network.go @@ -35,30 +35,29 @@ type externalInterface struct { // A container network is a set of endpoints allowed to communicate with each other. type network struct { - Id string - HnsId string `json:",omitempty"` - Mode string - VlanId int - Subnets []SubnetInfo - Endpoints map[string]*endpoint - extIf *externalInterface - DNS DNSInfo - EnableSnatOnHost bool - AllowInboundFromHostToNC bool + Id string + HnsId string `json:",omitempty"` + Mode string + VlanId int + Subnets []SubnetInfo + Endpoints map[string]*endpoint + extIf *externalInterface + DNS DNSInfo + EnableSnatOnHost bool + SnatBridgeIP string } // NetworkInfo contains read-only information about a container network. type NetworkInfo struct { - MasterIfName string - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - EnableSnatOnHost bool - AllowInboundFromHostToNC bool - Options map[string]interface{} + MasterIfName string + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + Options map[string]interface{} } // SubnetInfo contains subnet information for a container network. diff --git a/network/network_linux.go b/network/network_linux.go index c5f8b7bd67..9929861758 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -52,6 +52,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt if opt != nil && opt[VlanIDKey] != nil { vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string)) } + case opModeTransparent: break default: @@ -60,14 +61,13 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // Create the network object. nw := &network{ - Id: nwInfo.Id, - Mode: nwInfo.Mode, - Endpoints: make(map[string]*endpoint), - extIf: extIf, - VlanId: vlanid, - DNS: nwInfo.DNS, - EnableSnatOnHost: nwInfo.EnableSnatOnHost, - AllowInboundFromHostToNC: nwInfo.AllowInboundFromHostToNC, + Id: nwInfo.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: vlanid, + DNS: nwInfo.DNS, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, } return nw, nil @@ -77,14 +77,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient - nwInfo, err := nm.GetNetworkInfo(nw.Id) - if err != nil { - log.Printf("Error while getting network info: %v", err) - return nil - } - if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nwInfo) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name) } else { networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) } @@ -202,13 +196,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { - snatBridgeIP := "" - - if opt != nil && opt[SnatBridgeIPKey] != nil { - snatBridgeIP, _ = opt[SnatBridgeIPKey].(string) - } - - networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo) + networkClient = NewOVSClient(bridgeName, extIf.Name) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 47c9c8d3d5..b7dd3b9c12 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -6,47 +6,48 @@ import ( "github.com/Azure/azure-container-networking/network/ovssnat" ) -func NewSnatClient(client *OVSEndpointClient, epInfo *EndpointInfo) { - if client.enableSnatOnHost { - var localIP, snatBridgeIP string +func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) - if _, ok := epInfo.Data[LocalIPKey]; ok { - localIP = epInfo.Data[LocalIPKey].(string) - } - - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - snatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } - client.snatClient = ovssnat.NewSnatClient(hostIfName, contIfName, localIP, snatBridgeIP, epInfo.DNS.Servers) } } func AddSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost { - return client.snatClient.CreateSnatEndpoint(client.bridgeName) + if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { + return err + } } return nil } func AddSnatEndpointRules(client *OVSEndpointClient) error { - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { if err := client.snatClient.AddPrivateIPBlockRule(); err != nil { return err } - return AddStaticRoute(ovssnat.ImdsIP, client.bridgeName) + if err := AddStaticRoute(ovssnat.ImdsIP, client.bridgeName); err != nil { + return err + } + } + + if client.allowInboundFromHostToNC { + if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { + return err + } } return nil } func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) } @@ -54,7 +55,7 @@ func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, } func SetupSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { return client.snatClient.SetupSnatContainerInterface() } @@ -62,7 +63,7 @@ func SetupSnatContainerInterface(client *OVSEndpointClient) error { } func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { return client.snatClient.ConfigureSnatContainerInterface() } @@ -70,9 +71,17 @@ func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { } func DeleteSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost { + if client.enableSnatOnHost || client.allowInboundFromHostToNC { return client.snatClient.DeleteSnatEndpoint() } return nil } + +func DeleteSnatEndpointRules(client *OVSEndpointClient) error { + if client.allowInboundFromHostToNC { + return client.snatClient.DeleteInboundFromHostToNC() + } + + return nil +} diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index b580bbd5dd..d3c04d0342 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -12,17 +12,18 @@ import ( ) type OVSEndpointClient struct { - bridgeName string - hostPrimaryIfName string - hostVethName string - hostPrimaryMac string - containerVethName string - containerMac string - snatClient ovssnat.OVSSnatClient - infraVnetClient ovsinfravnet.OVSInfraVnetClient - vlanID int - enableSnatOnHost bool - enableInfraVnet bool + bridgeName string + hostPrimaryIfName string + hostVethName string + hostPrimaryMac string + containerVethName string + containerMac string + snatClient ovssnat.OVSSnatClient + infraVnetClient ovsinfravnet.OVSInfraVnetClient + vlanID int + enableSnatOnHost bool + enableInfraVnet bool + allowInboundFromHostToNC bool } const ( @@ -31,26 +32,27 @@ const ( ) func NewOVSEndpointClient( - extIf *externalInterface, + nw *network, epInfo *EndpointInfo, hostVethName string, containerVethName string, vlanid int, -) *OVSEndpointClient { + localIP string) *OVSEndpointClient { client := &OVSEndpointClient{ - bridgeName: extIf.BridgeName, - hostPrimaryIfName: extIf.Name, - hostVethName: hostVethName, - hostPrimaryMac: extIf.MacAddress.String(), - containerVethName: containerVethName, - vlanID: vlanid, - enableSnatOnHost: epInfo.EnableSnatOnHost, - enableInfraVnet: epInfo.EnableInfraVnet, + bridgeName: nw.extIf.BridgeName, + hostPrimaryIfName: nw.extIf.Name, + hostVethName: hostVethName, + hostPrimaryMac: nw.extIf.MacAddress.String(), + containerVethName: containerVethName, + vlanID: vlanid, + enableSnatOnHost: epInfo.EnableSnatOnHost, + enableInfraVnet: epInfo.EnableInfraVnet, + allowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, } NewInfraVnetClient(client, epInfo.Id[:7]) - NewSnatClient(client, epInfo) + NewSnatClient(client, nw.SnatBridgeIP, localIP, epInfo) return client } @@ -155,6 +157,7 @@ func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { log.Printf("[ovs] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) ovsctl.DeletePortFromOVS(client.bridgeName, client.hostVethName) + DeleteSnatEndpointRules(client) DeleteInfraVnetEndpointRules(client, ep, hostPort) } @@ -212,5 +215,6 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { return err } + DeleteSnatEndpoint(client) return DeleteInfraVnetEndpoint(client, ep.Id[:7]) } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 6d2e6e5b98..d3bedd3957 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -6,18 +6,12 @@ import ( "strings" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network/epcommon" - "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" ) type OVSNetworkClient struct { - bridgeName string - hostInterfaceName string - snatBridgeIP string - skipAddressesFromBlock []string - enableSnatOnHost bool - allowInboundFromHostToNC bool + bridgeName string + hostInterfaceName string } const ( @@ -56,71 +50,34 @@ func updateOVSConfig(option string) error { return nil } -func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, nwInfo *NetworkInfo) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName string) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ - bridgeName: bridgeName, - hostInterfaceName: hostInterfaceName, - snatBridgeIP: snatBridgeIP, - skipAddressesFromBlock: nwInfo.DNS.Servers, - enableSnatOnHost: nwInfo.EnableSnatOnHost, - allowInboundFromHostToNC: nwInfo.AllowInboundFromHostToNC, + bridgeName: bridgeName, + hostInterfaceName: hostInterfaceName, } - log.Printf("allowInboundFromHostToNC %v", nwInfo.AllowInboundFromHostToNC) return ovsClient } func (client *OVSNetworkClient) CreateBridge() error { - if err := ovsctl.CreateOVSBridge(client.bridgeName); err != nil { - return err - } + var err error - if err := updateOVSConfig(ovsOpt); err != nil { + if err = ovsctl.CreateOVSBridge(client.bridgeName); err != nil { return err } - if client.enableSnatOnHost || client.allowInboundFromHostToNC { - if err := ovssnat.CreateSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { - log.Printf("[net] Creating snat bridge failed with erro %v", err) - return err - } - - if err := ovssnat.AddMasqueradeRule(client.snatBridgeIP); err != nil { - return err - } - - if err := ovssnat.AddVlanDropRule(); err != nil { - return err + defer func() { + if err != nil { + client.DeleteBridge() } - } - - log.Printf("allowInboundFromHostToNC %v", client.allowInboundFromHostToNC) - if client.allowInboundFromHostToNC { - return ovssnat.AllowInboundFromContainerHost(client.snatBridgeIP) - } + }() - return nil + return updateOVSConfig(ovsOpt) } func (client *OVSNetworkClient) DeleteBridge() error { if err := ovsctl.DeleteOVSBridge(client.bridgeName); err != nil { log.Printf("Deleting ovs bridge failed with error %v", err) - return err - } - - if client.enableSnatOnHost || client.allowInboundFromHostToNC { - ovssnat.DeleteMasqueradeRule() - - if err := ovssnat.DeleteSnatBridge(client.bridgeName); err != nil { - log.Printf("Deleting snat bridge failed with error %v", err) - return err - } - } - - if client.allowInboundFromHostToNC { - err := ovssnat.DeleteInboundFromContainerHost(client.snatBridgeIP) - log.Printf("Deleting allowInboundFromHostToNC rule failed with: %v", err) - return nil } return nil @@ -146,25 +103,12 @@ func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { return err } - if client.enableSnatOnHost { - if err := epcommon.AddOrDeletePrivateIPBlockRule(ovssnat.SnatBridgeName, client.skipAddressesFromBlock, "A"); err != nil { - return err - } - - return AddStaticRoute(ovssnat.ImdsIP, client.bridgeName) - } - return nil } func (client *OVSNetworkClient) DeleteL2Rules(extIf *externalInterface) { ovsctl.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) - if client.enableSnatOnHost { - if err := epcommon.AddOrDeletePrivateIPBlockRule(ovssnat.SnatBridgeName, client.skipAddressesFromBlock, "D"); err != nil { - log.Printf("Deleting PrivateIPBlock rules failed with error %v", err) - } - } } func (client *OVSNetworkClient) SetBridgeMasterToHostInterface() error { diff --git a/network/ovsinfravnet/infravnet_linux.go b/network/ovsinfravnet/infravnet.go similarity index 100% rename from network/ovsinfravnet/infravnet_linux.go rename to network/ovsinfravnet/infravnet.go diff --git a/network/ovsinfravnet/infravnet_windows.go b/network/ovsinfravnet/infravnet_windows.go deleted file mode 100644 index 512fc9de77..0000000000 --- a/network/ovsinfravnet/infravnet_windows.go +++ /dev/null @@ -1 +0,0 @@ -package ovsinfravnet diff --git a/network/ovssnat/ovssnat_linux.go b/network/ovssnat/ovssnat.go similarity index 70% rename from network/ovssnat/ovssnat_linux.go rename to network/ovssnat/ovssnat.go index 61ea3f2344..7ad3da43af 100644 --- a/network/ovssnat/ovssnat_linux.go +++ b/network/ovssnat/ovssnat.go @@ -5,6 +5,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/epcommon" @@ -26,6 +27,7 @@ type OVSSnatClient struct { localIP string snatBridgeIP string SkipAddressesFromBlock []string + containerSnatVethMac net.HardwareAddr } func NewSnatClient(hostIfName string, contIfName string, localIP string, snatBridgeIP string, skipAddressesFromBlock []string) OVSSnatClient { @@ -66,6 +68,9 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return err } + snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + client.containerSnatVethMac = snatContainerVeth.HardwareAddr + return netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) } @@ -93,6 +98,75 @@ func (client *OVSSnatClient) SetupSnatContainerInterface() error { return nil } +func (client *OVSSnatClient) AllowInboundFromHostToNC() error { + bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + containerIP, _, _ := net.ParseCIDR(client.localIP) + + // Create cnioutput chain + if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + // Forward from Output to cnioutput chain + if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + // Allow Host to NC traffic + matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) + err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) + return err + } + + // Create cniinput chain + if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + // Forward from Input to cniinput chain + if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + matchCondition = fmt.Sprintf(" -i %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) + err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) + return err + } + + err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, client.containerSnatVethMac) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s %v", err, containerIP, client.containerSnatVethMac.String()) + } + + return err +} + +func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { + bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + containerIP, _, _ := net.ParseCIDR(client.localIP) + + matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) + err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) + } + + err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, client.containerSnatVethMac) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s mac %s %v", err, containerIP, client.containerSnatVethMac.String()) + } + + return err +} + func (client *OVSSnatClient) ConfigureSnatContainerInterface() error { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) @@ -265,39 +339,3 @@ func AddVlanDropRule() error { _, err = platform.ExecuteCommand(cmd) return err } - -func AllowInboundFromContainerHost(snatBridgeIPWithPrefix string) error { - ip, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t filter -I OUTPUT 1 -s %s -d %s -j ACCEPT", ip.String(), ipNet.String()) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) - return err - } - - cmd = fmt.Sprintf("iptables -t filter -I INPUT 1 -i %s -m state --state ESTABLISHED,RELATED -j ACCEPT", SnatBridgeName) - _, err = platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) - } - - return err -} - -func DeleteInboundFromContainerHost(snatBridgeIPWithPrefix string) error { - ip, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t filter -D OUTPUT -s %s -d %s -j ACCEPT", ip.String(), ipNet.String()) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) - return err - } - - cmd = fmt.Sprintf("iptables -t filter -D INPUT -i %s -m state --state ESTABLISHED,RELATED -j ACCEPT", SnatBridgeName) - _, err = platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("AddInboundFromContainerHost: setting up output chain failed with %v", err) - } - - return err -} diff --git a/network/ovssnat/ovssnat_windows.go b/network/ovssnat/ovssnat_windows.go deleted file mode 100644 index 4978a6a744..0000000000 --- a/network/ovssnat/ovssnat_windows.go +++ /dev/null @@ -1 +0,0 @@ -package ovssnat From 0cd9e3b56eb3cf7566a10ec43096fcbe949b10c2 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 27 Mar 2019 18:36:29 -0700 Subject: [PATCH 03/11] fixed old changes --- cni/network/network.go | 9 ++++----- network/manager.go | 3 +-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index f7c88ac003..61e620effb 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -389,11 +389,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Gateway: gateway, }, }, - BridgeName: nwCfg.Bridge, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - AllowInboundFromHostToNC: true, - DNS: nwDNSInfo, - Policies: policies, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, + DNS: nwDNSInfo, + Policies: policies, } nwInfo.Options = make(map[string]interface{}) diff --git a/network/manager.go b/network/manager.go index 75c182efc7..69edba0755 100644 --- a/network/manager.go +++ b/network/manager.go @@ -287,8 +287,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) Mode: nw.Mode, EnableSnatOnHost: nw.EnableSnatOnHost, DNS: nw.DNS, - AllowInboundFromHostToNC: nw.AllowInboundFromHostToNC, - Options: make(map[string]interface{}), + Options: make(map[string]interface{}), } getNetworkInfoImpl(nwInfo, nw) From 6e144d54385f06a7260a768401c925c8205c0d02 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 2 Apr 2019 11:44:16 -0700 Subject: [PATCH 04/11] minor fixes --- network/bridge_endpointclient_linux.go | 3 +-- network/ovssnat/ovssnat.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 3b065a95f6..4527fb94a7 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -75,8 +75,7 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) if client.mode != opModeTunnel { log.Printf("[net] Adding static arp for IP address %v and MAC %v in VM", ipAddr.String(), client.containerMac.String()) - netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac) - if err != nil { + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac); err != nil { log.Printf("Failed setting arp in vm: %v", err) } } diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 7ad3da43af..2d47720bf8 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -68,9 +68,6 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return err } - snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) - client.containerSnatVethMac = snatContainerVeth.HardwareAddr - return netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) } @@ -141,9 +138,12 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { return err } - err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, client.containerSnatVethMac) + snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + + log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) + err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) if err != nil { - log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s %v", err, containerIP, client.containerSnatVethMac.String()) + log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) } return err @@ -159,9 +159,10 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) } - err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, client.containerSnatVethMac) + log.Printf("Removing static arp entry for ip %s ", containerIP) + err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) if err != nil { - log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s mac %s %v", err, containerIP, client.containerSnatVethMac.String()) + log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s: %v", containerIP, err) } return err From 1b51cb95b5f5020f21975d491d95604424f17465 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 9 Jul 2019 11:58:09 -0700 Subject: [PATCH 05/11] added NC to host commn support --- cni/network/network_linux.go | 2 +- cns/NetworkContainerContract.go | 1 - iptables/iptables.go | 25 +++--- network/endpoint.go | 3 + network/endpoint_linux.go | 2 + network/epcommon/endpoint_common.go | 47 +++++----- network/ovs_endpoint_snatroute_linux.go | 36 +++++--- network/ovs_endpointclient_linux.go | 2 + network/ovssnat/ovssnat.go | 109 +++++++++++++++++++----- 9 files changed, 158 insertions(+), 69 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 714ab0f990..a9c4b28877 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -55,7 +55,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication - + epInfo.AllowInboundFromNCToHost = true //cnsNwConfig.AllowNCToHostCommunication } epInfo.Data[network.OptVethName] = vethName diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 7510c17e66..263e19bf2d 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -134,7 +134,6 @@ type GetNetworkContainerResponse struct { MultiTenancyInfo MultiTenancyInfo PrimaryInterfaceIdentifier string LocalIPConfiguration IPConfiguration - AllowHostToNCCommunication bool Response Response } diff --git a/iptables/iptables.go b/iptables/iptables.go index f0d3b762fa..70882151b2 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -16,19 +16,24 @@ const ( // standard iptable chains const ( - Input = "INPUT" - Output = "OUTPUT" + Input = "INPUT" + Output = "OUTPUT" + Forward = "FORWARD" + Prerouting = "PREROUTING" + Postrouting = "POSTROUTING" ) // Standard Table names const ( Filter = "filter" + Nat = "nat" ) // target const ( - Accept = "ACCEPT" - Drop = "Drop" + Accept = "ACCEPT" + Drop = "Drop" + Masquerade = "MASQUERADE" ) // actions @@ -39,7 +44,7 @@ const ( ) func ChainExists(tableName, chainName string) bool { - cmd := fmt.Sprintf("iptables -t %s -L %s", tableName, chainName) + cmd := fmt.Sprintf("iptables -w 60 -t %s -L %s", tableName, chainName) if _, err := platform.ExecuteCommand(cmd); err != nil { return false } @@ -51,7 +56,7 @@ func CreateCNIChain(tableName, chainName string) error { var err error if !ChainExists(tableName, chainName) { - cmd := fmt.Sprintf("iptables -t %s -N %s", tableName, chainName) + cmd := fmt.Sprintf("iptables -w 60 -t %s -N %s", tableName, chainName) _, err = platform.ExecuteCommand(cmd) } else { log.Printf("%s Chain exists in table %s", chainName, tableName) @@ -61,7 +66,7 @@ func CreateCNIChain(tableName, chainName string) error { } func RuleExists(tableName, chainName, match, target string) bool { - cmd := fmt.Sprintf("iptables -t %s -C %s %s -j %s", tableName, chainName, match, target) + cmd := fmt.Sprintf("iptables -w 60 -t %s -C %s %s -j %s", tableName, chainName, match, target) _, err := platform.ExecuteCommand(cmd) if err != nil { return false @@ -76,7 +81,7 @@ func InsertIptableRule(tableName, chainName, match, target string) error { return nil } - cmd := fmt.Sprintf("iptables -t %s -I %s 1 %s -j %s", tableName, chainName, match, target) + cmd := fmt.Sprintf("iptables -w 60 -t %s -I %s 1 %s -j %s", tableName, chainName, match, target) _, err := platform.ExecuteCommand(cmd) return err } @@ -87,13 +92,13 @@ func AppendIptableRule(tableName, chainName, match, target string) error { return nil } - cmd := fmt.Sprintf("iptables -t %s -A %s %s -j %s", tableName, chainName, match, target) + cmd := fmt.Sprintf("iptables -w 60 -t %s -A %s %s -j %s", tableName, chainName, match, target) _, err := platform.ExecuteCommand(cmd) return err } func DeleteIptableRule(tableName, chainName, match, target string) error { - cmd := fmt.Sprintf("iptables -t %s -D %s %s -j %s", tableName, chainName, match, target) + cmd := fmt.Sprintf("iptables -w 60 -t %s -D %s %s -j %s", tableName, chainName, match, target) _, err := platform.ExecuteCommand(cmd) return err } diff --git a/network/endpoint.go b/network/endpoint.go index bafff5df21..d06f5f75bf 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -34,6 +34,7 @@ type endpoint struct { EnableInfraVnet bool EnableMultitenancy bool AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` @@ -60,6 +61,7 @@ type EndpointInfo struct { EnableInfraVnet bool EnableMultiTenancy bool AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool PODName string PODNameSpace string Data map[string]interface{} @@ -197,6 +199,7 @@ func (ep *endpoint) getInfo() *EndpointInfo { EnableInfraVnet: ep.EnableInfraVnet, EnableMultiTenancy: ep.EnableMultitenancy, AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, + AllowInboundFromNCToHost: ep.AllowInboundFromNCToHost, IfName: ep.IfName, ContainerID: ep.ContainerID, NetNsPath: ep.NetworkNameSpace, diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index e97e21598b..fabb522e4a 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -122,6 +122,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { EnableSnatOnHost: epInfo.EnableSnatOnHost, EnableMultitenancy: epInfo.EnableMultiTenancy, AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, } if containerIf != nil { @@ -203,6 +204,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { EnableInfraVnet: epInfo.EnableInfraVnet, EnableMultitenancy: epInfo.EnableMultiTenancy, AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, NetworkNameSpace: epInfo.NetNsPath, ContainerID: epInfo.ContainerID, PODName: epInfo.PODName, diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go index c166bc919c..19fcbaed47 100644 --- a/network/epcommon/endpoint_common.go +++ b/network/epcommon/endpoint_common.go @@ -6,9 +6,9 @@ import ( "fmt" "net" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/platform" ) /*RFC For Private Address Space: https://tools.ietf.org/html/rfc1918 @@ -99,40 +99,27 @@ func AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { } func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, chainName string, target string) error { - var cmd string + var err error option := "i" - if chainName == "OUTPUT" { + if chainName == iptables.Output { option = "o" } - if action != "D" { - cmd = fmt.Sprintf("iptables -t filter -C %v -%v %v -d %v -j %v", chainName, option, bridgeName, ipAddress, target) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("Iptable filter for private ipaddr %v on %v chain %v target rule already exists", ipAddress, chainName, target) - return nil - } - } - - if target != "ACCEPT" { - cmd = fmt.Sprintf("iptables -t filter -%v %v -%v %v -d %v -j %v", action, chainName, option, bridgeName, ipAddress, target) - } else { - action = "I" - cmd = fmt.Sprintf("iptables -t filter -%v %v 1 -%v %v -d %v -j %v", action, chainName, option, bridgeName, ipAddress, target) - } + matchCondition := fmt.Sprintf("-%s %s -d %s", option, bridgeName, ipAddress) - _, err := platform.ExecuteCommand(cmd) - if err != nil { - log.Printf("Iptable filter %v action for private ipaddr %v on %v chain %v target failed with %v", action, ipAddress, chainName, target, err) - return err + if action == iptables.Insert { + err = iptables.InsertIptableRule(iptables.Filter, chainName, matchCondition, target) + } else if action == iptables.Append { + err = iptables.AppendIptableRule(iptables.Filter, chainName, matchCondition, target) + } else if action == iptables.Delete { + err = iptables.DeleteIptableRule(iptables.Filter, chainName, matchCondition, target) } - return nil + return err } -func AddOrDeletePrivateIPBlockRule(bridgeName string, skipAddresses []string, action string) error { - privateIPAddresses := getPrivateIPSpace() +func AllowIPAddresses(bridgeName string, skipAddresses []string, action string) error { chains := getFilterChains() target := getFilterchainTarget() @@ -153,6 +140,16 @@ func AddOrDeletePrivateIPBlockRule(bridgeName string, skipAddresses []string, ac } + return nil +} + +func BlockIPAddresses(bridgeName string, action string) error { + privateIPAddresses := getPrivateIPSpace() + chains := getFilterChains() + target := getFilterchainTarget() + + log.Printf("[net] Addresses to block %v", privateIPAddresses) + for _, ipAddress := range privateIPAddresses { if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[0], target[1]); err != nil { return err diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index b7dd3b9c12..aeb26895b7 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -17,7 +17,7 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin } func AddSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { return err } @@ -27,27 +27,35 @@ func AddSnatEndpoint(client *OVSEndpointClient) error { } func AddSnatEndpointRules(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { - if err := client.snatClient.AddPrivateIPBlockRule(); err != nil { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { + if err := client.snatClient.AllowIPAddresses(); err != nil { return err } - if err := AddStaticRoute(ovssnat.ImdsIP, client.bridgeName); err != nil { + if err := client.snatClient.BlockIPAddresses(); err != nil { return err } - } - if client.allowInboundFromHostToNC { - if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { + if err := AddStaticRoute(ovssnat.ImdsIP, client.bridgeName); err != nil { return err } + + if client.allowInboundFromHostToNC { + if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { + return err + } + } + + if client.allowInboundFromNCToHost { + return client.snatClient.AllowInboundFromNCToHost() + } } return nil } func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) } @@ -55,7 +63,7 @@ func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, } func SetupSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { return client.snatClient.SetupSnatContainerInterface() } @@ -63,7 +71,7 @@ func SetupSnatContainerInterface(client *OVSEndpointClient) error { } func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { return client.snatClient.ConfigureSnatContainerInterface() } @@ -71,7 +79,7 @@ func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { } func DeleteSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { return client.snatClient.DeleteSnatEndpoint() } @@ -80,7 +88,11 @@ func DeleteSnatEndpoint(client *OVSEndpointClient) error { func DeleteSnatEndpointRules(client *OVSEndpointClient) error { if client.allowInboundFromHostToNC { - return client.snatClient.DeleteInboundFromHostToNC() + client.snatClient.DeleteInboundFromHostToNC() + } + + if client.allowInboundFromNCToHost { + client.snatClient.DeleteInboundFromNCToHost() } return nil diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index d3c04d0342..ae033b8dcf 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -24,6 +24,7 @@ type OVSEndpointClient struct { enableSnatOnHost bool enableInfraVnet bool allowInboundFromHostToNC bool + allowInboundFromNCToHost bool } const ( @@ -49,6 +50,7 @@ func NewOVSEndpointClient( enableSnatOnHost: epInfo.EnableSnatOnHost, enableInfraVnet: epInfo.EnableInfraVnet, allowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + allowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, } NewInfraVnetClient(client, epInfo.Id[:7]) diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 2d47720bf8..60cf7c11d0 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" ) @@ -17,6 +18,8 @@ const ( azureSnatVeth0 = "azSnatveth0" azureSnatVeth1 = "azSnatveth1" azureSnatIfName = "eth1" + cniOutputChain = "AZURECNIOUTPUT" + cniInputChain = "AZURECNIINPUT" SnatBridgeName = "azSnatbr" ImdsIP = "169.254.169.254/32" ) @@ -71,9 +74,18 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) } -func (client *OVSSnatClient) AddPrivateIPBlockRule() error { - if err := epcommon.AddOrDeletePrivateIPBlockRule(SnatBridgeName, client.SkipAddressesFromBlock, "A"); err != nil { - log.Printf("AddPrivateIPBlockRule failed with error %v", err) +func (client *OVSSnatClient) AllowIPAddresses() error { + if err := epcommon.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + log.Printf("AllowIPAddresses failed with error %v", err) + return err + } + + return nil +} + +func (client *OVSSnatClient) BlockIPAddresses() error { + if err := epcommon.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { + log.Printf("AllowIPAddresses failed with error %v", err) return err } @@ -99,13 +111,11 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) containerIP, _, _ := net.ParseCIDR(client.localIP) - // Create cnioutput chain if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return err } - // Forward from Output to cnioutput chain if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) return err @@ -168,6 +178,76 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { return err } +func (client *OVSSnatClient) AllowInboundFromNCToHost() error { + bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + containerIP, _, _ := net.ParseCIDR(client.localIP) + + if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + // Forward from Input to cniinput chain + if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + // Allow Host to NC traffic + matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) + err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) + return err + } + + if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + matchCondition = fmt.Sprintf(" -o %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) + err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) + return err + } + + snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + + log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) + err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) + if err != nil { + log.Printf("AllowInboundFromNCToHost: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) + } + + return err +} + +func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { + bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + containerIP, _, _ := net.ParseCIDR(client.localIP) + + matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) + err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err) + } + + log.Printf("Removing static arp entry for ip %s ", containerIP) + err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) + if err != nil { + log.Printf("DeleteInboundFromNCToHost: Error removing static arp entry for ip %s: %v", containerIP, err) + } + + return err +} + func (client *OVSSnatClient) ConfigureSnatContainerInterface() error { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) @@ -283,17 +363,8 @@ func DeleteSnatBridge(bridgeName string) error { func AddMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("iptable snat rule already exists") - return nil - } - - cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - log.Printf("Adding iptable snat rule %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err + matchCondition := fmt.Sprintf("-s %s", ipNet.String()) + return iptables.InsertIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) } func DeleteMasqueradeRule() error { @@ -311,10 +382,8 @@ func DeleteMasqueradeRule() error { } if ipAddr.To4() != nil { - cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - log.Printf("Deleting iptable snat rule %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err + matchCondition := fmt.Sprintf("-s %s", ipNet.String()) + return iptables.DeleteIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) } } From 963b8f4a9905357aca002cebfb8b37c475f80809 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 9 Jul 2019 13:13:11 -0700 Subject: [PATCH 06/11] added NCToHostCommunication field in cns contract --- cni/network/network_linux.go | 2 +- cns/NetworkContainerContract.go | 2 ++ cns/restserver/restserver.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index d53cbd2fe1..393c265586 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -55,7 +55,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication - epInfo.AllowInboundFromNCToHost = true //cnsNwConfig.AllowNCToHostCommunication + epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication } epInfo.Data[network.OptVethName] = vethName diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 9575a2829c..2a9f71c6d2 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -57,6 +57,7 @@ type CreateNetworkContainerRequest struct { CnetAddressSpace []IPSubnet // To setup SNAT (should include service endpoint vips). Routes []Route AllowHostToNCCommunication bool + AllowNCToHostCommunication bool } // ConfigureContainerNetworkingRequest - specifies request to attach/detach container to network. @@ -137,6 +138,7 @@ type GetNetworkContainerResponse struct { LocalIPConfiguration IPConfiguration Response Response AllowHostToNCCommunication bool + AllowNCToHostCommunication bool } // DeleteNetworkContainerRequest specifies the details about the request to delete a specifc network container. diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 586986a946..e22b67ad66 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1208,6 +1208,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo PrimaryInterfaceIdentifier: savedReq.PrimaryInterfaceIdentifier, LocalIPConfiguration: savedReq.LocalIPConfiguration, AllowHostToNCCommunication: savedReq.AllowHostToNCCommunication, + AllowNCToHostCommunication: savedReq.AllowNCToHostCommunication, } return getNetworkContainerResponse From 2af49dde268b1c7d88a535fa78efe32a789bd85a Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 12 Jul 2019 17:05:38 -0700 Subject: [PATCH 07/11] addressed review comments --- iptables/iptables.go | 2 +- network/ovs_endpoint_snatroute_linux.go | 4 +- network/ovs_networkclient_linux.go | 1 - network/ovssnat/ovssnat.go | 58 +++++++++++++------------ 4 files changed, 33 insertions(+), 32 deletions(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index 70882151b2..0ae83ba2f1 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -32,7 +32,7 @@ const ( // target const ( Accept = "ACCEPT" - Drop = "Drop" + Drop = "DROP" Masquerade = "MASQUERADE" ) diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index aeb26895b7..47449476cd 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -86,7 +86,7 @@ func DeleteSnatEndpoint(client *OVSEndpointClient) error { return nil } -func DeleteSnatEndpointRules(client *OVSEndpointClient) error { +func DeleteSnatEndpointRules(client *OVSEndpointClient) { if client.allowInboundFromHostToNC { client.snatClient.DeleteInboundFromHostToNC() } @@ -94,6 +94,4 @@ func DeleteSnatEndpointRules(client *OVSEndpointClient) error { if client.allowInboundFromNCToHost { client.snatClient.DeleteInboundFromNCToHost() } - - return nil } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index d3bedd3957..127605b6f0 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -108,7 +108,6 @@ func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { func (client *OVSNetworkClient) DeleteL2Rules(extIf *externalInterface) { ovsctl.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) - } func (client *OVSNetworkClient) SetBridgeMasterToHostInterface() error { diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 60cf7c11d0..550f0e0a2a 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -15,13 +15,17 @@ import ( ) const ( - azureSnatVeth0 = "azSnatveth0" - azureSnatVeth1 = "azSnatveth1" - azureSnatIfName = "eth1" - cniOutputChain = "AZURECNIOUTPUT" - cniInputChain = "AZURECNIINPUT" - SnatBridgeName = "azSnatbr" - ImdsIP = "169.254.169.254/32" + azureSnatVeth0 = "azSnatveth0" + azureSnatVeth1 = "azSnatveth1" + azureSnatIfName = "eth1" + cniOutputChain = "AZURECNIOUTPUT" + cniInputChain = "AZURECNIINPUT" + SnatBridgeName = "azSnatbr" + ImdsIP = "169.254.169.254/32" + vlanDropDeleteRule = "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" + vlanDropAddRule = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" + vlanDropMatch = "-p 802_1Q -j DROP" + l2PreroutingEntries = "ebtables -t nat -L PREROUTING" ) type OVSSnatClient struct { @@ -35,11 +39,12 @@ type OVSSnatClient struct { func NewSnatClient(hostIfName string, contIfName string, localIP string, snatBridgeIP string, skipAddressesFromBlock []string) OVSSnatClient { log.Printf("Initialize new snat client") - snatClient := OVSSnatClient{} - snatClient.hostSnatVethName = hostIfName - snatClient.containerSnatVethName = contIfName - snatClient.localIP = localIP - snatClient.snatBridgeIP = snatBridgeIP + snatClient := OVSSnatClient{ + hostSnatVethName: hostIfName, + containerSnatVethName: contIfName, + localIP: localIP, + snatBridgeIP: snatBridgeIP, + } for _, address := range skipAddressesFromBlock { snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, address) @@ -107,9 +112,14 @@ func (client *OVSSnatClient) SetupSnatContainerInterface() error { return nil } -func (client *OVSSnatClient) AllowInboundFromHostToNC() error { +func getNCLocalAndGatewayIP(client *OVSSnatClient) (net.IP, net.IP) { bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) containerIP, _, _ := net.ParseCIDR(client.localIP) + return bridgeIP, containerIP +} + +func (client *OVSSnatClient) AllowInboundFromHostToNC() error { + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) @@ -160,8 +170,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { } func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { - bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) - containerIP, _, _ := net.ParseCIDR(client.localIP) + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) @@ -179,8 +188,7 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { } func (client *OVSSnatClient) AllowInboundFromNCToHost() error { - bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) - containerIP, _, _ := net.ParseCIDR(client.localIP) + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) @@ -230,8 +238,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { } func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { - bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) - containerIP, _, _ := net.ParseCIDR(client.localIP) + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) @@ -338,8 +345,7 @@ func CreateSnatBridge(snatBridgeIP string, mainInterface string) error { } func DeleteSnatBridge(bridgeName string) error { - cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" - _, err := platform.ExecuteCommand(cmd) + _, err := platform.ExecuteCommand(vlanDropDeleteRule) if err != nil { log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) } @@ -391,21 +397,19 @@ func DeleteMasqueradeRule() error { } func AddVlanDropRule() error { - cmd := "ebtables -t nat -L PREROUTING" - out, err := platform.ExecuteCommand(cmd) + out, err := platform.ExecuteCommand(l2PreroutingEntries) if err != nil { log.Printf("Error while listing ebtable rules %v", err) return err } out = strings.TrimSpace(out) - if strings.Contains(out, "-p 802_1Q -j DROP") { + if strings.Contains(out, vlanDropMatch) { log.Printf("vlan drop rule already exists") return nil } - cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" - log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", cmd) - _, err = platform.ExecuteCommand(cmd) + log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", vlanDropAddRule) + _, err = platform.ExecuteCommand(vlanDropAddRule) return err } From a15e9e34003b262fd9475b82fbea59e07f5694e2 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 15 Jul 2019 14:47:36 -0700 Subject: [PATCH 08/11] addressed review comments --- iptables/iptables.go | 49 +++++++++++++++++++++++++------------- network/ovssnat/ovssnat.go | 29 ++++++++++++++++++++-- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index 0ae83ba2f1..dc72254590 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -43,21 +43,37 @@ const ( Delete = "D" ) -func ChainExists(tableName, chainName string) bool { - cmd := fmt.Sprintf("iptables -w 60 -t %s -L %s", tableName, chainName) +const ( + iptables = "iptables" + lockTimeout = 60 +) + +func runCmd(params string) error { + cmd := fmt.Sprintf("%s -w %d %s", iptables, lockTimeout, params) if _, err := platform.ExecuteCommand(cmd); err != nil { + return err + } + + return nil +} + +// check if iptable chain alreay exists +func ChainExists(tableName, chainName string) bool { + params := fmt.Sprintf("-t %s -L %s", tableName, chainName) + if err := runCmd(params); err != nil { return false } return true } +// create new iptable chain under specified table name func CreateCNIChain(tableName, chainName string) error { var err error if !ChainExists(tableName, chainName) { - cmd := fmt.Sprintf("iptables -w 60 -t %s -N %s", tableName, chainName) - _, err = platform.ExecuteCommand(cmd) + params := fmt.Sprintf("-t %s -N %s", tableName, chainName) + err = runCmd(params) } else { log.Printf("%s Chain exists in table %s", chainName, tableName) } @@ -65,40 +81,39 @@ func CreateCNIChain(tableName, chainName string) error { return err } +// check if iptable rule alreay exists func RuleExists(tableName, chainName, match, target string) bool { - cmd := fmt.Sprintf("iptables -w 60 -t %s -C %s %s -j %s", tableName, chainName, match, target) - _, err := platform.ExecuteCommand(cmd) - if err != nil { + params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target) + if err := runCmd(params); err != nil { return false } - return true } +// Insert iptable rule at beginning of iptable chain func InsertIptableRule(tableName, chainName, match, target string) error { if RuleExists(tableName, chainName, match, target) { log.Printf("Rule already exists") return nil } - cmd := fmt.Sprintf("iptables -w 60 -t %s -I %s 1 %s -j %s", tableName, chainName, match, target) - _, err := platform.ExecuteCommand(cmd) - return err + params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target) + return runCmd(params) } +// Append iptable rule at end of iptable chain func AppendIptableRule(tableName, chainName, match, target string) error { if RuleExists(tableName, chainName, match, target) { log.Printf("Rule already exists") return nil } - cmd := fmt.Sprintf("iptables -w 60 -t %s -A %s %s -j %s", tableName, chainName, match, target) - _, err := platform.ExecuteCommand(cmd) - return err + params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target) + return runCmd(params) } +// Delete matched iptable rule func DeleteIptableRule(tableName, chainName, match, target string) error { - cmd := fmt.Sprintf("iptables -w 60 -t %s -D %s %s -j %s", tableName, chainName, match, target) - _, err := platform.ExecuteCommand(cmd) - return err + params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target) + return runCmd(params) } diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 550f0e0a2a..4800178f0f 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -79,6 +79,9 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) } +/** + This fucntion adds iptables rules that allows only specific Private IPs via linux bridge +**/ func (client *OVSSnatClient) AllowIPAddresses() error { if err := epcommon.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) @@ -88,6 +91,9 @@ func (client *OVSSnatClient) AllowIPAddresses() error { return nil } +/** + This fucntion adds iptables rules that blocks all private IPs flowing via linux bridge +**/ func (client *OVSSnatClient) BlockIPAddresses() error { if err := epcommon.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) @@ -118,6 +124,9 @@ func getNCLocalAndGatewayIP(client *OVSSnatClient) (net.IP, net.IP) { return bridgeIP, containerIP } +/** + This function adds iptables rules that allows only host to NC communication and not the other way +**/ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) @@ -131,7 +140,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { return err } - // Allow Host to NC traffic + // Allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { @@ -151,6 +160,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { return err } + // Accept packets from NC only if established connection matchCondition = fmt.Sprintf(" -i %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { @@ -160,6 +170,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + // Add static arp entry for localIP to prevent arp going out of VM log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) if err != nil { @@ -187,6 +198,9 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { return err } +/** + This function adds iptables rules that allows only NC to Host communication and not the other way +**/ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) @@ -201,7 +215,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { return err } - // Allow Host to NC traffic + // Allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { @@ -219,6 +233,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { return err } + // Accept packets from Host only if established connection matchCondition = fmt.Sprintf(" -o %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { @@ -228,6 +243,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + // Add static arp entry for localIP to prevent arp going out of VM log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) if err != nil { @@ -272,6 +288,9 @@ func (client *OVSSnatClient) DeleteSnatEndpoint() error { return nil } +/** + This function creates linux bridge which will be used for outbound connectivity by NCs +**/ func CreateSnatBridge(snatBridgeIP string, mainInterface string) error { _, err := net.InterfaceByName(SnatBridgeName) if err == nil { @@ -367,6 +386,9 @@ func DeleteSnatBridge(bridgeName string) error { return err } +/** + This function adds iptable rules that will snat all traffic that has source ip in apipa range and coming via linux bridge +**/ func AddMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) @@ -396,6 +418,9 @@ func DeleteMasqueradeRule() error { return nil } +/** + Drop all vlan traffic on linux bridge +**/ func AddVlanDropRule() error { out, err := platform.ExecuteCommand(l2PreroutingEntries) if err != nil { From ba62b819c041eb4439bc270e0b6ded1dd6545d2b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 16 Jul 2019 16:40:00 -0700 Subject: [PATCH 09/11] addressed review comments --- iptables/iptables.go | 11 ++++++- network/ovs_endpoint_snatroute_linux.go | 7 ++-- network/ovssnat/ovssnat.go | 43 +++++++++++++++++++------ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index dc72254590..f035a6ff5c 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -1,5 +1,7 @@ package iptables +// This package contains wrapper functions to program iptables rules + import ( "fmt" @@ -43,11 +45,18 @@ const ( Delete = "D" ) +// states +const ( + Established = "ESTABLISHED" + Related = "RELATED" +) + const ( iptables = "iptables" lockTimeout = 60 ) +// Run iptables command func runCmd(params string) error { cmd := fmt.Sprintf("%s -w %d %s", iptables, lockTimeout, params) if _, err := platform.ExecuteCommand(cmd); err != nil { @@ -68,7 +77,7 @@ func ChainExists(tableName, chainName string) bool { } // create new iptable chain under specified table name -func CreateCNIChain(tableName, chainName string) error { +func CreateChain(tableName, chainName string) error { var err error if !ChainExists(tableName, chainName) { diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 47449476cd..b15f82518d 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -28,14 +28,17 @@ func AddSnatEndpoint(client *OVSEndpointClient) error { func AddSnatEndpointRules(client *OVSEndpointClient) error { if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost { - if err := client.snatClient.AllowIPAddresses(); err != nil { + // Allow specific Private IPs via Snat Bridge + if err := client.snatClient.AllowIPAddressesOnSnatBrdige(); err != nil { return err } - if err := client.snatClient.BlockIPAddresses(); err != nil { + // Block Private IPs via Snat Bridge + if err := client.snatClient.BlockIPAddressesOnSnatBrdige(); err != nil { return err } + // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge if err := AddStaticRoute(ovssnat.ImdsIP, client.bridgeName); err != nil { return err } diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 4800178f0f..63a8eaa7c3 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -56,21 +56,25 @@ func NewSnatClient(hostIfName string, contIfName string, localIP string, snatBri } func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { + // Create linux Bridge for outbound connectivity if err := CreateSnatBridge(client.snatBridgeIP, bridgeName); err != nil { log.Printf("creating snat bridge failed with error %v", err) return err } + // SNAT Rule to masquerade packets destined to non-vnet ip if err := AddMasqueradeRule(client.snatBridgeIP); err != nil { log.Printf("Adding snat rule failed with error %v", err) return err } + // Drop all vlan packets coming via linux bridge. if err := AddVlanDropRule(); err != nil { log.Printf("Adding vlan drop rule failed with error %v", err) return err } + // Create veth pair to tie one end to container and other end to linux bridge if err := epcommon.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { log.Printf("Creating Snat Endpoint failed with error %v", err) return err @@ -82,7 +86,7 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { /** This fucntion adds iptables rules that allows only specific Private IPs via linux bridge **/ -func (client *OVSSnatClient) AllowIPAddresses() error { +func (client *OVSSnatClient) AllowIPAddressesOnSnatBrdige() error { if err := epcommon.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return err @@ -94,7 +98,7 @@ func (client *OVSSnatClient) AllowIPAddresses() error { /** This fucntion adds iptables rules that blocks all private IPs flowing via linux bridge **/ -func (client *OVSSnatClient) BlockIPAddresses() error { +func (client *OVSSnatClient) BlockIPAddressesOnSnatBrdige() error { if err := epcommon.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return err @@ -103,11 +107,17 @@ func (client *OVSSnatClient) BlockIPAddresses() error { return nil } +/** + Move container veth inside container network namespace +**/ func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) return netlink.SetLinkNetNs(client.containerSnatVethName, nsID) } +/** + Configure Routes and setup name for container veth +**/ func (client *OVSSnatClient) SetupSnatContainerInterface() error { if err := epcommon.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { return err @@ -130,11 +140,13 @@ func getNCLocalAndGatewayIP(client *OVSSnatClient) (net.IP, net.IP) { func (client *OVSSnatClient) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + // Create CNI Ouptut chain + if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return err } + // Forward traffic from Ouptut chain to CNI Output chain if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) return err @@ -149,7 +161,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { } // Create cniinput chain - if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIInputChain); err != nil { + if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) return err } @@ -161,7 +173,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { } // Accept packets from NC only if established connection - matchCondition = fmt.Sprintf(" -i %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) + matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) @@ -183,12 +195,14 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + // Delete allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) } + // Remove static arp entry added for container local IP log.Printf("Removing static arp entry for ip %s ", containerIP) err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) if err != nil { @@ -204,12 +218,13 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { func (client *OVSSnatClient) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIInputChain); err != nil { + // Create CNI Input chain + if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) return err } - // Forward from Input to cniinput chain + // Forward traffic from Input to cniinput chain if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) return err @@ -223,18 +238,20 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { return err } - if err := iptables.CreateCNIChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + // Create CNI output chain + if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return err } + // Forward traffic from Output to CNI Output chain if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) return err } // Accept packets from Host only if established connection - matchCondition = fmt.Sprintf(" -o %s -m state --state ESTABLISHED,RELATED", SnatBridgeName) + matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) @@ -256,12 +273,14 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + // Delete allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err) } + // Remove static arp entry added for container local IP log.Printf("Removing static arp entry for ip %s ", containerIP) err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) if err != nil { @@ -271,6 +290,10 @@ func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { return err } +/** + Configures Local IP Address for container Veth +**/ + func (client *OVSSnatClient) ConfigureSnatContainerInterface() error { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) @@ -311,6 +334,8 @@ func CreateSnatBridge(snatBridgeIP string, mainInterface string) error { return err } + // Create a veth pair. One end of veth will be attached to ovs bridge and other end + // of veth will be attached to linux bridge _, err = net.InterfaceByName(azureSnatVeth0) if err == nil { log.Printf("Azure snat veth already exists") From 66f0a5bf467bcf81273e853110c7a04af5a34346 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 16 Jul 2019 17:01:45 -0700 Subject: [PATCH 10/11] address comments --- iptables/iptables.go | 1 - network/epcommon/endpoint_common.go | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index f035a6ff5c..b1988e50f3 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/platform" ) diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go index 19fcbaed47..7b75a0283a 100644 --- a/network/epcommon/endpoint_common.go +++ b/network/epcommon/endpoint_common.go @@ -108,11 +108,12 @@ func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, c matchCondition := fmt.Sprintf("-%s %s -d %s", option, bridgeName, ipAddress) - if action == iptables.Insert { + switch action { + case iptables.Insert: err = iptables.InsertIptableRule(iptables.Filter, chainName, matchCondition, target) - } else if action == iptables.Append { + case iptables.Append: err = iptables.AppendIptableRule(iptables.Filter, chainName, matchCondition, target) - } else if action == iptables.Delete { + case iptables.Delete: err = iptables.DeleteIptableRule(iptables.Filter, chainName, matchCondition, target) } From 1b6c562773d9d911af8fe2610a963ec22857a340 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 16 Jul 2019 17:06:49 -0700 Subject: [PATCH 11/11] fixed space damage --- network/endpoint_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 6339457009..453691bffb 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -190,11 +190,11 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Create the endpoint object. ep = &endpoint{ - Id: epInfo.Id, - IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. - HostIfName: hostIfName, - MacAddress: containerIf.HardwareAddr, - InfraVnetIP: epInfo.InfraVnetIP, + Id: epInfo.Id, + IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. + HostIfName: hostIfName, + MacAddress: containerIf.HardwareAddr, + InfraVnetIP: epInfo.InfraVnetIP, LocalIP: localIP, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{nw.extIf.IPv4Gateway},