From 7b04b862307d36b3f5f6f217381c9fa669f7a1b8 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 28 Nov 2023 09:29:05 -0800 Subject: [PATCH 01/13] Move network utils functions with iptables to new file --- network/networkutils/iptables.go | 16 ++++ network/networkutils/iptableutils_linux.go | 93 ++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 network/networkutils/iptables.go create mode 100644 network/networkutils/iptableutils_linux.go diff --git a/network/networkutils/iptables.go b/network/networkutils/iptables.go new file mode 100644 index 0000000000..cf5b1ee57b --- /dev/null +++ b/network/networkutils/iptables.go @@ -0,0 +1,16 @@ +package networkutils + +import "github.com/Azure/azure-container-networking/iptables" + +type IPTablesClientInterface interface { + RunCmd(version, params string) error + ChainExists(version, tableName, chainName string) bool + GetCreateChainCmd(version, tableName, chainName string) iptables.IPTableEntry + CreateChain(version, tableName, chainName string) error + RuleExists(version, tableName, chainName, match, target string) bool + GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) iptables.IPTableEntry + InsertIptableRule(version, tableName, chainName, match, target string) error + GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) iptables.IPTableEntry + AppendIptableRule(version, tableName, chainName, match, target string) error + DeleteIptableRule(version, tableName, chainName, match, target string) error +} diff --git a/network/networkutils/iptableutils_linux.go b/network/networkutils/iptableutils_linux.go new file mode 100644 index 0000000000..cc4022f9cc --- /dev/null +++ b/network/networkutils/iptableutils_linux.go @@ -0,0 +1,93 @@ +//go:build linux +// +build linux + +package networkutils + +import ( + "fmt" + "net" + + "github.com/Azure/azure-container-networking/iptables" + "go.uber.org/zap" +) + +func addOrDeleteFilterRule(iptablesClient IPTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { + var err error + option := "i" + + if chainName == iptables.Output { + option = "o" + } + + matchCondition := fmt.Sprintf("-%s %s -d %s", option, bridgeName, ipAddress) + + switch action { + case iptables.Insert: + err = iptablesClient.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + case iptables.Append: + err = iptablesClient.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + case iptables.Delete: + err = iptablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + } + + return err +} + +func AllowIPAddresses(iptablesClient IPTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { + chains := getFilterChains() + target := getFilterchainTarget() + + logger.Info("Addresses to allow", zap.Any("skipAddresses", skipAddresses)) + + for _, address := range skipAddresses { + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[0], target[0]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[1], target[0]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[2], target[0]); err != nil { + return err + } + + } + + return nil +} + +func BlockIPAddresses(iptablesClient IPTablesClientInterface, bridgeName, action string) error { + privateIPAddresses := getPrivateIPSpace() + chains := getFilterChains() + target := getFilterchainTarget() + + logger.Info("Addresses to block", zap.Any("privateIPAddresses", privateIPAddresses)) + + for _, ipAddress := range privateIPAddresses { + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[0], target[1]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[1], target[1]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[2], target[1]); err != nil { + return err + } + } + + return nil +} + +// This fucntion adds rule which snat to ip passed filtered by match string. +func AddSnatRule(iptablesClient IPTablesClientInterface, match string, ip net.IP) error { + version := iptables.V4 + if ip.To4() == nil { + version = iptables.V6 + } + + target := fmt.Sprintf("SNAT --to %s", ip.String()) + return iptablesClient.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) +} From 263ea03464a5820ee77833bb7647336a3f911bf0 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 30 Nov 2023 13:36:52 -0800 Subject: [PATCH 02/13] Add receiver to iptables and create interface --- cni/network/invoker_cns.go | 21 +++-- cni/network/network.go | 2 +- cnm/network/network.go | 3 +- cnms/service/networkmonitor.go | 3 +- iptables/iptables.go | 50 +++++----- network/endpoint.go | 8 +- network/endpoint_linux.go | 12 ++- network/endpoint_snatroute_linux.go | 2 +- network/endpoint_test.go | 23 ++--- network/manager.go | 13 ++- network/network_linux.go | 13 +-- network/networkutils/iptableutils_linux.go | 93 ------------------- network/networkutils/networkutils_linux.go | 32 +++---- network/ovs_endpoint_snatroute_linux.go | 1 + network/ovs_endpointclient_linux.go | 3 + network/snat/snat_linux.go | 39 ++++---- network/snat/snat_linux_test.go | 5 + ...ansparent_vlan_endpoint_snatroute_linux.go | 1 + .../transparent_vlan_endpointclient_linux.go | 7 +- 19 files changed, 138 insertions(+), 193 deletions(-) delete mode 100644 network/networkutils/iptableutils_linux.go diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 7bf72779d4..323f818bac 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -227,25 +227,26 @@ func setHostOptions(ncSubnetPrefix *net.IPNet, options map[string]interface{}, i // we need to snat IMDS traffic to node IP, this sets up snat '--to' snatHostIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.hostPrimaryIP) + iptablesClient := iptables.NewIPTablesClient() var iptableCmds []iptables.IPTableEntry - if !iptables.ChainExists(iptables.V4, iptables.Nat, iptables.Swift) { - iptableCmds = append(iptableCmds, iptables.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift)) + if !iptablesClient.ChainExists(iptables.V4, iptables.Nat, iptables.Swift) { + iptableCmds = append(iptableCmds, iptablesClient.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift)) } - if !iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift) { - iptableCmds = append(iptableCmds, iptables.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift)) + if !iptablesClient.RuleExists(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift) { + iptableCmds = append(iptableCmds, iptablesClient.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift)) } - if !iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump) { - iptableCmds = append(iptableCmds, iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump)) + if !iptablesClient.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump) { + iptableCmds = append(iptableCmds, iptablesClient.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump)) } - if !iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump) { - iptableCmds = append(iptableCmds, iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump)) + if !iptablesClient.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump) { + iptableCmds = append(iptableCmds, iptablesClient.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump)) } - if !iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump) { - iptableCmds = append(iptableCmds, iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump)) + if !iptablesClient.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump) { + iptableCmds = append(iptableCmds, iptablesClient.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump)) } options[network.IPTablesKey] = iptableCmds diff --git a/cni/network/network.go b/cni/network/network.go index ba603a2268..4e62c7be41 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -117,7 +117,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}, network.NewNamespaceClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index faf6dd775f..75dfe240ab 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cnm" cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" @@ -53,7 +54,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index 545442c8f2..8b4d4915ec 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -10,6 +10,7 @@ import ( cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" acn "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" @@ -157,7 +158,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/iptables/iptables.go b/iptables/iptables.go index 2e80e30227..27db9a6d29 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -87,8 +87,14 @@ type IPTableEntry struct { Params string } +type IPTablesClient struct{} + +func NewIPTablesClient() *IPTablesClient { + return &IPTablesClient{} +} + // Run iptables command -func RunCmd(version, params string) error { +func (c *IPTablesClient) RunCmd(version, params string) error { var cmd string p := platform.NewExecClient(logger) @@ -111,16 +117,16 @@ func RunCmd(version, params string) error { } // check if iptable chain alreay exists -func ChainExists(version, tableName, chainName string) bool { +func (c *IPTablesClient) ChainExists(version, tableName, chainName string) bool { params := fmt.Sprintf("-t %s -L %s", tableName, chainName) - if err := RunCmd(version, params); err != nil { + if err := c.RunCmd(version, params); err != nil { return false } return true } -func GetCreateChainCmd(version, tableName, chainName string) IPTableEntry { +func (c *IPTablesClient) GetCreateChainCmd(version, tableName, chainName string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -N %s", tableName, chainName), @@ -128,12 +134,12 @@ func GetCreateChainCmd(version, tableName, chainName string) IPTableEntry { } // create new iptable chain under specified table name -func CreateChain(version, tableName, chainName string) error { +func (c *IPTablesClient) CreateChain(version, tableName, chainName string) error { var err error - if !ChainExists(version, tableName, chainName) { - cmd := GetCreateChainCmd(version, tableName, chainName) - err = RunCmd(version, cmd.Params) + if !c.ChainExists(version, tableName, chainName) { + cmd := c.GetCreateChainCmd(version, tableName, chainName) + err = c.RunCmd(version, cmd.Params) } else { logger.Info("Chain exists in table", zap.String("chainName", chainName), zap.String("tableName", tableName)) } @@ -142,15 +148,15 @@ func CreateChain(version, tableName, chainName string) error { } // check if iptable rule alreay exists -func RuleExists(version, tableName, chainName, match, target string) bool { +func (c *IPTablesClient) RuleExists(version, tableName, chainName, match, target string) bool { params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target) - if err := RunCmd(version, params); err != nil { + if err := c.RunCmd(version, params); err != nil { return false } return true } -func GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { +func (c *IPTablesClient) GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target), @@ -158,17 +164,17 @@ func GetInsertIptableRuleCmd(version, tableName, chainName, match, target string } // Insert iptable rule at beginning of iptable chain -func InsertIptableRule(version, tableName, chainName, match, target string) error { - if RuleExists(version, tableName, chainName, match, target) { +func (c *IPTablesClient) InsertIptableRule(version, tableName, chainName, match, target string) error { + if c.RuleExists(version, tableName, chainName, match, target) { logger.Info("Rule already exists") return nil } - cmd := GetInsertIptableRuleCmd(version, tableName, chainName, match, target) - return RunCmd(version, cmd.Params) + cmd := c.GetInsertIptableRuleCmd(version, tableName, chainName, match, target) + return c.RunCmd(version, cmd.Params) } -func GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { +func (c *IPTablesClient) GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target), @@ -176,18 +182,18 @@ func GetAppendIptableRuleCmd(version, tableName, chainName, match, target string } // Append iptable rule at end of iptable chain -func AppendIptableRule(version, tableName, chainName, match, target string) error { - if RuleExists(version, tableName, chainName, match, target) { +func (c *IPTablesClient) AppendIptableRule(version, tableName, chainName, match, target string) error { + if c.RuleExists(version, tableName, chainName, match, target) { logger.Info("Rule already exists") return nil } - cmd := GetAppendIptableRuleCmd(version, tableName, chainName, match, target) - return RunCmd(version, cmd.Params) + cmd := c.GetAppendIptableRuleCmd(version, tableName, chainName, match, target) + return c.RunCmd(version, cmd.Params) } // Delete matched iptable rule -func DeleteIptableRule(version, tableName, chainName, match, target string) error { +func (c *IPTablesClient) DeleteIptableRule(version, tableName, chainName, match, target string) error { params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target) - return RunCmd(version, params) + return c.RunCmd(version, params) } diff --git a/network/endpoint.go b/network/endpoint.go index d8d20553c3..b5e7ec83de 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "go.uber.org/zap" @@ -140,6 +141,7 @@ func (nw *network) newEndpoint( plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ep *endpoint @@ -153,7 +155,7 @@ func (nw *network) newEndpoint( // Call the platform implementation. // Pass nil for epClient and will be initialized in newendpointImpl - ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, epInfo) + ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, iptc, epInfo) if err != nil { return nil, err } @@ -164,7 +166,7 @@ func (nw *network) newEndpoint( } // DeleteEndpoint deletes an existing endpoint from the network. -func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, endpointID string) error { +func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface, endpointID string) error { var err error logger.Info("Deleting endpoint from network", zap.String("endpointID", endpointID), zap.String("id", nw.Id)) @@ -183,7 +185,7 @@ func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.Exec // Call the platform implementation. // Pass nil for epClient and will be initialized in deleteEndpointImpl - err = nw.deleteEndpointImpl(nl, plc, nil, nioc, nsc, ep) + err = nw.deleteEndpointImpl(nl, plc, nil, nioc, nsc, iptc, ep) if err != nil { return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 60cdf61744..59c06fff51 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -56,6 +56,7 @@ func (nw *network) newEndpointImpl( netioCli netio.NetIOInterface, testEpClient EndpointClient, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ( @@ -134,7 +135,7 @@ func (nw *network) newEndpointImpl( if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } - epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc, nsc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc, nsc, iptc) } else { logger.Info("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -150,7 +151,8 @@ func (nw *network) newEndpointImpl( localIP, nl, ovsctl.NewOvsctl(), - plc) + plc, + iptc) } } else if nw.Mode != opModeTransparent { logger.Info("Bridge client") @@ -255,7 +257,7 @@ func (nw *network) newEndpointImpl( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface, ep *endpoint) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. @@ -267,10 +269,10 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epInfo := ep.getInfo() if nw.Mode == opModeTransparentVlan { logger.Info("Transparent vlan client") - epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc, nsc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc, nsc, iptc) } else { - epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc, iptc) } } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 3f45a7f16e..772ddf0040 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -37,7 +37,7 @@ func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl n return errors.Wrap(err, "failed to block ip addresses on snat bridge") } nuc := networkutils.NewNetworkUtils(nl, plc) - if err := nuc.EnableIPForwarding(); err != nil { + if err := nuc.EnableIPForwarding(snatClient.IPTablesClient); err != nil { return errors.Wrap(err, "failed to enable ip forwarding") } diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 669b16804a..da53d956f8 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/platform" @@ -183,7 +184,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -195,7 +196,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -211,7 +212,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -221,17 +222,17 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) // Deleting the endpoint //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewIPTablesClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) // Deleting same endpoint with same id should not fail //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewIPTablesClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) }) }) @@ -252,11 +253,11 @@ var _ = Describe("Test Endpoint", func() { } return nil - }), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + }), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -282,14 +283,14 @@ var _ = Describe("Test Endpoint", func() { It("Should not endpoint to the network when there is an error", func() { secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) Expect(ep).To(BeNil()) secondaryEpInfo.MacAddress = netio.HwAddr ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) @@ -297,7 +298,7 @@ var _ = Describe("Test Endpoint", func() { It("Should add endpoint when there are no errors", func() { secondaryEpInfo.MacAddress = netio.HwAddr ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) diff --git a/network/manager.go b/network/manager.go index a36424ee05..302f9b0846 100644 --- a/network/manager.go +++ b/network/manager.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" @@ -81,6 +82,7 @@ type networkManager struct { netio netio.NetIOInterface plClient platform.ExecClient nsClient NamespaceClientInterface + iptablesClient networkutils.IPTablesClientInterface sync.Mutex } @@ -113,13 +115,14 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, plClient: plc, netio: netioCli, nsClient: nsc, + iptablesClient: iptc, } return nm, nil @@ -386,7 +389,9 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn epInfo[0].Data[VlanIDKey] = nw.VlanId } } - ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, epInfo) + + ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo) + if err != nil { return err } @@ -429,7 +434,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E return err } - err = nw.deleteEndpoint(nm.netlink, nm.plClient, nm.netio, nm.nsClient, endpointID) + err = nw.deleteEndpoint(nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, endpointID) if err != nil { return err } @@ -466,7 +471,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint NetworkContainerID: epInfo.Id, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) - return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, ep) + return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) } // GetEndpointInfo returns information about the given endpoint. diff --git a/network/network_linux.go b/network/network_linux.go index 503bbf1a09..a6aa65669c 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -611,7 +611,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI // unmark packet if set by kube-proxy to skip kube-postrouting rule and processed // by cni snat rule - if err = iptables.InsertIptableRule(iptables.V6, iptables.Mangle, iptables.Postrouting, "", "MARK --set-mark 0x0"); err != nil { + if err = nm.iptablesClient.InsertIptableRule(iptables.V6, iptables.Mangle, iptables.Postrouting, "", "MARK --set-mark 0x0"); err != nil { logger.Error("Adding Iptable mangle rule failed", zap.Error(err)) return err } @@ -651,10 +651,10 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface, logger.Info("Disconnected interface", zap.String("Name", extIf.Name)) } -func (*networkManager) addToIptables(cmds []iptables.IPTableEntry) error { +func (nm *networkManager) addToIptables(cmds []iptables.IPTableEntry) error { logger.Info("Adding additional iptable rules...") for _, cmd := range cmds { - err := iptables.RunCmd(cmd.Version, cmd.Params) + err := nm.iptablesClient.RunCmd(cmd.Version, cmd.Params) if err != nil { return err } @@ -684,7 +684,7 @@ func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { } // snat ipv6 traffic to secondary ipv6 ip before leaving VM -func (*networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { +func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { var ( ipv6SnatRuleSet bool ipv6SubnetPrefix net.IPNet @@ -705,8 +705,9 @@ func (*networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *Network if ipAddr.IP.To4() == nil { logger.Info("Adding ipv6 snat rule") matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) - if err := networkutils.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { - return fmt.Errorf("Adding iptable snat rule failed:%w", err) + nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + if err := nu.AddSnatRule(nm.iptablesClient, matchSrcPrefix, ipAddr.IP); err != nil { + return fmt.Errorf("adding iptable snat rule failed:%w", err) } ipv6SnatRuleSet = true } diff --git a/network/networkutils/iptableutils_linux.go b/network/networkutils/iptableutils_linux.go deleted file mode 100644 index cc4022f9cc..0000000000 --- a/network/networkutils/iptableutils_linux.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build linux -// +build linux - -package networkutils - -import ( - "fmt" - "net" - - "github.com/Azure/azure-container-networking/iptables" - "go.uber.org/zap" -) - -func addOrDeleteFilterRule(iptablesClient IPTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { - var err error - option := "i" - - if chainName == iptables.Output { - option = "o" - } - - matchCondition := fmt.Sprintf("-%s %s -d %s", option, bridgeName, ipAddress) - - switch action { - case iptables.Insert: - err = iptablesClient.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) - case iptables.Append: - err = iptablesClient.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) - case iptables.Delete: - err = iptablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) - } - - return err -} - -func AllowIPAddresses(iptablesClient IPTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { - chains := getFilterChains() - target := getFilterchainTarget() - - logger.Info("Addresses to allow", zap.Any("skipAddresses", skipAddresses)) - - for _, address := range skipAddresses { - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[0], target[0]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[1], target[0]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[2], target[0]); err != nil { - return err - } - - } - - return nil -} - -func BlockIPAddresses(iptablesClient IPTablesClientInterface, bridgeName, action string) error { - privateIPAddresses := getPrivateIPSpace() - chains := getFilterChains() - target := getFilterchainTarget() - - logger.Info("Addresses to block", zap.Any("privateIPAddresses", privateIPAddresses)) - - for _, ipAddress := range privateIPAddresses { - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[0], target[1]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[1], target[1]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[2], target[1]); err != nil { - return err - } - } - - return nil -} - -// This fucntion adds rule which snat to ip passed filtered by match string. -func AddSnatRule(iptablesClient IPTablesClientInterface, match string, ip net.IP) error { - version := iptables.V4 - if ip.To4() == nil { - version = iptables.V6 - } - - target := fmt.Sprintf("SNAT --to %s", ip.String()) - return iptablesClient.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) -} diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index ad1c1a83f2..30e3f410a9 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -130,7 +130,7 @@ func (nu NetworkUtils) AssignIPToInterface(interfaceName string, ipAddresses []n return nil } -func addOrDeleteFilterRule(bridgeName, action, ipAddress, chainName, target string) error { +func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient IPTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { var err error option := "i" @@ -142,32 +142,32 @@ func addOrDeleteFilterRule(bridgeName, action, ipAddress, chainName, target stri switch action { case iptables.Insert: - err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + err = iptablesClient.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) case iptables.Append: - err = iptables.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + err = iptablesClient.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) case iptables.Delete: - err = iptables.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) + err = iptablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) } return err } -func AllowIPAddresses(bridgeName string, skipAddresses []string, action string) error { +func (nu NetworkUtils) AllowIPAddresses(iptablesClient IPTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { chains := getFilterChains() target := getFilterchainTarget() logger.Info("Addresses to allow", zap.Any("skipAddresses", skipAddresses)) for _, address := range skipAddresses { - if err := addOrDeleteFilterRule(bridgeName, action, address, chains[0], target[0]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[0], target[0]); err != nil { return err } - if err := addOrDeleteFilterRule(bridgeName, action, address, chains[1], target[0]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[1], target[0]); err != nil { return err } - if err := addOrDeleteFilterRule(bridgeName, action, address, chains[2], target[0]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, address, chains[2], target[0]); err != nil { return err } @@ -182,7 +182,7 @@ func BlockEgressTrafficFromContainer(version, ipAddress, protocol string, port i return errors.Wrap(iptables.InsertIptableRule(version, iptables.Filter, iptables.Forward, dropTraffic, iptables.Drop), "iptables block traffic failed") } -func BlockIPAddresses(bridgeName, action string) error { +func (nu NetworkUtils) BlockIPAddresses(iptablesClient IPTablesClientInterface, bridgeName, action string) error { privateIPAddresses := getPrivateIPSpace() chains := getFilterChains() target := getFilterchainTarget() @@ -190,15 +190,15 @@ func BlockIPAddresses(bridgeName, action string) error { logger.Info("Addresses to block", zap.Any("privateIPAddresses", privateIPAddresses)) for _, ipAddress := range privateIPAddresses { - if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[0], target[1]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[0], target[1]); err != nil { return err } - if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[1], target[1]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[1], target[1]); err != nil { return err } - if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[2], target[1]); err != nil { + if err := nu.addOrDeleteFilterRule(iptablesClient, bridgeName, action, ipAddress, chains[2], target[1]); err != nil { return err } } @@ -207,7 +207,7 @@ func BlockIPAddresses(bridgeName, action string) error { } // This function enables ip forwarding in VM and allow forwarding packets from the interface -func (nu NetworkUtils) EnableIPForwarding() error { +func (nu NetworkUtils) EnableIPForwarding(iptablesClient IPTablesClientInterface) error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) @@ -218,7 +218,7 @@ func (nu NetworkUtils) EnableIPForwarding() error { } // Append a rule in forward chain to allow forwarding from bridge - if err := iptables.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { + if err := iptablesClient.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { logger.Error("Appending forward chain rule: allow traffic coming from snatbridge failed with", zap.Error(err)) return err @@ -261,14 +261,14 @@ func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { } // This fucntion adds rule which snat to ip passed filtered by match string. -func AddSnatRule(match string, ip net.IP) error { +func (nu NetworkUtils) AddSnatRule(iptablesClient IPTablesClientInterface, match string, ip net.IP) error { version := iptables.V4 if ip.To4() == nil { version = iptables.V6 } target := fmt.Sprintf("SNAT --to %s", ip.String()) - return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) + return iptablesClient.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) } func (nu NetworkUtils) DisableRAForInterface(ifName string) error { diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 7df00ffb21..3e10db761c 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -32,6 +32,7 @@ func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epI false, client.netlink, client.plClient, + client.iptablesClient, ) } } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 428e533ab6..5aedddbf1b 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -35,6 +35,7 @@ type OVSEndpointClient struct { netioshim netio.NetIOInterface ovsctlClient ovsctl.OvsInterface plClient platform.ExecClient + iptablesClient networkutils.IPTablesClientInterface } const ( @@ -52,6 +53,7 @@ func NewOVSEndpointClient( nl netlink.NetlinkInterface, ovs ovsctl.OvsInterface, plc platform.ExecClient, + iptc networkutils.IPTablesClientInterface, ) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, @@ -68,6 +70,7 @@ func NewOVSEndpointClient( netlink: nl, ovsctlClient: ovs, plClient: plc, + iptablesClient: iptc, netioshim: &netio.NetIO{}, } diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index e1968243e5..e77bbf09e9 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -46,6 +46,7 @@ type Client struct { enableProxyArpOnBridge bool netlink netlink.NetlinkInterface plClient platform.ExecClient + IPTablesClient networkutils.IPTablesClientInterface } func NewSnatClient(hostIfName string, @@ -57,6 +58,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge bool, nl netlink.NetlinkInterface, plClient platform.ExecClient, + IPTablesClient networkutils.IPTablesClientInterface, ) Client { logger.Info("Initialize new snat client") snatClient := Client{ @@ -68,6 +70,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge: enableProxyArpOnBridge, netlink: nl, plClient: plClient, + IPTablesClient: IPTablesClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -120,7 +123,8 @@ func (client *Client) CreateSnatEndpoint() error { // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge func (client *Client) AllowIPAddressesOnSnatBridge() error { - if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + nu := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nu.AllowIPAddresses(client.IPTablesClient, SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { logger.Error("AllowIPAddresses failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) } @@ -130,7 +134,8 @@ func (client *Client) AllowIPAddressesOnSnatBridge() error { // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge func (client *Client) BlockIPAddressesOnSnatBridge() error { - if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { + nu := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nu.BlockIPAddresses(client.IPTablesClient, SnatBridgeName, iptables.Append); err != nil { logger.Error("AllowIPAddresses failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) } @@ -172,40 +177,40 @@ func (client *Client) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Output chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Forward traffic from Ouptut chain to CNI Output chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting output rule failed with ", zap.Error(err)) return newErrorSnatClient(err.Error()) } // Create cniinput chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Forward from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Accept packets from NC only if established connection matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting input rule failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) @@ -237,7 +242,7 @@ func (client *Client) DeleteInboundFromHostToNC() error { // Delete allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := client.IPTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("DeleteInboundFromHostToNC: Error removing output rule", zap.Error(err)) } @@ -264,14 +269,14 @@ func (client *Client) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Input chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err } // Forward traffic from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err @@ -279,21 +284,21 @@ func (client *Client) AllowInboundFromNCToHost() error { // Allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting output rule failed with", zap.Error(err)) return err } // Create CNI output chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err } // Forward traffic from Output to CNI Output chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err @@ -301,7 +306,7 @@ func (client *Client) AllowInboundFromNCToHost() error { // Accept packets from Host only if established connection matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting input rule failed with", zap.Error(err)) return err @@ -331,7 +336,7 @@ func (client *Client) DeleteInboundFromNCToHost() error { // Delete allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := client.IPTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("DeleteInboundFromNCToHost: Error removing output rule", zap.Error(err)) } @@ -453,7 +458,7 @@ func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) erro func (client *Client) addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) + return client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) } // Drop all vlan traffic on linux bridge diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index 1838b6aab1..9f3ff8bca9 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/netlink" ) @@ -19,11 +20,13 @@ func TestMain(m *testing.M) { func TestAllowInboundFromHostToNC(t *testing.T) { nl := netlink.NewNetlink() + iptc := iptables.NewIPTablesClient() client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, + IPTablesClient: iptc, } if err := nl.AddLink(&netlink.DummyLink{ @@ -66,11 +69,13 @@ func TestAllowInboundFromHostToNC(t *testing.T) { func TestAllowInboundFromNCToHost(t *testing.T) { nl := netlink.NewNetlink() + iptc := iptables.NewIPTablesClient() client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, + IPTablesClient: iptc, } if err := nl.AddLink(&netlink.DummyLink{ diff --git a/network/transparent_vlan_endpoint_snatroute_linux.go b/network/transparent_vlan_endpoint_snatroute_linux.go index 609ec137df..b6f712758a 100644 --- a/network/transparent_vlan_endpoint_snatroute_linux.go +++ b/network/transparent_vlan_endpoint_snatroute_linux.go @@ -20,6 +20,7 @@ func (client *TransparentVlanEndpointClient) NewSnatClient(snatBridgeIP, localIP true, client.netlink, client.plClient, + client.iptablesClient, ) } } diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 2ec756ad0d..5f0c3d2220 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -66,6 +66,7 @@ type TransparentVlanEndpointClient struct { plClient platform.ExecClient netUtilsClient networkutils.NetworkUtils nsClient NamespaceClientInterface + iptablesClient networkutils.IPTablesClientInterface } func NewTransparentVlanEndpointClient( @@ -78,6 +79,7 @@ func NewTransparentVlanEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, ) *TransparentVlanEndpointClient { vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) @@ -100,6 +102,7 @@ func NewTransparentVlanEndpointClient( plClient: plc, netUtilsClient: networkutils.NewNetworkUtils(nl, plc), nsClient: nsc, + iptablesClient: iptc, } client.NewSnatClient(nw.SnatBridgeIP, localIP, ep) @@ -396,12 +399,12 @@ func (client *TransparentVlanEndpointClient) AddEndpointRules(epInfo *EndpointIn func (client *TransparentVlanEndpointClient) AddVnetRules(epInfo *EndpointInfo) error { // iptables -t mangle -I PREROUTING -j MARK --set-mark markOption := fmt.Sprintf("MARK --set-mark %d", tunnelingMark) - if err := iptables.InsertIptableRule(iptables.V4, "mangle", "PREROUTING", "", markOption); err != nil { + if err := client.iptablesClient.InsertIptableRule(iptables.V4, "mangle", "PREROUTING", "", markOption); err != nil { return errors.Wrap(err, "unable to insert iptables rule mark all packets not entering on vlan interface") } // iptables -t mangle -I PREROUTING -j ACCEPT -i match := fmt.Sprintf("-i %s", client.vlanIfName) - if err := iptables.InsertIptableRule(iptables.V4, "mangle", "PREROUTING", match, "ACCEPT"); err != nil { + if err := client.iptablesClient.InsertIptableRule(iptables.V4, "mangle", "PREROUTING", match, "ACCEPT"); err != nil { return errors.Wrap(err, "unable to insert iptables rule accept all incoming from vlan interface") } // Blocks wireserver traffic from customer vnet nic From 8c57dbaf65c46b5aa06af03bf853774375c1a4c5 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 30 Nov 2023 15:45:38 -0800 Subject: [PATCH 03/13] Resolve conflicts from rebasing --- network/network_linux.go | 2 +- network/networkutils/networkutils_linux.go | 4 ++-- network/transparent_vlan_endpointclient_linux.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/network/network_linux.go b/network/network_linux.go index a6aa65669c..39721430cc 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -103,7 +103,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt } logger.Info("Disabled ipv6") // Blocks wireserver traffic from apipa nic - if err := networkutils.BlockEgressTrafficFromContainer(iptables.V4, networkutils.AzureDNS, iptables.TCP, iptables.HTTPPort); err != nil { + if err := nu.BlockEgressTrafficFromContainer(nm.iptablesClient, iptables.V4, networkutils.AzureDNS, iptables.TCP, iptables.HTTPPort); err != nil { return nil, errors.Wrap(err, "unable to insert vm iptables rule drop wireserver packets") } logger.Info("Block wireserver traffic rule added") diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index 30e3f410a9..cbc30594e6 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -176,10 +176,10 @@ func (nu NetworkUtils) AllowIPAddresses(iptablesClient IPTablesClientInterface, return nil } -func BlockEgressTrafficFromContainer(version, ipAddress, protocol string, port int) error { +func (nu NetworkUtils) BlockEgressTrafficFromContainer(iptablesClient IPTablesClientInterface, version, ipAddress, protocol string, port int) error { // iptables -t filter -I FORWARD -j DROP -d -p -m --dport dropTraffic := fmt.Sprintf("-d %s -p %s -m %s --dport %d", ipAddress, protocol, protocol, port) - return errors.Wrap(iptables.InsertIptableRule(version, iptables.Filter, iptables.Forward, dropTraffic, iptables.Drop), "iptables block traffic failed") + return errors.Wrap(iptablesClient.InsertIptableRule(version, iptables.Filter, iptables.Forward, dropTraffic, iptables.Drop), "iptables block traffic failed") } func (nu NetworkUtils) BlockIPAddresses(iptablesClient IPTablesClientInterface, bridgeName, action string) error { diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 5f0c3d2220..2a3030f36b 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -408,7 +408,7 @@ func (client *TransparentVlanEndpointClient) AddVnetRules(epInfo *EndpointInfo) return errors.Wrap(err, "unable to insert iptables rule accept all incoming from vlan interface") } // Blocks wireserver traffic from customer vnet nic - if err := networkutils.BlockEgressTrafficFromContainer(iptables.V4, networkutils.AzureDNS, iptables.TCP, iptables.HTTPPort); err != nil { + if err := client.netUtilsClient.BlockEgressTrafficFromContainer(client.iptablesClient, iptables.V4, networkutils.AzureDNS, iptables.TCP, iptables.HTTPPort); err != nil { return errors.Wrap(err, "unable to insert iptables rule to drop wireserver packets") } From d303cc705def67bb1c93c3c3d46e9d3cdcd01ecc Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 1 Dec 2023 11:56:18 -0800 Subject: [PATCH 04/13] Add changes for building on windows --- network/endpoint_windows.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 4c17db94fb..e17332d2ea 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" @@ -71,6 +72,7 @@ func (nw *network) newEndpointImpl( _ netio.NetIOInterface, _ EndpointClient, _ NamespaceClientInterface, + _ networkutils.IPTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future @@ -409,7 +411,8 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ netio.NetIOInterface, _ NamespaceClientInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ netio.NetIOInterface, _ NamespaceClientInterface, + _ networkutils.IPTablesClientInterface, ep *endpoint) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { return err From 80dc635e25512a01d719dc2b0015f645c8375524 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Fri, 1 Dec 2023 11:58:16 -0800 Subject: [PATCH 05/13] Address linter issues --- iptables/iptables.go | 26 +++++++++++----------- network/endpoint.go | 4 +++- network/endpoint_linux.go | 4 +++- network/manager.go | 4 +++- network/networkutils/networkutils_linux.go | 4 ++-- network/snat/snat_linux.go | 15 +++++++------ 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index 27db9a6d29..22458d2cdc 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -87,14 +87,14 @@ type IPTableEntry struct { Params string } -type IPTablesClient struct{} +type Client struct{} -func NewIPTablesClient() *IPTablesClient { - return &IPTablesClient{} +func NewIPTablesClient() *Client { + return &Client{} } // Run iptables command -func (c *IPTablesClient) RunCmd(version, params string) error { +func (c *Client) RunCmd(version, params string) error { var cmd string p := platform.NewExecClient(logger) @@ -117,7 +117,7 @@ func (c *IPTablesClient) RunCmd(version, params string) error { } // check if iptable chain alreay exists -func (c *IPTablesClient) ChainExists(version, tableName, chainName string) bool { +func (c *Client) ChainExists(version, tableName, chainName string) bool { params := fmt.Sprintf("-t %s -L %s", tableName, chainName) if err := c.RunCmd(version, params); err != nil { return false @@ -126,7 +126,7 @@ func (c *IPTablesClient) ChainExists(version, tableName, chainName string) bool return true } -func (c *IPTablesClient) GetCreateChainCmd(version, tableName, chainName string) IPTableEntry { +func (c *Client) GetCreateChainCmd(version, tableName, chainName string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -N %s", tableName, chainName), @@ -134,7 +134,7 @@ func (c *IPTablesClient) GetCreateChainCmd(version, tableName, chainName string) } // create new iptable chain under specified table name -func (c *IPTablesClient) CreateChain(version, tableName, chainName string) error { +func (c *Client) CreateChain(version, tableName, chainName string) error { var err error if !c.ChainExists(version, tableName, chainName) { @@ -148,7 +148,7 @@ func (c *IPTablesClient) CreateChain(version, tableName, chainName string) error } // check if iptable rule alreay exists -func (c *IPTablesClient) RuleExists(version, tableName, chainName, match, target string) bool { +func (c *Client) RuleExists(version, tableName, chainName, match, target string) bool { params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target) if err := c.RunCmd(version, params); err != nil { return false @@ -156,7 +156,7 @@ func (c *IPTablesClient) RuleExists(version, tableName, chainName, match, target return true } -func (c *IPTablesClient) GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { +func (c *Client) GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target), @@ -164,7 +164,7 @@ func (c *IPTablesClient) GetInsertIptableRuleCmd(version, tableName, chainName, } // Insert iptable rule at beginning of iptable chain -func (c *IPTablesClient) InsertIptableRule(version, tableName, chainName, match, target string) error { +func (c *Client) InsertIptableRule(version, tableName, chainName, match, target string) error { if c.RuleExists(version, tableName, chainName, match, target) { logger.Info("Rule already exists") return nil @@ -174,7 +174,7 @@ func (c *IPTablesClient) InsertIptableRule(version, tableName, chainName, match, return c.RunCmd(version, cmd.Params) } -func (c *IPTablesClient) GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { +func (c *Client) GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry { return IPTableEntry{ Version: version, Params: fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target), @@ -182,7 +182,7 @@ func (c *IPTablesClient) GetAppendIptableRuleCmd(version, tableName, chainName, } // Append iptable rule at end of iptable chain -func (c *IPTablesClient) AppendIptableRule(version, tableName, chainName, match, target string) error { +func (c *Client) AppendIptableRule(version, tableName, chainName, match, target string) error { if c.RuleExists(version, tableName, chainName, match, target) { logger.Info("Rule already exists") return nil @@ -193,7 +193,7 @@ func (c *IPTablesClient) AppendIptableRule(version, tableName, chainName, match, } // Delete matched iptable rule -func (c *IPTablesClient) DeleteIptableRule(version, tableName, chainName, match, target string) error { +func (c *Client) DeleteIptableRule(version, tableName, chainName, match, target string) error { params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target) return c.RunCmd(version, params) } diff --git a/network/endpoint.go b/network/endpoint.go index b5e7ec83de..41a6778aa4 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -166,7 +166,9 @@ func (nw *network) newEndpoint( } // DeleteEndpoint deletes an existing endpoint from the network. -func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface, endpointID string) error { +func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, endpointID string, +) error { var err error logger.Info("Deleting endpoint from network", zap.String("endpointID", endpointID), zap.String("id", nw.Id)) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 59c06fff51..5ce860f462 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -257,7 +257,9 @@ func (nw *network) newEndpointImpl( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, ep *endpoint, +) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. diff --git a/network/manager.go b/network/manager.go index 302f9b0846..815d2856c6 100644 --- a/network/manager.go +++ b/network/manager.go @@ -115,7 +115,9 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, iptc networkutils.IPTablesClientInterface) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, + iptc networkutils.IPTablesClientInterface, +) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index cbc30594e6..e1c46f0484 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -260,7 +260,7 @@ func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { return err } -// This fucntion adds rule which snat to ip passed filtered by match string. +// This function adds rule which snat to ip passed filtered by match string. func (nu NetworkUtils) AddSnatRule(iptablesClient IPTablesClientInterface, match string, ip net.IP) error { version := iptables.V4 if ip.To4() == nil { @@ -268,7 +268,7 @@ func (nu NetworkUtils) AddSnatRule(iptablesClient IPTablesClientInterface, match } target := fmt.Sprintf("SNAT --to %s", ip.String()) - return iptablesClient.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) + return errors.Wrap(iptablesClient.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target), "failed to add snat rule") } func (nu NetworkUtils) DisableRAForInterface(ifName string) error { diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index e77bbf09e9..0465091776 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -58,7 +58,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge bool, nl netlink.NetlinkInterface, plClient platform.ExecClient, - IPTablesClient networkutils.IPTablesClientInterface, + ipTablesClient networkutils.IPTablesClientInterface, ) Client { logger.Info("Initialize new snat client") snatClient := Client{ @@ -70,7 +70,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge: enableProxyArpOnBridge, netlink: nl, plClient: plClient, - IPTablesClient: IPTablesClient, + IPTablesClient: ipTablesClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -197,13 +197,13 @@ func (client *Client) AllowInboundFromHostToNC() error { } // Create cniinput chain - if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + if err = client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Forward from Input to cniinput chain - if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } @@ -291,14 +291,14 @@ func (client *Client) AllowInboundFromNCToHost() error { } // Create CNI output chain - if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + if err = client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err } // Forward traffic from Output to CNI Output chain - if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err @@ -458,7 +458,8 @@ func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) erro func (client *Client) addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) + return errors.Wrap(client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade), + "failed to add masquerade rule") } // Drop all vlan traffic on linux bridge From dd4c48c314dabad453ac63e8d3757a0e4fc1fe95 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 4 Dec 2023 11:52:51 -0800 Subject: [PATCH 06/13] Address windows linter issues --- network/endpoint_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index e17332d2ea..d53486019b 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -412,7 +412,8 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ netio.NetIOInterface, _ NamespaceClientInterface, - _ networkutils.IPTablesClientInterface, ep *endpoint) error { + _ networkutils.IPTablesClientInterface, ep *endpoint, +) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { return err From b107304d275499ab0ad8b29bef868fd266e531fa Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 4 Dec 2023 12:10:34 -0800 Subject: [PATCH 07/13] Invert if condition for linter nesting --- network/network_linux.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/network/network_linux.go b/network/network_linux.go index 39721430cc..c66e6263c8 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -702,15 +702,16 @@ func (nm *networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *Netw } for _, ipAddr := range extIf.IPAddresses { - if ipAddr.IP.To4() == nil { - logger.Info("Adding ipv6 snat rule") - matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) - nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) - if err := nu.AddSnatRule(nm.iptablesClient, matchSrcPrefix, ipAddr.IP); err != nil { - return fmt.Errorf("adding iptable snat rule failed:%w", err) - } - ipv6SnatRuleSet = true + if ipAddr.IP.To4() != nil { + continue + } + logger.Info("Adding ipv6 snat rule") + matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) + nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + if err := nu.AddSnatRule(nm.iptablesClient, matchSrcPrefix, ipAddr.IP); err != nil { + return fmt.Errorf("adding iptable snat rule failed:%w", err) } + ipv6SnatRuleSet = true } if !ipv6SnatRuleSet { From 925011fe9a885817118cec30101565cf306d2e6a Mon Sep 17 00:00:00 2001 From: QxBytes Date: Mon, 4 Dec 2023 13:34:39 -0800 Subject: [PATCH 08/13] Scope iptables interfaces to package --- network/endpoint.go | 5 ++--- network/endpoint_linux.go | 4 ++-- network/endpoint_windows.go | 5 ++--- network/iptables.go | 9 +++++++++ network/manager.go | 5 ++--- network/networkutils/iptables.go | 16 ---------------- network/networkutils/networkutils_linux.go | 18 ++++++++++++------ network/ovs_endpointclient_linux.go | 4 ++-- network/snat/snat_linux.go | 11 +++++++++-- .../transparent_vlan_endpointclient_linux.go | 4 ++-- 10 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 network/iptables.go delete mode 100644 network/networkutils/iptables.go diff --git a/network/endpoint.go b/network/endpoint.go index 41a6778aa4..0e12d8b764 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "go.uber.org/zap" @@ -141,7 +140,7 @@ func (nw *network) newEndpoint( plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, + iptc ipTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ep *endpoint @@ -167,7 +166,7 @@ func (nw *network) newEndpoint( // DeleteEndpoint deletes an existing endpoint from the network. func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, endpointID string, + iptc ipTablesClientInterface, endpointID string, ) error { var err error diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 5ce860f462..b8c7e63635 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -56,7 +56,7 @@ func (nw *network) newEndpointImpl( netioCli netio.NetIOInterface, testEpClient EndpointClient, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, + iptc ipTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ( @@ -258,7 +258,7 @@ func (nw *network) newEndpointImpl( // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, ep *endpoint, + iptc ipTablesClientInterface, ep *endpoint, ) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index d53486019b..f62a54850c 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,7 +11,6 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" @@ -72,7 +71,7 @@ func (nw *network) newEndpointImpl( _ netio.NetIOInterface, _ EndpointClient, _ NamespaceClientInterface, - _ networkutils.IPTablesClientInterface, + _ ipTablesClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future @@ -412,7 +411,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ netio.NetIOInterface, _ NamespaceClientInterface, - _ networkutils.IPTablesClientInterface, ep *endpoint, + _ ipTablesClientInterface, ep *endpoint, ) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { diff --git a/network/iptables.go b/network/iptables.go new file mode 100644 index 0000000000..c292df4536 --- /dev/null +++ b/network/iptables.go @@ -0,0 +1,9 @@ +package network + +type ipTablesClientInterface interface { + InsertIptableRule(version, tableName, chainName, match, target string) error + AppendIptableRule(version, tableName, chainName, match, target string) error + DeleteIptableRule(version, tableName, chainName, match, target string) error + CreateChain(version, tableName, chainName string) error + RunCmd(version, params string) error +} diff --git a/network/manager.go b/network/manager.go index 815d2856c6..2195bb7be4 100644 --- a/network/manager.go +++ b/network/manager.go @@ -15,7 +15,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" @@ -82,7 +81,7 @@ type networkManager struct { netio netio.NetIOInterface plClient platform.ExecClient nsClient NamespaceClientInterface - iptablesClient networkutils.IPTablesClientInterface + iptablesClient ipTablesClientInterface sync.Mutex } @@ -116,7 +115,7 @@ type NetworkManager interface { // Creates a new network manager. func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, + iptc ipTablesClientInterface, ) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), diff --git a/network/networkutils/iptables.go b/network/networkutils/iptables.go deleted file mode 100644 index cf5b1ee57b..0000000000 --- a/network/networkutils/iptables.go +++ /dev/null @@ -1,16 +0,0 @@ -package networkutils - -import "github.com/Azure/azure-container-networking/iptables" - -type IPTablesClientInterface interface { - RunCmd(version, params string) error - ChainExists(version, tableName, chainName string) bool - GetCreateChainCmd(version, tableName, chainName string) iptables.IPTableEntry - CreateChain(version, tableName, chainName string) error - RuleExists(version, tableName, chainName, match, target string) bool - GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) iptables.IPTableEntry - InsertIptableRule(version, tableName, chainName, match, target string) error - GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) iptables.IPTableEntry - AppendIptableRule(version, tableName, chainName, match, target string) error - DeleteIptableRule(version, tableName, chainName, match, target string) error -} diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index e1c46f0484..d9e29c1a5a 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -41,6 +41,12 @@ const ( var logger = log.CNILogger.With(zap.String("component", "net-utils")) +type ipTablesClientInterface interface { + InsertIptableRule(version, tableName, chainName, match, target string) error + AppendIptableRule(version, tableName, chainName, match, target string) error + DeleteIptableRule(version, tableName, chainName, match, target string) error +} + var errorNetworkUtils = errors.New("NetworkUtils Error") func newErrorNetworkUtils(errStr string) error { @@ -130,7 +136,7 @@ func (nu NetworkUtils) AssignIPToInterface(interfaceName string, ipAddresses []n return nil } -func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient IPTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { +func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient ipTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { var err error option := "i" @@ -152,7 +158,7 @@ func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient IPTablesClientInterf return err } -func (nu NetworkUtils) AllowIPAddresses(iptablesClient IPTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { +func (nu NetworkUtils) AllowIPAddresses(iptablesClient ipTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { chains := getFilterChains() target := getFilterchainTarget() @@ -176,13 +182,13 @@ func (nu NetworkUtils) AllowIPAddresses(iptablesClient IPTablesClientInterface, return nil } -func (nu NetworkUtils) BlockEgressTrafficFromContainer(iptablesClient IPTablesClientInterface, version, ipAddress, protocol string, port int) error { +func (nu NetworkUtils) BlockEgressTrafficFromContainer(iptablesClient ipTablesClientInterface, version, ipAddress, protocol string, port int) error { // iptables -t filter -I FORWARD -j DROP -d -p -m --dport dropTraffic := fmt.Sprintf("-d %s -p %s -m %s --dport %d", ipAddress, protocol, protocol, port) return errors.Wrap(iptablesClient.InsertIptableRule(version, iptables.Filter, iptables.Forward, dropTraffic, iptables.Drop), "iptables block traffic failed") } -func (nu NetworkUtils) BlockIPAddresses(iptablesClient IPTablesClientInterface, bridgeName, action string) error { +func (nu NetworkUtils) BlockIPAddresses(iptablesClient ipTablesClientInterface, bridgeName, action string) error { privateIPAddresses := getPrivateIPSpace() chains := getFilterChains() target := getFilterchainTarget() @@ -207,7 +213,7 @@ func (nu NetworkUtils) BlockIPAddresses(iptablesClient IPTablesClientInterface, } // This function enables ip forwarding in VM and allow forwarding packets from the interface -func (nu NetworkUtils) EnableIPForwarding(iptablesClient IPTablesClientInterface) error { +func (nu NetworkUtils) EnableIPForwarding(iptablesClient ipTablesClientInterface) error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) @@ -261,7 +267,7 @@ func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { } // This function adds rule which snat to ip passed filtered by match string. -func (nu NetworkUtils) AddSnatRule(iptablesClient IPTablesClientInterface, match string, ip net.IP) error { +func (nu NetworkUtils) AddSnatRule(iptablesClient ipTablesClientInterface, match string, ip net.IP) error { version := iptables.V4 if ip.To4() == nil { version = iptables.V6 diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 5aedddbf1b..0798403c06 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -35,7 +35,7 @@ type OVSEndpointClient struct { netioshim netio.NetIOInterface ovsctlClient ovsctl.OvsInterface plClient platform.ExecClient - iptablesClient networkutils.IPTablesClientInterface + iptablesClient ipTablesClientInterface } const ( @@ -53,7 +53,7 @@ func NewOVSEndpointClient( nl netlink.NetlinkInterface, ovs ovsctl.OvsInterface, plc platform.ExecClient, - iptc networkutils.IPTablesClientInterface, + iptc ipTablesClientInterface, ) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 0465091776..70ce2b33e3 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -30,6 +30,13 @@ const ( var logger = log.CNILogger.With(zap.String("component", "net")) +type ipTablesClientInterface interface { + InsertIptableRule(version, tableName, chainName, match, target string) error + AppendIptableRule(version, tableName, chainName, match, target string) error + DeleteIptableRule(version, tableName, chainName, match, target string) error + CreateChain(version, tableName, chainName string) error +} + var errorSnatClient = errors.New("SnatClient Error") func newErrorSnatClient(errStr string) error { @@ -46,7 +53,7 @@ type Client struct { enableProxyArpOnBridge bool netlink netlink.NetlinkInterface plClient platform.ExecClient - IPTablesClient networkutils.IPTablesClientInterface + IPTablesClient ipTablesClientInterface } func NewSnatClient(hostIfName string, @@ -58,7 +65,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge bool, nl netlink.NetlinkInterface, plClient platform.ExecClient, - ipTablesClient networkutils.IPTablesClientInterface, + ipTablesClient ipTablesClientInterface, ) Client { logger.Info("Initialize new snat client") snatClient := Client{ diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 2a3030f36b..50bf838436 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -66,7 +66,7 @@ type TransparentVlanEndpointClient struct { plClient platform.ExecClient netUtilsClient networkutils.NetworkUtils nsClient NamespaceClientInterface - iptablesClient networkutils.IPTablesClientInterface + iptablesClient ipTablesClientInterface } func NewTransparentVlanEndpointClient( @@ -79,7 +79,7 @@ func NewTransparentVlanEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, nsc NamespaceClientInterface, - iptc networkutils.IPTablesClientInterface, + iptc ipTablesClientInterface, ) *TransparentVlanEndpointClient { vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) From b79066cecf8e141de00a5e67dd169dccd12eb031 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 5 Dec 2023 10:43:30 -0800 Subject: [PATCH 09/13] Rename iptables client to avoid stuttering --- cni/network/invoker_cns.go | 2 +- cni/network/network.go | 2 +- cnm/network/network.go | 2 +- cnms/service/networkmonitor.go | 2 +- iptables/iptables.go | 2 +- network/endpoint_test.go | 22 +++++++++++----------- network/snat/snat_linux_test.go | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 323f818bac..f3e75690f7 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -227,7 +227,7 @@ func setHostOptions(ncSubnetPrefix *net.IPNet, options map[string]interface{}, i // we need to snat IMDS traffic to node IP, this sets up snat '--to' snatHostIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.hostPrimaryIP) - iptablesClient := iptables.NewIPTablesClient() + iptablesClient := iptables.NewClient() var iptableCmds []iptables.IPTableEntry if !iptablesClient.ChainExists(iptables.V4, iptables.Nat, iptables.Swift) { iptableCmds = append(iptableCmds, iptablesClient.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift)) diff --git a/cni/network/network.go b/cni/network/network.go index 4e62c7be41..2381b8ffc4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -117,7 +117,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewClient()) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index 75dfe240ab..4358d1a485 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -54,7 +54,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewClient()) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index 8b4d4915ec..96c790a667 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -158,7 +158,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewIPTablesClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewClient()) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/iptables/iptables.go b/iptables/iptables.go index 22458d2cdc..2d3fdf2e22 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -89,7 +89,7 @@ type IPTableEntry struct { type Client struct{} -func NewIPTablesClient() *Client { +func NewClient() *Client { return &Client{} } diff --git a/network/endpoint_test.go b/network/endpoint_test.go index da53d956f8..0fc0cf829e 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -184,7 +184,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -196,7 +196,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -212,7 +212,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -222,17 +222,17 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) // Deleting the endpoint //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewIPTablesClient(), ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) // Deleting same endpoint with same id should not fail //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewIPTablesClient(), ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, netio.NewMockNetIO(false, 0), NewMockNamespaceClient(), iptables.NewClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) }) }) @@ -253,11 +253,11 @@ var _ = Describe("Test Endpoint", func() { } return nil - }), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + }), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -283,14 +283,14 @@ var _ = Describe("Test Endpoint", func() { It("Should not endpoint to the network when there is an error", func() { secondaryEpInfo.MacAddress = netio.BadHwAddr // mock netlink will fail to set link state on bad eth ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("SecondaryEndpointClient Error: " + netlink.ErrorMockNetlink.Error())) Expect(ep).To(BeNil()) secondaryEpInfo.MacAddress = netio.HwAddr ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) @@ -298,7 +298,7 @@ var _ = Describe("Test Endpoint", func() { It("Should add endpoint when there are no errors", func() { secondaryEpInfo.MacAddress = netio.HwAddr ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewIPTablesClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) + netio.NewMockNetIO(false, 0), nil, NewMockNamespaceClient(), iptables.NewClient(), []*EndpointInfo{epInfo, secondaryEpInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep.Id).To(Equal(epInfo.Id)) }) diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index 9f3ff8bca9..3f3b1ad3b5 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -20,7 +20,7 @@ func TestMain(m *testing.M) { func TestAllowInboundFromHostToNC(t *testing.T) { nl := netlink.NewNetlink() - iptc := iptables.NewIPTablesClient() + iptc := iptables.NewClient() client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", @@ -69,7 +69,7 @@ func TestAllowInboundFromHostToNC(t *testing.T) { func TestAllowInboundFromNCToHost(t *testing.T) { nl := netlink.NewNetlink() - iptc := iptables.NewIPTablesClient() + iptc := iptables.NewClient() client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", From 848c50d816c1be9840192db9041c736517a9f873 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 7 Dec 2023 14:07:08 -0800 Subject: [PATCH 10/13] Move EnableIPForwarding to snat linux --- network/endpoint_snatroute_linux.go | 4 +- network/networkutils/networkutils_linux.go | 22 -------- network/snat/snat_linux.go | 60 +++++++++++++++------- network/snat/snat_linux_test.go | 4 +- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 772ddf0040..0f8de4e1ac 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" @@ -36,8 +35,7 @@ func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl n if err := snatClient.BlockIPAddressesOnSnatBridge(); err != nil { return errors.Wrap(err, "failed to block ip addresses on snat bridge") } - nuc := networkutils.NewNetworkUtils(nl, plc) - if err := nuc.EnableIPForwarding(snatClient.IPTablesClient); err != nil { + if err := snatClient.EnableIPForwarding(); err != nil { return errors.Wrap(err, "failed to enable ip forwarding") } diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index d9e29c1a5a..2a0edcdbea 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -31,7 +31,6 @@ RFC for Link Local Addresses: https://tools.ietf.org/html/rfc3927 */ const ( - enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1" toggleIPV6Cmd = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d" enableIPV6ForwardCmd = "sysctl -w net.ipv6.conf.all.forwarding=1" enableIPV4ForwardCmd = "sysctl -w net.ipv4.conf.all.forwarding=1" @@ -212,27 +211,6 @@ func (nu NetworkUtils) BlockIPAddresses(iptablesClient ipTablesClientInterface, return nil } -// This function enables ip forwarding in VM and allow forwarding packets from the interface -func (nu NetworkUtils) EnableIPForwarding(iptablesClient ipTablesClientInterface) error { - // Enable ip forwading on linux vm. - // sysctl -w net.ipv4.ip_forward=1 - cmd := fmt.Sprint(enableIPForwardCmd) - _, err := nu.plClient.ExecuteCommand(cmd) - if err != nil { - logger.Error("Enable ipforwarding failed with", zap.Error(err)) - return err - } - - // Append a rule in forward chain to allow forwarding from bridge - if err := iptablesClient.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { - logger.Error("Appending forward chain rule: allow traffic coming from snatbridge failed with", - zap.Error(err)) - return err - } - - return nil -} - func (nu NetworkUtils) EnableIPV4Forwarding() error { _, err := nu.plClient.ExecuteCommand(enableIPV4ForwardCmd) if err != nil { diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 70ce2b33e3..ab65115fee 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -26,6 +26,7 @@ const ( vlanDropAddRule = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" vlanDropMatch = "-p 802_1Q -j DROP" l2PreroutingEntries = "ebtables -t nat -L PREROUTING" + enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1" ) var logger = log.CNILogger.With(zap.String("component", "net")) @@ -53,7 +54,7 @@ type Client struct { enableProxyArpOnBridge bool netlink netlink.NetlinkInterface plClient platform.ExecClient - IPTablesClient ipTablesClientInterface + ipTablesClient ipTablesClientInterface } func NewSnatClient(hostIfName string, @@ -77,7 +78,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge: enableProxyArpOnBridge, netlink: nl, plClient: plClient, - IPTablesClient: ipTablesClient, + ipTablesClient: ipTablesClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -131,7 +132,7 @@ func (client *Client) CreateSnatEndpoint() error { // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge func (client *Client) AllowIPAddressesOnSnatBridge() error { nu := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := nu.AllowIPAddresses(client.IPTablesClient, SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + if err := nu.AllowIPAddresses(client.ipTablesClient, SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { logger.Error("AllowIPAddresses failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) } @@ -142,7 +143,7 @@ func (client *Client) AllowIPAddressesOnSnatBridge() error { // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge func (client *Client) BlockIPAddressesOnSnatBridge() error { nu := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := nu.BlockIPAddresses(client.IPTablesClient, SnatBridgeName, iptables.Append); err != nil { + if err := nu.BlockIPAddresses(client.ipTablesClient, SnatBridgeName, iptables.Append); err != nil { logger.Error("AllowIPAddresses failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) } @@ -184,40 +185,40 @@ func (client *Client) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Output chain - if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + if err := client.ipTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Forward traffic from Ouptut chain to CNI Output chain - if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err := client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting output rule failed with ", zap.Error(err)) return newErrorSnatClient(err.Error()) } // Create cniinput chain - if err = client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + if err = client.ipTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Forward from Input to cniinput chain - if err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err = client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.Any("CNIOutputChain", iptables.CNIOutputChain), zap.Error(err)) return newErrorSnatClient(err.Error()) } // Accept packets from NC only if established connection matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err = client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting input rule failed with", zap.Error(err)) return newErrorSnatClient(err.Error()) @@ -249,7 +250,7 @@ func (client *Client) DeleteInboundFromHostToNC() error { // Delete allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := client.IPTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := client.ipTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("DeleteInboundFromHostToNC: Error removing output rule", zap.Error(err)) } @@ -276,14 +277,14 @@ func (client *Client) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Input chain - if err := client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + if err := client.ipTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err } // Forward traffic from Input to cniinput chain - if err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err := client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err @@ -291,21 +292,21 @@ func (client *Client) AllowInboundFromNCToHost() error { // Allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting output rule failed with", zap.Error(err)) return err } // Create CNI output chain - if err = client.IPTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + if err = client.ipTablesClient.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Creating failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err } // Forward traffic from Output to CNI Output chain - if err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err = client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { logger.Error("AllowInboundFromHostToNC: Inserting forward rule to failed with", zap.String("CNIInputChain", iptables.CNIInputChain), zap.Error(err)) return err @@ -313,7 +314,7 @@ func (client *Client) AllowInboundFromNCToHost() error { // Accept packets from Host only if established connection matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err = client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("AllowInboundFromHostToNC: Inserting input rule failed with", zap.Error(err)) return err @@ -343,7 +344,7 @@ func (client *Client) DeleteInboundFromNCToHost() error { // Delete allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := client.IPTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := client.ipTablesClient.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { logger.Error("DeleteInboundFromNCToHost: Error removing output rule", zap.Error(err)) } @@ -465,7 +466,7 @@ func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) erro func (client *Client) addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return errors.Wrap(client.IPTablesClient.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade), + return errors.Wrap(client.ipTablesClient.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade), "failed to add masquerade rule") } @@ -487,3 +488,24 @@ func (client *Client) addVlanDropRule() error { _, err = client.plClient.ExecuteCommand(vlanDropAddRule) return err } + +// This function enables ip forwarding in VM and allow forwarding packets from the interface +func (client *Client) EnableIPForwarding() error { + // Enable ip forwading on linux vm. + // sysctl -w net.ipv4.ip_forward=1 + cmd := fmt.Sprint(enableIPForwardCmd) + _, err := client.plClient.ExecuteCommand(cmd) + if err != nil { + logger.Error("Enable ipforwarding failed with", zap.Error(err)) + return err + } + + // Append a rule in forward chain to allow forwarding from bridge + if err := client.ipTablesClient.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { + logger.Error("Appending forward chain rule: allow traffic coming from snatbridge failed with", + zap.Error(err)) + return err + } + + return nil +} diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index 3f3b1ad3b5..0ffee1ebf2 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -26,7 +26,7 @@ func TestAllowInboundFromHostToNC(t *testing.T) { localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, - IPTablesClient: iptc, + ipTablesClient: iptc, } if err := nl.AddLink(&netlink.DummyLink{ @@ -75,7 +75,7 @@ func TestAllowInboundFromNCToHost(t *testing.T) { localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, - IPTablesClient: iptc, + ipTablesClient: iptc, } if err := nl.AddLink(&netlink.DummyLink{ From 8f025745f7f63f5ff50717feb4d4ee9832ef68b0 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 7 Dec 2023 15:20:55 -0800 Subject: [PATCH 11/13] Rename ipTablesClientInterface to ipTablesClient --- network/endpoint.go | 4 ++-- network/endpoint_linux.go | 4 ++-- network/iptables.go | 2 +- network/manager.go | 4 ++-- network/ovs_endpointclient_linux.go | 4 ++-- network/snat/snat_linux.go | 8 ++++---- network/transparent_vlan_endpointclient_linux.go | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 0e12d8b764..0adf857ea9 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -140,7 +140,7 @@ func (nw *network) newEndpoint( plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, + iptc ipTablesClient, epInfo []*EndpointInfo, ) (*endpoint, error) { var ep *endpoint @@ -166,7 +166,7 @@ func (nw *network) newEndpoint( // DeleteEndpoint deletes an existing endpoint from the network. func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, endpointID string, + iptc ipTablesClient, endpointID string, ) error { var err error diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b8c7e63635..f603f389fb 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -56,7 +56,7 @@ func (nw *network) newEndpointImpl( netioCli netio.NetIOInterface, testEpClient EndpointClient, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, + iptc ipTablesClient, epInfo []*EndpointInfo, ) (*endpoint, error) { var ( @@ -258,7 +258,7 @@ func (nw *network) newEndpointImpl( // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nioc netio.NetIOInterface, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, ep *endpoint, + iptc ipTablesClient, ep *endpoint, ) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require diff --git a/network/iptables.go b/network/iptables.go index c292df4536..c572198453 100644 --- a/network/iptables.go +++ b/network/iptables.go @@ -1,6 +1,6 @@ package network -type ipTablesClientInterface interface { +type ipTablesClient interface { InsertIptableRule(version, tableName, chainName, match, target string) error AppendIptableRule(version, tableName, chainName, match, target string) error DeleteIptableRule(version, tableName, chainName, match, target string) error diff --git a/network/manager.go b/network/manager.go index 2195bb7be4..c411b532d0 100644 --- a/network/manager.go +++ b/network/manager.go @@ -81,7 +81,7 @@ type networkManager struct { netio netio.NetIOInterface plClient platform.ExecClient nsClient NamespaceClientInterface - iptablesClient ipTablesClientInterface + iptablesClient ipTablesClient sync.Mutex } @@ -115,7 +115,7 @@ type NetworkManager interface { // Creates a new network manager. func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, + iptc ipTablesClient, ) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 0798403c06..569232eb9a 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -35,7 +35,7 @@ type OVSEndpointClient struct { netioshim netio.NetIOInterface ovsctlClient ovsctl.OvsInterface plClient platform.ExecClient - iptablesClient ipTablesClientInterface + iptablesClient ipTablesClient } const ( @@ -53,7 +53,7 @@ func NewOVSEndpointClient( nl netlink.NetlinkInterface, ovs ovsctl.OvsInterface, plc platform.ExecClient, - iptc ipTablesClientInterface, + iptc ipTablesClient, ) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index ab65115fee..45a51acbff 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -31,7 +31,7 @@ const ( var logger = log.CNILogger.With(zap.String("component", "net")) -type ipTablesClientInterface interface { +type ipTablesClient interface { InsertIptableRule(version, tableName, chainName, match, target string) error AppendIptableRule(version, tableName, chainName, match, target string) error DeleteIptableRule(version, tableName, chainName, match, target string) error @@ -54,7 +54,7 @@ type Client struct { enableProxyArpOnBridge bool netlink netlink.NetlinkInterface plClient platform.ExecClient - ipTablesClient ipTablesClientInterface + ipTablesClient ipTablesClient } func NewSnatClient(hostIfName string, @@ -66,7 +66,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge bool, nl netlink.NetlinkInterface, plClient platform.ExecClient, - ipTablesClient ipTablesClientInterface, + iptc ipTablesClient, ) Client { logger.Info("Initialize new snat client") snatClient := Client{ @@ -78,7 +78,7 @@ func NewSnatClient(hostIfName string, enableProxyArpOnBridge: enableProxyArpOnBridge, netlink: nl, plClient: plClient, - ipTablesClient: ipTablesClient, + ipTablesClient: iptc, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 50bf838436..6fcf9719e9 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -66,7 +66,7 @@ type TransparentVlanEndpointClient struct { plClient platform.ExecClient netUtilsClient networkutils.NetworkUtils nsClient NamespaceClientInterface - iptablesClient ipTablesClientInterface + iptablesClient ipTablesClient } func NewTransparentVlanEndpointClient( @@ -79,7 +79,7 @@ func NewTransparentVlanEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, nsc NamespaceClientInterface, - iptc ipTablesClientInterface, + iptc ipTablesClient, ) *TransparentVlanEndpointClient { vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) From 4a667dac97f58a1b62fe8f9f5b0b366923c1e1b3 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 7 Dec 2023 15:38:56 -0800 Subject: [PATCH 12/13] Address linter issues from moving enable ip forwarding function --- network/endpoint_windows.go | 4 ++-- network/snat/snat_linux.go | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index f62a54850c..244ab6f234 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -71,7 +71,7 @@ func (nw *network) newEndpointImpl( _ netio.NetIOInterface, _ EndpointClient, _ NamespaceClientInterface, - _ ipTablesClientInterface, + _ ipTablesClient, epInfo []*EndpointInfo, ) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future @@ -411,7 +411,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ netio.NetIOInterface, _ NamespaceClientInterface, - _ ipTablesClientInterface, ep *endpoint, + _ ipTablesClient, ep *endpoint, ) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 45a51acbff..fb7f348b78 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -493,18 +493,14 @@ func (client *Client) addVlanDropRule() error { func (client *Client) EnableIPForwarding() error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 - cmd := fmt.Sprint(enableIPForwardCmd) - _, err := client.plClient.ExecuteCommand(cmd) + _, err := client.plClient.ExecuteCommand(enableIPForwardCmd) if err != nil { - logger.Error("Enable ipforwarding failed with", zap.Error(err)) - return err + return errors.Wrap(err, "enable ipforwarding command failed") } // Append a rule in forward chain to allow forwarding from bridge if err := client.ipTablesClient.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { - logger.Error("Appending forward chain rule: allow traffic coming from snatbridge failed with", - zap.Error(err)) - return err + return errors.Wrap(err, "appending forward chain rule to allow traffic from snat bridge failed") } return nil From d621c22e84ab85b8d7c096b80acae1575a936606 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Tue, 12 Dec 2023 15:39:12 -0800 Subject: [PATCH 13/13] Rename after rebase --- network/manager.go | 1 - network/networkutils/networkutils_linux.go | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/network/manager.go b/network/manager.go index c411b532d0..65e45c2440 100644 --- a/network/manager.go +++ b/network/manager.go @@ -392,7 +392,6 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } ep, err := nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, nm.iptablesClient, epInfo) - if err != nil { return err } diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index 2a0edcdbea..519da94f9c 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -40,7 +40,7 @@ const ( var logger = log.CNILogger.With(zap.String("component", "net-utils")) -type ipTablesClientInterface interface { +type ipTablesClient interface { InsertIptableRule(version, tableName, chainName, match, target string) error AppendIptableRule(version, tableName, chainName, match, target string) error DeleteIptableRule(version, tableName, chainName, match, target string) error @@ -135,7 +135,7 @@ func (nu NetworkUtils) AssignIPToInterface(interfaceName string, ipAddresses []n return nil } -func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient ipTablesClientInterface, bridgeName, action, ipAddress, chainName, target string) error { +func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient ipTablesClient, bridgeName, action, ipAddress, chainName, target string) error { var err error option := "i" @@ -157,7 +157,7 @@ func (nu NetworkUtils) addOrDeleteFilterRule(iptablesClient ipTablesClientInterf return err } -func (nu NetworkUtils) AllowIPAddresses(iptablesClient ipTablesClientInterface, bridgeName string, skipAddresses []string, action string) error { +func (nu NetworkUtils) AllowIPAddresses(iptablesClient ipTablesClient, bridgeName string, skipAddresses []string, action string) error { chains := getFilterChains() target := getFilterchainTarget() @@ -181,13 +181,13 @@ func (nu NetworkUtils) AllowIPAddresses(iptablesClient ipTablesClientInterface, return nil } -func (nu NetworkUtils) BlockEgressTrafficFromContainer(iptablesClient ipTablesClientInterface, version, ipAddress, protocol string, port int) error { +func (nu NetworkUtils) BlockEgressTrafficFromContainer(iptablesClient ipTablesClient, version, ipAddress, protocol string, port int) error { // iptables -t filter -I FORWARD -j DROP -d -p -m --dport dropTraffic := fmt.Sprintf("-d %s -p %s -m %s --dport %d", ipAddress, protocol, protocol, port) return errors.Wrap(iptablesClient.InsertIptableRule(version, iptables.Filter, iptables.Forward, dropTraffic, iptables.Drop), "iptables block traffic failed") } -func (nu NetworkUtils) BlockIPAddresses(iptablesClient ipTablesClientInterface, bridgeName, action string) error { +func (nu NetworkUtils) BlockIPAddresses(iptablesClient ipTablesClient, bridgeName, action string) error { privateIPAddresses := getPrivateIPSpace() chains := getFilterChains() target := getFilterchainTarget() @@ -245,7 +245,7 @@ func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { } // This function adds rule which snat to ip passed filtered by match string. -func (nu NetworkUtils) AddSnatRule(iptablesClient ipTablesClientInterface, match string, ip net.IP) error { +func (nu NetworkUtils) AddSnatRule(iptablesClient ipTablesClient, match string, ip net.IP) error { version := iptables.V4 if ip.To4() == nil { version = iptables.V6