From 4260d8cb0ae7712e87f85fc6faae06bf8577482f Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 18 Oct 2021 14:27:35 -0700 Subject: [PATCH 1/2] handle errors in setting up routes and iptables for AKS-Swift --- network/network_linux.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/network/network_linux.go b/network/network_linux.go index df7355e235..5e18ed6ac3 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -65,10 +65,6 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt switch nwInfo.Mode { case opModeTunnel: - err := nm.handleCommonOptions(extIf.Name, nwInfo) - if err != nil { - log.Printf("tunnel handleCommonOptions failed with error %s", err.Error()) - } fallthrough case opModeBridge: log.Printf("create bridge") @@ -79,10 +75,6 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt if opt != nil && opt[VlanIDKey] != nil { vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string)) } - err := nm.handleCommonOptions(extIf.BridgeName, nwInfo) - if err != nil { - log.Printf("bridge handleCommonOptions failed with error %s", err.Error()) - } case opModeTransparent: log.Printf("Transparent mode") if nwInfo.IPV6Mode != "" { @@ -95,6 +87,12 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, errNetworkModeInvalid } + err := nm.handleCommonOptions(extIf.BridgeName, nwInfo) + if err != nil { + log.Printf("handleCommonOptions failed with error %s", err.Error()) + return nil, err + } + // Create the network object. nw := &network{ Id: nwInfo.Id, @@ -109,6 +107,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nw, nil } +// RoutesKey and IPTablesKey is set only in AKS swift func (nm *networkManager) handleCommonOptions(ifname string, nwInfo *NetworkInfo) error { var err error if routes, exists := nwInfo.Options[RoutesKey]; exists { From 2a6a8c77b1ae3ede71f1c1a1f11e3efc6339e963 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 18 Oct 2021 14:55:54 -0700 Subject: [PATCH 2/2] added netio interface in networkmanager --- cni/network/network.go | 3 ++- cnm/network/network.go | 3 ++- cnms/service/networkmonitor.go | 3 ++- network/manager.go | 5 +++- network/network_linux.go | 42 ++++++++-------------------------- 5 files changed, 19 insertions(+), 37 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 92c9e0494f..6b0b9def05 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -19,6 +19,7 @@ import ( "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" @@ -114,7 +115,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(), &netio.NetIO{}) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index e811f4d098..f95b31666f 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -12,6 +12,7 @@ import ( cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" @@ -52,7 +53,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(), &netio.NetIO{}) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index 6341bc06ee..8020bc3a53 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -11,6 +11,7 @@ import ( cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" @@ -148,7 +149,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(), &netio.NetIO{}) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/network/manager.go b/network/manager.go index 4ae1829924..44739f9e24 100644 --- a/network/manager.go +++ b/network/manager.go @@ -11,6 +11,7 @@ import ( cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" @@ -58,6 +59,7 @@ type networkManager struct { ExternalInterfaces map[string]*externalInterface store store.KeyValueStore netlink netlink.NetlinkInterface + netio netio.NetIOInterface plClient platform.ExecClient sync.Mutex } @@ -86,11 +88,12 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, plClient: plc, + netio: netioCli, } return nm, nil diff --git a/network/network_linux.go b/network/network_linux.go index 5e18ed6ac3..2afb4a5a0f 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -59,7 +59,10 @@ type route netlink.Route // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Connect the external interface. - var vlanid int + var ( + vlanid int + ifName string + ) opt, _ := nwInfo.Options[genericData].(map[string]interface{}) log.Printf("opt %+v options %+v", opt, nwInfo.Options) @@ -68,6 +71,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt fallthrough case opModeBridge: log.Printf("create bridge") + ifName = extIf.BridgeName if err := nm.connectExternalInterface(extIf, nwInfo); err != nil { return nil, err } @@ -77,6 +81,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt } case opModeTransparent: log.Printf("Transparent mode") + ifName = extIf.Name if nwInfo.IPV6Mode != "" { nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) if err := nu.EnableIPV6Forwarding(); err != nil { @@ -87,7 +92,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, errNetworkModeInvalid } - err := nm.handleCommonOptions(extIf.BridgeName, nwInfo) + err := nm.handleCommonOptions(ifName, nwInfo) if err != nil { log.Printf("handleCommonOptions failed with error %s", err.Error()) return nil, err @@ -107,11 +112,10 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nw, nil } -// RoutesKey and IPTablesKey is set only in AKS swift -func (nm *networkManager) handleCommonOptions(ifname string, nwInfo *NetworkInfo) error { +func (nm *networkManager) handleCommonOptions(ifName string, nwInfo *NetworkInfo) error { var err error if routes, exists := nwInfo.Options[RoutesKey]; exists { - err = nm.addBridgeRoutes(ifname, routes.([]RouteInfo)) + err = addRoutes(nm.netlink, nm.netio, ifName, routes.([]RouteInfo)) if err != nil { return err } @@ -570,34 +574,6 @@ func (*networkManager) addToIptables(cmds []iptables.IPTableEntry) error { return nil } -func (nm *networkManager) addBridgeRoutes(bridgeName string, routes []RouteInfo) error { - log.Printf("Adding routes...") - for _, route := range routes { - route.DevName = bridgeName - devIf, _ := net.InterfaceByName(route.DevName) - ifIndex := devIf.Index - gwfamily := netlink.GetIPAddressFamily(route.Gw) - - nlRoute := &netlink.Route{ - Family: gwfamily, - Dst: &route.Dst, - Gw: route.Gw, - LinkIndex: ifIndex, - } - - if err := nm.netlink.AddIPRoute(nlRoute); err != nil { - if !strings.Contains(strings.ToLower(err.Error()), "file exists") { - return fmt.Errorf("Failed to add %+v to host interface with error: %v", nlRoute, err) - } - log.Printf("[cni-net] route already exists: dst %+v, gw %+v, interfaceName %v", nlRoute.Dst, nlRoute.Gw, route.DevName) - } - - log.Printf("[cni-net] Added route %+v", route) - } - - return nil -} - // Add ipv6 nat gateway IP on bridge func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { log.Printf("[net] Adding ipv6 nat gateway on azure bridge")