diff --git a/Makefile b/Makefile index eb667c2450..bc6b8950e1 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,7 @@ ROOT_DIR = $(shell pwd) CNM_DIR = cnm/plugin CNI_NET_DIR = cni/network/plugin CNI_IPAM_DIR = cni/ipam/plugin +CNI_IPAMV6_DIR = cni/ipam/pluginv6 CNI_TELEMETRY_DIR = cni/telemetry/service TELEMETRY_CONF_DIR = telemetry CNS_DIR = cns/service @@ -144,7 +145,8 @@ ENSURE_OUTPUT_DIR_EXISTS := $(shell mkdir -p $(OUTPUT_DIR)) azure-cnm-plugin: $(CNM_BUILD_DIR)/azure-vnet-plugin$(EXE_EXT) cnm-archive azure-vnet: $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) azure-vnet-ipam: $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -azure-cni-plugin: azure-vnet azure-vnet-ipam azure-vnet-telemetry cni-archive +azure-vnet-ipamv6: $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT) +azure-cni-plugin: azure-vnet azure-vnet-ipam azure-vnet-ipamv6 azure-vnet-telemetry cni-archive azure-cns: $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) cns-archive azure-vnet-telemetry: $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) @@ -184,6 +186,10 @@ $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT): $(CNIFILES) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES) go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_IPAM_DIR)/*.go +# Build the Azure CNI IPAMV6 plugin. +$(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT): $(CNIFILES) + go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_IPAMV6_DIR)/*.go + # Build the Azure CNI telemetry plugin. $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES) go build -v -o $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(ACN_PACKAGE_PATH)/telemetry.aiMetadata=$(CNI_AI_ID) -s -w" $(CNI_TELEMETRY_DIR)/*.go @@ -314,8 +320,8 @@ publish-azure-cns-image: cni-archive: cp cni/azure-$(GOOS).conflist $(CNI_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_BUILD_DIR)/azure-vnet-telemetry.config - chmod 0755 $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) - cd $(CNI_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config + chmod 0755 $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) + cd $(CNI_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-ipamv6$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config chown $(BUILD_USER):$(BUILD_USER) $(CNI_BUILD_DIR)/$(CNI_ARCHIVE_NAME) mkdir -p $(CNI_MULTITENANCY_BUILD_DIR) cp cni/azure-$(GOOS)-multitenancy.conflist $(CNI_MULTITENANCY_BUILD_DIR)/10-azure.conflist @@ -371,5 +377,6 @@ test-all: ./cnm/network/ \ ./cni/ipam/ \ ./cns/ipamclient/ \ + ./cnms/service/ \ ./npm/iptm/ \ - ./npm/ipsm/ \ No newline at end of file + ./npm/ipsm/ diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index 0c1f3f70e8..6b551155f2 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -62,6 +62,7 @@ func TestMain(m *testing.M) { } log.Close() + hostAgent.Stop() os.Exit(exitCode) } diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index 2c17ef92b1..49e8e5c52c 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -19,6 +19,10 @@ import ( cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) +const ( + ipamV6 = "azure-vnet-ipamv6" +) + var ( ipv4DefaultRouteDstPrefix = net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)} ) @@ -154,8 +158,13 @@ func (plugin *ipamPlugin) Add(args *cniSkel.CmdArgs) error { options := make(map[string]string) options[ipam.OptInterfaceName] = nwCfg.Master + isIpv6 := false + if nwCfg.Ipam.Type == ipamV6 { + isIpv6 = true + } + // Allocate an address pool. - poolID, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", options, false) + poolID, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", options, isIpv6) if err != nil { err = plugin.Errorf("Failed to allocate pool: %v", err) return err diff --git a/cni/ipam/pluginv6/main.go b/cni/ipam/pluginv6/main.go new file mode 100644 index 0000000000..cb93640c48 --- /dev/null +++ b/cni/ipam/pluginv6/main.go @@ -0,0 +1,80 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package main + +import ( + "fmt" + "os" + + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cni/ipam" + "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" +) + +const ( + name = "azure-vnet-ipamv6" +) + +// Version is populated by make during build. +var version string + +// Main is the entry point for CNI IPAM plugin. +func main() { + var config common.PluginConfig + config.Version = version + logDirectory := "" // Sets the current location as log directory + + log.SetName(name) + log.SetLevel(log.LevelInfo) + if err := log.SetTargetLogDirectory(log.TargetLogfile, logDirectory); err != nil { + fmt.Printf("Failed to setup cni logging: %v\n", err) + return + } + + defer log.Close() + + ipamPlugin, err := ipam.NewPlugin(name, &config) + if err != nil { + fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err) + os.Exit(1) + } + + if err := ipamPlugin.Plugin.InitializeKeyValueStore(&config); err != nil { + fmt.Printf("Failed to initialize key-value store of ipam plugin, err:%v.\n", err) + + if isSafe, _ := ipamPlugin.Plugin.IsSafeToRemoveLock(ipamPlugin.Plugin.Name); isSafe { + log.Printf("[IPAM] Removing lock file as process holding lock exited") + if errUninit := ipamPlugin.Plugin.UninitializeKeyValueStore(true); errUninit != nil { + log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit) + } + } + + os.Exit(1) + } + + defer func() { + if errUninit := ipamPlugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil { + fmt.Printf("Failed to uninitialize key-value store of ipam plugin, err:%v.\n", err) + } + + if recover() != nil { + os.Exit(1) + } + }() + + err = ipamPlugin.Start(&config) + if err != nil { + fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err) + panic("ipam plugin fatal error") + } + + err = ipamPlugin.Execute(cni.PluginApi(ipamPlugin)) + + ipamPlugin.Stop() + + if err != nil { + panic("ipam plugin fatal error") + } +} diff --git a/cni/netconfig.go b/cni/netconfig.go index 54a1c13995..3faeef8451 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -51,6 +51,9 @@ type NetworkConfig struct { LogLevel string `json:"logLevel,omitempty"` LogTarget string `json:"logTarget,omitempty"` InfraVnetAddressSpace string `json:"infraVnetAddressSpace,omitempty"` + IPV6Mode string `json:"ipv6Mode,omitempty"` + ServiceCidrs string `json:"serviceCidrs,omitempty"` + VnetCidrs string `json:"vnetCidrs,omitempty"` PodNamespaceForDualNetwork []string `json:"podNamespaceForDualNetwork,omitempty"` IPsToRouteViaHost []string `json:"ipsToRouteViaHost,omitempty"` MultiTenancy bool `json:"multiTenancy,omitempty"` diff --git a/cni/network/network.go b/cni/network/network.go index 35a7f0881c..b6b3829abd 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -33,6 +33,7 @@ const ( opModeTransparent = "transparent" // Supported IP version. Currently support only IPv4 ipVersion = "4" + ipamV6 = "azure-vnet-ipamv6" ) // CNI Operation Types @@ -226,6 +227,93 @@ func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType st plugin.report.InterfaceDetails.SecondaryCAUsedCount = plugin.nm.GetNumberOfEndpoints("", nwCfg.Name) } +func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig, + resultV6 *cniTypesCurr.Result, + nwInfo *network.NetworkInfo) { + if nwCfg.IPV6Mode == network.IPV6Nat { + ipv6Subnet := resultV6.IPs[0].Address + ipv6Subnet.IP = ipv6Subnet.IP.Mask(ipv6Subnet.Mask) + ipv6SubnetInfo := network.SubnetInfo{ + Family: platform.AfINET6, + Prefix: ipv6Subnet, + Gateway: resultV6.IPs[0].Gateway, + } + log.Printf("[net] ipv6 subnet info:%+v", ipv6SubnetInfo) + nwInfo.Subnets = append(nwInfo.Subnets, ipv6SubnetInfo) + } +} + +func (plugin *netPlugin) invokeIpamDel( + result *cniTypesCurr.Result, + ipamType string, + nwCfg cni.NetworkConfig, + isDeletePoolOnError bool) { + + if result != nil { + if ipamType == ipamV6 { + nwCfg.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam + nwCfg.Ipam.Type = ipamType + } + + _, subnet, _ := net.ParseCIDR(result.IPs[0].Address.String()) + nwCfg.Ipam.Subnet = subnet.String() + nwCfg.Ipam.Address = result.IPs[0].Address.IP.String() + plugin.DelegateDel(ipamType, &nwCfg) + + // Release pool + if isDeletePoolOnError { + nwCfg.Ipam.Address = "" + plugin.DelegateDel(ipamType, &nwCfg) + } + } +} + +func (plugin *netPlugin) invokeIpamAdd( + nwCfg cni.NetworkConfig, + nwInfo network.NetworkInfo, + isDeletePoolOnError bool) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) { + + var ( + result *cniTypesCurr.Result + resultV6 *cniTypesCurr.Result + err error + ) + + if len(nwInfo.Subnets) > 0 { + nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() + } + + // Call into IPAM plugin to allocate an address pool for the network. + result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, &nwCfg) + if err != nil { + err = plugin.Errorf("Failed to allocate pool: %v", err) + return result, resultV6, err + } + + defer func() { + if err != nil { + plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, isDeletePoolOnError) + } + }() + + if nwCfg.IPV6Mode != "" { + nwCfg6 := nwCfg + nwCfg6.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam + nwCfg6.Ipam.Type = ipamV6 + + if len(nwInfo.Subnets) > 1 { + nwCfg6.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String() + } + + resultV6, err = plugin.DelegateAdd(ipamV6, &nwCfg6) + if err != nil { + err = plugin.Errorf("Failed to allocate v6 pool: %v", err) + } + } + + return result, resultV6, err +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md @@ -235,6 +323,7 @@ func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType st func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var ( result *cniTypesCurr.Result + resultV6 *cniTypesCurr.Result azIpamResult *cniTypesCurr.Result err error vethName string @@ -291,8 +380,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { iface = &cniTypesCurr.Interface{ Name: args.IfName, } - result.Interfaces = append(result.Interfaces, iface) + + if resultV6 != nil { + result.IPs = append(result.IPs, resultV6.IPs...) + } + addSnatInterface(nwCfg, result) // Convert result to the requested CNI version. res, vererr := result.GetAsVersion(nwCfg.CNIVersion) @@ -398,34 +491,22 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Creating network %v.", networkId) if !nwCfg.MultiTenancy { - // Call into IPAM plugin to allocate an address pool for the network. - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + result, resultV6, err = plugin.invokeIpamAdd(*nwCfg, nwInfo, true) if err != nil { - err = plugin.Errorf("Failed to allocate pool: %v", err) return err } - // Derive the subnet prefix from allocated IP address. + defer func() { + if err != nil { + plugin.invokeIpamDel(result, nwCfg.Ipam.Type, *nwCfg, true) + plugin.invokeIpamDel(resultV6, ipamV6, *nwCfg, true) + } + }() + subnetPrefix = result.IPs[0].Address - iface := &cniTypesCurr.Interface{Name: args.IfName} - result.Interfaces = append(result.Interfaces, iface) } - ipconfig := result.IPs[0] - gateway := ipconfig.Gateway - - // On failure, call into IPAM plugin to release the address and address pool. - defer func() { - if err != nil { - nwCfg.Ipam.Subnet = subnetPrefix.String() - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) - - nwCfg.Ipam.Address = "" - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) - } - }() - + gateway := result.IPs[0].Gateway subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) // Find the master interface. masterIfName := plugin.findMasterInterface(nwCfg, &subnetPrefix) @@ -461,23 +542,27 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Mode: nwCfg.Mode, MasterIfName: masterIfName, Subnets: []network.SubnetInfo{ - network.SubnetInfo{ + { Family: platform.AfINET, Prefix: subnetPrefix, Gateway: gateway, }, }, - BridgeName: nwCfg.Bridge, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - DNS: nwDNSInfo, - Policies: policies, - NetNs: args.Netns, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, + DNS: nwDNSInfo, + Policies: policies, + NetNs: args.Netns, DisableHairpinOnHostInterface: nwCfg.DisableHairpinOnHostInterface, + IPV6Mode: nwCfg.IPV6Mode, + ServiceCidrs: nwCfg.ServiceCidrs, } nwInfo.Options = make(map[string]interface{}) setNetworkOptions(cnsNetworkConfig, &nwInfo) + addNatIPV6SubnetInfo(nwCfg, resultV6, &nwInfo) + err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { err = plugin.Errorf("Failed to create network: %v", err) @@ -488,26 +573,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } else { if !nwCfg.MultiTenancy { // Network already exists. - subnetPrefix := nwInfo.Subnets[0].Prefix.String() - log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix) - nwCfg.Ipam.Subnet = subnetPrefix + log.Printf("[cni-net] Found network %v with subnet %v.", networkId, nwInfo.Subnets[0].Prefix.String()) - // Call into IPAM plugin to allocate an address for the endpoint. - result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + result, resultV6, err = plugin.invokeIpamAdd(*nwCfg, nwInfo, false) if err != nil { - err = plugin.Errorf("Failed to allocate address: %v", err) return err } - ipconfig := result.IPs[0] - iface := &cniTypesCurr.Interface{Name: args.IfName} - result.Interfaces = append(result.Interfaces, iface) - - // On failure, call into IPAM plugin to release the address. defer func() { if err != nil { - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + plugin.invokeIpamDel(result, nwCfg.Ipam.Type, *nwCfg, false) + plugin.invokeIpamDel(resultV6, ipamV6, *nwCfg, false) } }() } @@ -535,6 +611,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { PODName: k8sPodName, PODNameSpace: k8sNamespace, SkipHotAttachEp: false, // Hot attach at the time of endpoint creation + IPV6Mode: nwCfg.IPV6Mode, + VnetCidrs: nwCfg.VnetCidrs, + ServiceCidrs: nwCfg.ServiceCidrs, } epPolicies := getPoliciesFromRuntimeCfg(nwCfg) @@ -547,6 +626,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } + if resultV6 != nil { + for _, ipconfig := range resultV6.IPs { + epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) + } + } + // Populate routes. for _, route := range result.Routes { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) @@ -694,9 +779,10 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { k8sPodName string k8sNamespace string networkId string - nwInfo *network.NetworkInfo + nwInfo network.NetworkInfo epInfo *network.EndpointInfo cniMetric telemetry.AIMetric + msg string ) startTime := time.Now() @@ -704,7 +790,9 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v, StdinData:%s}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - defer func() { log.Printf("[cni-net] DEL command completed with err:%v.", err) }() + defer func() { + log.Printf("[cni-net] DEL command completed with err:%v.", err) + }() // Parse network configuration from stdin. if nwCfg, err = cni.ParseNetworkConfig(args.StdinData); err != nil { @@ -714,14 +802,14 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) - iptables.DisableIPTableLock = nwCfg.DisableIPTableLock - plugin.setCNIReportDetails(nwCfg, CNI_DEL, "") - // Parse Pod arguments. if k8sPodName, k8sNamespace, err = plugin.getPodInfo(args.Args); err != nil { log.Printf("[cni-net] Failed to get POD info due to error: %v", err) } + plugin.setCNIReportDetails(nwCfg, CNI_DEL, "") + iptables.DisableIPTableLock = nwCfg.DisableIPTableLock + if nwCfg.MultiTenancy { // Initialize CNSClient cnsclient.InitCnsClient(nwCfg.CNSUrl) @@ -750,6 +838,17 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { return err } + defer func() { + operationTimeMs := time.Since(startTime).Milliseconds() + cniMetric.Metric = aitelemetry.Metric{ + Name: telemetry.CNIDelTimeMetricStr, + Value: float64(operationTimeMs), + CustomDimensions: make(map[string]string), + } + SetCustomDimensions(&cniMetric, nwCfg, err) + telemetry.SendCNIMetric(&cniMetric, plugin.tb) + }() + // Delete the endpoint. if err = plugin.nm.DeleteEndpoint(networkId, endpointId); err != nil { err = plugin.Errorf("Failed to delete endpoint: %v", err) @@ -758,13 +857,30 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { if !nwCfg.MultiTenancy { // Call into IPAM plugin to release the endpoint's addresses. - nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() for _, address := range epInfo.IPAddresses { nwCfg.Ipam.Address = address.IP.String() - err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) - if err != nil { - err = plugin.Errorf("Failed to release address: %v", err) - return err + if address.IP.To4() != nil { + nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() + log.Printf("Releasing ipv4 address :%s pool: %s", + nwCfg.Ipam.Address, nwCfg.Ipam.Subnet) + if err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg); err != nil { + log.Printf("Failed to release ipv4 address: %v", err) + err = plugin.Errorf("Failed to release ipv4 address: %v", err) + } + } else { + nwCfgIpv6 := *nwCfg + nwCfgIpv6.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam + nwCfgIpv6.Ipam.Type = ipamV6 + if len(nwInfo.Subnets) > 1 { + nwCfgIpv6.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String() + } + + log.Printf("Releasing ipv6 address :%s pool: %s", + nwCfgIpv6.Ipam.Address, nwCfgIpv6.Ipam.Subnet) + if err = plugin.DelegateDel(nwCfgIpv6.Ipam.Type, &nwCfgIpv6); err != nil { + log.Printf("Failed to release ipv6 address: %v", err) + err = plugin.Errorf("Failed to release ipv6 address: %v", err) + } } } } else if epInfo.EnableInfraVnet { @@ -772,24 +888,15 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { nwCfg.Ipam.Address = epInfo.InfraVnetIP.IP.String() err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) if err != nil { + log.Printf("Failed to release address: %v", err) err = plugin.Errorf("Failed to release address: %v", err) - return err } } - msg := fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace) + msg = fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace) plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg) - operationTimeMs := time.Since(startTime).Milliseconds() - cniMetric.Metric = aitelemetry.Metric{ - Name: telemetry.CNIDelTimeMetricStr, - Value: float64(operationTimeMs), - CustomDimensions: make(map[string]string), - } - SetCustomDimensions(&cniMetric, nwCfg, nil) - telemetry.SendCNIMetric(&cniMetric, plugin.tb) - - return nil + return err } // Update handles CNI update commands. diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 263a30ae31..e4a7f72757 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -25,7 +25,7 @@ const ( ) // handleConsecutiveAdd is a dummy function for Linux platform. -func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 615c871496..2802cadb42 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -33,7 +33,7 @@ var ( * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ -func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { // Return in case of HNSv2 as consecutive add call doesn't need to be handled if useHnsV2, err := network.UseHnsV2(args.Netns); useHnsV2 { return nil, err diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index 119c7e494f..02d0c4cc27 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -18,10 +18,18 @@ const ( // Ebtable tables. Nat = "nat" Broute = "broute" + Filter = "filter" // Ebtable chains. PreRouting = "PREROUTING" PostRouting = "POSTROUTING" Brouting = "BROUTING" + Forward = "FORWARD" + // Ebtable Protocols + IPV4 = "IPv4" + IPV6 = "IPv6" + // Ebtable Targets + Accept = "ACCEPT" + RedirectAccept = "redirect --redirect-target ACCEPT" ) // SetSnatForInterface sets a MAC SNAT rule for an interface. @@ -84,10 +92,28 @@ func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAd // SetDnatForIPAddress sets a MAC DNAT rule for an IP address. func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error { + protocol := "IPv4" + dst := "--ip-dst" + if ipAddress.To4() == nil { + protocol = "IPv6" + dst = "--ip6-dst" + } + table := Nat chain := PreRouting - rule := fmt.Sprintf("-p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", - interfaceName, ipAddress.String(), macAddress.String()) + rule := fmt.Sprintf("-p %s -i %s %s %s -j dnat --to-dst %s --dnat-target ACCEPT", + protocol, interfaceName, dst, ipAddress.String(), macAddress.String()) + + return runEbCmd(table, action, chain, rule) +} + +// Drop Icmpv6 discovery messages going out of interface +func DropICMPv6Solicitation(interfaceName string, action string) error { + table := Filter + chain := Forward + + rule := fmt.Sprintf("-p IPv6 --ip6-proto ipv6-icmp --ip6-icmp-type neighbour-solicitation -o %s -j DROP", + interfaceName) return runEbCmd(table, action, chain, rule) } @@ -133,6 +159,41 @@ func GetEbtableRules(tableName, chainName string) ([]string, error) { return rules, nil } +// SetBrouteAcceptCidr - broute chain MAC redirect rule. Will change mac target address to bridge port +// that receives the frame. +func SetBrouteAcceptByCidr(ipNet *net.IPNet, protocol, action, target string) error { + dst := "--ip-dst" + if protocol == IPV6 { + dst = "--ip6-dst" + } + + var rule string + table := Broute + chain := Brouting + if ipNet != nil { + rule = fmt.Sprintf("-p %s %s %s -j %s", + protocol, dst, ipNet.String(), target) + } else { + rule = fmt.Sprintf("-p %s -j %s", + protocol, target) + } + + return runEbCmd(table, action, chain, rule) +} + +func SetBrouteAcceptByInterface(ifName string, protocol, action, target string) error { + var rule string + table := Broute + chain := Brouting + if protocol == "" { + rule = fmt.Sprintf("-i %s -j %s", ifName, target) + } else { + rule = fmt.Sprintf("-p %s -i %s -j %s", protocol, ifName, target) + } + + return runEbCmd(table, action, chain, rule) +} + // EbTableRuleExists checks if eb rule exists in table and chain. func EbTableRuleExists(tableName, chainName, matchSet string) (bool, error) { rules, err := GetEbtableRules(tableName, chainName) diff --git a/iptables/iptables.go b/iptables/iptables.go index a3d8d73373..98c5ee7bdb 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -28,6 +28,7 @@ const ( const ( Filter = "filter" Nat = "nat" + Mangle = "mangle" ) // target @@ -52,21 +53,32 @@ const ( const ( iptables = "iptables" + ip6tables = "ip6tables" lockTimeout = 60 ) +const ( + V4 = "4" + V6 = "6" +) + var ( DisableIPTableLock bool ) // Run iptables command -func runCmd(params string) error { +func runCmd(version, params string) error { var cmd string + iptCmd := iptables + if version == V6 { + iptCmd = ip6tables + } + if DisableIPTableLock { - cmd = fmt.Sprintf("%s %s", iptables, params) + cmd = fmt.Sprintf("%s %s", iptCmd, params) } else { - cmd = fmt.Sprintf("%s -w %d %s", iptables, lockTimeout, params) + cmd = fmt.Sprintf("%s -w %d %s", iptCmd, lockTimeout, params) } if _, err := platform.ExecuteCommand(cmd); err != nil { @@ -77,9 +89,9 @@ func runCmd(params string) error { } // check if iptable chain alreay exists -func ChainExists(tableName, chainName string) bool { +func ChainExists(version, tableName, chainName string) bool { params := fmt.Sprintf("-t %s -L %s", tableName, chainName) - if err := runCmd(params); err != nil { + if err := runCmd(version, params); err != nil { return false } @@ -87,12 +99,12 @@ func ChainExists(tableName, chainName string) bool { } // create new iptable chain under specified table name -func CreateChain(tableName, chainName string) error { +func CreateChain(version, tableName, chainName string) error { var err error - if !ChainExists(tableName, chainName) { + if !ChainExists(version, tableName, chainName) { params := fmt.Sprintf("-t %s -N %s", tableName, chainName) - err = runCmd(params) + err = runCmd(version, params) } else { log.Printf("%s Chain exists in table %s", chainName, tableName) } @@ -101,38 +113,38 @@ func CreateChain(tableName, chainName string) error { } // check if iptable rule alreay exists -func RuleExists(tableName, chainName, match, target string) bool { +func RuleExists(version, tableName, chainName, match, target string) bool { params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target) - if err := runCmd(params); err != nil { + if err := runCmd(version, params); err != nil { return false } return true } // Insert iptable rule at beginning of iptable chain -func InsertIptableRule(tableName, chainName, match, target string) error { - if RuleExists(tableName, chainName, match, target) { +func InsertIptableRule(version, tableName, chainName, match, target string) error { + if RuleExists(version, tableName, chainName, match, target) { log.Printf("Rule already exists") return nil } params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target) - return runCmd(params) + return runCmd(version, params) } // Append iptable rule at end of iptable chain -func AppendIptableRule(tableName, chainName, match, target string) error { - if RuleExists(tableName, chainName, match, target) { +func AppendIptableRule(version, tableName, chainName, match, target string) error { + if RuleExists(version, tableName, chainName, match, target) { log.Printf("Rule already exists") return nil } params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target) - return runCmd(params) + return runCmd(version, params) } // Delete matched iptable rule -func DeleteIptableRule(tableName, chainName, match, target string) error { +func DeleteIptableRule(version, tableName, chainName, match, target string) error { params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target) - return runCmd(params) + return runCmd(version, params) } diff --git a/netlink/link.go b/netlink/link.go index dfbdbaaa48..3c899a42e7 100644 --- a/netlink/link.go +++ b/netlink/link.go @@ -391,7 +391,7 @@ func SetLinkHairpin(bridgeName string, on bool) error { } // AddOrRemoveStaticArp sets/removes static arp entry based on mode -func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr) error { +func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr, isProxy bool) error { s, err := getSocket() if err != nil { return err @@ -413,13 +413,23 @@ func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.Hardware } msg := neighMsg{ - Family: uint8(unix.AF_INET), + Family: uint8(GetIpAddressFamily(ipaddr)), Index: uint32(iface.Index), State: uint16(state), } + + // NTF_PROXY is for setting neighbor proxy + if isProxy { + msg.Flags = msg.Flags | NTF_PROXY + } + req.addPayload(&msg) ipData := ipaddr.To4() + if ipData == nil { + ipData = ipaddr.To16() + } + dstData := newRtAttr(NDA_DST, ipData) req.addPayload(dstData) diff --git a/netlink/netlink_test.go b/netlink/netlink_test.go index 622db1d279..c4630b7eff 100644 --- a/netlink/netlink_test.go +++ b/netlink/netlink_test.go @@ -244,12 +244,12 @@ func TestAddRemoveStaticArp(t *testing.T) { ip := net.ParseIP("192.168.0.2") mac, _ := net.ParseMAC("aa:b3:4d:5e:e2:4a") - err = AddOrRemoveStaticArp(ADD, ifName, ip, mac) + err = AddOrRemoveStaticArp(ADD, ifName, ip, mac, false) if err != nil { t.Errorf("ret val %v", err) } - err = AddOrRemoveStaticArp(REMOVE, ifName, ip, mac) + err = AddOrRemoveStaticArp(REMOVE, ifName, ip, mac, false) if err != nil { t.Errorf("ret val %v", err) } diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 5ecbb86e6b..05397af117 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -10,6 +10,12 @@ import ( "github.com/Azure/azure-container-networking/network/epcommon" ) +const ( + defaultV6VnetCidr = "2001:1234:5678:9abc::/64" + defaultV6HostGw = "fe80::1234:5678:9abc" + defaultHostGwMac = "12:34:56:78:9a:bc" +) + type LinuxBridgeEndpointClient struct { bridgeName string hostPrimaryIfName string @@ -17,6 +23,7 @@ type LinuxBridgeEndpointClient struct { containerVethName string hostPrimaryMac net.HardwareAddr containerMac net.HardwareAddr + hostIPAddresses []*net.IPNet mode string } @@ -33,9 +40,14 @@ func NewLinuxBridgeEndpointClient( hostVethName: hostVethName, containerVethName: containerVethName, hostPrimaryMac: extIf.MacAddress, + hostIPAddresses: []*net.IPNet{}, mode: mode, } + for _, ipAddr := range extIf.IPAddresses { + client.hostIPAddresses = append(client.hostIPAddresses, ipAddr) + } + return client } @@ -62,10 +74,12 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) } for _, ipAddr := range epInfo.IPAddresses { - // Add ARP reply rule. - log.Printf("[net] Adding ARP reply rule for IP address %v", ipAddr.String()) - if err = ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(client.containerMac), ebtables.Append); err != nil { - return err + if ipAddr.IP.To4() != nil { + // Add ARP reply rule. + log.Printf("[net] Adding ARP reply rule for IP address %v", ipAddr.String()) + if err = ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(client.containerMac), ebtables.Append); err != nil { + return err + } } // Add MAC address translation rule. @@ -74,9 +88,9 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) return err } - if client.mode != opModeTunnel { + if client.mode != opModeTunnel && ipAddr.IP.To4() != nil { log.Printf("[net] Adding static arp for IP address %v and MAC %v in VM", ipAddr.String(), client.containerMac.String()) - if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac); err != nil { + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac, false); err != nil { log.Printf("Failed setting arp in vm: %v", err) } } @@ -96,23 +110,25 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) func (client *LinuxBridgeEndpointClient) DeleteEndpointRules(ep *endpoint) { // Delete rules for IP addresses on the container interface. for _, ipAddr := range ep.IPAddresses { - // Delete ARP reply rule. - log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err := ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(ep.MacAddress), ebtables.Delete) - if err != nil { - log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) + if ipAddr.IP.To4() != nil { + // Delete ARP reply rule. + log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id) + err := ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(ep.MacAddress), ebtables.Delete) + if err != nil { + log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err) + } } // Delete MAC address translation rule. log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id) - err = ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, ep.MacAddress, ebtables.Delete) + err := ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, ep.MacAddress, ebtables.Delete) if err != nil { log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err) } - if client.mode != opModeTunnel { + if client.mode != opModeTunnel && ipAddr.IP.To4() != nil { log.Printf("[net] Removing static arp for IP address %v and MAC %v from VM", ipAddr.String(), ep.MacAddress.String()) - netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress) + err := netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress, false) if err != nil { log.Printf("Failed removing arp from vm: %v", err) } @@ -156,6 +172,13 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if epInfo.IPV6Mode != "" { + // Enable ipv6 setting in container + if err := epcommon.UpdateIPV6Setting(0); err != nil { + return err + } + } + if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } @@ -164,6 +187,14 @@ func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(e return err } + if err := client.setupIPV6Routes(epInfo); err != nil { + return err + } + + if err := client.setIPV6NeighEntry(epInfo); err != nil { + return err + } + return nil } @@ -207,3 +238,63 @@ func addRuleToRouteViaHost(epInfo *EndpointInfo) error { return nil } + +func (client *LinuxBridgeEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) error { + if epInfo.IPV6Mode != "" { + if epInfo.VnetCidrs == "" { + epInfo.VnetCidrs = defaultV6VnetCidr + } + + routes := []RouteInfo{} + _, v6IpNet, _ := net.ParseCIDR(epInfo.VnetCidrs) + v6Gw := net.ParseIP(defaultV6HostGw) + vnetRoute := RouteInfo{ + Dst: *v6IpNet, + Gw: v6Gw, + Priority: 101, + } + + var vmV6Route RouteInfo + + for _, ipAddr := range client.hostIPAddresses { + if ipAddr.IP.To4() == nil { + vmV6Route = RouteInfo{ + Dst: *ipAddr, + Priority: 100, + } + } + } + + _, defIPNet, _ := net.ParseCIDR("::/0") + defaultV6Route := RouteInfo{ + Dst: *defIPNet, + Gw: v6Gw, + } + + routes = append(routes, vnetRoute) + routes = append(routes, vmV6Route) + routes = append(routes, defaultV6Route) + + log.Printf("[net] Adding ipv6 routes in container %+v", routes) + if err := addRoutes(client.containerVethName, routes); err != nil { + return nil + } + } + + return nil +} + +func (client *LinuxBridgeEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) error { + if epInfo.IPV6Mode != "" { + log.Printf("[net] Add neigh entry for host gw ip") + hardwareAddr, _ := net.ParseMAC(defaultHostGwMac) + hostGwIp := net.ParseIP(defaultV6HostGw) + if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, + hostGwIp, hardwareAddr, false); err != nil { + log.Printf("Failed setting neigh entry in container: %v", err) + return err + } + } + + return nil +} diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index 24ebb0b79d..1c72f3b848 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -6,18 +6,23 @@ import ( "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" +) + +const ( + multicastSolicitPrefix = "ff02::1:ff00:0/104" ) type LinuxBridgeClient struct { bridgeName string hostInterfaceName string - mode string + nwInfo NetworkInfo } -func NewLinuxBridgeClient(bridgeName string, hostInterfaceName string, mode string) *LinuxBridgeClient { +func NewLinuxBridgeClient(bridgeName string, hostInterfaceName string, nwInfo NetworkInfo) *LinuxBridgeClient { client := &LinuxBridgeClient{ bridgeName: bridgeName, - mode: mode, + nwInfo: nwInfo, hostInterfaceName: hostInterfaceName, } @@ -80,8 +85,32 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { return err } + if client.nwInfo.IPV6Mode != "" { + // for ipv6 node cidr set broute accept + if err := ebtables.SetBrouteAcceptByCidr(&client.nwInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { + return err + } + + _, mIpNet, _ := net.ParseCIDR(multicastSolicitPrefix) + if err := ebtables.SetBrouteAcceptByCidr(mIpNet, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil { + return err + } + + if err := ebtables.DropICMPv6Solicitation(client.hostInterfaceName, ebtables.Append); err != nil { + return err + } + + if err := client.setBrouteRedirect(ebtables.Append); err != nil { + return err + } + + if err := epcommon.EnableIPV6Forwarding(); err != nil { + return err + } + } + // Enable VEPA for host policy enforcement if necessary. - if client.mode == opModeTunnel { + if client.nwInfo.Mode == opModeTunnel { log.Printf("[net] Enabling VEPA mode for %v.", client.hostInterfaceName) if err := ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append); err != nil { return err @@ -96,6 +125,15 @@ func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) { ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete) ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) + if client.nwInfo.IPV6Mode != "" { + if len(extIf.IPAddresses) > 1 { + ebtables.SetBrouteAcceptByCidr(extIf.IPAddresses[1], ebtables.IPV6, ebtables.Delete, ebtables.Accept) + } + _, mIpNet, _ := net.ParseCIDR(multicastSolicitPrefix) + ebtables.SetBrouteAcceptByCidr(mIpNet, ebtables.IPV6, ebtables.Delete, ebtables.Accept) + client.setBrouteRedirect(ebtables.Delete) + ebtables.DropICMPv6Solicitation(extIf.Name, ebtables.Delete) + } } func (client *LinuxBridgeClient) SetBridgeMasterToHostInterface() error { @@ -105,3 +143,17 @@ func (client *LinuxBridgeClient) SetBridgeMasterToHostInterface() error { func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error { return netlink.SetLinkHairpin(client.hostInterfaceName, enable) } + +func (client *LinuxBridgeClient) setBrouteRedirect(action string) error { + if client.nwInfo.ServiceCidrs != "" { + if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV4, ebtables.Append, ebtables.RedirectAccept); err != nil { + return err + } + + if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV6, ebtables.Append, ebtables.RedirectAccept); err != nil { + return err + } + } + + return nil +} diff --git a/network/endpoint.go b/network/endpoint.go index 529092234b..bf4d19cad1 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -72,6 +72,9 @@ type EndpointInfo struct { Data map[string]interface{} InfraVnetAddressSpace string SkipHotAttachEp bool + IPV6Mode string + VnetCidrs string + ServiceCidrs string } // RouteInfo contains information about an IP route. @@ -82,6 +85,7 @@ type RouteInfo struct { Protocol int DevName string Scope int + Priority int } // NewEndpoint creates a new endpoint in the network. diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 453691bffb..5ea6d9345d 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -258,11 +258,17 @@ func addRoutes(interfaceName string, routes []RouteInfo) error { ifIndex = interfaceIf.Index } + family := netlink.GetIpAddressFamily(route.Gw) + if route.Gw == nil { + family = netlink.GetIpAddressFamily(route.Dst.IP) + } + nlRoute := &netlink.Route{ - Family: netlink.GetIpAddressFamily(route.Gw), + Family: family, Dst: &route.Dst, Gw: route.Gw, LinkIndex: ifIndex, + Priority: route.Priority, } if err := netlink.AddIpRoute(nlRoute); err != nil { diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go index 5d7c9d5065..923f918447 100644 --- a/network/epcommon/endpoint_common.go +++ b/network/epcommon/endpoint_common.go @@ -28,7 +28,9 @@ RFC for Link Local Addresses: https://tools.ietf.org/html/rfc3927 */ const ( - enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1" + enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1" + toggleIPV6Cmd = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d" + enableIPV6ForwardCmd = "sysctl -w net.ipv6.conf.all.forwarding=1" ) func getPrivateIPSpace() []string { @@ -115,11 +117,11 @@ func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, c switch action { case iptables.Insert: - err = iptables.InsertIptableRule(iptables.Filter, chainName, matchCondition, target) + err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) case iptables.Append: - err = iptables.AppendIptableRule(iptables.Filter, chainName, matchCondition, target) + err = iptables.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) case iptables.Delete: - err = iptables.DeleteIptableRule(iptables.Filter, chainName, matchCondition, target) + err = iptables.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target) } return err @@ -173,9 +175,7 @@ func BlockIPAddresses(bridgeName string, action string) error { return nil } -/** - This fucntion enables ip forwarding in VM and allow forwarding packets from the interface -**/ +// This fucntion enables ip forwarding in VM and allow forwarding packets from the interface func EnableIPForwarding(ifName string) error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 @@ -187,10 +187,44 @@ func EnableIPForwarding(ifName string) error { } // Append a rule in forward chain to allow forwarding from bridge - if err := iptables.AppendIptableRule(iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { + if err := iptables.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil { log.Printf("[net] Appending forward chain rule: allow traffic coming from snatbridge failed with: %v", err) return err } return nil } + +func EnableIPV6Forwarding() error { + cmd := fmt.Sprintf(enableIPV6ForwardCmd) + _, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("[net] Enable ipv6 forwarding failed with: %v", err) + return err + } + + return nil +} + +// This functions enables/disables ipv6 setting based on enable parameter passed. +func UpdateIPV6Setting(disable int) error { + // sysctl -w net.ipv6.conf.all.disable_ipv6=0/1 + cmd := fmt.Sprintf(toggleIPV6Cmd, disable) + _, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("[net] Update IPV6 Setting failed with: %v", err) + } + + return err +} + +// This fucntion adds rule which snat to ip passed filtered by match string. +func AddSnatRule(match string, ip net.IP) error { + version := iptables.V4 + if ip.To4() == nil { + version = iptables.V6 + } + + target := fmt.Sprintf("SNAT --to %s", ip.String()) + return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) +} diff --git a/network/manager.go b/network/manager.go index 18252be530..d5468f50da 100644 --- a/network/manager.go +++ b/network/manager.go @@ -58,7 +58,7 @@ type NetworkManager interface { CreateNetwork(nwInfo *NetworkInfo) error DeleteNetwork(networkId string) error - GetNetworkInfo(networkId string) (*NetworkInfo, error) + GetNetworkInfo(networkId string) (NetworkInfo, error) CreateEndpoint(networkId string, epInfo *EndpointInfo) error DeleteEndpoint(networkId string, endpointId string) error @@ -174,7 +174,7 @@ func (nm *networkManager) restore() error { extIf.BridgeName = "" - _, err = nm.newNetworkImpl(nwInfo, extIf) + _, err = nm.newNetworkImpl(&nwInfo, extIf) if err != nil { log.Printf("[net] Restoring network failed for nwInfo %v extif %v. This should not happen %v", nwInfo, extIf, err) return err @@ -274,16 +274,16 @@ func (nm *networkManager) DeleteNetwork(networkId string) error { } // GetNetworkInfo returns information about the given network. -func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) { +func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) { nm.Lock() defer nm.Unlock() nw, err := nm.getNetwork(networkId) if err != nil { - return nil, err + return NetworkInfo{}, err } - nwInfo := &NetworkInfo{ + nwInfo := NetworkInfo{ Id: networkId, Subnets: nw.Subnets, Mode: nw.Mode, @@ -292,7 +292,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) Options: make(map[string]interface{}), } - getNetworkInfoImpl(nwInfo, nw) + getNetworkInfoImpl(&nwInfo, nw) if nw.extIf != nil { nwInfo.BridgeName = nw.extIf.BridgeName diff --git a/network/network.go b/network/network.go index bdaff589b7..db39bb1882 100644 --- a/network/network.go +++ b/network/network.go @@ -20,6 +20,11 @@ const ( opModeDefault = opModeTunnel ) +const ( + // ipv6 modes + IPV6Nat = "ipv6nat" +) + // ExternalInterface is a host network interface that bridges containers to external networks. type externalInterface struct { Name string @@ -62,6 +67,8 @@ type NetworkInfo struct { NetNs string Options map[string]interface{} DisableHairpinOnHostInterface bool + IPV6Mode string + ServiceCidrs string } // SubnetInfo contains subnet information for a container network. diff --git a/network/network_linux.go b/network/network_linux.go index 5c7e3531d6..be0dfad817 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -9,8 +9,10 @@ import ( "strconv" "strings" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" "github.com/Azure/azure-container-networking/platform" "golang.org/x/sys/unix" ) @@ -90,7 +92,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { if nw.VlanId != 0 { networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name) } else { - networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) + networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}) } // Disconnect the interface if this was the last network using it. @@ -287,8 +289,11 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { // ConnectExternalInterface connects the given host interface to a bridge. func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error { - var err error - var networkClient NetworkClient + var ( + err error + networkClient NetworkClient + ) + log.Printf("[net] Connecting interface %v.", extIf.Name) defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }() @@ -314,25 +319,18 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI if opt != nil && opt[VlanIDKey] != nil { networkClient = NewOVSClient(bridgeName, extIf.Name) } else { - networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo) } // Check if the bridge already exists. bridge, err := net.InterfaceByName(bridgeName) if err != nil { // Create the bridge. - if err := networkClient.CreateBridge(); err != nil { + if err = networkClient.CreateBridge(); err != nil { log.Printf("Error while creating bridge %+v", err) return err } - // On failure, delete the bridge. - defer func() { - if err != nil { - networkClient.DeleteBridge() - } - }() - bridge, err = net.InterfaceByName(bridgeName) if err != nil { return err @@ -342,6 +340,13 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI log.Printf("[net] Found existing bridge %v.", bridgeName) } + defer func() { + if err != nil { + log.Printf("[net] cleanup network") + nm.disconnectExternalInterface(extIf, networkClient) + } + }() + // Save host IP configuration. err = nm.saveIPConfig(hostIf, extIf) if err != nil { @@ -351,7 +356,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI isGreaterOrEqualUbuntu17 := isGreaterOrEqaulUbuntuVersion(ubuntuVersion17) if isGreaterOrEqualUbuntu17 { log.Printf("[net] Saving dns config from %v", extIf.Name) - if err := saveDnsConfig(extIf); err != nil { + if err = saveDnsConfig(extIf); err != nil { log.Printf("[net] Failed to save dns config: %v", err) return err } @@ -366,7 +371,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI // Connect the external interface to the bridge. log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName) - if err := networkClient.SetBridgeMasterToHostInterface(); err != nil { + if err = networkClient.SetBridgeMasterToHostInterface(); err != nil { return err } @@ -393,7 +398,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI // External interface hairpin on. if !nwInfo.DisableHairpinOnHostInterface { log.Printf("[net] Setting link %v hairpin on.", hostIf.Name) - if err := networkClient.SetHairpinOnHostInterface(true); err != nil { + if err = networkClient.SetHairpinOnHostInterface(true); err != nil { return err } } @@ -408,7 +413,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI if isGreaterOrEqualUbuntu17 { log.Printf("[net] Applying dns config on %v", bridgeName) - if err := applyDnsConfig(extIf, bridgeName); err != nil { + if err = applyDnsConfig(extIf, bridgeName); err != nil { log.Printf("[net] Failed to apply DNS configuration: %v.", err) return err } @@ -416,6 +421,26 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI log.Printf("[net] Applied dns config %v on %v", extIf.DNSInfo, bridgeName) } + if nwInfo.IPV6Mode == IPV6Nat { + // adds pod cidr gateway ip to bridge + if err = addIpv6NatGateway(nwInfo); err != nil { + log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err) + return err + } + + if err = addIpv6SnatRule(extIf, nwInfo); err != nil { + log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err) + return err + } + + // unmark packet if set by kube-proxy to skip kube-postrouting rule and processed + // by cni snat rule + if err = iptables.InsertIptableRule(iptables.V6, iptables.Mangle, iptables.Postrouting, "", "MARK --set-mark 0x0"); err != nil { + log.Errorf("[net] Adding Iptable mangle rule failed:%v", err) + return err + } + } + extIf.BridgeName = bridgeName log.Printf("[net] Connected interface %v to bridge %v.", extIf.Name, extIf.BridgeName) @@ -450,6 +475,36 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface, log.Printf("[net] Disconnected interface %v.", extIf.Name) } +// Add ipv6 nat gateway IP on bridge +func addIpv6NatGateway(nwInfo *NetworkInfo) error { + log.Printf("[net] Adding ipv6 nat gateway on azure bridge") + for _, subnetInfo := range nwInfo.Subnets { + if subnetInfo.Family == platform.AfINET6 { + ipAddr := []net.IPNet{{ + IP: subnetInfo.Gateway, + Mask: subnetInfo.Prefix.Mask, + }} + return epcommon.AssignIPToInterface(nwInfo.BridgeName, ipAddr) + } + } + + return nil +} + +// snat ipv6 traffic to secondary ipv6 ip before leaving VM +func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { + log.Printf("[net] Adding ipv6 snat rule") + for _, ipAddr := range extIf.IPAddresses { + if ipAddr.IP.To4() == nil { + if err := epcommon.AddSnatRule("", ipAddr.IP); err != nil { + return err + } + } + } + + return nil +} + func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { if nw.VlanId != 0 { vlanMap := make(map[string]interface{}) diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index 63a8eaa7c3..9891189597 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -141,40 +141,40 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Ouptut chain - if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return err } // Forward traffic from Ouptut chain to CNI Output chain - if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) return err } // Allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) return err } // Create cniinput chain - if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil { + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) return err } // Forward from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) return err } // Accept packets from NC only if established connection matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) return err @@ -184,7 +184,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { // Add static arp entry for localIP to prevent arp going out of VM log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) - err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) + err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) if err != nil { log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) } @@ -197,14 +197,14 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { // Delete allow connection from Host to NC matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) } // Remove static arp entry added for container local IP log.Printf("Removing static arp entry for ip %s ", containerIP) - err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) + err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) if err != nil { log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s: %v", containerIP, err) } @@ -219,40 +219,40 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Input chain - if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil { + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) return err } // Forward traffic from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) return err } // Allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) return err } // Create CNI output chain - if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil { + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return err } // Forward traffic from Output to CNI Output chain - if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) return err } // Accept packets from Host only if established connection matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) return err @@ -262,7 +262,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { // Add static arp entry for localIP to prevent arp going out of VM log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) - err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr) + err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) if err != nil { log.Printf("AllowInboundFromNCToHost: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) } @@ -275,14 +275,14 @@ func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { // Delete allow NC to Host connection matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err) } // Remove static arp entry added for container local IP log.Printf("Removing static arp entry for ip %s ", containerIP) - err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil) + err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) if err != nil { log.Printf("DeleteInboundFromNCToHost: Error removing static arp entry for ip %s: %v", containerIP, err) } @@ -417,7 +417,7 @@ func DeleteSnatBridge(bridgeName string) error { func AddMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return iptables.InsertIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) + return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) } func DeleteMasqueradeRule() error { @@ -436,7 +436,7 @@ func DeleteMasqueradeRule() error { if ipAddr.To4() != nil { matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return iptables.DeleteIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) + return iptables.DeleteIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) } } diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go index 463ff2301c..56b8d5ba14 100644 --- a/telemetry/telemetry_test.go +++ b/telemetry/telemetry_test.go @@ -124,6 +124,7 @@ func TestMain(m *testing.M) { } tb.Cleanup(FdName) + ipamAgent.Stop() os.Exit(exitCode) }