diff --git a/Makefile b/Makefile index c87533d682..bc59dcf4ab 100644 --- a/Makefile +++ b/Makefile @@ -460,7 +460,8 @@ PRETTYGOTEST := $(shell command -v gotest 2> /dev/null) # run all tests .PHONY: test-all test-all: - go test -coverpkg=./... -v -race -covermode atomic -failfast -coverprofile=coverage.out ./... + go test -tags "unit" -coverpkg=./... -v -race -covermode atomic -failfast -coverprofile=coverage.out ./... + # run all tests .PHONY: test-integration diff --git a/cni/api/api.go b/cni/api/api.go new file mode 100644 index 0000000000..0d062331f0 --- /dev/null +++ b/cni/api/api.go @@ -0,0 +1,41 @@ +package api + +import ( + "encoding/json" + "net" + "os" + + "github.com/Azure/azure-container-networking/log" +) + +type PodNetworkInterfaceInfo struct { + PodName string + PodNamespace string + PodEndpointId string + ContainerID string + IPAddresses []net.IPNet +} + +type CNIState interface { + PrintResult() error +} + +type AzureCNIState struct { + ContainerInterfaces map[string]PodNetworkInterfaceInfo +} + +func (a *AzureCNIState) PrintResult() error { + b, err := json.MarshalIndent(a, "", " ") + if err != nil { + log.Errorf("Failed to unmarshall Azure CNI state, err:%v.\n", err) + } + + // write result to stdout to be captured by caller + _, err = os.Stdout.Write(b) + if err != nil { + log.Printf("Failed to write response to stdout %v", err) + return err + } + + return nil +} diff --git a/cni/client/client.go b/cni/client/client.go new file mode 100644 index 0000000000..bce421b1c5 --- /dev/null +++ b/cni/client/client.go @@ -0,0 +1,54 @@ +package client + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cni/api" + "github.com/Azure/azure-container-networking/log" + utilexec "k8s.io/utils/exec" +) + +const ( + azureVnetBinName = "./azure-vnet" + azureVnetBinDirectory = "/opt/cni/bin" +) + +type CNIClient interface { + GetEndpointState() (api.CNIState, error) +} + +type AzureCNIClient struct { + exec utilexec.Interface +} + +func NewCNIClient(exec utilexec.Interface) *AzureCNIClient { + return &AzureCNIClient{ + exec: exec, + } +} + +func (c *AzureCNIClient) GetEndpointState() (api.CNIState, error) { + cmd := c.exec.Command(azureVnetBinName) + cmd.SetDir(azureVnetBinDirectory) + + envs := os.Environ() + cmdenv := fmt.Sprintf("%s=%s", cni.Cmd, cni.CmdGetEndpointsState) + log.Printf("Setting cmd to %s", cmdenv) + envs = append(envs, cmdenv) + cmd.SetEnv(envs) + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to call Azure CNI bin with err: [%w], output: [%s]", err, string(output)) + } + + state := &api.AzureCNIState{} + if err := json.Unmarshal(output, state); err != nil { + return nil, fmt.Errorf("failed to decode response from Azure CNI when retrieving state: [%w], response from CNI: [%s]", err, string(output)) + } + + return state, nil +} diff --git a/cni/client/client_common_test.go b/cni/client/client_common_test.go new file mode 100644 index 0000000000..bb2539a446 --- /dev/null +++ b/cni/client/client_common_test.go @@ -0,0 +1,23 @@ +// +build unit integration + +package client + +import ( + "net" + + "github.com/Azure/azure-container-networking/cni/api" +) + +func testGetPodNetworkInterfaceInfo(podendpointid, podname, podnamespace, containerid, ipwithcidr string) api.PodNetworkInterfaceInfo { + ip, ipnet, _ := net.ParseCIDR(ipwithcidr) + ipnet.IP = ip + return api.PodNetworkInterfaceInfo{ + PodName: podname, + PodNamespace: podnamespace, + PodEndpointId: podendpointid, + ContainerID: containerid, + IPAddresses: []net.IPNet{ + *ipnet, + }, + } +} diff --git a/cni/client/client_integration_test.go b/cni/client/client_integration_test.go new file mode 100644 index 0000000000..9af07595e6 --- /dev/null +++ b/cni/client/client_integration_test.go @@ -0,0 +1,54 @@ +// +build linux +// +build integration + +package client + +import ( + "io" + "os" + "testing" + + "github.com/Azure/azure-container-networking/cni/api" + testutils "github.com/Azure/azure-container-networking/test/utils" + "github.com/stretchr/testify/require" + "k8s.io/utils/exec" +) + +// todo: enable this test in CI, requires built azure vnet +func TestGetStateFromAzureCNI(t *testing.T) { + testutils.RequireRootforTest(t) + + // copy test state file to /var/run/azure-vnet.json + in, err := os.Open("./testresources/azure-vnet-test.json") + require.NoError(t, err) + + defer in.Close() + + out, err := os.Create("/var/run/azure-vnet.json") + require.NoError(t, err) + + defer func() { + out.Close() + err := os.Remove("/var/run/azure-vnet.json") + require.NoError(t, err) + }() + + _, err = io.Copy(out, in) + require.NoError(t, err) + + out.Close() + + realexec := exec.New() + c := NewCNIClient(realexec) + state, err := c.GetEndpointState() + require.NoError(t, err) + + res := &api.AzureCNIState{ + ContainerInterfaces: map[string]api.PodNetworkInterfaceInfo{ + "3f813b02-eth0": testGetPodNetworkInterfaceInfo("3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system", "3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "10.241.0.17/16"), + "6e688597-eth0": testGetPodNetworkInterfaceInfo("6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system", "6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "10.241.0.13/16"), + }, + } + + require.Exactly(t, res, state) +} diff --git a/cni/client/client_unit_test.go b/cni/client/client_unit_test.go new file mode 100644 index 0000000000..dc39ebf591 --- /dev/null +++ b/cni/client/client_unit_test.go @@ -0,0 +1,32 @@ +// +build unit + +package client + +import ( + "testing" + + "github.com/Azure/azure-container-networking/cni/api" + testutils "github.com/Azure/azure-container-networking/test/utils" + "github.com/stretchr/testify/require" +) + +func TestGetState(t *testing.T) { + calls := []testutils.TestCmd{ + {Cmd: []string{"./azure-vnet"}, Stdout: `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`}, + } + + fakeexec, _ := testutils.GetFakeExecWithScripts(calls) + + c := NewCNIClient(fakeexec) + state, err := c.GetEndpointState() + require.NoError(t, err) + + res := &api.AzureCNIState{ + ContainerInterfaces: map[string]api.PodNetworkInterfaceInfo{ + "3f813b02-eth0": testGetPodNetworkInterfaceInfo("3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system", "3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "10.241.0.17/16"), + "6e688597-eth0": testGetPodNetworkInterfaceInfo("6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system", "6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "10.241.0.13/16"), + }, + } + + require.Equal(t, res, state) +} diff --git a/cni/client/testresources/azure-vnet-test.json b/cni/client/testresources/azure-vnet-test.json new file mode 100644 index 0000000000..dceefe71a6 --- /dev/null +++ b/cni/client/testresources/azure-vnet-test.json @@ -0,0 +1,155 @@ +{ + "Network": { + "Version": "v1.2.6", + "TimeStamp": "2021-06-04T17:15:58.638215441Z", + "ExternalInterfaces": { + "eth0": { + "Name": "eth0", + "Networks": { + "azure": { + "Id": "azure", + "Mode": "transparent", + "VlanId": 0, + "Subnets": [ + { + "Family": 2, + "Prefix": { + "IP": "10.240.0.0", + "Mask": "//8AAA==" + }, + "Gateway": "10.241.0.1", + "PrimaryIP": "" + } + ], + "Endpoints": { + "3f813b02-eth0": { + "Id": "3f813b02-eth0", + "SandboxKey": "", + "IfName": "azvd805fb1b0f82", + "HostIfName": "azvd805fb1b0f8", + "MacAddress": "ipNAs8jK", + "InfraVnetIP": { + "IP": "", + "Mask": null + }, + "LocalIP": "", + "IPAddresses": [ + { + "IP": "10.241.0.17", + "Mask": "//8AAA==" + } + ], + "Gateways": [ + "0.0.0.0" + ], + "DNS": { + "Suffix": "", + "Servers": null, + "Options": null + }, + "Routes": [ + { + "Dst": { + "IP": "0.0.0.0", + "Mask": "AAAAAA==" + }, + "Src": "", + "Gw": "10.241.0.1", + "Protocol": 0, + "DevName": "", + "Scope": 0, + "Priority": 0 + } + ], + "VlanID": 0, + "EnableSnatOnHost": false, + "EnableInfraVnet": false, + "EnableMultitenancy": false, + "AllowInboundFromHostToNC": false, + "AllowInboundFromNCToHost": false, + "NetworkContainerID": "", + "NetworkNameSpace": "/var/run/netns/cni-e66c52d6-44be-555b-fb65-0b36296b6861", + "ContainerID": "3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", + "PODName": "metrics-server-77c8679d7d-6ksdh", + "PODNameSpace": "kube-system" + }, + "6e688597-eth0": { + "Id": "6e688597-eth0", + "SandboxKey": "", + "IfName": "azvc214c1237ce2", + "HostIfName": "azvc214c1237ce", + "MacAddress": "ZjL4HSJ+", + "InfraVnetIP": { + "IP": "", + "Mask": null + }, + "LocalIP": "", + "IPAddresses": [ + { + "IP": "10.241.0.13", + "Mask": "//8AAA==" + } + ], + "Gateways": [ + "0.0.0.0" + ], + "DNS": { + "Suffix": "", + "Servers": null, + "Options": null + }, + "Routes": [ + { + "Dst": { + "IP": "0.0.0.0", + "Mask": "AAAAAA==" + }, + "Src": "", + "Gw": "10.241.0.1", + "Protocol": 0, + "DevName": "", + "Scope": 0, + "Priority": 0 + } + ], + "VlanID": 0, + "EnableSnatOnHost": false, + "EnableInfraVnet": false, + "EnableMultitenancy": false, + "AllowInboundFromHostToNC": false, + "AllowInboundFromNCToHost": false, + "NetworkContainerID": "", + "NetworkNameSpace": "/var/run/netns/cni-56872dcd-b3ab-fc90-ccd8-6a6dd11d56cf", + "ContainerID": "6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", + "PODName": "tunnelfront-5d96f9b987-65xbn", + "PODNameSpace": "kube-system" + } + }, + "DNS": { + "Suffix": "", + "Servers": null, + "Options": null + }, + "EnableSnatOnHost": false, + "NetNs": "", + "SnatBridgeIP": "" + } + }, + "Subnets": [ + "10.240.0.0/16" + ], + "BridgeName": "", + "DNSInfo": { + "Suffix": "", + "Servers": null, + "Options": null + }, + "MacAddress": "AA06xXdb", + "IPAddresses": null, + "Routes": null, + "IPv4Gateway": "0.0.0.0", + "IPv6Gateway": "::" + } + } + } +} diff --git a/cni/cni.go b/cni/cni.go index 239eb6301d..bfcd1b1854 100644 --- a/cni/cni.go +++ b/cni/cni.go @@ -15,6 +15,9 @@ const ( CmdDel = "DEL" CmdUpdate = "UPDATE" + // nonstandard CNI spec command, used to dump CNI state to stdout + CmdGetEndpointsState = "GET_ENDPOINT_STATE" + // CNI errors. ErrRuntime = 100 diff --git a/cni/ipam/ipam_test.go b/cni/ipam/ipam_test.go index d480e31b1e..cc6c32ef17 100644 --- a/cni/ipam/ipam_test.go +++ b/cni/ipam/ipam_test.go @@ -6,17 +6,18 @@ package ipam import ( "encoding/json" "fmt" - cniSkel "github.com/containernetworking/cni/pkg/skel" - cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" - "github.com/google/uuid" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "net" "net/http" "net/url" "testing" "time" + cniSkel "github.com/containernetworking/cni/pkg/skel" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/Azure/azure-container-networking/common" ) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 3afc4a95d8..aceb3e242a 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -4,7 +4,7 @@ import ( "net" "github.com/Azure/azure-container-networking/cni" - "github.com/Azure/azure-container-networking/network" + cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -14,8 +14,8 @@ import ( type IPAMInvoker interface { //Add returns two results, one IPv4, the other IPv6. - Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) + Add(nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) //Delete calls to the invoker source, and returns error. Returning an error here will fail the CNI Delete call. - Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, epInfo *network.EndpointInfo, options map[string]interface{}) error + Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, options map[string]interface{}) error } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index bb3c99361c..0850755977 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" + cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -24,7 +25,7 @@ func NewAzureIpamInvoker(plugin *netPlugin, nwInfo *network.NetworkInfo) *AzureI } } -func (invoker *AzureIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) { +func (invoker *AzureIPAMInvoker) Add(nwCfg *cni.NetworkConfig, _ *cniSkel.CmdArgs, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) { var ( result *cniTypesCurr.Result resultV6 *cniTypesCurr.Result @@ -44,13 +45,15 @@ func (invoker *AzureIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net result, err = invoker.plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) if err != nil { err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) - return nil, nil, err + return nil, nil, err } defer func() { if err != nil { if len(result.IPs) > 0 { - invoker.plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, nil, options) + if er := invoker.plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, nil, options); er != nil { + err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) + } } else { err = fmt.Errorf("No IP's to delete on error: %v", err) } @@ -79,7 +82,7 @@ func (invoker *AzureIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net return result, resultV6, err } -func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, _ *network.EndpointInfo, options map[string]interface{}) error { +func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, _ *cniSkel.CmdArgs, options map[string]interface{}) error { if nwCfg == nil { return invoker.plugin.Errorf("nil nwCfg passed to CNI ADD, stack: %+v", string(debug.Stack())) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 89a173cde4..2c593de6a4 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" + cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -48,7 +49,7 @@ func NewCNSInvoker(podName, namespace string) (*CNSIPAMInvoker, error) { } //Add uses the requestipconfig API in cns, and returns ipv4 and a nil ipv6 as CNS doesn't support IPv6 yet -func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) { +func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, hostSubnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) { // Parse Pod arguments. podInfo := cns.KubernetesPodInfo{PodName: invoker.podName, PodNamespace: invoker.podNamespace} @@ -57,8 +58,15 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *n return nil, nil, err } - log.Printf("Requesting IP for pod %v", podInfo) - response, err := invoker.cnsClient.RequestIPAddress(orchestratorContext) + endpointId := GetEndpointID(args) + ipconfig := cns.IPConfigRequest{ + OrchestratorContext: orchestratorContext, + PodInterfaceID: endpointId, + InfraContainerID: args.ContainerID, + } + + log.Printf("Requesting IP for pod %+v using ipconfig %+v", podInfo, ipconfig) + response, err := invoker.cnsClient.RequestIPAddress(&ipconfig) if err != nil { log.Printf("Failed to get IP address from CNS with error %v, response: %v", err, response) return nil, nil, err @@ -77,7 +85,7 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *n // set the NC Primary IP in options options[network.SNATIPKey] = info.ncPrimaryIP - log.Printf("[cni-invoker-cns] Received info %v for pod %v", info, podInfo) + log.Printf("[cni-invoker-cns] Received info %+v for pod %v", info, podInfo) ncgw := net.ParseIP(info.ncGatewayIPAddress) if ncgw == nil { @@ -168,7 +176,7 @@ func setHostOptions(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, ncSub } // Delete calls into the releaseipconfiguration API in CNS -func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, epInfo *network.EndpointInfo, options map[string]interface{}) error { +func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, options map[string]interface{}) error { // Parse Pod arguments. podInfo := cns.KubernetesPodInfo{PodName: invoker.podName, PodNamespace: invoker.podNamespace} @@ -178,8 +186,11 @@ func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConf return err } + endpointId := GetEndpointID(args) req := cns.IPConfigRequest{ OrchestratorContext: orchestratorContext, + PodInterfaceID: endpointId, + InfraContainerID: args.ContainerID, } if address != nil { @@ -188,12 +199,5 @@ func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConf log.Printf("CNS invoker called with empty IP address") } - if epInfo != nil { - req.PodInterfaceID = epInfo.Id - req.InfraContainerID = epInfo.ContainerID - } else { - log.Printf("CNS invoker called with empty endpoint information") - } - return invoker.cnsClient.ReleaseIPAddress(&req) } diff --git a/cni/network/network.go b/cni/network/network.go index 6a91f9913b..6456142f35 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/cni/utils" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/cnsclient" @@ -154,6 +155,32 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { return nil } +func (plugin *netPlugin) GetAllEndpointState(networkid string) (api.CNIState, error) { + st := api.AzureCNIState{ + ContainerInterfaces: make(map[string]api.PodNetworkInterfaceInfo), + } + + eps, err := plugin.nm.GetAllEndpoints(networkid) + if err != nil { + return nil, err + } + + for _, ep := range eps { + id := ep.Id + info := api.PodNetworkInterfaceInfo{ + PodName: ep.PODName, + PodNamespace: ep.PODNameSpace, + PodEndpointId: ep.Id, + ContainerID: ep.ContainerID, + IPAddresses: ep.IPAddresses, + } + + st.ContainerInterfaces[id] = info + } + + return &st, err +} + // Stops the plugin. func (plugin *netPlugin) Stop() { plugin.nm.Uninitialize() @@ -474,7 +501,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Creating network %v.", networkId) if !nwCfg.MultiTenancy { - result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, &subnetPrefix, options) + result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, args, &subnetPrefix, options) if err != nil { return err } @@ -482,10 +509,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { if err != nil { if result != nil && len(result.IPs) > 0 { - plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, epInfo, options) + if er := plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, args, options); er != nil { + err = plugin.Errorf("Failed to cleanup when NwInfo was not nil with error %v, after Add failed with error %w", er, err) + } } if resultV6 != nil && len(resultV6.IPs) > 0 { - plugin.ipamInvoker.Delete(&resultV6.IPs[0].Address, nwCfg, epInfo, options) + if er := plugin.ipamInvoker.Delete(&resultV6.IPs[0].Address, nwCfg, args, options); er != nil { + err = plugin.Errorf("Failed to cleanup when NwInfo was not nil with error %v, after Add failed with error %w", er, err) + } } } }() @@ -576,7 +607,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { if !nwCfg.MultiTenancy { // Network already exists. log.Printf("[cni-net] Found network %v with subnet %v.", networkId, nwInfo.Subnets[0].Prefix.String()) - result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, &subnetPrefix, nwInfo.Options) + result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, args, &subnetPrefix, nwInfo.Options) if err != nil { return err } @@ -586,10 +617,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { if err != nil { if result != nil && len(result.IPs) > 0 { - plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, epInfo, nwInfo.Options) + if er := plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, args, nwInfo.Options); er != nil { + err = plugin.Errorf("Failed to cleanup when NwInfo was nil with error %v, after Add failed with error %w", er, err) + } } if resultV6 != nil && len(resultV6.IPs) > 0 { - plugin.ipamInvoker.Delete(&resultV6.IPs[0].Address, nwCfg, epInfo, nwInfo.Options) + if er := plugin.ipamInvoker.Delete(&resultV6.IPs[0].Address, nwCfg, args, nwInfo.Options); er != nil { + err = plugin.Errorf("Failed to cleanup when NwInfo was nil with error %v, after Add failed with error %w", er, err) + } } } }() @@ -889,7 +924,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { if !nwCfg.MultiTenancy { // attempt to release address associated with this Endpoint id // This is to ensure clean up is done even in failure cases - err = plugin.ipamInvoker.Delete(nil, nwCfg, epInfo, nwInfo.Options) + err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options) if err != nil { log.Printf("Network not found, attempted to release address with error: %v", err) } @@ -908,7 +943,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { // attempt to release address associated with this Endpoint id // This is to ensure clean up is done even in failure cases log.Printf("release ip ep not found") - if err = plugin.ipamInvoker.Delete(nil, nwCfg, epInfo, nwInfo.Options); err != nil { + if err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options); err != nil { log.Printf("Endpoint not found, attempted to release address with error: %v", err) } } @@ -931,7 +966,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { // Call into IPAM plugin to release the endpoint's addresses. for _, address := range epInfo.IPAddresses { log.Printf("release ip:%s", address.IP.String()) - err = plugin.ipamInvoker.Delete(&address, nwCfg, epInfo, nwInfo.Options) + err = plugin.ipamInvoker.Delete(&address, nwCfg, args, nwInfo.Options) if err != nil { err = plugin.Errorf("Failed to release address %v with error: %v", address, err) return err @@ -940,7 +975,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { } else if epInfo.EnableInfraVnet { nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() nwCfg.Ipam.Address = epInfo.InfraVnetIP.IP.String() - err = plugin.ipamInvoker.Delete(nil, nwCfg, epInfo, nwInfo.Options) + err = plugin.ipamInvoker.Delete(nil, nwCfg, args, nwInfo.Options) if err != nil { log.Printf("Failed to release address: %v", err) err = plugin.Errorf("Failed to release address %v with error: %v", nwCfg.Ipam.Address, err) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index d58a6db03f..308b125101 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -6,26 +6,31 @@ import ( "testing" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/network" acnnetwork "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/nns" "github.com/Azure/azure-container-networking/telemetry" cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/stretchr/testify/require" ) -// the Add/Delete methods in Plugin require refactoring to have UT's written for them, -// but the mocks in this test are a start -func TestPlugin(t *testing.T) { - config := &common.PluginConfig{} +func getTestResources() (*netPlugin, *acnnetwork.MockNetworkManager) { pluginName := "testplugin" - - mockNetworkManager := acnnetwork.NewMockNetworkmanager() - + config := &common.PluginConfig{} grpcClient := &nns.MockGrpcClient{} plugin, _ := NewPlugin(pluginName, config, grpcClient) plugin.report = &telemetry.CNIReport{} + mockNetworkManager := acnnetwork.NewMockNetworkmanager() plugin.nm = mockNetworkManager + return plugin, mockNetworkManager +} + +// the Add/Delete methods in Plugin require refactoring to have UT's written for them, +// but the mocks in this test are a start +func TestPlugin(t *testing.T) { + plugin, _ := getTestResources() nwCfg := cni.NetworkConfig{ Name: "test-nwcfg", @@ -69,5 +74,59 @@ func TestPlugin(t *testing.T) { Options: make(map[string]interface{}), } plugin.nm.CreateNetwork(nwInfo) - plugin.Delete(args) + // plugin.Delete(args) +} + +func getTestEndpoint(podname, podnamespace, ipwithcidr, podinterfaceid, infracontainerid string) *acnnetwork.EndpointInfo { + ip, ipnet, _ := net.ParseCIDR(ipwithcidr) + ipnet.IP = ip + ep := acnnetwork.EndpointInfo{ + PODName: podname, + PODNameSpace: podnamespace, + Id: podinterfaceid, + ContainerID: infracontainerid, + IPAddresses: []net.IPNet{ + *ipnet, + }, + } + + return &ep +} + +func TestGetAllEndpointState(t *testing.T) { + plugin, mockNetworkManager := getTestResources() + networkid := "azure" + + ep1 := getTestEndpoint("podname1", "podnamespace1", "10.0.0.1/24", "podinterfaceid1", "testcontainerid1") + ep2 := getTestEndpoint("podname2", "podnamespace2", "10.0.0.2/24", "podinterfaceid2", "testcontainerid2") + + err := mockNetworkManager.CreateEndpoint(networkid, ep1) + require.NoError(t, err) + + err = mockNetworkManager.CreateEndpoint(networkid, ep2) + require.NoError(t, err) + + state, err := plugin.GetAllEndpointState(networkid) + require.NoError(t, err) + + res := &api.AzureCNIState{ + ContainerInterfaces: map[string]api.PodNetworkInterfaceInfo{ + ep1.Id: { + PodEndpointId: ep1.Id, + PodName: ep1.PODName, + PodNamespace: ep1.PODNameSpace, + ContainerID: ep1.ContainerID, + IPAddresses: ep1.IPAddresses, + }, + ep2.Id: { + PodEndpointId: ep2.Id, + PodName: ep2.PODName, + PodNamespace: ep2.PODNameSpace, + ContainerID: ep2.ContainerID, + IPAddresses: ep2.IPAddresses, + }, + }, + } + + require.Exactly(t, res, state) } diff --git a/cni/network/plugin/main.go b/cni/network/plugin/main.go index 7f9abdee15..d3cf147c13 100644 --- a/cni/network/plugin/main.go +++ b/cni/network/plugin/main.go @@ -6,12 +6,13 @@ package main import ( "encoding/json" "fmt" - "github.com/Azure/azure-container-networking/nns" "io/ioutil" "os" "reflect" "time" + "github.com/Azure/azure-container-networking/nns" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/network" "github.com/Azure/azure-container-networking/common" @@ -230,8 +231,25 @@ func main() { cniCmd := os.Getenv(cni.Cmd) log.Printf("CNI_COMMAND environment variable set to %s", cniCmd) + // used to dump state + if cniCmd == cni.CmdGetEndpointsState { + log.Printf("Retrieving state") + simpleState, err := netPlugin.GetAllEndpointState("azure") + if err != nil { + log.Errorf("Failed to get Azure CNI state, err:%v.\n", err) + return + } + + err = simpleState.PrintResult() + if err != nil { + log.Errorf("Failed to print state result to stdout with err %v\n", err) + } + + return + } + handled, err := handleIfCniUpdate(netPlugin.Update) - if handled == true { + if handled { log.Printf("CNI UPDATE finished.") } else if err = netPlugin.Execute(cni.PluginApi(netPlugin)); err != nil { log.Errorf("Failed to execute network plugin, err:%v.\n", err) diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index e59c6c6a4e..71867cccaf 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -211,7 +211,7 @@ func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) } // RequestIPAddress calls the requestIPAddress in CNS -func (cnsClient *CNSClient) RequestIPAddress(orchestratorContext []byte) (*cns.IPConfigResponse, error) { +func (cnsClient *CNSClient) RequestIPAddress(ipconfig *cns.IPConfigRequest) (*cns.IPConfigResponse, error) { var ( err error res *http.Response @@ -222,17 +222,15 @@ func (cnsClient *CNSClient) RequestIPAddress(orchestratorContext []byte) (*cns.I url := cnsClient.connectionURL + cns.RequestIPConfig - payload := &cns.IPConfigRequest{ - OrchestratorContext: orchestratorContext, - } - defer func() { if err != nil { - cnsClient.ReleaseIPAddress(payload) + if er := cnsClient.ReleaseIPAddress(ipconfig); er != nil { + log.Errorf("failed to release IP address [%v] after failed add [%v]", er, err) + } } }() - err = json.NewEncoder(&body).Encode(payload) + err = json.NewEncoder(&body).Encode(ipconfig) if err != nil { log.Errorf("encoding json failed with %v", err) return response, err diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index 0185baeb85..4de17f8bdc 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -233,7 +233,7 @@ func TestCNSClientRequestAndRelease(t *testing.T) { } // request IP address - resp, err := cnsClient.RequestIPAddress(orchestratorContext) + resp, err := cnsClient.RequestIPAddress(&cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) if err != nil { t.Fatalf("get IP from CNS failed with %+v", err) } @@ -301,7 +301,7 @@ func TestCNSClientPodContextApi(t *testing.T) { } // request IP address - _, err = cnsClient.RequestIPAddress(orchestratorContext) + _, err = cnsClient.RequestIPAddress(&cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) if err != nil { t.Fatalf("get IP from CNS failed with %+v", err) } @@ -341,7 +341,7 @@ func TestCNSClientDebugAPI(t *testing.T) { } // request IP address - _, err1 := cnsClient.RequestIPAddress(orchestratorContext) + _, err1 := cnsClient.RequestIPAddress(&cns.IPConfigRequest{OrchestratorContext: orchestratorContext}) if err1 != nil { t.Fatalf("get IP from CNS failed with %+v", err1) } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 25e34d04f0..0891e35bc8 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -31,7 +31,7 @@ func generateVethName(key string) string { return hex.EncodeToString(h.Sum(nil))[:11] } -func ConstructEndpointID(containerID string, netNsPath string, ifName string) (string, string) { +func ConstructEndpointID(containerID string, _ string, ifName string) (string, string) { if len(containerID) > 8 { containerID = containerID[:8] } else { diff --git a/network/manager.go b/network/manager.go index 42d3f4ed69..221953d430 100644 --- a/network/manager.go +++ b/network/manager.go @@ -72,6 +72,7 @@ type NetworkManager interface { CreateEndpoint(networkId string, epInfo *EndpointInfo) error DeleteEndpoint(networkId string, endpointId string) error GetEndpointInfo(networkId string, endpointId string) (*EndpointInfo, error) + GetAllEndpoints(networkId string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkId string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkId string, endpointId string, sandboxKey string) (*endpoint, error) DetachEndpoint(networkId string, endpointId string) error @@ -379,6 +380,24 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) ( return ep.getInfo(), nil } +func (nm *networkManager) GetAllEndpoints(networkId string) (map[string]*EndpointInfo, error) { + nm.Lock() + defer nm.Unlock() + + nw, err := nm.getNetwork(networkId) + if err != nil { + return nil, err + } + + eps := make(map[string]*EndpointInfo) + + for epid, ep := range nw.Endpoints { + eps[epid] = ep.getInfo() + } + + return eps, nil +} + // GetEndpointInfoBasedOnPODDetails returns information about the given endpoint. // It returns an error if a single pod has multiple endpoints. func (nm *networkManager) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) { diff --git a/network/manager_mock.go b/network/manager_mock.go index e097ad4968..6d49722591 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -9,15 +9,15 @@ import ( //MockNetworkManager is a mock structure for Network Manager type MockNetworkManager struct { - NetworkInfo map[string]*NetworkInfo - EndpointInfo map[string]*EndpointInfo + TestNetworkInfoMap map[string]*NetworkInfo + TestEndpointInfoMap map[string]*EndpointInfo } //NewMockNetworkmanager returns a new mock func NewMockNetworkmanager() *MockNetworkManager { return &MockNetworkManager{ - NetworkInfo: make(map[string]*NetworkInfo), - EndpointInfo: make(map[string]*EndpointInfo), + TestNetworkInfoMap: make(map[string]*NetworkInfo), + TestEndpointInfoMap: make(map[string]*EndpointInfo), } } @@ -36,7 +36,7 @@ func (nm *MockNetworkManager) AddExternalInterface(ifName string, subnet string) //CreateNetwork mock func (nm *MockNetworkManager) CreateNetwork(nwInfo *NetworkInfo) error { - nm.NetworkInfo[nwInfo.Id] = nwInfo + nm.TestNetworkInfoMap[nwInfo.Id] = nwInfo return nil } @@ -47,7 +47,7 @@ func (nm *MockNetworkManager) DeleteNetwork(networkID string) error { //GetNetworkInfo mock func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, error) { - if info, exists := nm.NetworkInfo[networkID]; exists { + if info, exists := nm.TestNetworkInfoMap[networkID]; exists { return *info, nil } return NetworkInfo{}, fmt.Errorf("Not found") @@ -55,7 +55,7 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err //CreateEndpoint mock func (nm *MockNetworkManager) CreateEndpoint(networkID string, epInfo *EndpointInfo) error { - nm.EndpointInfo[networkID] = epInfo + nm.TestEndpointInfoMap[epInfo.Id] = epInfo return nil } @@ -64,9 +64,13 @@ func (nm *MockNetworkManager) DeleteEndpoint(networkID string, endpointID string return nil } +func (nm *MockNetworkManager) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) { + return nm.TestEndpointInfoMap, nil +} + //GetEndpointInfo mock func (nm *MockNetworkManager) GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) { - return nm.EndpointInfo[networkID], nil + return nm.TestEndpointInfoMap[networkID], nil } //GetEndpointInfoBasedOnPODDetails mock diff --git a/network/network.go b/network/network.go index 50c4da7784..68b440dec8 100644 --- a/network/network.go +++ b/network/network.go @@ -25,7 +25,7 @@ const ( IPV6Nat = "ipv6nat" ) -// ExternalInterface is a host network interface that bridges containers to external networks. +// externalInterface is a host network interface that bridges containers to external networks. type externalInterface struct { Name string Networks map[string]*network diff --git a/npm/ipsm/ipsm_test.go b/npm/ipsm/ipsm_test.go index ec09c4b225..1cec012688 100644 --- a/npm/ipsm/ipsm_test.go +++ b/npm/ipsm/ipsm_test.go @@ -104,10 +104,10 @@ func TestDeleteFromList(t *testing.T) { {Cmd: []string{"ipset", "test", "-exist", util.GetHashedName("test-list"), util.GetHashedName("test-set")}}, {Cmd: []string{"ipset", "-D", "-exist", util.GetHashedName("test-list"), util.GetHashedName("test-set")}}, {Cmd: []string{"ipset", "-X", "-exist", util.GetHashedName("test-list")}}, - {Cmd: []string{"ipset", "test", "-exist", util.GetHashedName("test-list"), util.GetHashedName("test-set")}, Stderr: "ipset still exists", ExitCode: 2}, - {Cmd: []string{"ipset", "list", "-exist", util.GetHashedName("test-list")}, Stderr: "ipset still exists", ExitCode: 2}, + {Cmd: []string{"ipset", "test", "-exist", util.GetHashedName("test-list"), util.GetHashedName("test-set")}, Stdout: "ipset still exists", ExitCode: 2}, + {Cmd: []string{"ipset", "list", "-exist", util.GetHashedName("test-list")}, Stdout: "ipset still exists", ExitCode: 2}, {Cmd: []string{"ipset", "-X", "-exist", util.GetHashedName("test-set")}}, - {Cmd: []string{"ipset", "list", "-exist", util.GetHashedName("test-set")}, Stderr: "ipset still exists", ExitCode: 2}, + {Cmd: []string{"ipset", "list", "-exist", util.GetHashedName("test-set")}, Stdout: "ipset still exists", ExitCode: 2}, } fexec, fcmd := testutils.GetFakeExecWithScripts(calls) @@ -570,7 +570,7 @@ func TestRunError(t *testing.T) { setname := "test-set" var calls = []testutils.TestCmd{ - {Cmd: []string{"ipset", "-N", "-exist", util.GetHashedName(setname), "nethash"}, Stderr: "test failure", ExitCode: 2}, + {Cmd: []string{"ipset", "-N", "-exist", util.GetHashedName(setname), "nethash"}, Stdout: "test failure", ExitCode: 2}, } fexec, fcmd := testutils.GetFakeExecWithScripts(calls) diff --git a/test/utils/utils.go b/test/utils/utils.go index 0d33b491ea..59adc9f0c6 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,6 +1,9 @@ package testingutils import ( + "log" + "os" + "os/user" "testing" "github.com/stretchr/testify/require" @@ -11,7 +14,7 @@ import ( type TestCmd struct { Cmd []string - Stderr string + Stdout string ExitCode int } @@ -21,12 +24,12 @@ func GetFakeExecWithScripts(calls []TestCmd) (*fakeexec.FakeExec, *fakeexec.Fake fcmd := &fakeexec.FakeCmd{} for _, call := range calls { - if call.Stderr != "" || call.ExitCode != 0 { - stderr := call.Stderr + stdout := call.Stdout + if call.ExitCode != 0 { err := &fakeexec.FakeExitError{Status: call.ExitCode} - fcmd.CombinedOutputScript = append(fcmd.CombinedOutputScript, func() ([]byte, []byte, error) { return []byte(stderr), nil, err }) + fcmd.CombinedOutputScript = append(fcmd.CombinedOutputScript, func() ([]byte, []byte, error) { return []byte(stdout), nil, err }) } else { - fcmd.CombinedOutputScript = append(fcmd.CombinedOutputScript, func() ([]byte, []byte, error) { return []byte{}, nil, nil }) + fcmd.CombinedOutputScript = append(fcmd.CombinedOutputScript, func() ([]byte, []byte, error) { return []byte(stdout), nil, nil }) } } @@ -44,3 +47,27 @@ func VerifyCallsMatch(t *testing.T, calls []TestCmd, fexec *fakeexec.FakeExec, f require.Equalf(t, call.Cmd, fcmd.CombinedOutputLog[i], "Call [%d] doesn't match expected", i) } } + +func isCurrentUserRoot() bool { + currentUser, err := user.Current() + if err != nil { + log.Printf("Failed to get current user") + return false + } else if currentUser.Username == "root" { + return true + } + return false +} + +func RequireRootforTest(t *testing.T) { + if !isCurrentUserRoot() { + t.Fatalf("Test [%s] requires root!", t.Name()) + } +} + +func RequireRootforTestMain(m *testing.M) { + if !isCurrentUserRoot() { + log.Printf("These tests require root!") + os.Exit(1) + } +}