From 7ae9b5d6ea1c5dbd95fb4503de76e7df643000f6 Mon Sep 17 00:00:00 2001 From: t-qini Date: Thu, 1 Aug 2019 21:03:55 +0800 Subject: [PATCH 1/2] Enable e2e test for cross groups nodes. --- tests/e2e/network/ensureloadbalancer.go | 8 +- tests/e2e/network/functions.go | 129 ++++++++++++++++++++ tests/e2e/network/network_security_group.go | 2 +- tests/e2e/network/service_annotations.go | 40 +++--- tests/e2e/utils/azure_test_client.go | 5 + tests/e2e/utils/network_utils.go | 19 +++ tests/e2e/utils/node_utils.go | 16 ++- tests/e2e/utils/resource_group_utils.go | 16 +++ 8 files changed, 204 insertions(+), 31 deletions(-) create mode 100644 tests/e2e/network/functions.go diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go index 23fea8b496..0d96918a32 100644 --- a/tests/e2e/network/ensureloadbalancer.go +++ b/tests/e2e/network/ensureloadbalancer.go @@ -87,7 +87,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() { } ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4] - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(service) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) @@ -131,7 +131,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() { ip1, err := utils.SelectAvailablePrivateIP(tc) Expect(err).NotTo(HaveOccurred()) - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) service = updateServiceBalanceIP(service, true, ip1) _, err = cs.CoreV1().Services(ns.Name).Create(service) Expect(err).NotTo(HaveOccurred()) @@ -173,7 +173,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() { } ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4] - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) _, err := cs.CoreV1().Services(ns.Name).Create(service) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) @@ -222,7 +222,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() { Expect(err).NotTo(HaveOccurred()) targetIP := to.String(pip.IPAddress) - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) service = updateServiceBalanceIP(service, false, targetIP) _, err = cs.CoreV1().Services(ns.Name).Create(service) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/network/functions.go b/tests/e2e/network/functions.go new file mode 100644 index 0000000000..1f77f3262f --- /dev/null +++ b/tests/e2e/network/functions.go @@ -0,0 +1,129 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/cloud-provider-azure/tests/e2e/utils" +) + +var _ = Describe("Cloud Provider Azure", func() { + basename := "service-lb" + serviceName := "servicelb-test" + + var cs clientset.Interface + var ns *v1.Namespace + //var tc *utils.AzureTestClient + + labels := map[string]string{ + "app": serviceName, + } + ports := []v1.ServicePort{{ + Port: nginxPort, + TargetPort: intstr.FromInt(nginxPort), + }} + + BeforeEach(func() { + var err error + cs, err = utils.CreateKubeClientSet() + Expect(err).NotTo(HaveOccurred()) + + ns, err = utils.CreateTestingNamespace(basename, cs) + Expect(err).NotTo(HaveOccurred()) + + //tc, err = utils.CreateAzureTestClient() + //Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Creating deployment " + serviceName) + deployment := createNginxDeploymentManifest(serviceName, labels) + _, err = cs.AppsV1().Deployments(ns.Name).Create(deployment) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := cs.AppsV1().Deployments(ns.Name).Delete(serviceName, nil) + Expect(err).NotTo(HaveOccurred()) + + err = utils.DeleteNamespace(cs, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + cs = nil + ns = nil + //tc = nil + }) + + It("should support nodes crossing resource groups [multi-group] [availabilitySet]", func() { + master, err := utils.GetMaster(cs) + Expect(err).NotTo(HaveOccurred()) + + var rgMaster string + rgMaster, err = utils.GetNodeResourceGroup(master) + utils.Logf("found master resource group %s", rgMaster) + Expect(err).NotTo(HaveOccurred()) + + nodes, err := utils.GetAgentNodes(cs) + Expect(err).NotTo(HaveOccurred()) + var nodeNotInRGMaster v1.Node + var nodeNotInRGMAsterCount int + for _, node := range nodes { + if rg, err := utils.GetNodeResourceGroup(&node); err == nil && rg != rgMaster { + utils.Logf("rg of node %s is %s", node.Name, rg) + nodeNotInRGMaster = node + nodeNotInRGMAsterCount++ + } else if err != nil { + Fail("cannot obtain the node's resource group") + } + } + utils.Logf("found node %s in another resource group", nodeNotInRGMaster.Name) + + if nodeNotInRGMAsterCount == 0 { + Skip("cannot find a second resource group, skipping") + } + labels := nodeNotInRGMaster.Labels + Expect(labels).NotTo(BeNil()) + excludeLB, ok := labels[`alpha.service-controller.kubernetes.io/exclude-balancer`] + Expect(ok).To(BeTrue()) + Expect(excludeLB).To(Equal("true")) + + clusterRG, ok := labels[`kubernetes.azure.com/cluster`] + Expect(ok).To(BeTrue()) + Expect(clusterRG).To(Equal(rgMaster)) + + nodeRG, ok := labels[`kubernetes.azure.com/resource-group`] + Expect(ok).To(BeTrue()) + Expect(nodeRG).NotTo(Equal(rgMaster)) + + publicIP := createServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) + lb := getAzureLoadBalancerFromPIP(publicIP, rgMaster, rgMaster) + + for _, backendPool := range *lb.BackendAddressPools { + if backendPool.BackendIPConfigurations != nil { + if *backendPool.BackendIPConfigurations != nil { + for _, ipConfig := range *backendPool.BackendIPConfigurations { + matches := backendASIPConfigurationRE.FindStringSubmatch(*ipConfig.ID) + Expect(len(matches)).To(Equal(2)) + Expect(matches[1]).NotTo(Equal(nodeNotInRGMaster.Name)) + } + } + } + } + }) +}) diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go index 0da061bb68..080c48a9fc 100644 --- a/tests/e2e/network/network_security_group.go +++ b/tests/e2e/network/network_security_group.go @@ -224,7 +224,7 @@ func validateSharedSecurityRuleExists(nsg *aznetwork.SecurityGroup, ips []string func createAndWaitServiceExposure(cs clientset.Interface, ns string, serviceName string, annotation map[string]string, labels map[string]string, ports []v1.ServicePort) (string, error) { ip := "" - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports) if _, err := cs.CoreV1().Services(ns).Create(service); err != nil { return ip, err } diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index c9a8e325ac..ea53e52725 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -42,9 +42,10 @@ import ( ) var ( - scalesetRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) - lbNameRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`) - backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) + scalesetRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) + lbNameRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`) + backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) + backendASIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)/virtualMachines(?:.*)`) ) const ( @@ -240,7 +241,7 @@ var _ = FDescribe("Service with annotation", func() { azure.ServiceAnnotationLoadBalancerResourceGroup: to.String(rg.Name), } By("Creating service " + serviceName + " in namespace " + ns.Name) - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports) service.Spec.LoadBalancerIP = *pip.IPAddress _, err = cs.CoreV1().Services(ns.Name).Create(service) Expect(err).NotTo(HaveOccurred()) @@ -251,7 +252,7 @@ var _ = FDescribe("Service with annotation", func() { _, err = utils.WaitServiceExposure(cs, ns.Name, serviceName) Expect(err).NotTo(HaveOccurred()) - lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name) + lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name, "") Expect(lb).NotTo(BeNil()) }) }) @@ -373,13 +374,13 @@ func getAzureLoadBalancer(pip string) *network.LoadBalancer { return &lb } -func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBalancer { +func getAzureLoadBalancerFromPIP(pip, pipResourceGroup, lbResourceGroup string) *network.LoadBalancer { By("Creating Azure clients") azureTestClient, err := utils.CreateAzureTestClient() Expect(err).NotTo(HaveOccurred()) - By("Getting public IPs in the resourceGroup " + resourceGroupName) - pipList, err := azureTestClient.ListPublicIPs(resourceGroupName) + By("Getting public IPs in the resourceGroup " + pipResourceGroup) + pipList, err := azureTestClient.ListPublicIPs(pipResourceGroup) Expect(err).NotTo(HaveOccurred()) By("Getting public IP frontend configuration ID") @@ -405,7 +406,10 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal utils.Logf("Got loadBalancerName %q", loadBalancerName) By("Getting loadBalancer") - lb, err := azureTestClient.GetLoadBalancer(azureTestClient.GetResourceGroup(), loadBalancerName) + if lbResourceGroup == "" { + lbResourceGroup = azureTestClient.GetResourceGroup() + } + lb, err := azureTestClient.GetLoadBalancer(lbResourceGroup, loadBalancerName) Expect(err).NotTo(HaveOccurred()) Expect(lb.LoadBalancingRules).NotTo(BeNil()) @@ -414,7 +418,7 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) string { By("Creating service " + serviceName + " in namespace " + nsName) - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports) _, err := cs.CoreV1().Services(nsName).Create(service) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + nsName) @@ -427,20 +431,6 @@ func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName str return publicIP } -func createLoadBalancerServiceManifest(c clientset.Interface, name string, annotation map[string]string, labels map[string]string, namespace string, ports []v1.ServicePort) *v1.Service { - return &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Annotations: annotation, - }, - Spec: v1.ServiceSpec{ - Selector: labels, - Ports: ports, - Type: "LoadBalancer", - }, - } -} - // defaultDeployment returns a default deployment // running nginx image which exposes port 80 func createNginxDeploymentManifest(name string, labels map[string]string) (result *appsv1.Deployment) { @@ -554,7 +544,7 @@ func validateLoadBalancerBackendPools(vmssName string, cs clientset.Interface, s annotation := map[string]string{ azure.ServiceAnnotationLoadBalancerMode: vmssName, } - service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports) + service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports) _, err := cs.CoreV1().Services(ns).Create(service) Expect(err).NotTo(HaveOccurred()) utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns) diff --git a/tests/e2e/utils/azure_test_client.go b/tests/e2e/utils/azure_test_client.go index 758bdae9d8..9324c6a770 100644 --- a/tests/e2e/utils/azure_test_client.go +++ b/tests/e2e/utils/azure_test_client.go @@ -133,6 +133,11 @@ func getResourceGroupFromProviderID(providerID string) (string, error) { return matches[1], nil } +// GetNodeResourceGroup returns the resource group of the given node +func GetNodeResourceGroup(node *v1.Node) (string, error) { + return getResourceGroupFromProviderID(node.Spec.ProviderID) +} + func getLocationFromNodeLabels(node *v1.Node) string { if location, ok := node.Labels[nodeLabelLocation]; ok { return location diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go index 53bc1a305d..abf352b6a8 100644 --- a/tests/e2e/utils/network_utils.go +++ b/tests/e2e/utils/network_utils.go @@ -24,7 +24,11 @@ import ( aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network" "github.com/Azure/go-autorest/autorest/to" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" ) const k8sVNetPrefix = "k8s-vnet-" @@ -146,6 +150,21 @@ func (azureTestClient *AzureTestClient) GetClusterSecurityGroup() (ret *aznetwor return } +// CreateLoadBalancerServiceManifest return the specific service to be created +func CreateLoadBalancerServiceManifest(c clientset.Interface, name string, annotation map[string]string, labels map[string]string, namespace string, ports []v1.ServicePort) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: annotation, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Ports: ports, + Type: "LoadBalancer", + }, + } +} + // WaitCreatePIP waits to create a public ip resource func WaitCreatePIP(azureTestClient *AzureTestClient, ipName string, ipParameter aznetwork.PublicIPAddress) (aznetwork.PublicIPAddress, error) { Logf("Creating public IP resource named %s", ipName) diff --git a/tests/e2e/utils/node_utils.go b/tests/e2e/utils/node_utils.go index 6cc9ebe624..6883306299 100644 --- a/tests/e2e/utils/node_utils.go +++ b/tests/e2e/utils/node_utils.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,6 +47,20 @@ func GetAgentNodes(cs clientset.Interface) ([]v1.Node, error) { return ret, nil } +// GetMaster returns the master node +func GetMaster(cs clientset.Interface) (*v1.Node, error) { + nodesList, err := getNodeList(cs) + if err != nil { + return nil, err + } + for _, node := range nodesList.Items { + if isMasterNode(&node) { + return &node, nil + } + } + return nil, fmt.Errorf("cannot obtain the master node") +} + // GetNodeList is a wapper around listing nodes func getNodeList(cs clientset.Interface) (*v1.NodeList, error) { var nodes *v1.NodeList diff --git a/tests/e2e/utils/resource_group_utils.go b/tests/e2e/utils/resource_group_utils.go index 81cfd96223..49cc8b89dc 100644 --- a/tests/e2e/utils/resource_group_utils.go +++ b/tests/e2e/utils/resource_group_utils.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils import ( From 723eec6a6cc3ebe90e28daae1a79edec7750b2e4 Mon Sep 17 00:00:00 2001 From: t-qini Date: Fri, 2 Aug 2019 15:43:06 +0800 Subject: [PATCH 2/2] Modify code logic. --- .../{functions.go => cross_rg_nodes.go} | 57 +++++++++---- tests/e2e/utils/azure_test_client.go | 5 ++ tests/e2e/utils/network_interface_utils.go | 79 +++++++++++++++++++ 3 files changed, 127 insertions(+), 14 deletions(-) rename tests/e2e/network/{functions.go => cross_rg_nodes.go} (64%) create mode 100644 tests/e2e/utils/network_interface_utils.go diff --git a/tests/e2e/network/functions.go b/tests/e2e/network/cross_rg_nodes.go similarity index 64% rename from tests/e2e/network/functions.go rename to tests/e2e/network/cross_rg_nodes.go index 1f77f3262f..7935f851b0 100644 --- a/tests/e2e/network/functions.go +++ b/tests/e2e/network/cross_rg_nodes.go @@ -17,21 +17,27 @@ limitations under the License. package network import ( + "fmt" + "regexp" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" "k8s.io/cloud-provider-azure/tests/e2e/utils" ) -var _ = Describe("Cloud Provider Azure", func() { +var vmNameRE = regexp.MustCompile(`(k8s-.+-\d+)-\d+`) + +var _ = Describe("Cloud Provider Azure cross resource group nodes", func() { basename := "service-lb" serviceName := "servicelb-test" var cs clientset.Interface var ns *v1.Namespace - //var tc *utils.AzureTestClient + var tc *utils.AzureTestClient labels := map[string]string{ "app": serviceName, @@ -49,8 +55,8 @@ var _ = Describe("Cloud Provider Azure", func() { ns, err = utils.CreateTestingNamespace(basename, cs) Expect(err).NotTo(HaveOccurred()) - //tc, err = utils.CreateAzureTestClient() - //Expect(err).NotTo(HaveOccurred()) + tc, err = utils.CreateAzureTestClient() + Expect(err).NotTo(HaveOccurred()) utils.Logf("Creating deployment " + serviceName) deployment := createNginxDeploymentManifest(serviceName, labels) @@ -67,14 +73,13 @@ var _ = Describe("Cloud Provider Azure", func() { cs = nil ns = nil - //tc = nil }) It("should support nodes crossing resource groups [multi-group] [availabilitySet]", func() { master, err := utils.GetMaster(cs) Expect(err).NotTo(HaveOccurred()) - var rgMaster string + var rgMaster, rgNotMaster string rgMaster, err = utils.GetNodeResourceGroup(master) utils.Logf("found master resource group %s", rgMaster) Expect(err).NotTo(HaveOccurred()) @@ -88,6 +93,7 @@ var _ = Describe("Cloud Provider Azure", func() { utils.Logf("rg of node %s is %s", node.Name, rg) nodeNotInRGMaster = node nodeNotInRGMAsterCount++ + rgNotMaster = rg } else if err != nil { Fail("cannot obtain the node's resource group") } @@ -95,7 +101,7 @@ var _ = Describe("Cloud Provider Azure", func() { utils.Logf("found node %s in another resource group", nodeNotInRGMaster.Name) if nodeNotInRGMAsterCount == 0 { - Skip("cannot find a second resource group, skipping") + Skip("cannot find a second resource group, skip the case") } labels := nodeNotInRGMaster.Labels Expect(labels).NotTo(BeNil()) @@ -114,16 +120,39 @@ var _ = Describe("Cloud Provider Azure", func() { publicIP := createServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports) lb := getAzureLoadBalancerFromPIP(publicIP, rgMaster, rgMaster) - for _, backendPool := range *lb.BackendAddressPools { - if backendPool.BackendIPConfigurations != nil { - if *backendPool.BackendIPConfigurations != nil { - for _, ipConfig := range *backendPool.BackendIPConfigurations { - matches := backendASIPConfigurationRE.FindStringSubmatch(*ipConfig.ID) - Expect(len(matches)).To(Equal(2)) - Expect(matches[1]).NotTo(Equal(nodeNotInRGMaster.Name)) + utils.Logf("finding NIC of the node %s, assuming it's in the same rg as master", nodeNotInRGMaster.Name) + nodeNamePrefix, err := getNodeNamePrefix(nodeNotInRGMaster.Name) + Expect(err).NotTo(HaveOccurred()) + NICList, err := utils.ListNICs(tc, rgMaster) + Expect(err).NotTo(HaveOccurred()) + targetNIC, err := utils.GetTargetNICFromList(NICList, nodeNamePrefix) + Expect(err).NotTo(HaveOccurred()) + if targetNIC == nil { + utils.Logf("finding NIC of the node %s in another resource group", nodeNotInRGMaster.Name) + NICList, err = utils.ListNICs(tc, rgNotMaster) + Expect(err).NotTo(HaveOccurred()) + targetNIC, err = utils.GetTargetNICFromList(NICList, nodeNamePrefix) + Expect(err).NotTo(HaveOccurred()) + } + Expect(targetNIC).NotTo(BeNil()) + utils.Logf("found NIC %s of node %s", *targetNIC.Name, nodeNotInRGMaster.Name) + + for _, nicIPConfigs := range *targetNIC.IPConfigurations { + if nicIPConfigs.LoadBalancerBackendAddressPools != nil { + for _, nicBackendPool := range *nicIPConfigs.LoadBalancerBackendAddressPools { + for _, lbBackendPool := range *lb.BackendAddressPools { + Expect(*lbBackendPool.ID).NotTo(Equal(*nicBackendPool.ID)) } } } } }) }) + +func getNodeNamePrefix(nodeName string) (string, error) { + nodeNameMatches := vmNameRE.FindStringSubmatch(nodeName) + if len(nodeNameMatches) != 2 { + return "", fmt.Errorf("cannot obtain the prefix from given node name %s", nodeName) + } + return nodeNameMatches[1], nil +} diff --git a/tests/e2e/utils/azure_test_client.go b/tests/e2e/utils/azure_test_client.go index 9324c6a770..3b4db3fd02 100644 --- a/tests/e2e/utils/azure_test_client.go +++ b/tests/e2e/utils/azure_test_client.go @@ -118,6 +118,11 @@ func (tc *AzureTestClient) creteLoadBalancerClient() *aznetwork.LoadBalancersCli return &aznetwork.LoadBalancersClient{BaseClient: tc.networkClient} } +// createInterfacesClient generates network interface client with the same baseclient as azure test client +func (tc *AzureTestClient) createInterfacesClient() *aznetwork.InterfacesClient { + return &aznetwork.InterfacesClient{BaseClient: tc.networkClient} +} + // createResourceGroupClient generates resource group client with the same baseclient as azure test client func (tc *AzureTestClient) createResourceGroupClient() *azresources.GroupsClient { return &azresources.GroupsClient{BaseClient: tc.resourceClient} diff --git a/tests/e2e/utils/network_interface_utils.go b/tests/e2e/utils/network_interface_utils.go new file mode 100644 index 0000000000..3b523c4ea4 --- /dev/null +++ b/tests/e2e/utils/network_interface_utils.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "regexp" + + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network" +) + +var ( + nicIDConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/networkInterfaces/(.+)`) + vmNamePrefixRE = regexp.MustCompile(`(.+)-nic-\d+`) +) + +// ListNICs returns the NIC list in the given resource group +func ListNICs(tc *AzureTestClient, rgName string) (*[]network.Interface, error) { + Logf("Getting network interfaces list in resource group %s", rgName) + + ic := tc.createInterfacesClient() + + list, err := ic.List(context.Background(), rgName) + if err != nil { + return nil, err + } + if len(list.Values()) == 0 { + Logf("cannot find the corresponding NIC in resource group %s", rgName) + return nil, nil + } + value := list.Values() + return &value, err +} + +func getVMNamePrefixFromNICID(nicID string) (string, error) { + nicMatches := nicIDConfigurationRE.FindStringSubmatch(nicID) + if len(nicMatches) != 2 { + return "", fmt.Errorf("cannot obtain the name of virtual machine from nicID") + } + vmMatches := vmNamePrefixRE.FindStringSubmatch(nicMatches[1]) + if len(vmMatches) != 2 { + return "", fmt.Errorf("cannot obtain the name of virtual machine from nicID") + } + return vmMatches[1], nil +} + +// GetTargetNICFromList pick the target virtual machine's NIC from the given NIC list +func GetTargetNICFromList(list *[]network.Interface, targetVMNamePrefix string) (*network.Interface, error) { + if list == nil { + Logf("empty list given, skip finding target NIC") + return nil, nil + } + for _, nic := range *list { + vmNamePrefix, err := getVMNamePrefixFromNICID(*nic.ID) + if err != nil { + return nil, err + } + if vmNamePrefix == targetVMNamePrefix { + return &nic, err + } + } + Logf("cannot find the target NIC in the given NIC list") + return nil, nil +}