Enable e2e test for cross groups nodes.

tests/e2e/network/cross_rg_nodes.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"fmt"
+	"regexp"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/cloud-provider-azure/tests/e2e/utils"
+)
+
+var vmNameRE = regexp.MustCompile(`(k8s-.+-\d+)-\d+`)
+
+var _ = Describe("Cloud Provider Azure cross resource group nodes", func() {
+	basename := "service-lb"
+	serviceName := "servicelb-test"
+
+	var cs clientset.Interface
+	var ns *v1.Namespace
+	var tc *utils.AzureTestClient
+
+	labels := map[string]string{
+		"app": serviceName,
+	}
+	ports := []v1.ServicePort{{
+		Port:       nginxPort,
+		TargetPort: intstr.FromInt(nginxPort),
+	}}
+
+	BeforeEach(func() {
+		var err error
+		cs, err = utils.CreateKubeClientSet()
+		Expect(err).NotTo(HaveOccurred())
+
+		ns, err = utils.CreateTestingNamespace(basename, cs)
+		Expect(err).NotTo(HaveOccurred())
+
+		tc, err = utils.CreateAzureTestClient()
+		Expect(err).NotTo(HaveOccurred())
+
+		utils.Logf("Creating deployment " + serviceName)
+		deployment := createNginxDeploymentManifest(serviceName, labels)
+		_, err = cs.AppsV1().Deployments(ns.Name).Create(deployment)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		err := cs.AppsV1().Deployments(ns.Name).Delete(serviceName, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = utils.DeleteNamespace(cs, ns.Name)
+		Expect(err).NotTo(HaveOccurred())
+
+		cs = nil
+		ns = nil
+	})
+
+	It("should support nodes crossing resource groups [multi-group] [availabilitySet]", func() {
+		master, err := utils.GetMaster(cs)
+		Expect(err).NotTo(HaveOccurred())
+
+		var rgMaster, rgNotMaster string
+		rgMaster, err = utils.GetNodeResourceGroup(master)
+		utils.Logf("found master resource group %s", rgMaster)
+		Expect(err).NotTo(HaveOccurred())
+
+		nodes, err := utils.GetAgentNodes(cs)
+		Expect(err).NotTo(HaveOccurred())
+		var nodeNotInRGMaster v1.Node
+		var nodeNotInRGMAsterCount int
+		for _, node := range nodes {
+			if rg, err := utils.GetNodeResourceGroup(&node); err == nil && rg != rgMaster {
+				utils.Logf("rg of node %s is %s", node.Name, rg)
+				nodeNotInRGMaster = node
+				nodeNotInRGMAsterCount++
+				rgNotMaster = rg
+			} else if err != nil {
+				Fail("cannot obtain the node's resource group")
+			}
+		}
+		utils.Logf("found node %s in another resource group", nodeNotInRGMaster.Name)
+
+		if nodeNotInRGMAsterCount == 0 {
+			Skip("cannot find a second resource group, skip the case")
+		}
+		labels := nodeNotInRGMaster.Labels
+		Expect(labels).NotTo(BeNil())
+		excludeLB, ok := labels[`alpha.service-controller.kubernetes.io/exclude-balancer`]
+		Expect(ok).To(BeTrue())
+		Expect(excludeLB).To(Equal("true"))
+
+		clusterRG, ok := labels[`kubernetes.azure.com/cluster`]
+		Expect(ok).To(BeTrue())
+		Expect(clusterRG).To(Equal(rgMaster))
+
+		nodeRG, ok := labels[`kubernetes.azure.com/resource-group`]
+		Expect(ok).To(BeTrue())
+		Expect(nodeRG).NotTo(Equal(rgMaster))
+
+		publicIP := createServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports)
+		lb := getAzureLoadBalancerFromPIP(publicIP, rgMaster, rgMaster)
+
+		utils.Logf("finding NIC of the node %s, assuming it's in the same rg as master", nodeNotInRGMaster.Name)
+		nodeNamePrefix, err := getNodeNamePrefix(nodeNotInRGMaster.Name)
+		Expect(err).NotTo(HaveOccurred())
+		NICList, err := utils.ListNICs(tc, rgMaster)
+		Expect(err).NotTo(HaveOccurred())
+		targetNIC, err := utils.GetTargetNICFromList(NICList, nodeNamePrefix)
+		Expect(err).NotTo(HaveOccurred())
+		if targetNIC == nil {
+			utils.Logf("finding NIC of the node %s in another resource group", nodeNotInRGMaster.Name)
+			NICList, err = utils.ListNICs(tc, rgNotMaster)
+			Expect(err).NotTo(HaveOccurred())
+			targetNIC, err = utils.GetTargetNICFromList(NICList, nodeNamePrefix)
+			Expect(err).NotTo(HaveOccurred())
+		}
+		Expect(targetNIC).NotTo(BeNil())
+		utils.Logf("found NIC %s of node %s", *targetNIC.Name, nodeNotInRGMaster.Name)
+
+		for _, nicIPConfigs := range *targetNIC.IPConfigurations {
+			if nicIPConfigs.LoadBalancerBackendAddressPools != nil {
+				for _, nicBackendPool := range *nicIPConfigs.LoadBalancerBackendAddressPools {
+					for _, lbBackendPool := range *lb.BackendAddressPools {
+						Expect(*lbBackendPool.ID).NotTo(Equal(*nicBackendPool.ID))
+					}
+				}
+			}
+		}
+	})
+})
+
+func getNodeNamePrefix(nodeName string) (string, error) {
+	nodeNameMatches := vmNameRE.FindStringSubmatch(nodeName)
+	if len(nodeNameMatches) != 2 {
+		return "", fmt.Errorf("cannot obtain the prefix from given node name %s", nodeName)
+	}
+	return nodeNameMatches[1], nil
+}

tests/e2e/network/ensureloadbalancer.go

@@ -87,7 +87,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		}
 		ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4]
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		_, err := cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
@@ -131,7 +131,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		ip1, err := utils.SelectAvailablePrivateIP(tc)
 		Expect(err).NotTo(HaveOccurred())
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service = updateServiceBalanceIP(service, true, ip1)
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
@@ -173,7 +173,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		}
 		ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4]
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		_, err := cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
@@ -222,7 +222,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service = updateServiceBalanceIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())

tests/e2e/network/network_security_group.go

@@ -224,7 +224,7 @@ func validateSharedSecurityRuleExists(nsg *aznetwork.SecurityGroup, ips []string
 
 func createAndWaitServiceExposure(cs clientset.Interface, ns string, serviceName string, annotation map[string]string, labels map[string]string, ports []v1.ServicePort) (string, error) {
 	ip := ""
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
 	if _, err := cs.CoreV1().Services(ns).Create(service); err != nil {
 		return ip, err
 	}

tests/e2e/network/service_annotations.go

@@ -42,9 +42,10 @@ import (
 )
 
 var (
-	scalesetRE               = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
-	lbNameRE                 = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`)
-	backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	scalesetRE                 = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	lbNameRE                   = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`)
+	backendIPConfigurationRE   = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	backendASIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)/virtualMachines(?:.*)`)
 )
 
 const (
@@ -240,7 +241,7 @@ var _ = FDescribe("Service with annotation", func() {
 			azure.ServiceAnnotationLoadBalancerResourceGroup: to.String(rg.Name),
 		}
 		By("Creating service " + serviceName + " in namespace " + ns.Name)
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service.Spec.LoadBalancerIP = *pip.IPAddress
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
@@ -251,7 +252,7 @@ var _ = FDescribe("Service with annotation", func() {
 		_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName)
 		Expect(err).NotTo(HaveOccurred())
 
-		lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name)
+		lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name, "")
 		Expect(lb).NotTo(BeNil())
 	})
 })
@@ -373,13 +374,13 @@ func getAzureLoadBalancer(pip string) *network.LoadBalancer {
 	return &lb
 }
 
-func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBalancer {
+func getAzureLoadBalancerFromPIP(pip, pipResourceGroup, lbResourceGroup string) *network.LoadBalancer {
 	By("Creating Azure clients")
 	azureTestClient, err := utils.CreateAzureTestClient()
 	Expect(err).NotTo(HaveOccurred())
 
-	By("Getting public IPs in the resourceGroup " + resourceGroupName)
-	pipList, err := azureTestClient.ListPublicIPs(resourceGroupName)
+	By("Getting public IPs in the resourceGroup " + pipResourceGroup)
+	pipList, err := azureTestClient.ListPublicIPs(pipResourceGroup)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("Getting public IP frontend configuration ID")
@@ -405,7 +406,10 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal
 	utils.Logf("Got loadBalancerName %q", loadBalancerName)
 
 	By("Getting loadBalancer")
-	lb, err := azureTestClient.GetLoadBalancer(azureTestClient.GetResourceGroup(), loadBalancerName)
+	if lbResourceGroup == "" {
+		lbResourceGroup = azureTestClient.GetResourceGroup()
+	}
+	lb, err := azureTestClient.GetLoadBalancer(lbResourceGroup, loadBalancerName)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(lb.LoadBalancingRules).NotTo(BeNil())
 
@@ -414,7 +418,7 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal
 
 func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) string {
 	By("Creating service " + serviceName + " in namespace " + nsName)
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports)
 	_, err := cs.CoreV1().Services(nsName).Create(service)
 	Expect(err).NotTo(HaveOccurred())
 	utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + nsName)
@@ -427,20 +431,6 @@ func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName str
 	return publicIP
 }
 
-func createLoadBalancerServiceManifest(c clientset.Interface, name string, annotation map[string]string, labels map[string]string, namespace string, ports []v1.ServicePort) *v1.Service {
-	return &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        name,
-			Annotations: annotation,
-		},
-		Spec: v1.ServiceSpec{
-			Selector: labels,
-			Ports:    ports,
-			Type:     "LoadBalancer",
-		},
-	}
-}
-
 // defaultDeployment returns a default deployment
 // running nginx image which exposes port 80
 func createNginxDeploymentManifest(name string, labels map[string]string) (result *appsv1.Deployment) {
@@ -554,7 +544,7 @@ func validateLoadBalancerBackendPools(vmssName string, cs clientset.Interface, s
 	annotation := map[string]string{
 		azure.ServiceAnnotationLoadBalancerMode: vmssName,
 	}
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
 	_, err := cs.CoreV1().Services(ns).Create(service)
 	Expect(err).NotTo(HaveOccurred())
 	utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns)

tests/e2e/utils/azure_test_client.go

@@ -118,6 +118,11 @@ func (tc *AzureTestClient) creteLoadBalancerClient() *aznetwork.LoadBalancersCli
 	return &aznetwork.LoadBalancersClient{BaseClient: tc.networkClient}
 }
 
+// createInterfacesClient generates network interface client with the same baseclient as azure test client
+func (tc *AzureTestClient) createInterfacesClient() *aznetwork.InterfacesClient {
+	return &aznetwork.InterfacesClient{BaseClient: tc.networkClient}
+}
+
 // createResourceGroupClient generates resource group client with the same baseclient as azure test client
 func (tc *AzureTestClient) createResourceGroupClient() *azresources.GroupsClient {
 	return &azresources.GroupsClient{BaseClient: tc.resourceClient}
@@ -133,6 +138,11 @@ func getResourceGroupFromProviderID(providerID string) (string, error) {
 	return matches[1], nil
 }
 
+// GetNodeResourceGroup returns the resource group of the given node
+func GetNodeResourceGroup(node *v1.Node) (string, error) {
+	return getResourceGroupFromProviderID(node.Spec.ProviderID)
+}
+
 func getLocationFromNodeLabels(node *v1.Node) string {
 	if location, ok := node.Labels[nodeLabelLocation]; ok {
 		return location