Enable e2e test for cross groups nodes.

tests/e2e/network/ensureloadbalancer.go

@@ -87,7 +87,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		}
 		ipName := basename + "-public-none-IP" + string(uuid.NewUUID())[0:4]
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		_, err := cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
@@ -131,7 +131,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		ip1, err := utils.SelectAvailablePrivateIP(tc)
 		Expect(err).NotTo(HaveOccurred())
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service = updateServiceBalanceIP(service, true, ip1)
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
@@ -173,7 +173,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		}
 		ipName := basename + "-internal-none-public-IP" + string(uuid.NewUUID())[0:4]
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		_, err := cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
@@ -222,7 +222,7 @@ var _ = FDescribe("Ensure LoadBalancer", func() {
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
 
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service = updateServiceBalanceIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())

tests/e2e/network/functions.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/cloud-provider-azure/tests/e2e/utils"
+)
+
+var _ = Describe("Cloud Provider Azure", func() {
+	basename := "service-lb"
+	serviceName := "servicelb-test"
+
+	var cs clientset.Interface
+	var ns *v1.Namespace
+	//var tc *utils.AzureTestClient
+
+	labels := map[string]string{
+		"app": serviceName,
+	}
+	ports := []v1.ServicePort{{
+		Port:       nginxPort,
+		TargetPort: intstr.FromInt(nginxPort),
+	}}
+
+	BeforeEach(func() {
+		var err error
+		cs, err = utils.CreateKubeClientSet()
+		Expect(err).NotTo(HaveOccurred())
+
+		ns, err = utils.CreateTestingNamespace(basename, cs)
+		Expect(err).NotTo(HaveOccurred())
+
+		//tc, err = utils.CreateAzureTestClient()
+		//Expect(err).NotTo(HaveOccurred())
+
+		utils.Logf("Creating deployment " + serviceName)
+		deployment := createNginxDeploymentManifest(serviceName, labels)
+		_, err = cs.AppsV1().Deployments(ns.Name).Create(deployment)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		err := cs.AppsV1().Deployments(ns.Name).Delete(serviceName, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = utils.DeleteNamespace(cs, ns.Name)
+		Expect(err).NotTo(HaveOccurred())
+
+		cs = nil
+		ns = nil
+		//tc = nil
+	})
+
+	It("should support nodes crossing resource groups [multi-group] [availabilitySet]", func() {
+		master, err := utils.GetMaster(cs)
+		Expect(err).NotTo(HaveOccurred())
+
+		var rgMaster string
+		rgMaster, err = utils.GetNodeResourceGroup(master)
+		utils.Logf("found master resource group %s", rgMaster)
+		Expect(err).NotTo(HaveOccurred())
+
+		nodes, err := utils.GetAgentNodes(cs)
+		Expect(err).NotTo(HaveOccurred())
+		var nodeNotInRGMaster v1.Node
+		var nodeNotInRGMAsterCount int
+		for _, node := range nodes {
+			if rg, err := utils.GetNodeResourceGroup(&node); err == nil && rg != rgMaster {
+				utils.Logf("rg of node %s is %s", node.Name, rg)
+				nodeNotInRGMaster = node
+				nodeNotInRGMAsterCount++
+			} else if err != nil {
+				Fail("cannot obtain the node's resource group")
+			}
+		}
+		utils.Logf("found node %s in another resource group", nodeNotInRGMaster.Name)
+
+		if nodeNotInRGMAsterCount == 0 {
+			Skip("cannot find a second resource group, skipping")
+		}
+		labels := nodeNotInRGMaster.Labels
+		Expect(labels).NotTo(BeNil())
+		excludeLB, ok := labels[`alpha.service-controller.kubernetes.io/exclude-balancer`]
+		Expect(ok).To(BeTrue())
+		Expect(excludeLB).To(Equal("true"))
+
+		clusterRG, ok := labels[`kubernetes.azure.com/cluster`]
+		Expect(ok).To(BeTrue())
+		Expect(clusterRG).To(Equal(rgMaster))
+
+		nodeRG, ok := labels[`kubernetes.azure.com/resource-group`]
+		Expect(ok).To(BeTrue())
+		Expect(nodeRG).NotTo(Equal(rgMaster))
+
+		publicIP := createServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports)
+		lb := getAzureLoadBalancerFromPIP(publicIP, rgMaster, rgMaster)
+
+		for _, backendPool := range *lb.BackendAddressPools {
+			if backendPool.BackendIPConfigurations != nil {
+				if *backendPool.BackendIPConfigurations != nil {
+					for _, ipConfig := range *backendPool.BackendIPConfigurations {
+						matches := backendASIPConfigurationRE.FindStringSubmatch(*ipConfig.ID)
+						Expect(len(matches)).To(Equal(2))
+						Expect(matches[1]).NotTo(Equal(nodeNotInRGMaster.Name))
+					}
+				}
+			}
+		}
+	})
+})

tests/e2e/network/network_security_group.go

@@ -224,7 +224,7 @@ func validateSharedSecurityRuleExists(nsg *aznetwork.SecurityGroup, ips []string
 
 func createAndWaitServiceExposure(cs clientset.Interface, ns string, serviceName string, annotation map[string]string, labels map[string]string, ports []v1.ServicePort) (string, error) {
 	ip := ""
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
 	if _, err := cs.CoreV1().Services(ns).Create(service); err != nil {
 		return ip, err
 	}

tests/e2e/network/service_annotations.go

@@ -42,9 +42,10 @@ import (
 )
 
 var (
-	scalesetRE               = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
-	lbNameRE                 = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`)
-	backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	scalesetRE                 = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	lbNameRE                   = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`)
+	backendIPConfigurationRE   = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+	backendASIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)/virtualMachines(?:.*)`)
 )
 
 const (
@@ -240,7 +241,7 @@ var _ = FDescribe("Service with annotation", func() {
 			azure.ServiceAnnotationLoadBalancerResourceGroup: to.String(rg.Name),
 		}
 		By("Creating service " + serviceName + " in namespace " + ns.Name)
-		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
 		service.Spec.LoadBalancerIP = *pip.IPAddress
 		_, err = cs.CoreV1().Services(ns.Name).Create(service)
 		Expect(err).NotTo(HaveOccurred())
@@ -251,7 +252,7 @@ var _ = FDescribe("Service with annotation", func() {
 		_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName)
 		Expect(err).NotTo(HaveOccurred())
 
-		lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name)
+		lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name, "")
 		Expect(lb).NotTo(BeNil())
 	})
 })
@@ -373,13 +374,13 @@ func getAzureLoadBalancer(pip string) *network.LoadBalancer {
 	return &lb
 }
 
-func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBalancer {
+func getAzureLoadBalancerFromPIP(pip, pipResourceGroup, lbResourceGroup string) *network.LoadBalancer {
 	By("Creating Azure clients")
 	azureTestClient, err := utils.CreateAzureTestClient()
 	Expect(err).NotTo(HaveOccurred())
 
-	By("Getting public IPs in the resourceGroup " + resourceGroupName)
-	pipList, err := azureTestClient.ListPublicIPs(resourceGroupName)
+	By("Getting public IPs in the resourceGroup " + pipResourceGroup)
+	pipList, err := azureTestClient.ListPublicIPs(pipResourceGroup)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("Getting public IP frontend configuration ID")
@@ -405,7 +406,10 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal
 	utils.Logf("Got loadBalancerName %q", loadBalancerName)
 
 	By("Getting loadBalancer")
-	lb, err := azureTestClient.GetLoadBalancer(azureTestClient.GetResourceGroup(), loadBalancerName)
+	if lbResourceGroup == "" {
+		lbResourceGroup = azureTestClient.GetResourceGroup()
+	}
+	lb, err := azureTestClient.GetLoadBalancer(lbResourceGroup, loadBalancerName)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(lb.LoadBalancingRules).NotTo(BeNil())
 
@@ -414,7 +418,7 @@ func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBal
 
 func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) string {
 	By("Creating service " + serviceName + " in namespace " + nsName)
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports)
 	_, err := cs.CoreV1().Services(nsName).Create(service)
 	Expect(err).NotTo(HaveOccurred())
 	utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + nsName)
@@ -427,20 +431,6 @@ func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName str
 	return publicIP
 }
 
-func createLoadBalancerServiceManifest(c clientset.Interface, name string, annotation map[string]string, labels map[string]string, namespace string, ports []v1.ServicePort) *v1.Service {
-	return &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        name,
-			Annotations: annotation,
-		},
-		Spec: v1.ServiceSpec{
-			Selector: labels,
-			Ports:    ports,
-			Type:     "LoadBalancer",
-		},
-	}
-}
-
 // defaultDeployment returns a default deployment
 // running nginx image which exposes port 80
 func createNginxDeploymentManifest(name string, labels map[string]string) (result *appsv1.Deployment) {
@@ -554,7 +544,7 @@ func validateLoadBalancerBackendPools(vmssName string, cs clientset.Interface, s
 	annotation := map[string]string{
 		azure.ServiceAnnotationLoadBalancerMode: vmssName,
 	}
-	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
+	service := utils.CreateLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns, ports)
 	_, err := cs.CoreV1().Services(ns).Create(service)
 	Expect(err).NotTo(HaveOccurred())
 	utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns)

tests/e2e/utils/azure_test_client.go

@@ -133,6 +133,11 @@ func getResourceGroupFromProviderID(providerID string) (string, error) {
 	return matches[1], nil
 }
 
+// GetNodeResourceGroup returns the resource group of the given node
+func GetNodeResourceGroup(node *v1.Node) (string, error) {
+	return getResourceGroupFromProviderID(node.Spec.ProviderID)
+}
+
 func getLocationFromNodeLabels(node *v1.Node) string {
 	if location, ok := node.Labels[nodeLabelLocation]; ok {
 		return location

tests/e2e/utils/network_utils.go

@@ -24,7 +24,11 @@ import (
 
 	aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network"
 	"github.com/Azure/go-autorest/autorest/to"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 )
 
 const k8sVNetPrefix = "k8s-vnet-"
@@ -146,6 +150,21 @@ func (azureTestClient *AzureTestClient) GetClusterSecurityGroup() (ret *aznetwor
 	return
 }
 
+// CreateLoadBalancerServiceManifest return the specific service to be created
+func CreateLoadBalancerServiceManifest(c clientset.Interface, name string, annotation map[string]string, labels map[string]string, namespace string, ports []v1.ServicePort) *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: annotation,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: labels,
+			Ports:    ports,
+			Type:     "LoadBalancer",
+		},
+	}
+}
+
 // WaitCreatePIP waits to create a public ip resource
 func WaitCreatePIP(azureTestClient *AzureTestClient, ipName string, ipParameter aznetwork.PublicIPAddress) (aznetwork.PublicIPAddress, error) {
 	Logf("Creating public IP resource named %s", ipName)

tests/e2e/utils/node_utils.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,6 +47,20 @@ func GetAgentNodes(cs clientset.Interface) ([]v1.Node, error) {
 	return ret, nil
 }
 
+// GetMaster returns the master node
+func GetMaster(cs clientset.Interface) (*v1.Node, error) {
+	nodesList, err := getNodeList(cs)
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range nodesList.Items {
+		if isMasterNode(&node) {
+			return &node, nil
+		}
+	}
+	return nil, fmt.Errorf("cannot obtain the master node")
+}
+
 // GetNodeList is a wapper around listing nodes
 func getNodeList(cs clientset.Interface) (*v1.NodeList, error) {
 	var nodes *v1.NodeList

tests/e2e/utils/resource_group_utils.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package utils
 
 import (