Merge pull request #200 from nilo19/t-qini-add_e2e_test_annotation_re…

…sourcegroup

Add e2e test for service annotation 'ServiceAnnotationLoadBalancerResourceGroup'

tests/e2e/network/service_annotations.go

@@ -24,6 +24,8 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network"
+	"github.com/Azure/go-autorest/autorest/to"
+
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -219,6 +221,39 @@ var _ = FDescribe("Service with annotation", func() {
 		}
 		Expect(len(existingProtocols)).To(Equal(2))
 	})
+
+	It("should support load balancer annotation 'ServiceAnnotationLoadBalancerResourceGroup'", func() {
+		By("Creating Azure clients")
+		tc, err := utils.CreateAzureTestClient()
+		Expect(err).NotTo(HaveOccurred())
+
+		By("creating a test resource group")
+		rg, cleanup := utils.CreateTestResourceGroup(tc)
+		defer cleanup(to.String(rg.Name))
+
+		By("creating test PIP in the test resource group")
+		testPIPName := "testPIP" + string(uuid.NewUUID())[0:4]
+		pip, err := utils.WaitCreateNewPIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName))
+		Expect(err).NotTo(HaveOccurred())
+
+		annotation := map[string]string{
+			azure.ServiceAnnotationLoadBalancerResourceGroup: to.String(rg.Name),
+		}
+		By("Creating service " + serviceName + " in namespace " + ns.Name)
+		service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, ns.Name, ports)
+		service.Spec.LoadBalancerIP = *pip.IPAddress
+		_, err = cs.CoreV1().Services(ns.Name).Create(service)
+		Expect(err).NotTo(HaveOccurred())
+		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
+
+		//wait and get service's public IP Address
+		By("Waiting service to expose...")
+		_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName)
+		Expect(err).NotTo(HaveOccurred())
+
+		lb := getAzureLoadBalancerFromPIP(*pip.IPAddress, *rg.Name)
+		Expect(lb).NotTo(BeNil())
+	})
 })
 
 var _ = Describe("[MultipleAgentPools][VMSS]", func() {
@@ -338,6 +373,45 @@ func getAzureLoadBalancer(pip string) *network.LoadBalancer {
 	return &lb
 }
 
+func getAzureLoadBalancerFromPIP(pip, resourceGroupName string) *network.LoadBalancer {
+	By("Creating Azure clients")
+	azureTestClient, err := utils.CreateAzureTestClient()
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Getting public IPs in the resourceGroup " + resourceGroupName)
+	pipList, err := azureTestClient.ListPublicIPs(resourceGroupName)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Getting public IP frontend configuration ID")
+	var pipFrontendConfigurationID string
+	for _, ip := range pipList {
+		if ip.PublicIPAddressPropertiesFormat != nil &&
+			ip.PublicIPAddressPropertiesFormat.IPAddress != nil &&
+			ip.PublicIPAddressPropertiesFormat.IPConfiguration != nil &&
+			ip.PublicIPAddressPropertiesFormat.IPConfiguration.ID != nil &&
+			*ip.PublicIPAddressPropertiesFormat.IPAddress == pip {
+			pipFrontendConfigurationID = *ip.PublicIPAddressPropertiesFormat.IPConfiguration.ID
+			break
+		}
+	}
+	Expect(pipFrontendConfigurationID).NotTo(Equal(""))
+	By(fmt.Sprintf("Successfully obtained PIP front config id: %v", pipFrontendConfigurationID))
+
+	By("Getting loadBalancer name from pipFrontendConfigurationID")
+	match := lbNameRE.FindStringSubmatch(pipFrontendConfigurationID)
+	Expect(len(match)).To(Equal(2))
+	loadBalancerName := match[1]
+	Expect(loadBalancerName).NotTo(Equal(""))
+	utils.Logf("Got loadBalancerName %q", loadBalancerName)
+
+	By("Getting loadBalancer")
+	lb, err := azureTestClient.GetLoadBalancer(azureTestClient.GetResourceGroup(), loadBalancerName)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(lb.LoadBalancingRules).NotTo(BeNil())
+
+	return &lb
+}
+
 func createServiceWithAnnotation(cs clientset.Interface, serviceName, nsName string, labels, annotation map[string]string, ports []v1.ServicePort) string {
 	By("Creating service " + serviceName + " in namespace " + nsName)
 	service := createLoadBalancerServiceManifest(cs, serviceName, annotation, labels, nsName, ports)

tests/e2e/utils/azure_test_client.go

@@ -20,19 +20,26 @@ import (
 	"fmt"
 	"regexp"
 
+	v1 "k8s.io/api/core/v1"
+
 	aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network"
+	azresources "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 )
 
 var (
 	azureResourceGroupNameRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/(?:.*)`)
+	nodeLabelLocation        = "failure-domain.beta.kubernetes.io/region"
+	defaultLocation          = "eastus2"
 )
 
 // AzureTestClient configs Azure specific clients
 type AzureTestClient struct {
-	resourceGroup string
-	networkClient aznetwork.BaseClient
+	location       string
+	resourceGroup  string
+	networkClient  aznetwork.BaseClient
+	resourceClient azresources.BaseClient
 }
 
 // CreateAzureTestClient makes a new AzureTestClient
@@ -49,6 +56,9 @@ func CreateAzureTestClient() (*AzureTestClient, error) {
 	baseClient := aznetwork.NewWithBaseURI(azure.PublicCloud.TokenAudience, authconfig.SubscriptionID)
 	baseClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipleToken)
 
+	resourceBaseClient := azresources.BaseClient(baseClient)
+	resourceBaseClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipleToken)
+
 	kubeClient, err := CreateKubeClientSet()
 	if err != nil {
 		return nil, err
@@ -66,10 +76,13 @@ func CreateAzureTestClient() (*AzureTestClient, error) {
 	if err != nil {
 		return nil, err
 	}
+	location := getLocationFromNodeLabels(&nodes[0])
 
 	c := &AzureTestClient{
-		resourceGroup: resourceGroup,
-		networkClient: baseClient,
+		location:       location,
+		resourceGroup:  resourceGroup,
+		networkClient:  baseClient,
+		resourceClient: resourceBaseClient,
 	}
 
 	return c, nil
@@ -105,6 +118,11 @@ func (tc *AzureTestClient) creteLoadBalancerClient() *aznetwork.LoadBalancersCli
 	return &aznetwork.LoadBalancersClient{BaseClient: tc.networkClient}
 }
 
+// createResourceGroupClient generates resource group client with the same baseclient as azure test client
+func (tc *AzureTestClient) createResourceGroupClient() *azresources.GroupsClient {
+	return &azresources.GroupsClient{BaseClient: tc.resourceClient}
+}
+
 // getResourceGroupFromProviderID gets the resource group name in the provider ID.
 func getResourceGroupFromProviderID(providerID string) (string, error) {
 	matches := azureResourceGroupNameRE.FindStringSubmatch(providerID)
@@ -114,3 +132,10 @@ func getResourceGroupFromProviderID(providerID string) (string, error) {
 
 	return matches[1], nil
 }
+
+func getLocationFromNodeLabels(node *v1.Node) string {
+	if location, ok := node.Labels[nodeLabelLocation]; ok {
+		return location
+	}
+	return defaultLocation
+}

tests/e2e/utils/network_utils.go

@@ -168,6 +168,28 @@ func WaitCreatePIP(azureTestClient *AzureTestClient, ipName string, ipParameter
 	return pip, err
 }
 
+// WaitCreateNewPIP waits to create a public ip resource in a specific resource group
+func WaitCreateNewPIP(azureTestClient *AzureTestClient, ipName, rgName string, ipParameter aznetwork.PublicIPAddress) (aznetwork.PublicIPAddress, error) {
+	Logf("Creating public IP resource named %s", ipName)
+	pipClient := azureTestClient.createPublicIPAddressesClient()
+	_, err := pipClient.CreateOrUpdate(context.Background(), rgName, ipName, ipParameter)
+	var pip aznetwork.PublicIPAddress
+	if err != nil {
+		return pip, err
+	}
+	err = wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
+		pip, err = pipClient.Get(context.Background(), rgName, ipName, "")
+		if err != nil {
+			if !IsRetryableAPIError(err) {
+				return false, err
+			}
+			return false, nil
+		}
+		return pip.IPAddress != nil, nil
+	})
+	return pip, err
+}
+
 // DeletePIPWithRetry tries to delete a pulic ip resourc
 func DeletePIPWithRetry(azureTestClient *AzureTestClient, ipName string) error {
 	Logf("Deleting public IP resource named %s", ipName)

tests/e2e/utils/resource_group_utils.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"k8s.io/apimachinery/pkg/util/uuid"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// CreateTestResourceGroup create a test rg
+func CreateTestResourceGroup(tc *AzureTestClient) (*resources.Group, func(string)) {
+	gc := tc.createResourceGroupClient()
+	rgName := to.StringPtr("e2e-" + string(uuid.NewUUID())[0:4])
+	rg, err := gc.CreateOrUpdate(context.Background(), *rgName, createTestTemplate(tc, rgName))
+	Expect(err).NotTo(HaveOccurred())
+	By(fmt.Sprintf("resource group %s created", *rgName))
+
+	return &rg, func(rgName string) {
+		Logf("cleaning up test resource group %s", rgName)
+		_, err := gc.Delete(context.Background(), rgName)
+		Expect(err).NotTo(HaveOccurred())
+		return
+	}
+}
+
+func createTestTemplate(tc *AzureTestClient, name *string) resources.Group {
+	return resources.Group{
+		Name:     name,
+		Location: to.StringPtr(tc.location),
+	}
+}

tests/e2e/utils/utils.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"