diff --git a/tests/e2e/network/cloud_config.go b/tests/e2e/network/cloud_config.go new file mode 100644 index 0000000000..2e6c2fa81b --- /dev/null +++ b/tests/e2e/network/cloud_config.go @@ -0,0 +1,305 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "context" + "os" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + + "sigs.k8s.io/cloud-provider-azure/pkg/consts" + "sigs.k8s.io/cloud-provider-azure/tests/e2e/utils" + + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + cloudConfigNamespace = "kube-system" + cloudConfigName = "azure-cloud-provider" + cloudConfigKey = "cloud-config" +) + +var _ = Describe("Cloud Config", Label(utils.TestSuiteLabelCloudConfig, utils.TestSuiteLabelSerial), func() { + basename := "cloudconfig-service" + serviceName := "cloudconfig-test" + + var ( + cs clientset.Interface + ns *v1.Namespace + tc *utils.AzureTestClient + ) + + labels := map[string]string{ + "app": serviceName, + } + ports := []v1.ServicePort{{ + Port: serverPort, + TargetPort: intstr.FromInt(serverPort), + }} + + BeforeEach(func() { + if !strings.EqualFold(os.Getenv(utils.CloudConfigFromSecret), "true") { + Skip("Testing cloud config needs reading config from secret") + } + var err error + cs, err = utils.CreateKubeClientSet() + Expect(err).NotTo(HaveOccurred()) + + ns, err = utils.CreateTestingNamespace(basename, cs) + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Creating Azure clients") + tc, err = utils.CreateAzureTestClient() + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Creating deployment " + serviceName) + deployment := createServerDeploymentManifest(serviceName, labels) + _, err = cs.AppsV1().Deployments(ns.Name).Create(context.TODO(), deployment, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + utils.Logf("Waiting for backend pods to be ready") + err = utils.WaitPodsToBeReady(cs, ns.Name) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := cs.AppsV1().Deployments(ns.Name).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + err = utils.DeleteNamespace(cs, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + cs = nil + ns = nil + tc = nil + }) + + It("should support cloud config `Tags`, `SystemTags` and `TagsMap`", func() { + By("Updating Tags and TagsMap in Cloudconfig") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + // Make sure no other existing tags + // TODO: don't clean up the default tags of NSG + config.SystemTags = "a, c, e, m, g=h" + config.Tags = "a=b,c= d,e =, =f" + config.TagsMap = map[string]string{"m": "n", "g=h": "i,j"} + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + + defer func() { + // TODO: clean Tags for NSG, route table and other resources completely in the future + By("Cleaning up Tags, SystemTags and TagsMap in cloud config secret") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.Tags = "" + config.SystemTags = "" + config.TagsMap = map[string]string{} + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + }() + + By("Creating a service") + service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, labels, ns.Name, ports) + _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting service to expose...") + ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) + + defer func() { + By("Cleaning up test service") + err := utils.DeleteServiceIfExists(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) + }() + + expectedTags := map[string]*string{ + "a": to.StringPtr("b"), + "c": to.StringPtr("d"), + "e": to.StringPtr(""), + "m": to.StringPtr("n"), + "g=h": to.StringPtr("i,j"), + } + + By("Checking tags on the loadbalancer") + err = waitCompareLBTags(tc, expectedTags, ip) + Expect(err).NotTo(HaveOccurred()) + + By("Checking tags of security group") + err = waitCompareNsgTags(tc, expectedTags) + Expect(err).NotTo(HaveOccurred()) + + By("Checking tags of route tables") + err = waitCompareRouteTableTags(tc, expectedTags) + Expect(err).NotTo(HaveOccurred()) + + By("Updating SystemTags in Cloudconfig") + config, err = utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.SystemTags = "a" + config.Tags = "u=w" + config.TagsMap = map[string]string{} + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + + expectedTags = map[string]*string{ + "a": to.StringPtr("b"), + "u": to.StringPtr("w"), + } + + By("Checking tags on the loadbalancer") + err = waitCompareLBTags(tc, expectedTags, ip) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should support cloud config `PrimaryScaleSetName` and `NodePoolsWithoutDedicatedSLB`", Label(utils.TestSuiteLabelMultiNodePools), func() { + // Get all the vmss names from all node's providerIDs + By("Get vmss names from node providerIDs") + vmssNames, resourceGroupName, err := utils.GetAllVMSSNamesAndResourceGroup(cs) + Expect(err).NotTo(HaveOccurred()) + + // Skip if there're less than two vmss + if len(vmssNames) < 2 { + Skip("nodePoolsWithoutDedicatedSLB tests only works for cluster with multiple vmss agent pools") + } + + vmssList := vmssNames.List()[:2] + + if !strings.EqualFold(os.Getenv(utils.LoadBalancerSkuEnv), string(network.PublicIPAddressSkuNameStandard)) { + Skip("nodePoolsWithoutDedicatedSLB tests only works for standard lb") + } + + By("Updating PrimaryScaleSetName in cloud config") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.EnableMultipleStandardLoadBalancers = true + config.NodePoolsWithoutDedicatedSLB = "" + originalPrimaryScaleSetName := config.PrimaryScaleSetName + config.PrimaryScaleSetName = vmssList[0] + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + + service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, labels, ns.Name, ports) + _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + // wait and get service's public IP Address + By("Waiting for service exposure") + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) + + defer func() { + By("Cleaning up test service") + err := utils.DeleteServiceIfExists(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) + }() + + // validate load balancer backend pools for PrimaryScaleSetName + backendPoolVMSSNames := getVMSSNamesInLoadBalancerBackendPools(tc, publicIP, tc.GetResourceGroup(), resourceGroupName) + Expect(backendPoolVMSSNames.Equal(sets.NewString(vmssList[0]))).To(Equal(true)) + + By("Updating NodePoolsWithoutDedicatedSLB in cloud config") + config, err = utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.NodePoolsWithoutDedicatedSLB = strings.Join(vmssList, consts.VMSetNamesSharingPrimarySLBDelimiter) + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + + // wait and validate load balancer backend pools for NodePoolsWithoutDedicatedSLB + err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (done bool, err error) { + backendPoolVMSSNames := getVMSSNamesInLoadBalancerBackendPools(tc, publicIP, tc.GetResourceGroup(), resourceGroupName) + return backendPoolVMSSNames.Equal(sets.NewString(vmssList...)), nil + }) + Expect(err).NotTo(HaveOccurred()) + + defer func() { + By("Cleaning up EnableMultipleStandardLoadBalancers in cloud config secret") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.EnableMultipleStandardLoadBalancers = false + config.NodePoolsWithoutDedicatedSLB = "" + config.PrimaryScaleSetName = originalPrimaryScaleSetName + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + }() + + }) +}) + +func waitCompareLBTags(tc *utils.AzureTestClient, expectedTags map[string]*string, ip string) error { + err := wait.PollImmediate(10*time.Second, 2*time.Minute, func() (done bool, err error) { + lb := getAzureLoadBalancerFromPIP(tc, ip, tc.GetResourceGroup(), "") + return compareTags(lb.Tags, expectedTags), nil + }) + return err +} + +func waitCompareRouteTableTags(tc *utils.AzureTestClient, expectedTags map[string]*string) error { + err := wait.PollImmediate(10*time.Second, 2*time.Minute, func() (done bool, err error) { + routeTables, err := utils.ListRouteTables(tc) + if err != nil { + return false, err + } + + rightTagFlag := true + for _, routeTable := range *routeTables { + utils.Logf("Checking tags for route table: " + *routeTable.Name) + if !compareTags(routeTable.Tags, expectedTags) { + rightTagFlag = false + break + } + } + return rightTagFlag, nil + }) + return err +} + +func waitCompareNsgTags(tc *utils.AzureTestClient, expectedTags map[string]*string) error { + err := wait.PollImmediate(10*time.Second, 2*time.Minute, func() (done bool, err error) { + nsgs, err := tc.GetClusterSecurityGroups() + if err != nil { + return false, err + } + + rightTagFlag := true + for _, nsg := range nsgs { + if !strings.Contains(*nsg.Name, "node") { + continue + } + utils.Logf("Checking tags for nsg: " + *nsg.Name) + tags := nsg.Tags + if !compareTags(tags, expectedTags) { + rightTagFlag = false + break + } + } + return rightTagFlag, nil + }) + return err +} diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go index 0d0ca67c3a..a9957d59f1 100644 --- a/tests/e2e/network/service_annotations.go +++ b/tests/e2e/network/service_annotations.go @@ -646,40 +646,98 @@ var _ = Describe("Multiple VMSS", Label(utils.TestSuiteLabelMultiNodePools, util tc = nil }) - It("should support service annotation `service.beta.kubernetes.io/azure-load-balancer-mode`", func() { - //get nodelist and providerID specific to an agentnodes - By("Getting agent nodes list") - nodes, err := utils.GetAgentNodes(cs) - Expect(err).NotTo(HaveOccurred()) - + It("should support service annotation `service.beta.kubernetes.io/azure-load-balancer-mode`", Label(utils.TestSuiteLabelSerial, utils.TestSuiteLabelSlow, utils.TestSuiteLabelCloudConfig), func() { // Get all the vmss names from all node's providerIDs By("Get vmss names from node providerIDs") - vmssNames := sets.NewString() - var resourceGroupName string - for i, node := range nodes { - if utils.IsControlPlaneNode(&nodes[i]) { - continue - } - providerID := node.Spec.ProviderID - matches := scalesetRE.FindStringSubmatch(providerID) - if len(matches) != 3 { - Skip("azure-load-balancer-mode tests only works for vmss cluster") - } - resourceGroupName = matches[1] - vmssNames.Insert(matches[2]) - } - Expect(resourceGroupName).NotTo(Equal("")) - utils.Logf("Got vmss names %v", vmssNames.List()) + vmssNames, resourceGroupName, err := utils.GetAllVMSSNamesAndResourceGroup(cs) + Expect(err).NotTo(HaveOccurred()) //Skip if there're less than two vmss if len(vmssNames) < 2 { Skip("azure-load-balancer-mode tests only works for cluster with multiple vmss agent pools") } + if strings.EqualFold(os.Getenv(utils.LoadBalancerSkuEnv), string(network.PublicIPAddressSkuNameStandard)) { + if strings.EqualFold(os.Getenv(utils.CloudConfigFromSecret), "true") { + By("Updating EnableMultipleStandardLoadBalancers to true in cloud config") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.EnableMultipleStandardLoadBalancers = true + config.NodePoolsWithoutDedicatedSLB = "" + config.PrimaryScaleSetName = "" + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + + defer func() { + By("Cleaning up EnableMultipleStandardLoadBalancers in cloud config secret") + config, err := utils.GetConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey) + Expect(err).NotTo(HaveOccurred()) + config.EnableMultipleStandardLoadBalancers = false + err = utils.UpdateConfigFromSecret(cs, cloudConfigNamespace, cloudConfigName, cloudConfigKey, config) + Expect(err).NotTo(HaveOccurred()) + }() + } else { + Skip("azure-load-balancer-mode only works for standard lb when EnableMultipleStandardLoadBalancers is true") + } + } + + utils.Logf("Updating deployment %s", serviceName) + tcpPort := int32(serverPort) + udpPort := int32(testingPort) + deployment := createDeploymentManifest(serviceName, labels, &tcpPort, &udpPort) + _, err = cs.AppsV1().Deployments(ns.Name).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + vmssList := vmssNames.List()[:2] - for _, vmss := range vmssList { - validateLoadBalancerBackendPools(tc, vmss, cs, serviceName, labels, ns.Name, ports, resourceGroupName) + // Make sure that the lb of first vmss has minimal rules + // This helps test the auto value for this annotation + ports2 := []v1.ServicePort{ + { + Name: "tcp", + Port: serverPort, + TargetPort: intstr.FromInt(serverPort), + Protocol: v1.ProtocolTCP, + }, + { + Name: "udp", + Port: testingPort, + TargetPort: intstr.FromInt(testingPort), + Protocol: v1.ProtocolUDP, + }, + } + for i, vmss := range vmssList { + if i == 0 { + validateLoadBalancerBackendPools(tc, vmss, cs, serviceName, labels, ns.Name, ports, resourceGroupName) + } else { + validateLoadBalancerBackendPools(tc, vmss, cs, serviceName, labels, ns.Name, ports2, resourceGroupName) + } } + + // Test auto value for service.beta.kubernetes.io/azure-load-balancer-mode annotation + // create annotation with auto value for LoadBalancer service + By("Creating service " + serviceName + " in namespace " + ns.Name) + annotation := map[string]string{ + consts.ServiceAnnotationLoadBalancerMode: consts.ServiceAnnotationLoadBalancerAutoModeValue, + } + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports) + _, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name) + + defer func() { + By("Cleaning up service") + err := utils.DeleteService(cs, ns.Name, serviceName) + Expect(err).NotTo(HaveOccurred()) + }() + + //wait and get service's public IP Address + By("Waiting for service exposure") + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "") + Expect(err).NotTo(HaveOccurred()) + + By("Checking if lbName of service with auto mode annotation equals " + vmssList[0]) + lb := getAzureLoadBalancerFromPIP(tc, publicIP, resourceGroupName, "") + Expect(*lb.Name).To(Equal(vmssList[0])) }) }) @@ -876,6 +934,19 @@ var _ = Describe("Multi-ports service", Label(utils.TestSuiteLabelMultiPorts), f }) }) +func compareTags(tags, expectedTags map[string]*string) bool { + printTags := func(name string, ts map[string]*string) { + msg := "" + for t := range ts { + msg += fmt.Sprintf("%s:%s ", t, *ts[t]) + } + utils.Logf("%s: [%s]", name, msg) + } + printTags("tags", tags) + printTags("expectedTags", expectedTags) + return reflect.DeepEqual(tags, expectedTags) +} + func waitComparePIPTags(tc *utils.AzureTestClient, expectedTags map[string]*string, pipName string) error { err := wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) { pip, err := utils.WaitGetPIP(tc, pipName) @@ -888,16 +959,7 @@ func waitComparePIPTags(tc *utils.AzureTestClient, expectedTags map[string]*stri delete(tags, consts.ServiceTagKey) delete(tags, consts.LegacyServiceTagKey) - printTags := func(name string, ts map[string]*string) { - msg := "" - for t := range ts { - msg += fmt.Sprintf("%s:%s ", t, *ts[t]) - } - utils.Logf("%s: [%s]", name, msg) - } - printTags("tags", tags) - printTags("expectedTags", expectedTags) - return reflect.DeepEqual(tags, expectedTags), nil + return compareTags(tags, expectedTags), nil }) return err } @@ -1015,62 +1077,17 @@ func createDeploymentManifest(name string, labels map[string]string, tcpPort, ud } } -func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string, cs clientset.Interface, serviceName string, labels map[string]string, ns string, ports []v1.ServicePort, resourceGroupName string) { - serviceName = fmt.Sprintf("%s-%s", serviceName, vmssName) - - // create annotation for LoadBalancer service - By("Creating service " + serviceName + " in namespace " + ns) - annotation := map[string]string{ - consts.ServiceAnnotationLoadBalancerMode: vmssName, - } - service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns, ports) - _, err := cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns) - - //wait and get service's public IP Address - By("Waiting for service exposure") - publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns, serviceName, "") - Expect(err).NotTo(HaveOccurred()) - +func getVMSSNamesInLoadBalancerBackendPools(tc *utils.AzureTestClient, publicIP, pipResourceGroup, lbResourceGroupName string) sets.String { // Invoking azure network client to get list of public IP Addresses - By("Getting public IPs in the resourceGroup " + resourceGroupName) - pipList, err := tc.ListPublicIPs(resourceGroupName) - Expect(err).NotTo(HaveOccurred()) - - By("Getting public IP frontend configuration ID") - var pipFrontendConfigurationID string - for _, ip := range pipList { - if ip.PublicIPAddressPropertiesFormat != nil && - ip.PublicIPAddressPropertiesFormat.IPAddress != nil && - ip.PublicIPAddressPropertiesFormat.IPConfiguration != nil && - ip.PublicIPAddressPropertiesFormat.IPConfiguration.ID != nil && - *ip.PublicIPAddressPropertiesFormat.IPAddress == publicIP { - pipFrontendConfigurationID = *ip.PublicIPAddressPropertiesFormat.IPConfiguration.ID - break - } - } + pipFrontendConfigurationID := getPIPFrontendConfigurationID(tc, publicIP, pipResourceGroup, "") Expect(pipFrontendConfigurationID).NotTo(Equal("")) - //Get Azure loadBalancer Name - By("Getting loadBalancer name from pipFrontendConfigurationID") - match := lbNameRE.FindStringSubmatch(pipFrontendConfigurationID) - Expect(len(match)).To(Equal(2)) - loadBalancerName := match[1] - Expect(loadBalancerName).NotTo(Equal("")) - utils.Logf("Got loadBalancerName %q", loadBalancerName) - //Get backendpools list By("Getting loadBalancer") - lb, err := tc.GetLoadBalancer(resourceGroupName, loadBalancerName) - Expect(err).NotTo(HaveOccurred()) + lb := getAzureLoadBalancerFromPIP(tc, publicIP, pipResourceGroup, lbResourceGroupName) Expect(lb.BackendAddressPools).NotTo(BeNil()) Expect(lb.LoadBalancingRules).NotTo(BeNil()) - if lb.Sku != nil && lb.Sku.Name == network.LoadBalancerSkuNameStandard { - Skip("azure-load-balancer-mode is not working for standard load balancer") - } - By("Getting loadBalancer backendPoolID") backendPoolID := "" for _, rule := range *lb.LoadBalancingRules { @@ -1084,6 +1101,8 @@ func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string } Expect(backendPoolID).NotTo(Equal("")) + backendPoolVMSSNames := sets.NewString() + By("Validating loadBalancer backendPool") for _, pool := range *lb.BackendAddressPools { if pool.ID == nil || pool.BackendIPConfigurations == nil || !strings.EqualFold(*pool.ID, backendPoolID) { @@ -1097,7 +1116,32 @@ func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string matches := backendIPConfigurationRE.FindStringSubmatch(*ipConfig.ID) Expect(len(matches)).To(Equal(2)) - Expect(matches[1]).To(Equal(vmssName)) + backendPoolVMSSNames.Insert(matches[1]) } } + + return backendPoolVMSSNames +} + +func validateLoadBalancerBackendPools(tc *utils.AzureTestClient, vmssName string, cs clientset.Interface, serviceName string, labels map[string]string, ns string, ports []v1.ServicePort, resourceGroupName string) { + serviceName = fmt.Sprintf("%s-%s", serviceName, vmssName) + + // create annotation for LoadBalancer service + By("Creating service " + serviceName + " in namespace " + ns) + annotation := map[string]string{ + consts.ServiceAnnotationLoadBalancerMode: vmssName, + } + service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns, ports) + _, err := cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns) + + //wait and get service's public IP Address + By("Waiting for service exposure") + publicIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns, serviceName, "") + Expect(err).NotTo(HaveOccurred()) + + // validate load balancer backend pools + backendPoolVMSSNames := getVMSSNamesInLoadBalancerBackendPools(tc, publicIP, tc.GetResourceGroup(), resourceGroupName) + Expect(backendPoolVMSSNames.Equal(sets.NewString(vmssName))).To(Equal(true)) } diff --git a/tests/e2e/utils/azure_auth.go b/tests/e2e/utils/azure_auth.go index 9a799d8c9d..7dcc4b95b6 100644 --- a/tests/e2e/utils/azure_auth.go +++ b/tests/e2e/utils/azure_auth.go @@ -44,6 +44,10 @@ const ( // If "E2E_ON_AKS_CLUSTER" is true, the test is running on a AKS cluster. AKSTestCCM = "E2E_ON_AKS_CLUSTER" AKSClusterType = "CLUSTER_TYPE" + // If "LOAD_CLOUD_CONFIG_FROM_SECRET" is true, ccm will read cloud-config from secret + // Otherwise, ccm will read cloud-config from azure.json + // This is a prerequisite for testing features in ckoud config + CloudConfigFromSecret = "LOAD_CLOUD_CONFIG_FROM_SECRET" ) // AzureAuthConfig holds auth related part of cloud config diff --git a/tests/e2e/utils/config_util.go b/tests/e2e/utils/config_util.go new file mode 100644 index 0000000000..ac8780e8fc --- /dev/null +++ b/tests/e2e/utils/config_util.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + + providerazure "sigs.k8s.io/cloud-provider-azure/pkg/provider" +) + +// GetConfigFromSecret gets cloud config from secret +func GetConfigFromSecret(cs clientset.Interface, ns, secretName, secretKey string) (*providerazure.Config, error) { + secret, err := cs.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %w", ns, secretName, err) + } + + cloudConfigData, ok := secret.Data[secretKey] + if !ok { + return nil, fmt.Errorf("cloud-config is not set in the secret (%s/%s)", ns, secretName) + } + + cloudConfig := providerazure.Config{} + err = json.Unmarshal(cloudConfigData, &cloudConfig) + if err != nil { + return nil, fmt.Errorf("failed to parse Azure cloud-config: %w", err) + } + return &cloudConfig, nil +} + +// UpdateConfigFromSecret updates cloud config from secret +func UpdateConfigFromSecret(cs clientset.Interface, ns, secretName, secretKey string, config *providerazure.Config) error { + cloudConfigData, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal Azure cloud-config: %w", err) + } + + secret, err := cs.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get secret %s/%s: %w", ns, secretName, err) + } + + secret.Data[secretKey] = cloudConfigData + _, err = cs.CoreV1().Secrets(ns).Update(context.TODO(), secret, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update secret %s/%s: %w", ns, secretName, err) + } + + return nil +} diff --git a/tests/e2e/utils/consts.go b/tests/e2e/utils/consts.go index ab9b60985b..d0e402813a 100644 --- a/tests/e2e/utils/consts.go +++ b/tests/e2e/utils/consts.go @@ -36,4 +36,5 @@ const ( TestSuiteLabelLB = "LB" TestSuiteLabelMultiPorts = "Multi-Ports" TestSuiteLabelNSG = "NSG" + TestSuiteLabelCloudConfig = "CloudConfig" ) diff --git a/tests/e2e/utils/vmss_utils.go b/tests/e2e/utils/vmss_utils.go index 93a7c0369a..7bce1d188b 100644 --- a/tests/e2e/utils/vmss_utils.go +++ b/tests/e2e/utils/vmss_utils.go @@ -27,10 +27,12 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" ) var ( - vmssVMRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines/(?:\d+)`) + scalesetRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) + vmssVMRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines/(?:\d+)`) ) // FindTestVMSS returns the first VMSS in the resource group, @@ -306,3 +308,33 @@ func GetVMSSVMComputerName(vm azcompute.VirtualMachineScaleSetVM) (string, error func IsSpotVMSS(vmss azcompute.VirtualMachineScaleSet) bool { return vmss.VirtualMachineProfile.Priority == azcompute.VirtualMachinePriorityTypesSpot } + +// GetAllVMSSNames gets all the vmss names and reource group from all node's providerIDs +func GetAllVMSSNamesAndResourceGroup(cs clientset.Interface) (sets.String, string, error) { + //get nodelist and providerID specific to an agentnodes + nodes, err := GetAgentNodes(cs) + if err != nil { + return nil, "", err + } + + // Get all the vmss names from all node's providerIDs + vmssNames := sets.NewString() + var resourceGroupName string + for i, node := range nodes { + if IsControlPlaneNode(&nodes[i]) { + continue + } + providerID := node.Spec.ProviderID + matches := scalesetRE.FindStringSubmatch(providerID) + if len(matches) != 3 { + continue + } + resourceGroupName = matches[1] + vmssNames.Insert(matches[2]) + } + if len(vmssNames) > 0 && resourceGroupName == "" { + return nil, "", fmt.Errorf("resource group name should not be empty") + } + Logf("Got vmss names %v", vmssNames.List()) + return vmssNames, resourceGroupName, nil +}