Merge pull request #1857 from lodrem/feat/support-pip-prefix-for-rele…

…ase-1.1

[release-1.1] feat(load-balancer): support specifying Public IP address prefix to produce IP of Load Balancer

pkg/consts/consts.go

@@ -232,6 +232,9 @@ const (
 	// ServiceAnnotationPIPName specifies the pip that will be applied to load balancer
 	ServiceAnnotationPIPName = "service.beta.kubernetes.io/azure-pip-name"
 
+	// ServiceAnnotationPIPPrefixID specifies the pip prefix that will be applied to the load balancer.
+	ServiceAnnotationPIPPrefixID = "service.beta.kubernetes.io/azure-pip-prefix-id"
+
 	// ServiceAnnotationIPTagsForPublicIP specifies the iptags used when dynamically creating a public ip
 	ServiceAnnotationIPTagsForPublicIP = "service.beta.kubernetes.io/azure-pip-ip-tags"
 

pkg/provider/azure_loadbalancer.go

@@ -48,7 +48,7 @@ import (
 // if so, what its status is.
 func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) {
 	// Since public IP is not a part of the load balancer on Azure,
-	// there is a chance that we could orphan public IP resources while we delete the load blanacer (kubernetes/kubernetes#80571).
+	// there is a chance that we could orphan public IP resources while we delete the load balancer (kubernetes/kubernetes#80571).
 	// We need to make sure the existence of the load balancer depends on the load balancer resource and public IP resource on Azure.
 	existsPip := func() bool {
 		pipName, _, err := az.determinePublicIPName(clusterName, service, nil)
@@ -943,9 +943,12 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L
 
 func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service, pips *[]network.PublicIPAddress) (string, bool, error) {
 	var shouldPIPExisted bool
+
 	if name, found := service.Annotations[consts.ServiceAnnotationPIPName]; found && name != "" {
-		shouldPIPExisted = true
-		return name, shouldPIPExisted, nil
+		return name, true, nil
+	}
+	if ipPrefix, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && ipPrefix != "" {
+		return az.getPublicIPName(clusterName, service), false, nil
 	}
 
 	pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
@@ -1127,6 +1130,9 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai
 			pip.Sku = &network.PublicIPAddressSku{
 				Name: network.PublicIPAddressSkuNameStandard,
 			}
+			if pipPrefixName, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && pipPrefixName != "" {
+				pip.PublicIPPrefix = &network.SubResource{ID: to.StringPtr(pipPrefixName)}
+			}
 
 			// skip adding zone info since edge zones doesn't support multiple availability zones.
 			if !az.HasExtendedLocation() {
@@ -3006,7 +3012,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa
 		lb = &loadBalancer
 	}
 
-	discoveredDesiredPublicIP, pipsToBeDeleted, deletedDesiredPublicIP, pipsToBeUpdated, err := az.getPublicIPUpdates(clusterName, service, pips, wantLb, isInternal, desiredPipName, serviceName, serviceIPTagRequest, shouldPIPExisted)
+	discoveredDesiredPublicIP, pipsToBeDeleted, deletedDesiredPublicIP, pipsToBeUpdated, err := az.getPublicIPUpdates(
+		clusterName, service, pips, wantLb, isInternal, desiredPipName, serviceName, serviceIPTagRequest, shouldPIPExisted)
 	if err != nil {
 		return nil, err
 	}
@@ -3049,7 +3056,17 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa
 	return nil, nil
 }
 
-func (az *Cloud) getPublicIPUpdates(clusterName string, service *v1.Service, pips []network.PublicIPAddress, wantLb bool, isInternal bool, desiredPipName string, serviceName string, serviceIPTagRequest serviceIPTagRequest, serviceAnnotationRequestsNamedPublicIP bool) (bool, []*network.PublicIPAddress, bool, []*network.PublicIPAddress, error) {
+func (az *Cloud) getPublicIPUpdates(
+	clusterName string,
+	service *v1.Service,
+	pips []network.PublicIPAddress,
+	wantLb bool,
+	isInternal bool,
+	desiredPipName string,
+	serviceName string,
+	serviceIPTagRequest serviceIPTagRequest,
+	serviceAnnotationRequestsNamedPublicIP bool,
+) (bool, []*network.PublicIPAddress, bool, []*network.PublicIPAddress, error) {
 	var (
 		err                       error
 		discoveredDesiredPublicIP bool

pkg/provider/azure_standard.go

@@ -323,7 +323,15 @@ func (az *Cloud) getRulePrefix(service *v1.Service) string {
 }
 
 func (az *Cloud) getPublicIPName(clusterName string, service *v1.Service) string {
-	return fmt.Sprintf("%s-%s", clusterName, az.GetLoadBalancerName(context.TODO(), clusterName, service))
+	pipName := fmt.Sprintf("%s-%s", clusterName, az.GetLoadBalancerName(context.TODO(), clusterName, service))
+	if prefixID, ok := service.Annotations[consts.ServiceAnnotationPIPPrefixID]; ok && prefixID != "" {
+		prefixName, err := getLastSegment(prefixID, "/")
+		if err != nil {
+			return pipName
+		}
+		pipName = fmt.Sprintf("%s-%s", pipName, prefixName)
+	}
+	return pipName
 }
 
 func (az *Cloud) serviceOwnsRule(service *v1.Service, rule string) bool {

site/content/en/topics/loadbalancer.md

@@ -27,8 +27,9 @@ Below is a list of annotations supported for Kubernetes services with type `Load
 | `service.beta.kubernetes.io/azure-load-balancer-resource-group` | Name of the PIP resource group   | Specify the resource group of the service's PIP that are not in the same resource group as the cluster. | v1.10.0 and later |
 | `service.beta.kubernetes.io/azure-allowed-service-tags`      | List of allowed service tags | Specify a list of allowed [service tags](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#service-tags) separated by comma. | v1.11.0 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-tcp-idle-timeout` | TCP idle timeouts in minutes | Specify the time, in minutes, for TCP connection idle timeouts to occur on the load balancer. Default and minimum value is 4. Maximum value is 30. Must be an integer. |  v1.11.4, v1.12.0 and later |
-| `service.beta.kubernetes.io/azure-pip-name` | Name of PIP | Specify the PIP that will be applied to load balancer | v1.16 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset` | `true` | Disable `enableTcpReset` for SLB | v1.16-v1.18. The annotation has been deprecated and would be removed in a future release. |
+| `service.beta.kubernetes.io/azure-pip-name` | Name of PIP | Specify the PIP that will be applied to load balancer. | v1.16 and later |
+| `service.beta.kubernetes.io/azure-pip-prefix-id` | ID of Public IP Prefix | Specify the Public IP Prefix that will be applied to load balancer. | v1.21 and later with out-of-tree cloud provider |
 | `service.beta.kubernetes.io/azure-pip-tags` | Tags of the PIP | Specify the tags of the PIP that will be associated to the load balancer typed service. [Doc](../tagging-resources) | v1.20 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` | Health probe protocol of the load balancer typed service | Refer the detailed docs [here](../custom-health-probe) | v1.20 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | Request path of the health probe | Refer the detailed docs [here](../custom-health-probe) | v1.20 and later |

tests/e2e/network/ensureloadbalancer.go

@@ -571,3 +571,16 @@ func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress {
 		},
 	}
 }
+
+func defaultPublicIPPrefix(name string) aznetwork.PublicIPPrefix {
+	return aznetwork.PublicIPPrefix{
+		Name:     to.StringPtr(name),
+		Location: to.StringPtr(os.Getenv(utils.ClusterLocationEnv)),
+		Sku: &aznetwork.PublicIPPrefixSku{
+			Name: aznetwork.PublicIPPrefixSkuNameStandard,
+		},
+		PublicIPPrefixPropertiesFormat: &aznetwork.PublicIPPrefixPropertiesFormat{
+			PrefixLength: to.Int32Ptr(28),
+		},
+	}
+}

tests/e2e/network/service_annotations.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"reflect"
 	"regexp"
 	"strings"
@@ -426,6 +427,108 @@ var _ = Describe("Service with annotation", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	It("should support service annotation `service.beta.kubernetes.io/azure-pip-prefix-id`", func() {
+		if skuEnv := os.Getenv(utils.LoadBalancerSkuEnv); skuEnv != "" {
+			if !strings.EqualFold(skuEnv, string(network.PublicIPAddressSkuNameStandard)) {
+				Skip("pip-prefix-id only work with Standard Load Balancer")
+			}
+		}
+
+		const (
+			prefix1Name = "prefix1"
+			prefix2Name = "prefix2"
+		)
+
+		By("Creating two test PIPPrefix")
+		prefix1, err := utils.WaitCreatePIPPrefix(tc, prefix1Name, tc.GetResourceGroup(), defaultPublicIPPrefix(prefix1Name))
+		Expect(err).NotTo(HaveOccurred())
+		prefix2, err := utils.WaitCreatePIPPrefix(tc, prefix2Name, tc.GetResourceGroup(), defaultPublicIPPrefix(prefix2Name))
+		Expect(err).NotTo(HaveOccurred())
+
+		defer func() {
+			By("Cleaning up test service")
+			{
+				err := utils.DeleteServiceIfExists(cs, ns.Name, serviceName)
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			// TODO: clean up PIPPrefix
+		}()
+
+		By("Creating a service referring to the prefix")
+		{
+			annotation := map[string]string{
+				consts.ServiceAnnotationPIPPrefixID: to.String(prefix1.ID),
+			}
+			service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
+			_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		}
+
+		By("Waiting for the service to expose")
+		{
+			ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
+			Expect(err).NotTo(HaveOccurred())
+
+			var pip network.PublicIPAddress
+			// wait until ip created by prefix
+			for i := 0; i < 30; i++ {
+				time.Sleep(10 * time.Second)
+				prefix, err := utils.WaitGetPIPPrefix(tc, prefix1Name)
+				if err != nil || prefix.PublicIPAddresses == nil || len(*prefix.PublicIPAddresses) != 1 {
+					continue
+				}
+
+				pipID := to.String((*prefix.PublicIPAddresses)[0].ID)
+				parts := strings.Split(pipID, "/")
+				pipName := parts[len(parts)-1]
+				pip, err = utils.WaitGetPIP(tc, pipName)
+				Expect(err).NotTo(HaveOccurred())
+
+				break
+			}
+			Expect(pip.IPAddress).NotTo(BeNil())
+			Expect(pip.PublicIPPrefix.ID).To(Equal(prefix1.ID))
+			Expect(ip).To(Equal(to.String(pip.IPAddress)))
+		}
+
+		By("Updating the service to refer to the second prefix")
+		{
+			service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			service.Annotations[consts.ServiceAnnotationPIPPrefixID] = to.String(prefix2.ID)
+			_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		}
+
+		By("Waiting for service IP to be updated")
+		{
+			var pip network.PublicIPAddress
+
+			// wait until ip created by prefix
+			for i := 0; i < 30; i++ {
+				time.Sleep(10 * time.Second)
+				prefix, err := utils.WaitGetPIPPrefix(tc, prefix2Name)
+				if err != nil || prefix.PublicIPAddresses == nil || len(*prefix.PublicIPAddresses) != 1 {
+					continue
+				}
+
+				pipID := to.String((*prefix.PublicIPAddresses)[0].ID)
+				parts := strings.Split(pipID, "/")
+				pipName := parts[len(parts)-1]
+				pip, err = utils.WaitGetPIP(tc, pipName)
+				Expect(err).NotTo(HaveOccurred())
+
+				break
+			}
+			Expect(pip.IPAddress).NotTo(BeNil())
+			Expect(pip.PublicIPPrefix.ID).To(Equal(prefix2.ID))
+
+			_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, to.String(pip.IPAddress))
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+
 	It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-health-probe-num-of-probe' and port specific configs", func() {
 		By("Creating a service with health probe annotations")
 		annotation := map[string]string{

tests/e2e/utils/azure_test_client.go

@@ -143,6 +143,11 @@ func (tc *AzureTestClient) createPublicIPAddressesClient() *aznetwork.PublicIPAd
 	return &aznetwork.PublicIPAddressesClient{BaseClient: tc.networkClient}
 }
 
+// createPublicIPPrefixesClient generates virtual network client with the same baseclient as azure test client
+func (tc *AzureTestClient) createPublicIPPrefixesClient() *aznetwork.PublicIPPrefixesClient {
+	return &aznetwork.PublicIPPrefixesClient{BaseClient: tc.networkClient}
+}
+
 // createLoadBalancerClient generates loadbalancer client with the same baseclient as azure test client
 func (tc *AzureTestClient) createLoadBalancerClient() *aznetwork.LoadBalancersClient {
 	return &aznetwork.LoadBalancersClient{BaseClient: tc.networkClient}

tests/e2e/utils/network_utils.go

@@ -207,6 +207,56 @@ func WaitCreatePIP(azureTestClient *AzureTestClient, ipName, rgName string, ipPa
 	return pip, err
 }
 
+func WaitCreatePIPPrefix(
+	cli *AzureTestClient,
+	name, rgName string,
+	parameter aznetwork.PublicIPPrefix,
+) (aznetwork.PublicIPPrefix, error) {
+	Logf("Creating PublicIPPrefix named %s", name)
+
+	resourceClient := cli.createPublicIPPrefixesClient()
+	_, err := resourceClient.CreateOrUpdate(context.Background(), rgName, name, parameter)
+	var prefix aznetwork.PublicIPPrefix
+	if err != nil {
+		return prefix, err
+	}
+	err = wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
+		prefix, err = resourceClient.Get(context.Background(), rgName, name, "")
+		if err != nil {
+			if !IsRetryableAPIError(err) {
+				return false, err
+			}
+			return false, nil
+		}
+		return prefix.IPPrefix != nil, nil
+	})
+	return prefix, err
+}
+
+func WaitGetPIPPrefix(
+	cli *AzureTestClient,
+	name string,
+) (aznetwork.PublicIPPrefix, error) {
+	Logf("Getting PublicIPPrefix named %s", name)
+
+	resourceClient := cli.createPublicIPPrefixesClient()
+	var (
+		prefix aznetwork.PublicIPPrefix
+		err    error
+	)
+	err = wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
+		prefix, err = resourceClient.Get(context.Background(), cli.GetResourceGroup(), name, "")
+		if err != nil {
+			if !IsRetryableAPIError(err) {
+				return false, err
+			}
+			return false, nil
+		}
+		return prefix.IPPrefix != nil, nil
+	})
+	return prefix, err
+}
+
 // DeletePIPWithRetry tries to delete a public ip resource
 func DeletePIPWithRetry(azureTestClient *AzureTestClient, ipName, rgName string) error {
 	if rgName == "" {