Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support sharing IP address acorss services by public IP name #4257

Merged
merged 2 commits into from
Jul 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
158 changes: 120 additions & 38 deletions pkg/provider/azure_loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"sort"
"strconv"
"strings"
"unicode"

"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"

Expand Down Expand Up @@ -809,7 +810,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service,

// For the services with loadBalancerIP set, an existing public IP is required, primary
// or secondary, or a public IP not found error would be reported.
pip, err := az.findMatchedPIPByLoadBalancerIP(service, loadBalancerIP, pipResourceGroup)
pip, err := az.findMatchedPIP(loadBalancerIP, "", pipResourceGroup)
if err != nil {
return "", false, err
}
Expand All @@ -821,39 +822,6 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service,
return "", false, fmt.Errorf("user supplied IP Address %s was not found in resource group %s", loadBalancerIP, pipResourceGroup)
}

func (az *Cloud) findMatchedPIPByLoadBalancerIP(service *v1.Service, loadBalancerIP, pipResourceGroup string) (*network.PublicIPAddress, error) {
pips, err := az.listPIP(pipResourceGroup, azcache.CacheReadTypeDefault)
if err != nil {
return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: failed to listPIP: %w", err)
}

pip, err := getExpectedPIPFromListByIPAddress(pips, loadBalancerIP)
if err != nil {
pips, err = az.listPIP(pipResourceGroup, azcache.CacheReadTypeForceRefresh)
if err != nil {
return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: failed to listPIP force refresh: %w", err)
}

pip, err = getExpectedPIPFromListByIPAddress(pips, loadBalancerIP)
if err != nil {
return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: cannot find public IP with IP address %s in resource group %s", loadBalancerIP, pipResourceGroup)
}
}

return pip, nil
}

func getExpectedPIPFromListByIPAddress(pips []network.PublicIPAddress, ip string) (*network.PublicIPAddress, error) {
for _, pip := range pips {
if pip.PublicIPAddressPropertiesFormat.IPAddress != nil &&
*pip.PublicIPAddressPropertiesFormat.IPAddress == ip {
return &pip, nil
}
}

return nil, fmt.Errorf("getExpectedPIPFromListByIPAddress: cannot find public IP with IP address %s", ip)
}

func flipServiceInternalAnnotation(service *v1.Service) *v1.Service {
copyService := service.DeepCopy()
if copyService.Annotations == nil {
Expand Down Expand Up @@ -3516,7 +3484,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus

// if there is no service tag on the pip, it is user-created pip
if serviceTag == "" {
return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), true
return isServiceSelectPIP(service, pip, isIPv6), true
}

// if there is service tag on the pip, it is system-created pip
Expand All @@ -3532,18 +3500,26 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus
}

// if the service is not included in the tags of the system-created pip, check the ip address
// this could happen for secondary services
return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), false
// or pip name, this could happen for secondary services
return isServiceSelectPIP(service, pip, isIPv6), false
}

// if the pip has no tags, it should be user-created
return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), true
return isServiceSelectPIP(service, pip, isIPv6), true
}

func isServiceLoadBalancerIPMatchesPIP(service *v1.Service, pip *network.PublicIPAddress, isIPV6 bool) bool {
return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, isIPV6))
}

func isServicePIPNameMatchesPIP(service *v1.Service, pip *network.PublicIPAddress, isIPV6 bool) bool {
return strings.EqualFold(pointer.StringDeref(pip.Name, ""), getServicePIPName(service, isIPV6))
}

func isServiceSelectPIP(service *v1.Service, pip *network.PublicIPAddress, isIPV6 bool) bool {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As for the name, how about using own instead of select since we already have serviceOwnsFrontendIP?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have this function.

return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPV6) || isServicePIPNameMatchesPIP(service, pip, isIPV6)
}

func isSVCNameInPIPTag(tag, svcName string) bool {
svcNames := parsePIPServiceTag(&tag)

Expand Down Expand Up @@ -3913,3 +3889,109 @@ func isLoadBalancerInUseByService(service *v1.Service, lbConfig MultipleStandard
}
return false
}

// There are two cases when a service owns the frontend IP config:
// 1. The primary service, which means the frontend IP config is created after the creation of the service.
// This means the name of the config can be tracked by the service UID.
// 2. The secondary services must have their loadBalancer IP set if they want to share the same config as the primary
// service. Hence, it can be tracked by the loadBalancer IP.
// If the IP version is not empty, which means it is the secondary Service, it returns IP version of the Service FIP.
func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, service *v1.Service) (bool, bool, network.IPVersion) {
var isPrimaryService bool
baseName := az.GetLoadBalancerName(context.TODO(), "", service)
if strings.HasPrefix(pointer.StringDeref(fip.Name, ""), baseName) {
klog.V(6).Infof("serviceOwnsFrontendIP: found primary service %s of the frontend IP config %s", service.Name, *fip.Name)
isPrimaryService = true
return true, isPrimaryService, ""
}

loadBalancerIPs := getServiceLoadBalancerIPs(service)
pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
var pipNames []string
if len(loadBalancerIPs) == 0 {
if !requiresInternalLoadBalancer(service) {
pipNames = getServicePIPNames(service)
for _, pipName := range pipNames {
if pipName != "" {
pip, err := az.findMatchedPIP("", pipName, pipResourceGroup)
if err != nil {
klog.Warningf("serviceOwnsFrontendIP: unexpected error when finding match public IP of the service %s with name %s: %v", service.Name, pipName, err)
return false, isPrimaryService, ""
}
if publicIPOwnsFrontendIP(service, &fip, pip) {
return true, isPrimaryService, pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion
}
}
}
}
// it is a must that the secondary services set the loadBalancer IP or pip name
return false, isPrimaryService, ""
}

// for external secondary service the public IP address should be checked
if !requiresInternalLoadBalancer(service) {
for _, loadBalancerIP := range loadBalancerIPs {
pip, err := az.findMatchedPIP(loadBalancerIP, "", pipResourceGroup)
if err != nil {
klog.Warningf("serviceOwnsFrontendIP: unexpected error when finding match public IP of the service %s with loadBalancerIP %s: %v", service.Name, loadBalancerIP, err)
return false, isPrimaryService, ""
}

if publicIPOwnsFrontendIP(service, &fip, pip) {
return true, isPrimaryService, pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion
}
klog.V(6).Infof("serviceOwnsFrontendIP: the public IP with ID %s is being referenced by other service with public IP address %s "+
"OR it is of incorrect IP version", *pip.ID, *pip.IPAddress)
}

return false, isPrimaryService, ""
}

// for internal secondary service the private IP address on the frontend IP config should be checked
if fip.PrivateIPAddress == nil {
return false, isPrimaryService, ""
}
privateIPAddrVersion := network.IPv4
if net.ParseIP(*fip.PrivateIPAddress).To4() == nil {
privateIPAddrVersion = network.IPv6
}

privateIPEquals := false
for _, loadBalancerIP := range loadBalancerIPs {
if strings.EqualFold(*fip.PrivateIPAddress, loadBalancerIP) {
privateIPEquals = true
break
}
}
return privateIPEquals, isPrimaryService, privateIPAddrVersion
}

func (az *Cloud) getFrontendIPConfigNames(service *v1.Service) map[bool]string {
isDualStack := isServiceDualStack(service)
defaultLBFrontendIPConfigName := az.getDefaultFrontendIPConfigName(service)
return map[bool]string{
consts.IPVersionIPv4: getResourceByIPFamily(defaultLBFrontendIPConfigName, isDualStack, consts.IPVersionIPv4),
consts.IPVersionIPv6: getResourceByIPFamily(defaultLBFrontendIPConfigName, isDualStack, consts.IPVersionIPv6),
}
}

func (az *Cloud) getDefaultFrontendIPConfigName(service *v1.Service) string {
baseName := az.GetLoadBalancerName(context.TODO(), "", service)
subnetName := getInternalSubnet(service)
if subnetName != nil {
ipcName := fmt.Sprintf("%s-%s", baseName, *subnetName)

// Azure lb front end configuration name must not exceed 80 characters
maxLength := consts.FrontendIPConfigNameMaxLength - consts.IPFamilySuffixLength
if len(ipcName) > maxLength {
ipcName = ipcName[:maxLength]
// Cutting the string may result in char like "-" as the string end.
// If the last char is not a letter or '_', replace it with "_".
if !unicode.IsLetter(rune(ipcName[len(ipcName)-1:][0])) && ipcName[len(ipcName)-1:] != "_" {
ipcName = ipcName[:len(ipcName)-1] + "_"
}
}
return ipcName
}
return baseName
}