Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[release-1.26][IPv6] Choose correct primary IP config #3770

Merged
merged 1 commit into from
Apr 21, 2023
Merged

[release-1.26][IPv6] Choose correct primary IP config #3770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 9 additions & 0 deletions pkg/provider/azure_standard.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ var (
vmasIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`)
)

const (
v6Suffix = "IPv6"
)

// returns the full identifier of an availabilitySet
func (az *Cloud) getAvailabilitySetID(resourceGroup, availabilitySetName string) string {
return fmt.Sprintf(
Expand Down Expand Up @@ -272,6 +276,11 @@ func getBackendPoolName(clusterName string, service *v1.Service) string {
return clusterName
}

// ifBackendPoolIPv6 checks if a backend pool is of IPv6 according to name/ID.
func isBackendPoolIPv6(name string) bool {
return strings.HasSuffix(name, fmt.Sprintf("-%s", v6Suffix))
}

func (az *Cloud) getLoadBalancerRuleName(service *v1.Service, protocol v1.Protocol, port int32) string {
prefix := az.getRulePrefix(service)
ruleName := fmt.Sprintf("%s-%s-%d", prefix, protocol, port)
Expand Down
16 changes: 16 additions & 0 deletions pkg/provider/azure_standard_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -923,6 +923,22 @@ func TestGetBackendPoolName(t *testing.T) {
}
}

func TestIsBackendPoolIPv6(t *testing.T) {
testcases := []struct {
name string
expectedIsIPv6 bool
}{
{"bp-IPv6", true},
{"bp-IPv4", false},
{"bp", false},
{"bp-ipv6", false},
}
for _, test := range testcases {
isIPv6 := isBackendPoolIPv6(test.name)
assert.Equal(t, test.expectedIsIPv6, isIPv6)
}
}

func TestGetStandardInstanceIDByNodeName(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
Expand Down
192 changes: 70 additions & 122 deletions pkg/provider/azure_vmss.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog/v2"
utilnet "k8s.io/utils/net"
"k8s.io/utils/pointer"

azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
Expand Down Expand Up @@ -1016,8 +1015,8 @@ func (ss *ScaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err
return nic, nil
}

// getPrimaryNetworkInterfaceConfiguration gets primary network interface configuration for scale set virtual machine.
func (ss *ScaleSet) getPrimaryNetworkInterfaceConfiguration(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
// getPrimaryNetworkInterfaceConfiguration gets primary network interface configuration for VMSS VM or VMSS.
func getPrimaryNetworkInterfaceConfiguration(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, resource string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
if len(networkConfigurations) == 1 {
return &networkConfigurations[0], nil
}
Expand All @@ -1029,58 +1028,38 @@ func (ss *ScaleSet) getPrimaryNetworkInterfaceConfiguration(networkConfiguration
}
}

return nil, fmt.Errorf("failed to find a primary network configuration for the scale set VM %q", nodeName)
return nil, fmt.Errorf("failed to find a primary network configuration for the VMSS VM or VMSS %q", resource)
}

// getPrimaryNetworkInterfaceConfigurationForScaleSet gets primary network interface configuration for scale set.
func getPrimaryNetworkInterfaceConfigurationForScaleSet(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, vmssName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
if len(networkConfigurations) == 1 {
return &networkConfigurations[0], nil
}

for idx := range networkConfigurations {
networkConfig := &networkConfigurations[idx]
if networkConfig.Primary != nil && *networkConfig.Primary {
return networkConfig, nil
}
}

return nil, fmt.Errorf("failed to find a primary network configuration for the scale set %q", vmssName)
}

func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScaleSetNetworkConfiguration) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScaleSetNetworkConfiguration, backendPoolID, resource string) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
ipConfigurations := *config.IPConfigurations
if len(ipConfigurations) == 1 {
return &ipConfigurations[0], nil
}

for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.Primary != nil && *ipConfig.Primary {
return ipConfig, nil
isIPv6 := isBackendPoolIPv6(backendPoolID)

if !isIPv6 {
// There should be exactly one primary IP config.
// https://learn.microsoft.com/en-us/azure/virtual-network/ip-services/virtual-network-network-interface-addresses?tabs=nic-address-portal#ip-configurations
if len(ipConfigurations) == 1 {
return &ipConfigurations[0], nil
}
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.Primary != nil && *ipConfig.Primary {
return ipConfig, nil
}
}
}

return nil, fmt.Errorf("failed to find a primary IP configuration")
}

func getConfigForScaleSetByIPFamily(config *compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string, IPv6 bool) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
ipConfigurations := *config.IPConfigurations

var ipVersion compute.IPVersion
if IPv6 {
ipVersion = compute.IPv6
} else {
ipVersion = compute.IPv4
}
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.PrivateIPAddressVersion == ipVersion {
return ipConfig, nil
// For IPv6 or dualstack service, we need to pick the right IP configuration based on the cluster ip family
// IPv6 configuration is only supported as non-primary, so we need to fetch the ip configuration where the
// privateIPAddressVersion matches the clusterIP family
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.PrivateIPAddressVersion == compute.IPv6 {
return ipConfig, nil
}
}
}

return nil, fmt.Errorf("failed to find a IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName)
return nil, fmt.Errorf("failed to find a primary IP configuration (IPv6=%t) for the VMSS VM or VMSS %q", isIPv6, resource)
}

// EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
Expand Down Expand Up @@ -1137,28 +1116,15 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
}

networkInterfaceConfigurations := *vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations
primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName)
primaryNetworkInterfaceConfiguration, err := getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName)
if err != nil {
return "", "", "", nil, err
}

var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration
ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
// Find primary network interface configuration.
if !ss.Cloud.ipv6DualStackEnabled && !ipv6 {
// Find primary IP configuration.
primaryIPConfiguration, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
if err != nil {
return "", "", "", nil, err
}
} else {
// For IPv6 or dualstack service, we need to pick the right IP configuration based on the cluster ip family
// IPv6 configuration is only supported as non-primary, so we need to fetch the ip configuration where the
// privateIPAddressVersion matches the clusterIP family
primaryIPConfiguration, err = getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6)
if err != nil {
return "", "", "", nil, err
}
primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration, backendPoolID, vmName)
if err != nil {
return "", "", "", nil, err
}

// Update primary IP configuration's LoadBalancerBackendAddressPools.
Expand Down Expand Up @@ -1312,24 +1278,14 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
continue
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, vmssName)
if err != nil {
return err
}
var primaryIPConfig *compute.VirtualMachineScaleSetIPConfiguration
ipv6 := utilnet.IsIPv6String(service.Spec.ClusterIP)
// Find primary network interface configuration.
if !ss.Cloud.ipv6DualStackEnabled && !ipv6 {
// Find primary IP configuration.
primaryIPConfig, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
if err != nil {
return err
}
} else {
primaryIPConfig, err = getConfigForScaleSetByIPFamily(primaryNIC, "", ipv6)
if err != nil {
return err
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, vmssName)
if err != nil {
return err
}

loadBalancerBackendAddressPools := []compute.SubResource{}
Expand Down Expand Up @@ -1601,40 +1557,20 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID str
return "", "", "", nil, nil
}
networkInterfaceConfigurations := *vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations
primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName)
primaryNetworkInterfaceConfiguration, err := getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName)
if err != nil {
return "", "", "", nil, err
}

// Find primary IP configuration.
primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
found, err := deleteBackendPoolFromIPConfig("ensureBackendPoolDeletedFromNode", backendPoolID, nodeName, primaryNetworkInterfaceConfiguration)
if err != nil {
return "", "", "", nil, err
}
if primaryIPConfiguration.LoadBalancerBackendAddressPools == nil || len(*primaryIPConfiguration.LoadBalancerBackendAddressPools) == 0 {
return "", "", "", nil, nil
}

// Construct new loadBalancerBackendAddressPools and remove backendAddressPools from primary IP configuration.
existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools
newBackendPools := []compute.SubResource{}
foundPool := false
for i := len(existingBackendPools) - 1; i >= 0; i-- {
curPool := existingBackendPools[i]
if strings.EqualFold(backendPoolID, *curPool.ID) {
klog.V(10).Infof("ensureBackendPoolDeletedFromNode gets unwanted backend pool %q for node %s", backendPoolID, nodeName)
foundPool = true
newBackendPools = append(existingBackendPools[:i], existingBackendPools[i+1:]...)
}
}

// Pool not found, assume it has been already removed.
if !foundPool {
if !found {
return "", "", "", nil, nil
}

// Compose a new vmssVM with added backendPoolID.
primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
newVM := &compute.VirtualMachineScaleSetVM{
Location: &vm.Location,
VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
Expand Down Expand Up @@ -1800,13 +1736,13 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromVmssUniform(backendPoolID, vmSet
return true
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, pointer.StringDeref(vmss.Name, ""))
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, pointer.StringDeref(vmss.Name, ""))
if err != nil {
klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to get the primary network interface config of the VMSS %s: %v", pointer.StringDeref(vmss.Name, ""), err)
errorList = append(errorList, err)
return true
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, pointer.StringDeref(vmss.Name, ""))
if err != nil {
klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to the primary IP config from the VMSS %s's network config : %v", pointer.StringDeref(vmss.Name, ""), err)
errorList = append(errorList, err)
Expand Down Expand Up @@ -2124,6 +2060,35 @@ func (ss *ScaleSet) GetNodeCIDRMasksByProviderID(providerID string) (int, int, e
return ipv4Mask, ipv6Mask, nil
}

// deleteBackendPoolFromIPConfig deletes the backend pool from the IP config.
func deleteBackendPoolFromIPConfig(msg, backendPoolID, resource string, primaryNIC *compute.VirtualMachineScaleSetNetworkConfiguration) (bool, error) {
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, resource)
if err != nil {
klog.Errorf("%s: failed to get the primary IP config from the VMSS %q's network config: %v", msg, resource, err)
return false, err
}
loadBalancerBackendAddressPools := []compute.SubResource{}
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}

var found bool
var newBackendPools []compute.SubResource
for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
curPool := loadBalancerBackendAddressPools[i]
if strings.EqualFold(backendPoolID, *curPool.ID) {
klog.V(10).Infof("%s gets unwanted backend pool %q for VMSS OR VMSS VM %q", msg, backendPoolID, resource)
found = true
newBackendPools = append(loadBalancerBackendAddressPools[:i], loadBalancerBackendAddressPools[i+1:]...)
}
}
if !found {
return false, nil
}
primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
return true, nil
}

// EnsureBackendPoolDeletedFromVMSets ensures the loadBalancer backendAddressPools deleted from the specified VMSS
func (ss *ScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]bool, backendPoolID string) error {
vmssUpdaters := make([]func() error, 0, len(vmssNamesMap))
Expand All @@ -2148,40 +2113,23 @@ func (ss *ScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]b
continue
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, vmssName)
if err != nil {
klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to get the primary network interface config of the VMSS %s: %v", vmssName, err)
errors = append(errors, err)
continue
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
found, err := deleteBackendPoolFromIPConfig("EnsureBackendPoolDeletedFromVMSets", backendPoolID, vmssName, primaryNIC)
if err != nil {
klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to the primary IP config from the VMSS %s's network config : %v", vmssName, err)
errors = append(errors, err)
continue
}
loadBalancerBackendAddressPools := []compute.SubResource{}
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}

var found bool
var newBackendPools []compute.SubResource
for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
curPool := loadBalancerBackendAddressPools[i]
if strings.EqualFold(backendPoolID, *curPool.ID) {
klog.V(10).Infof("EnsureBackendPoolDeletedFromVMSets gets unwanted backend pool %q for VMSS %s", backendPoolID, vmssName)
found = true
newBackendPools = append(loadBalancerBackendAddressPools[:i], loadBalancerBackendAddressPools[i+1:]...)
}
}
if !found {
continue
}

vmssUpdaters = append(vmssUpdaters, func() error {
// Compose a new vmss with added backendPoolID.
primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
newVMSS := compute.VirtualMachineScaleSet{
Location: vmss.Location,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
Expand Down