Skip to content

Commit

Permalink
[IPv6] Choose correct primary IP config
Browse files Browse the repository at this point in the history 
Regardless of IPv6 only or dualstack clusters, IPv4
IP config is always primary. So for IPv6 backend
address pool, IP config's IP version needs consideration.

Signed-off-by: Zhecheng Li <zhechengli@microsoft.com>
  • Loading branch information
@lzhecheng
lzhecheng committed Apr 14, 2023
1 parent 95d38de commit 0b77b12
Show file tree
Hide file tree
Showing 5 changed files with 382 additions and 326 deletions.
253 changes: 123 additions & 130 deletions pkg/provider/azure_vmss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1018,8 +1018,8 @@ func (ss *ScaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err
return nic, nil
}

// getPrimaryNetworkInterfaceConfiguration gets primary network interface configuration for scale set virtual machine.
func (ss *ScaleSet) getPrimaryNetworkInterfaceConfiguration(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
// getPrimaryNetworkInterfaceConfiguration gets primary network interface configuration for VMSS VM or VMSS.
func getPrimaryNetworkInterfaceConfiguration(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, resource string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
if len(networkConfigurations) == 1 {
return &networkConfigurations[0], nil
}
Expand All @@ -1031,58 +1031,38 @@ func (ss *ScaleSet) getPrimaryNetworkInterfaceConfiguration(networkConfiguration
}
}

return nil, fmt.Errorf("failed to find a primary network configuration for the scale set VM %q", nodeName)
return nil, fmt.Errorf("failed to find a primary network configuration for the VMSS VM or VMSS %q", resource)
}

// getPrimaryNetworkInterfaceConfigurationForScaleSet gets primary network interface configuration for scale set.
func getPrimaryNetworkInterfaceConfigurationForScaleSet(networkConfigurations []compute.VirtualMachineScaleSetNetworkConfiguration, vmssName string) (*compute.VirtualMachineScaleSetNetworkConfiguration, error) {
if len(networkConfigurations) == 1 {
return &networkConfigurations[0], nil
}

for idx := range networkConfigurations {
networkConfig := &networkConfigurations[idx]
if networkConfig.Primary != nil && *networkConfig.Primary {
return networkConfig, nil
}
}

return nil, fmt.Errorf("failed to find a primary network configuration for the scale set %q", vmssName)
}

func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScaleSetNetworkConfiguration) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScaleSetNetworkConfiguration, backendPoolID, resource string) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
ipConfigurations := *config.IPConfigurations
if len(ipConfigurations) == 1 {
return &ipConfigurations[0], nil
}

for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.Primary != nil && *ipConfig.Primary {
return ipConfig, nil
isIPv6 := isBackendPoolIPv6(backendPoolID)

if !isIPv6 {
// There should be one and only once primary IP config.
// https://learn.microsoft.com/en-us/azure/virtual-network/ip-services/virtual-network-network-interface-addresses?tabs=nic-address-portal#ip-configurations
if len(ipConfigurations) == 1 {
return &ipConfigurations[0], nil
}
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.Primary != nil && *ipConfig.Primary {
return ipConfig, nil
}
}
}

return nil, fmt.Errorf("failed to find a primary IP configuration")
}

func getConfigForScaleSetByIPFamily(config *compute.VirtualMachineScaleSetNetworkConfiguration, nodeName string, IPv6 bool) (*compute.VirtualMachineScaleSetIPConfiguration, error) {
ipConfigurations := *config.IPConfigurations

var ipVersion compute.IPVersion
if IPv6 {
ipVersion = compute.IPv6
} else {
ipVersion = compute.IPv4
}
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.PrivateIPAddressVersion == ipVersion {
return ipConfig, nil
// For IPv6 or dualstack service, we need to pick the right IP configuration based on the cluster ip family
// IPv6 configuration is only supported as non-primary, so we need to fetch the ip configuration where the
// privateIPAddressVersion matches the clusterIP family
for idx := range ipConfigurations {
ipConfig := &ipConfigurations[idx]
if ipConfig.PrivateIPAddressVersion == compute.IPv6 {
return ipConfig, nil
}
}
}

return nil, fmt.Errorf("failed to find a IPconfiguration(IPv6=%v) for the scale set VM %q", IPv6, nodeName)
return nil, fmt.Errorf("failed to find a primary IP configuration (IPv6=%v) for the VMSS VM or VMSS %q", isIPv6, resource)
}

// EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
Expand Down Expand Up @@ -1139,28 +1119,15 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
}

networkInterfaceConfigurations := *vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations
primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName)
primaryNetworkInterfaceConfiguration, err := getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName)
if err != nil {
return "", "", "", nil, err
}

var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration
ipv6 := isBackendPoolIPv6(backendPoolID)
// Find primary network interface configuration.
if !ss.Cloud.ipv6DualStackEnabled && !ipv6 {
// Find primary IP configuration.
primaryIPConfiguration, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
if err != nil {
return "", "", "", nil, err
}
} else {
// For IPv6 or dualstack service, we need to pick the right IP configuration based on the cluster ip family
// IPv6 configuration is only supported as non-primary, so we need to fetch the ip configuration where the
// privateIPAddressVersion matches the clusterIP family
primaryIPConfiguration, err = getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6)
if err != nil {
return "", "", "", nil, err
}
primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration, backendPoolID, vmName)
if err != nil {
return "", "", "", nil, err
}

// Update primary IP configuration's LoadBalancerBackendAddressPools.
Expand Down Expand Up @@ -1314,24 +1281,14 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
continue
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, vmssName)
if err != nil {
return err
}
var primaryIPConfig *compute.VirtualMachineScaleSetIPConfiguration
ipv6 := isBackendPoolIPv6(backendPoolID)
// Find primary network interface configuration.
if !ss.Cloud.ipv6DualStackEnabled && !ipv6 {
// Find primary IP configuration.
primaryIPConfig, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
if err != nil {
return err
}
} else {
primaryIPConfig, err = getConfigForScaleSetByIPFamily(primaryNIC, "", ipv6)
if err != nil {
return err
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, vmssName)
if err != nil {
return err
}

loadBalancerBackendAddressPools := []compute.SubResource{}
Expand Down Expand Up @@ -1603,42 +1560,60 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoo
return "", "", "", nil, nil
}
networkInterfaceConfigurations := *vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations
primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName)
primaryNetworkInterfaceConfiguration, err := getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName)
if err != nil {
return "", "", "", nil, err
}

// Find primary IP configuration.
primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
if err != nil {
return "", "", "", nil, err
}
if primaryIPConfiguration.LoadBalancerBackendAddressPools == nil || len(*primaryIPConfiguration.LoadBalancerBackendAddressPools) == 0 {
return "", "", "", nil, nil
}
handleBackendPool := func(backendPoolID string) (bool, error) {
// Find primary IP configuration.
newBackendPools := []compute.SubResource{}
foundPool := false
// Find primary IP configuration.
primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration, backendPoolID, nodeName)
if err != nil {
return false, err
}

// Construct new loadBalancerBackendAddressPools and remove backendAddressPools from primary IP configuration.
existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools
newBackendPools := []compute.SubResource{}
foundPool := false
for i := len(existingBackendPools) - 1; i >= 0; i-- {
curPool := existingBackendPools[i]
for _, backendPoolID := range backendPoolIDs {
if primaryIPConfiguration.LoadBalancerBackendAddressPools == nil || len(*primaryIPConfiguration.LoadBalancerBackendAddressPools) == 0 {
return false, nil
}

// Construct new loadBalancerBackendAddressPools and remove backendAddressPools from primary IP configuration.
existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools

for i := len(existingBackendPools) - 1; i >= 0; i-- {
curPool := existingBackendPools[i]
if strings.EqualFold(backendPoolID, *curPool.ID) {
klog.V(10).Infof("ensureBackendPoolDeletedFromNode gets unwanted backend pool %q for node %s", backendPoolID, nodeName)
foundPool = true
newBackendPools = append(existingBackendPools[:i], existingBackendPools[i+1:]...)
}
}
}

// Pool not found, assume it has been already removed.
if !foundPool {
// Pool not found, assume it has been already removed.
if !foundPool {
return false, nil
}

// Compose a new vmssVM with added backendPoolID.
primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
return true, nil
}
foundTotal := false
for _, backendPoolID := range backendPoolIDs {
found, err := handleBackendPool(backendPoolID)
if err != nil {
return "", "", "", nil, err
}
if found {
foundTotal = true
}
}
if !foundTotal {
return "", "", "", nil, nil
}

// Compose a new vmssVM with added backendPoolID.
primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
newVM := &compute.VirtualMachineScaleSetVM{
Location: &vm.Location,
VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
Expand Down Expand Up @@ -1803,30 +1778,38 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromVmssUniform(backendPoolIDs []str
return true
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, pointer.StringDeref(vmss.Name, ""))
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, pointer.StringDeref(vmss.Name, ""))
if err != nil {
klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to get the primary network interface config of the VMSS %s: %v", pointer.StringDeref(vmss.Name, ""), err)
errorList = append(errorList, err)
return true
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
if err != nil {
klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to the primary IP config from the VMSS %s's network config : %v", pointer.StringDeref(vmss.Name, ""), err)
errorList = append(errorList, err)
return true
}
loadBalancerBackendAddressPools := make([]compute.SubResource, 0)
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}
for _, loadBalancerBackendAddressPool := range loadBalancerBackendAddressPools {
klog.V(4).Infof("ensureBackendPoolDeletedFromVMSS: loadBalancerBackendAddressPool (%s) on vmss (%s)", pointer.StringDeref(loadBalancerBackendAddressPool.ID, ""), pointer.StringDeref(vmss.Name, ""))
for _, backendPoolID := range backendPoolIDs {

handleBackendPool := func(backendPoolID string) bool {
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, pointer.StringDeref(vmss.Name, ""))
if err != nil {
klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to find the primary IP config from the VMSS %s's network config : %v", pointer.StringDeref(vmss.Name, ""), err)
errorList = append(errorList, err)
return true
}

loadBalancerBackendAddressPools := make([]compute.SubResource, 0)
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}
for _, loadBalancerBackendAddressPool := range loadBalancerBackendAddressPools {
klog.V(4).Infof("ensureBackendPoolDeletedFromVMSS: loadBalancerBackendAddressPool (%s) on vmss (%s)", pointer.StringDeref(loadBalancerBackendAddressPool.ID, ""), pointer.StringDeref(vmss.Name, ""))
if strings.EqualFold(pointer.StringDeref(loadBalancerBackendAddressPool.ID, ""), backendPoolID) {
klog.V(4).Infof("ensureBackendPoolDeletedFromVMSS: found vmss %s with backend pool %s, removing it", pointer.StringDeref(vmss.Name, ""), backendPoolID)
vmssNamesMap[pointer.StringDeref(vmss.Name, "")] = true
}
}
return true
}
for _, backendPoolID := range backendPoolIDs {
if !handleBackendPool(backendPoolID) {
return false
}
}

return true
Expand Down Expand Up @@ -2169,42 +2152,52 @@ func (ss *ScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]b
continue
}
vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
primaryNIC, err := getPrimaryNetworkInterfaceConfiguration(vmssNIC, vmssName)
if err != nil {
klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to get the primary network interface config of the VMSS %s: %v", vmssName, err)
errors = append(errors, err)
continue
}
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
if err != nil {
klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to the primary IP config from the VMSS %s's network config : %v", vmssName, err)
errors = append(errors, err)
continue
}
loadBalancerBackendAddressPools := []compute.SubResource{}
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}
handleBackendPool := func(backendPoolID string) bool {
primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC, backendPoolID, vmssName)
if err != nil {
klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to get the primary IP config from the VMSS %s's network config : %v", vmssName, err)
errors = append(errors, err)
return false
}
loadBalancerBackendAddressPools := []compute.SubResource{}
if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
}

var found bool
var newBackendPools []compute.SubResource
for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
curPool := loadBalancerBackendAddressPools[i]
for _, backendPoolID := range backendPoolIDs {
var found bool
var newBackendPools []compute.SubResource
for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
curPool := loadBalancerBackendAddressPools[i]
if strings.EqualFold(backendPoolID, *curPool.ID) {
klog.V(10).Infof("EnsureBackendPoolDeletedFromVMSets gets unwanted backend pool %q for VMSS %s", backendPoolID, vmssName)
found = true
newBackendPools = append(loadBalancerBackendAddressPools[:i], loadBalancerBackendAddressPools[i+1:]...)
}
}
if !found {
return false
}
primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
return true
}
if !found {
foundTotal := false
for _, backendPoolID := range backendPoolIDs {
if changed := handleBackendPool(backendPoolID); changed {
foundTotal = true
}
}
if !foundTotal {
continue
}

vmssUpdaters = append(vmssUpdaters, func() error {
// Compose a new vmss with added backendPoolID.
primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
newVMSS := compute.VirtualMachineScaleSet{
Location: vmss.Location,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
Expand Down

0 comments on commit 0b77b12

Please sign in to comment.