Add assigned egress ips into capacity

This makes sure capacity field in cloud.network.openshift.io/egress-ipconfig
node annotation (which is introduced in 4.10) denotes correct value when
cluster upgrade happens for example 4.9 to 4.10 and node is already
assigned with egress ips.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>

cmd/cloud-network-config-controller/main.go

@@ -116,6 +116,7 @@ func main() {
 					kubeClient,
 					cloudProviderClient,
 					kubeInformerFactory.Core().V1().Nodes(),
+					cloudNetworkInformerFactory.Cloud().V1().CloudPrivateIPConfigs(),
 				)
 				secretController := secretcontroller.NewSecretController(
 					ctx,

pkg/cloudprivateipconfig/cloudprivateipconfig.go

@@ -0,0 +1,39 @@
+package cloudprivateipconfig
+
+import (
+	"errors"
+	"net"
+	"strings"
+)
+
+// IPFamily string representing ip family
+type IPFamily string
+
+const (
+	// IPv4 IPFamily constant ipv4 family
+	IPv4 IPFamily = "ipv4"
+	// IPv6 IPFamily constant ipv6 family
+	IPv6 IPFamily = "ipv6"
+)
+
+// NameToIP converts the resource name to net.IP. Given a
+// limitation in the Kubernetes API server (see:
+// https://github.com/kubernetes/kubernetes/pull/100950)
+// CloudPrivateIPConfig.metadata.name cannot represent an IPv6 address. To
+// work-around this limitation it was decided that the network plugin creating
+// the CR will fully expand the IPv6 address and replace all colons with dots,
+// Example: The IPv6 address fc00:f853:ccd:e793::54 will be represented
+// as: fc00.f853.0ccd.e793.0000.0000.0000.0054, We thus need to replace
+// every fifth character's dot with a colon.
+func NameToIP(name string) (net.IP, IPFamily, error) {
+	// handle IPv4: this is enough since it will be serialized just fine
+	if ip := net.ParseIP(name); ip != nil {
+		return ip, IPv4, nil
+	}
+	// handle IPv6
+	name = strings.ReplaceAll(name, ".", ":")
+	if ip := net.ParseIP(name); ip != nil {
+		return ip, IPv6, nil
+	}
+	return nil, "", errors.New("invalid ip family")
+}

pkg/cloudprivateipconfig/cloudprivateipconfig_test.go

@@ -0,0 +1,33 @@
+package cloudprivateipconfig
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCloudPrivateIPConfig(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Test CloudPrivateIPConfig")
+}
+
+var _ = Describe("CloudPrivateIPConfig", func() {
+	Context("Validate Name to IP", func() {
+		It("With valid names", func() {
+			ip, family, err := NameToIP("192.168.0.10")
+			Expect(err).To(BeNil())
+			Expect(ip.String()).To(Equal(("192.168.0.10")))
+			Expect(family).To(Equal(IPv4))
+			ip, family, err = NameToIP("fc00.f853.0ccd.e793.0000.0000.0000.0054")
+			Expect(err).To(BeNil())
+			Expect(ip.String()).To(Equal(("fc00:f853:ccd:e793::54")))
+			Expect(family).To(Equal(IPv6))
+		})
+		It("With invalid name", func() {
+			_, _, err := NameToIP("invalid_config")
+			Expect(err).NotTo(BeNil())
+		})
+	})
+
+})

pkg/cloudprovider/aws.go

@@ -13,6 +13,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	v1 "github.com/openshift/api/cloudnetwork/v1"
+	"github.com/openshift/cloud-network-config-controller/pkg/cloudprivateipconfig"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -177,7 +179,7 @@ func (a *AWS) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	}
 }
 
-func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	instance, err := a.getInstance(node)
 	if err != nil {
 		return nil, err
@@ -205,7 +207,10 @@ func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPCo
 	if v6Subnet != nil {
 		config.IFAddr.IPv6 = v6Subnet.String()
 	}
-	capV4, capV6 := a.getCapacity(instanceV4Capacity, instanceV6Capacity, networkInterface)
+	capV4, capV6, err := a.getCapacity(instanceV4Capacity, instanceV6Capacity, networkInterface, cloudPrivateIPConfigs)
+	if err != nil {
+		return nil, err
+	}
 	config.Capacity = capacity{
 		IPv4: capV4,
 		IPv6: capV6,
@@ -289,7 +294,7 @@ func (a *AWS) getSubnet(networkInterface *ec2.InstanceNetworkInterface) (*net.IP
 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI
 // Hence we need to retrieve that and then subtract the amount already assigned
 // by default.
-func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInterface *ec2.InstanceNetworkInterface) (int, int) {
+func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInterface *ec2.InstanceNetworkInterface, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) (int, int, error) {
 	currentIPv4Usage, currentIPv6Usage := 0, 0
 	for _, assignedIPv6 := range networkInterface.Ipv6Addresses {
 		if assignedIP := net.ParseIP(*assignedIPv6.Ipv6Address); assignedIP != nil {
@@ -301,7 +306,18 @@ func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInt
 			currentIPv4Usage++
 		}
 	}
-	return instanceV4Capacity - currentIPv4Usage, instanceV6Capacity - currentIPv6Usage
+	for _, cloudPrivateIPConfig := range cloudPrivateIPConfigs {
+		_, ipFamily, err := cloudprivateipconfig.NameToIP(cloudPrivateIPConfig.Name)
+		if err != nil {
+			return 0, 0, err
+		}
+		if ipFamily == cloudprivateipconfig.IPv4 {
+			instanceV4Capacity++
+		} else if ipFamily == cloudprivateipconfig.IPv6 {
+			instanceV6Capacity++
+		}
+	}
+	return instanceV4Capacity - currentIPv4Usage, instanceV6Capacity - currentIPv6Usage, nil
 }
 
 func (a *AWS) getNetworkInterfaces(instance *ec2.Instance) ([]*ec2.InstanceNetworkInterface, error) {

pkg/cloudprovider/azure.go

@@ -14,6 +14,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	azureapi "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	utilnet "k8s.io/utils/net"
@@ -179,7 +180,7 @@ func (a *Azure) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	return a.waitForCompletion(result)
 }
 
-func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	instance, err := a.getInstance(node)
 	if err != nil {
 		return nil, err
@@ -208,7 +209,7 @@ func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIP
 		config.IFAddr.IPv6 = v6Subnet.String()
 	}
 	config.Capacity = capacity{
-		IP: a.getCapacity(networkInterface),
+		IP: a.getCapacity(networkInterface, len(cloudPrivateIPConfigs)),
 	}
 	return []*NodeEgressIPConfiguration{config}, nil
 }
@@ -253,7 +254,7 @@ func (a *Azure) getSubnet(networkInterface network.Interface) (*net.IPNet, *net.
 // We need to retrieve the amounts assigned to the node by default and subtract
 // that from the default 256 value. Note: there is also a "Private IP addresses
 // per virtual network" quota, but that's 65.536, so we can skip that.
-func (a *Azure) getCapacity(networkInterface network.Interface) int {
+func (a *Azure) getCapacity(networkInterface network.Interface, cloudPrivateIPsCount int) int {
 	currentIPv4Usage, currentIPv6Usage := 0, 0
 	for _, ipConfiguration := range *networkInterface.IPConfigurations {
 		if assignedIP := net.ParseIP(*ipConfiguration.PrivateIPAddress); assignedIP != nil {
@@ -264,7 +265,7 @@ func (a *Azure) getCapacity(networkInterface network.Interface) int {
 			}
 		}
 	}
-	return defaultAzurePrivateIPCapacity - currentIPv4Usage - currentIPv6Usage
+	return defaultAzurePrivateIPCapacity + cloudPrivateIPsCount - currentIPv4Usage - currentIPv6Usage
 }
 
 // This is what the node's providerID looks like on Azure

pkg/cloudprovider/cloudprovider.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sync"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -54,10 +55,8 @@ type CloudProviderIntf interface {
 	// specifically that: the IP capacity can be either hard-coded and global
 	// for all instance types and IP families (GCP, Azure) or variable per
 	// instance and IP family (AWS), also: the interface is either keyed by name
-	// (GCP) or ID (Azure, AWS). Note: this function should only be called when
-	// no egress IPs have been added to the node, it will return an incorrect
-	// "egress IP capacity" otherwise
-	GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error)
+	// (GCP) or ID (Azure, AWS).
+	GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error)
 }
 
 // CloudProviderConfig is all the command-line options needed to initialize

pkg/cloudprovider/cloudprovider_fake.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"time"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -62,7 +63,7 @@ func (f *FakeCloudProvider) waitForCompletion() error {
 	return nil
 }
 
-func (f *FakeCloudProvider) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (f *FakeCloudProvider) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	if f.mockErrorOnGetNodeEgressIPConfiguration {
 		return nil, fmt.Errorf("Get node egress IP configuration failed")
 	}

pkg/cloudprovider/gcp.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"strings"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	google "google.golang.org/api/compute/v1"
 	"google.golang.org/api/option"
 	corev1 "k8s.io/api/core/v1"
@@ -128,7 +129,7 @@ func (g *GCP) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	return g.waitForCompletion(project, zone, operation.Name)
 }
 
-func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	project, _, instance, err := g.getInstance(node)
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving instance associated with node, err: %v", err)
@@ -155,7 +156,7 @@ func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPCo
 			config.IFAddr.IPv6 = v6Subnet.String()
 		}
 		config.Capacity = capacity{
-			IP: g.getCapacity(networkInterface),
+			IP: g.getCapacity(networkInterface, len(cloudPrivateIPConfigs)),
 		}
 		return []*NodeEgressIPConfiguration{config}, nil //nolint:staticcheck
 	}
@@ -204,7 +205,7 @@ func (g *GCP) getSubnet(project string, networkInterface *google.NetworkInterfac
 
 // Note: there is also a global "alias IP per VPC quota", but OpenShift clusters on
 // GCP seem to have that value defined to 15,000. So we can skip that.
-func (g *GCP) getCapacity(networkInterface *google.NetworkInterface) int {
+func (g *GCP) getCapacity(networkInterface *google.NetworkInterface, cloudPrivateIPsCount int) int {
 	currentIPv4Usage := 0
 	currentIPv6Usage := 0
 	for _, aliasIPRange := range networkInterface.AliasIpRanges {
@@ -222,7 +223,7 @@ func (g *GCP) getCapacity(networkInterface *google.NetworkInterface) int {
 			}
 		}
 	}
-	return defaultGCPPrivateIPCapacity - currentIPv4Usage - currentIPv6Usage
+	return defaultGCPPrivateIPCapacity + cloudPrivateIPsCount - currentIPv4Usage - currentIPv6Usage
 }
 
 // getInstance retrieves the GCP instance referrred by the Node object.

pkg/cloudprovider/openstack.go

@@ -21,6 +21,8 @@ import (
 	neutronsubnets "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
 	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/gophercloud/utils/openstack/clientconfig"
+	v1 "github.com/openshift/api/cloudnetwork/v1"
+	"github.com/openshift/cloud-network-config-controller/pkg/cloudprivateipconfig"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -387,7 +389,7 @@ func (o *OpenStack) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 // * The interface is keyed by a neutron UUID
 // This function should only be called when no egress IPs have been added to the node,
 // it will return an incorrect "egress IP capacity" otherwise.
-func (o *OpenStack) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (o *OpenStack) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	if node == nil {
 		return nil, fmt.Errorf("invalid nil pointer provided for node when trying to get node EgressIP configuration")
 	}
@@ -409,7 +411,7 @@ func (o *OpenStack) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgre
 	cidrs := make(map[string]struct{})
 	for _, p := range serverPorts {
 		// Retrieve configuration for this port.
-		config, err := o.getNeutronPortNodeEgressIPConfiguration(p)
+		config, err := o.getNeutronPortNodeEgressIPConfiguration(p, cloudPrivateIPConfigs)
 		if err != nil {
 			return nil, err
 		}
@@ -450,7 +452,7 @@ func (o *OpenStack) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgre
 // TODO: As a solution, we currently report the EgressIP configuration for every attached interface, but other plugins
 // do not do this. Is the upper layer compatible with that?
 // TODO: How to determine the primary AF?
-func (o *OpenStack) getNeutronPortNodeEgressIPConfiguration(p neutronports.Port) (*NodeEgressIPConfiguration, error) {
+func (o *OpenStack) getNeutronPortNodeEgressIPConfiguration(p neutronports.Port, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) (*NodeEgressIPConfiguration, error) {
 	var ipv4, ipv6 string
 	var err error
 	var ip net.IP
@@ -465,6 +467,7 @@ func (o *OpenStack) getNeutronPortNodeEgressIPConfiguration(p neutronports.Port)
 	// Loop over all subnets. OpenStack potentially has several IPv4 or IPv6 subnets per port, but the
 	// CloudPrivateIPConfig expects only a single subnet of each address family per port. Throw an error
 	// in such a case.
+	var cloudPrivateIPsCount int
 	for _, s := range subnets {
 		// Parse CIDR information into ip and ipnet.
 		ip, ipnet, err = net.ParseCIDR(s.CIDR)
@@ -484,10 +487,20 @@ func (o *OpenStack) getNeutronPortNodeEgressIPConfiguration(p neutronports.Port)
 			}
 			ipv6 = ipnet.String()
 		}
-
+		// Loop over all cloudPrivateIPConfigs and check if they are part of this ipnet.
+		// If the IP is contained in the ipnet, increase cloudPrivateIPsCount.
+		for _, cpic := range cloudPrivateIPConfigs {
+			cip, _, err := cloudprivateipconfig.NameToIP(cpic.Name)
+			if err != nil {
+				return nil, err
+			}
+			if ipnet.Contains(cip) {
+				cloudPrivateIPsCount++
+			}
+		}
 	}
 
-	c := openstackMaxCapacity - len(p.AllowedAddressPairs)
+	c := openstackMaxCapacity + cloudPrivateIPsCount - len(p.AllowedAddressPairs)
 	return &NodeEgressIPConfiguration{
 		Interface: p.ID,
 		IFAddr: ifAddr{