Add assigned egress ips into capacity

This makes sure capacity field in cloud.network.openshift.io/egress-ipconfig
node annotation (which is introduced in 4.10) denotes correct value when
cluster upgrade happens for example 4.9 to 4.10 and node is already
assigned with egress ips.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>
(cherry picked from commit 8ffb0c1)
(cherry picked from commit bfe1601)

cmd/cloud-network-config-controller/main.go

@@ -114,6 +114,7 @@ func main() {
 					kubeClient,
 					cloudProviderClient,
 					kubeInformerFactory.Core().V1().Nodes(),
+					cloudNetworkInformerFactory.Cloud().V1().CloudPrivateIPConfigs(),
 				)
 				secretController := secretcontroller.NewSecretController(
 					ctx,

pkg/cloudprivateipconfig/cloudprivateipconfig.go

@@ -0,0 +1,39 @@
+package cloudprivateipconfig
+
+import (
+	"errors"
+	"net"
+	"strings"
+)
+
+// IPFamily string representing ip family
+type IPFamily string
+
+const (
+	// IPv4 IPFamily constant ipv4 family
+	IPv4 IPFamily = "ipv4"
+	// IPv6 IPFamily constant ipv6 family
+	IPv6 IPFamily = "ipv6"
+)
+
+// NameToIP converts the resource name to net.IP. Given a
+// limitation in the Kubernetes API server (see:
+// https://github.com/kubernetes/kubernetes/pull/100950)
+// CloudPrivateIPConfig.metadata.name cannot represent an IPv6 address. To
+// work-around this limitation it was decided that the network plugin creating
+// the CR will fully expand the IPv6 address and replace all colons with dots,
+// Example: The IPv6 address fc00:f853:ccd:e793::54 will be represented
+// as: fc00.f853.0ccd.e793.0000.0000.0000.0054, We thus need to replace
+// every fifth character's dot with a colon.
+func NameToIP(name string) (net.IP, IPFamily, error) {
+	// handle IPv4: this is enough since it will be serialized just fine
+	if ip := net.ParseIP(name); ip != nil {
+		return ip, IPv4, nil
+	}
+	// handle IPv6
+	name = strings.ReplaceAll(name, ".", ":")
+	if ip := net.ParseIP(name); ip != nil {
+		return ip, IPv6, nil
+	}
+	return nil, "", errors.New("invalid ip family")
+}

pkg/cloudprivateipconfig/cloudprivateipconfig_test.go

@@ -0,0 +1,33 @@
+package cloudprivateipconfig
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCloudPrivateIPConfig(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Test CloudPrivateIPConfig")
+}
+
+var _ = Describe("CloudPrivateIPConfig", func() {
+	Context("Validate Name to IP", func() {
+		It("With valid names", func() {
+			ip, family, err := NameToIP("192.168.0.10")
+			Expect(err).To(BeNil())
+			Expect(ip.String()).To(Equal(("192.168.0.10")))
+			Expect(family).To(Equal(IPv4))
+			ip, family, err = NameToIP("fc00.f853.0ccd.e793.0000.0000.0000.0054")
+			Expect(err).To(BeNil())
+			Expect(ip.String()).To(Equal(("fc00:f853:ccd:e793::54")))
+			Expect(family).To(Equal(IPv6))
+		})
+		It("With invalid name", func() {
+			_, _, err := NameToIP("invalid_config")
+			Expect(err).NotTo(BeNil())
+		})
+	})
+
+})

pkg/cloudprovider/aws.go

@@ -13,6 +13,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	v1 "github.com/openshift/api/cloudnetwork/v1"
+	"github.com/openshift/cloud-network-config-controller/pkg/cloudprivateipconfig"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -177,7 +179,7 @@ func (a *AWS) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	}
 }
 
-func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	instance, err := a.getInstance(node)
 	if err != nil {
 		return nil, err
@@ -205,7 +207,10 @@ func (a *AWS) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPCo
 	if v6Subnet != nil {
 		config.IFAddr.IPv6 = v6Subnet.String()
 	}
-	capV4, capV6 := a.getCapacity(instanceV4Capacity, instanceV6Capacity, networkInterface)
+	capV4, capV6, err := a.getCapacity(instanceV4Capacity, instanceV6Capacity, networkInterface, cloudPrivateIPConfigs)
+	if err != nil {
+		return nil, err
+	}
 	config.Capacity = capacity{
 		IPv4: capV4,
 		IPv6: capV6,
@@ -289,7 +294,7 @@ func (a *AWS) getSubnet(networkInterface *ec2.InstanceNetworkInterface) (*net.IP
 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI
 // Hence we need to retrieve that and then subtract the amount already assigned
 // by default.
-func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInterface *ec2.InstanceNetworkInterface) (int, int) {
+func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInterface *ec2.InstanceNetworkInterface, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) (int, int, error) {
 	currentIPv4Usage, currentIPv6Usage := 0, 0
 	for _, assignedIPv6 := range networkInterface.Ipv6Addresses {
 		if assignedIP := net.ParseIP(*assignedIPv6.Ipv6Address); assignedIP != nil {
@@ -301,7 +306,18 @@ func (a *AWS) getCapacity(instanceV4Capacity, instanceV6Capacity int, networkInt
 			currentIPv4Usage++
 		}
 	}
-	return instanceV4Capacity - currentIPv4Usage, instanceV6Capacity - currentIPv6Usage
+	for _, cloudPrivateIPConfig := range cloudPrivateIPConfigs {
+		_, ipFamily, err := cloudprivateipconfig.NameToIP(cloudPrivateIPConfig.Name)
+		if err != nil {
+			return 0, 0, err
+		}
+		if ipFamily == cloudprivateipconfig.IPv4 {
+			instanceV4Capacity++
+		} else if ipFamily == cloudprivateipconfig.IPv6 {
+			instanceV6Capacity++
+		}
+	}
+	return instanceV4Capacity - currentIPv4Usage, instanceV6Capacity - currentIPv6Usage, nil
 }
 
 func (a *AWS) getNetworkInterfaces(instance *ec2.Instance) ([]*ec2.InstanceNetworkInterface, error) {

pkg/cloudprovider/azure.go

@@ -14,6 +14,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	azureapi "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	utilnet "k8s.io/utils/net"
@@ -179,7 +180,7 @@ func (a *Azure) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	return a.waitForCompletion(result)
 }
 
-func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	instance, err := a.getInstance(node)
 	if err != nil {
 		return nil, err
@@ -208,7 +209,7 @@ func (a *Azure) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIP
 		config.IFAddr.IPv6 = v6Subnet.String()
 	}
 	config.Capacity = capacity{
-		IP: a.getCapacity(networkInterface),
+		IP: a.getCapacity(networkInterface, len(cloudPrivateIPConfigs)),
 	}
 	return []*NodeEgressIPConfiguration{config}, nil
 }
@@ -253,7 +254,7 @@ func (a *Azure) getSubnet(networkInterface network.Interface) (*net.IPNet, *net.
 // We need to retrieve the amounts assigned to the node by default and subtract
 // that from the default 256 value. Note: there is also a "Private IP addresses
 // per virtual network" quota, but that's 65.536, so we can skip that.
-func (a *Azure) getCapacity(networkInterface network.Interface) int {
+func (a *Azure) getCapacity(networkInterface network.Interface, cloudPrivateIPsCount int) int {
 	currentIPv4Usage, currentIPv6Usage := 0, 0
 	for _, ipConfiguration := range *networkInterface.IPConfigurations {
 		if assignedIP := net.ParseIP(*ipConfiguration.PrivateIPAddress); assignedIP != nil {
@@ -264,7 +265,7 @@ func (a *Azure) getCapacity(networkInterface network.Interface) int {
 			}
 		}
 	}
-	return defaultAzurePrivateIPCapacity - currentIPv4Usage - currentIPv6Usage
+	return defaultAzurePrivateIPCapacity + cloudPrivateIPsCount - currentIPv4Usage - currentIPv6Usage
 }
 
 // This is what the node's providerID looks like on Azure

pkg/cloudprovider/cloudprovider.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sync"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -54,10 +55,8 @@ type CloudProviderIntf interface {
 	// specifically that: the IP capacity can be either hard-coded and global
 	// for all instance types and IP families (GCP, Azure) or variable per
 	// instance and IP family (AWS), also: the interface is either keyed by name
-	// (GCP) or ID (Azure, AWS). Note: this function should only be called when
-	// no egress IPs have been added to the node, it will return an incorrect
-	// "egress IP capacity" otherwise
-	GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error)
+	// (GCP) or ID (Azure, AWS).
+	GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error)
 }
 
 // CloudProviderConfig is all the command-line options needed to initialize

pkg/cloudprovider/cloudprovider_fake.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"time"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -62,7 +63,7 @@ func (f *FakeCloudProvider) waitForCompletion() error {
 	return nil
 }
 
-func (f *FakeCloudProvider) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (f *FakeCloudProvider) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	if f.mockErrorOnGetNodeEgressIPConfiguration {
 		return nil, fmt.Errorf("Get node egress IP configuration failed")
 	}

pkg/cloudprovider/gcp.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"strings"
 
+	v1 "github.com/openshift/api/cloudnetwork/v1"
 	google "google.golang.org/api/compute/v1"
 	"google.golang.org/api/option"
 	corev1 "k8s.io/api/core/v1"
@@ -128,7 +129,7 @@ func (g *GCP) ReleasePrivateIP(ip net.IP, node *corev1.Node) error {
 	return g.waitForCompletion(project, zone, operation.Name)
 }
 
-func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPConfiguration, error) {
+func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node, cloudPrivateIPConfigs []*v1.CloudPrivateIPConfig) ([]*NodeEgressIPConfiguration, error) {
 	project, _, instance, err := g.getInstance(node)
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving instance associated with node, err: %v", err)
@@ -155,7 +156,7 @@ func (g *GCP) GetNodeEgressIPConfiguration(node *corev1.Node) ([]*NodeEgressIPCo
 			config.IFAddr.IPv6 = v6Subnet.String()
 		}
 		config.Capacity = capacity{
-			IP: g.getCapacity(networkInterface),
+			IP: g.getCapacity(networkInterface, len(cloudPrivateIPConfigs)),
 		}
 		return []*NodeEgressIPConfiguration{config}, nil
 	}
@@ -204,7 +205,7 @@ func (g *GCP) getSubnet(project string, networkInterface *google.NetworkInterfac
 
 // Note: there is also a global "alias IP per VPC quota", but OpenShift clusters on
 // GCP seem to have that value defined to 15,000. So we can skip that.
-func (g *GCP) getCapacity(networkInterface *google.NetworkInterface) int {
+func (g *GCP) getCapacity(networkInterface *google.NetworkInterface, cloudPrivateIPsCount int) int {
 	currentIPv4Usage := 0
 	currentIPv6Usage := 0
 	for _, aliasIPRange := range networkInterface.AliasIpRanges {
@@ -222,7 +223,7 @@ func (g *GCP) getCapacity(networkInterface *google.NetworkInterface) int {
 			}
 		}
 	}
-	return defaultGCPPrivateIPCapacity - currentIPv4Usage - currentIPv6Usage
+	return defaultGCPPrivateIPCapacity + cloudPrivateIPsCount - currentIPv4Usage - currentIPv6Usage
 }
 
 // getInstance retrieves the GCP instance referrred by the Node object.

pkg/controller/cloudprivateipconfig/cloudprivateipconfig_controller.go

@@ -5,15 +5,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net"
 	"reflect"
-	"strings"
 
 	cloudnetworkv1 "github.com/openshift/api/cloudnetwork/v1"
 	cloudnetworkclientset "github.com/openshift/client-go/cloudnetwork/clientset/versioned"
 	cloudnetworkscheme "github.com/openshift/client-go/cloudnetwork/clientset/versioned/scheme"
 	cloudnetworkinformers "github.com/openshift/client-go/cloudnetwork/informers/externalversions/cloudnetwork/v1"
 	cloudnetworklisters "github.com/openshift/client-go/cloudnetwork/listers/cloudnetwork/v1"
+	"github.com/openshift/cloud-network-config-controller/pkg/cloudprivateipconfig"
 	cloudprovider "github.com/openshift/cloud-network-config-controller/pkg/cloudprovider"
 	controller "github.com/openshift/cloud-network-config-controller/pkg/controller"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -175,7 +174,10 @@ func (c *CloudPrivateIPConfigController) SyncHandler(key string) error {
 		return nil
 	}
 
-	ip := cloudPrivateIPConfigNameToIP(cloudPrivateIPConfig.Name)
+	ip, _, err := cloudprivateipconfig.NameToIP(cloudPrivateIPConfig.Name)
+	if err != nil {
+		return err
+	}
 
 	// At most one of nodeToAdd or nodeToDel will be set
 	nodeToAdd, nodeToDel := c.computeOp(cloudPrivateIPConfig)
@@ -444,25 +446,3 @@ func (c *CloudPrivateIPConfigController) computeOp(cloudPrivateIPConfig *cloudne
 	// Default to NOOP
 	return "", ""
 }
-
-// cloudPrivateIPConfigNameToIP converts the resource name to net.IP. Given a
-// limitation in the Kubernetes API server (see:
-// https://github.com/kubernetes/kubernetes/pull/100950)
-// CloudPrivateIPConfig.metadata.name cannot represent an IPv6 address. To
-// work-around this limitation it was decided that the network plugin creating
-// the CR will fully expand the IPv6 address and replace all colons with dots,
-// ex:
-
-// The IPv6 address fc00:f853:ccd:e793::54 will be represented
-// as: fc00.f853.0ccd.e793.0000.0000.0000.0054
-
-// We thus need to replace every fifth character's dot with a colon.
-func cloudPrivateIPConfigNameToIP(name string) net.IP {
-	// handle IPv4: this is enough since it will be serialized just fine
-	if ip := net.ParseIP(name); ip != nil {
-		return ip
-	}
-	// handle IPv6
-	name = strings.ReplaceAll(name, ".", ":")
-	return net.ParseIP(name)
-}

pkg/controller/cloudprivateipconfig/cloudprivateipconfig_controller_test.go

@@ -11,6 +11,7 @@ import (
 	cloudnetworkv1 "github.com/openshift/api/cloudnetwork/v1"
 	fakecloudnetworkclientset "github.com/openshift/client-go/cloudnetwork/clientset/versioned/fake"
 	cloudnetworkinformers "github.com/openshift/client-go/cloudnetwork/informers/externalversions"
+	"github.com/openshift/cloud-network-config-controller/pkg/cloudprivateipconfig"
 	cloudprovider "github.com/openshift/cloud-network-config-controller/pkg/cloudprovider"
 	controller "github.com/openshift/cloud-network-config-controller/pkg/controller"
 	corev1 "k8s.io/api/core/v1"
@@ -1086,7 +1087,10 @@ func TestCloudPrivateIPConfigNameToIP(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
-		actualIP := cloudPrivateIPConfigNameToIP(test.name)
+		actualIP, _, err := cloudprivateipconfig.NameToIP(test.name)
+		if err != nil {
+			t.Fatalf("CloudPrivateIPConfigName expected no error, but got err: %v", err)
+		}
 		if !test.exectedIP.Equal(actualIP) {
 			t.Fatalf("Expected CloudPrivateIPConfigName %s to match IP: %v, but got: %v", test.name, test.exectedIP, actualIP)
 		}