Overlay config option in CNS

cns/api.go

@@ -271,7 +271,7 @@ type NodeConfiguration struct {
 
 type IPAMPoolMonitor interface {
 	Start(ctx context.Context) error
-	Update(nnc *v1alpha.NodeNetworkConfig)
+	Update(nnc *v1alpha.NodeNetworkConfig) error
 	GetStateSnapshot() IpamPoolMonitorStateSnapshot
 }
 

cns/configuration/configuration.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/pkg/errors"
 )
 
@@ -20,9 +21,10 @@ const (
 
 type CNSConfig struct {
 	ChannelMode                 string
+	Debug                       bool
+	IPAMMode                    v1alpha.Mode
 	InitializeFromCNI           bool
 	ManagedSettings             ManagedSettings
-	Debug                       bool
 	SyncHostNCTimeoutMs         int
 	SyncHostNCVersionIntervalMs int
 	TLSCertificatePath          string

cns/fakes/monitor.go

@@ -19,8 +19,9 @@ func (*MonitorFake) Start(ctx context.Context) error {
 	return nil
 }
 
-func (f *MonitorFake) Update(nnc *v1alpha.NodeNetworkConfig) {
+func (f *MonitorFake) Update(nnc *v1alpha.NodeNetworkConfig) error {
 	f.NodeNetworkConfig = nnc
+	return nil
 }
 
 func (*MonitorFake) Reconcile() error {

cns/ipampool/monitor.go

@@ -367,7 +367,7 @@ func GenerateARMID(nc *v1alpha.NetworkContainer) string {
 // the pool reconcile loop.
 // If the Monitor has not been Started, this will block until Start() is called, which will
 // immediately read this passed NNC and start the pool reconcile loop.
-func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) {
+func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) error {
 	pm.clampScaler(&nnc.Status.Scaler)
 
 	// if the nnc has converged, observe the pool scaling latency (if any).
@@ -377,6 +377,7 @@ func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) {
 		metric.ObserverPoolScaleLatency()
 	}
 	pm.nncSource <- *nnc
+	return nil
 }
 
 // clampScaler makes sure that the values stored in the scaler are sane.

cns/ipampool/monitor_test.go

@@ -26,10 +26,11 @@ type directUpdatePoolMonitor struct {
 	cns.IPAMPoolMonitor
 }
 
-func (d *directUpdatePoolMonitor) Update(nnc *v1alpha.NodeNetworkConfig) {
+func (d *directUpdatePoolMonitor) Update(nnc *v1alpha.NodeNetworkConfig) error {
 	scaler := nnc.Status.Scaler
 	d.m.spec = nnc.Spec
 	d.m.metastate.minFreeCount, d.m.metastate.maxFreeCount = CalculateMinFreeIPs(scaler), CalculateMaxFreeIPs(scaler)
+	return nil
 }
 
 type testState struct {

cns/service/main.go

@@ -865,7 +865,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter,
 	}
 
 	// Convert to CreateNetworkContainerRequest
-	ncRequest, err := kubecontroller.CRDStatusToNCRequest(&nnc.Status)
+	ncRequest, err := kubecontroller.CreateNCRequestFromDynamicNNC(nnc)
 	if err != nil {
 		return errors.Wrap(err, "failed to convert NNC status to network container request")
 	}
@@ -1010,7 +1010,16 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		return errors.Wrapf(err, "failed to get node %s", nodeName)
 	}
 
-	reconciler := kubecontroller.NewReconciler(nnccli, httpRestServiceImplementation, poolMonitor)
+	// set up the reconciler for the IPAM mode that we are running in
+	var reconciler *kubecontroller.Reconciler
+	switch cnsconfig.IPAMMode {
+	case v1alpha.Static:
+		// don't run the pool monitor in Overlay mode
+		reconciler = kubecontroller.NewReconciler(nnccli, kubecontroller.OverlayNodeNetworkConfigListener(httpRestServiceImplementation))
+	case v1alpha.Dynamic:
+		// run the pool monitor to autoscale the IPs on the node NNC when in Swift mode
+		reconciler = kubecontroller.NewReconciler(nnccli, kubecontroller.SwiftNodeNetworkConfigListener(httpRestServiceImplementation), poolMonitor)
+	}
 	// pass Node to the Reconciler for Controller xref
 	if err := reconciler.SetupWithManager(manager, node); err != nil {
 		return errors.Wrapf(err, "failed to setup reconciler with manager")
@@ -1038,8 +1047,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	mux.Handle("/healthz", http.StripPrefix("/healthz", &healthzhandler))
 
 	// Start the Manager which starts the reconcile loop.
-	// The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the
-	// Monitor's internal loop.
 	go func() {
 		logger.Printf("Starting NodeNetworkConfig reconciler.")
 		for {

cns/singletenantcontroller/conversion.go

@@ -7,6 +7,9 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	cnstypes "github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/pkg/errors"
 )
@@ -20,19 +23,55 @@ var (
 	ErrUnsupportedNCQuantity = errors.New("unsupported number of network containers")
 )
 
-// CRDStatusToNCRequest translates a crd status to createnetworkcontainer request
-func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNetworkContainerRequest, error) {
+type cnsClient interface {
+	CreateOrUpdateNetworkContainerInternal(*cns.CreateNetworkContainerRequest) cnstypes.ResponseCode
+}
+
+var _ nodeNetworkConfigListener = (NodeNetworkConfigListenerFunc)(nil) //nolint:gocritic // clarity
+
+type NodeNetworkConfigListenerFunc func(*v1alpha.NodeNetworkConfig) error
+
+func (f NodeNetworkConfigListenerFunc) Update(nnc *v1alpha.NodeNetworkConfig) error {
+	return f(nnc)
+}
+
+// SwiftNodeNetworkConfigListener return a function which satisfies the NodeNetworkConfigListener
+// interface. It accepts a CreateOrUpdateNetworkContainerInternal implementation, and when Update
+// is called, transforms the dynamic NNC in to an NC Request and calls the CNS Service implementation with
+// that request.
+func SwiftNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerFunc {
+	return func(nnc *v1alpha.NodeNetworkConfig) error {
+		// Create NC request and hand it off to CNS
+		ncRequest, err := CreateNCRequestFromDynamicNNC(nnc)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert NNC status to network container request")
+		}
+		responseCode := cnscli.CreateOrUpdateNetworkContainerInternal(&ncRequest)
+		err = restserver.ResponseCodeToError(responseCode)
+		if err != nil {
+			logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err)
+			return errors.Wrap(err, "failed to create or update network container")
+		}
+
+		// record assigned IPs metric
+		allocatedIPs.Set(float64(len(nnc.Status.NetworkContainers[0].IPAssignments)))
+		return nil
+	}
+}
+
+// CreateNCRequestFromDynamicNNC translates a crd status to createnetworkcontainer request for dynamic NNC (swift)
+func CreateNCRequestFromDynamicNNC(nnc *v1alpha.NodeNetworkConfig) (cns.CreateNetworkContainerRequest, error) {
 	// if NNC has no NC, return an empty request
-	if len(status.NetworkContainers) == 0 {
+	if len(nnc.Status.NetworkContainers) == 0 {
 		return cns.CreateNetworkContainerRequest{}, nil
 	}
 
 	// only support a single NC per node, error on more
-	if len(status.NetworkContainers) > 1 {
-		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(status.NetworkContainers))
+	if len(nnc.Status.NetworkContainers) > 1 {
+		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(nnc.Status.NetworkContainers))
 	}
 
-	nc := status.NetworkContainers[0]
+	nc := nnc.Status.NetworkContainers[0]
 
 	primaryIP := nc.PrimaryIP
 	// if the PrimaryIP is not a CIDR, append a /32
@@ -45,14 +84,14 @@ func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNe
 		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "IP: %s", primaryIP)
 	}
 
-	secondaryPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace)
+	subnetPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace)
 	if err != nil {
 		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "invalid SubnetAddressSpace %s", nc.SubnetAddressSpace)
 	}
 
 	subnet := cns.IPSubnet{
 		IPAddress:    primaryPrefix.Addr().String(),
-		PrefixLength: uint8(secondaryPrefix.Bits()),
+		PrefixLength: uint8(subnetPrefix.Bits()),
 	}
 
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
@@ -70,7 +109,87 @@ func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNe
 		SecondaryIPConfigs:   secondaryIPConfigs,
 		NetworkContainerid:   nc.ID,
 		NetworkContainerType: cns.Docker,
-		Version:              strconv.FormatInt(nc.Version, 10),
+		Version:              strconv.FormatInt(nc.Version, 10), //nolint:gomnd // it's decimal
+		IPConfiguration: cns.IPConfiguration{
+			IPSubnet:         subnet,
+			GatewayIPAddress: nc.DefaultGateway,
+		},
+	}, nil
+}
+
+// OverlayNodeNetworkConfigListener returns a function which satisfies the NodeNetworkConfigListener
+// interface. It accepts a CreateOrUpdateNetworkContainerInternal implementation, and when Update
+// is called, transforms the static NNC in to an NC Request and calls the CNS Service implementation with
+// that request.
+func OverlayNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerFunc {
+	return func(nnc *v1alpha.NodeNetworkConfig) error {
+		// Create NC request and hand it off to CNS
+		ncRequest, err := CreateNCRequestFromDynamicNNC(nnc)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert NNC status to network container request")
+		}
+		responseCode := cnscli.CreateOrUpdateNetworkContainerInternal(&ncRequest)
+		err = restserver.ResponseCodeToError(responseCode)
+		if err != nil {
+			logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err)
+			return errors.Wrap(err, "failed to create or update network container")
+		}
+
+		// record assigned IPs metric
+		allocatedIPs.Set(float64(len(nnc.Status.NetworkContainers[0].IPAssignments)))
+		return nil
+	}
+}
+
+// CreateNCRequestFromStaticNNC translates a crd status to createnetworkcontainer request for static NNC (overlay)
+func CreateNCRequestFromStaticNNC(nnc *v1alpha.NodeNetworkConfig) (cns.CreateNetworkContainerRequest, error) {
+	// if NNC has no NC, return an empty request
+	if len(nnc.Status.NetworkContainers) == 0 {
+		return cns.CreateNetworkContainerRequest{}, nil
+	}
+
+	// only support a single NC per node, error on more
+	if len(nnc.Status.NetworkContainers) > 1 {
+		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(nnc.Status.NetworkContainers))
+	}
+
+	nc := nnc.Status.NetworkContainers[0]
+
+	primaryPrefix, err := netip.ParsePrefix(nc.PrimaryIP)
+	if err != nil {
+		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "IP: %s", nc.PrimaryIP)
+	}
+
+	subnetPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace)
+	if err != nil {
+		return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "invalid SubnetAddressSpace %s", nc.SubnetAddressSpace)
+	}
+
+	subnet := cns.IPSubnet{
+		IPAddress:    primaryPrefix.Addr().String(),
+		PrefixLength: uint8(subnetPrefix.Bits()),
+	}
+
+	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
+
+	// iterate through all IP addresses in the subnet described by primaryPrefix and
+	// add them to the request as secondary IPConfigs. Skip the specific IP of the
+	// primaryPrefix; that is the gateway.
+	zeroAddr := primaryPrefix.Masked().Addr() // the masked address is the 0th IP in the subnet
+	for addr := zeroAddr.Next(); primaryPrefix.Contains(addr); addr = addr.Next() {
+		if addr == primaryPrefix.Addr() {
+			continue
+		}
+		secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{
+			IPAddress: addr.String(),
+			NCVersion: int(nc.Version),
+		}
+	}
+	return cns.CreateNetworkContainerRequest{
+		SecondaryIPConfigs:   secondaryIPConfigs,
+		NetworkContainerid:   nc.ID,
+		NetworkContainerType: cns.Docker,
+		Version:              strconv.FormatInt(nc.Version, 10), //nolint:gomnd // it's decimal
 		IPConfiguration: cns.IPConfiguration{
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,