From 6809ccdeebc835c4a5906e184388cc59922eb774 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Wed, 6 Apr 2022 20:42:57 +0000 Subject: [PATCH 1/3] create nnc listener concept and adapt existing poolmonitor and swift service to it Signed-off-by: Evan Baker --- cns/api.go | 2 +- cns/fakes/monitor.go | 3 +- cns/ipampool/monitor.go | 3 +- cns/ipampool/monitor_test.go | 3 +- cns/service/main.go | 2 +- cns/singletenantcontroller/conversion.go | 39 +++++++++++++ cns/singletenantcontroller/reconciler.go | 58 +++++++------------ cns/singletenantcontroller/reconciler_test.go | 12 ++-- 8 files changed, 74 insertions(+), 48 deletions(-) diff --git a/cns/api.go b/cns/api.go index c080cb6193..1e2b8349db 100644 --- a/cns/api.go +++ b/cns/api.go @@ -271,7 +271,7 @@ type NodeConfiguration struct { type IPAMPoolMonitor interface { Start(ctx context.Context) error - Update(nnc *v1alpha.NodeNetworkConfig) + Update(nnc *v1alpha.NodeNetworkConfig) error GetStateSnapshot() IpamPoolMonitorStateSnapshot } diff --git a/cns/fakes/monitor.go b/cns/fakes/monitor.go index c41f1c2f77..e3dd49b2a8 100644 --- a/cns/fakes/monitor.go +++ b/cns/fakes/monitor.go @@ -19,8 +19,9 @@ func (*MonitorFake) Start(ctx context.Context) error { return nil } -func (f *MonitorFake) Update(nnc *v1alpha.NodeNetworkConfig) { +func (f *MonitorFake) Update(nnc *v1alpha.NodeNetworkConfig) error { f.NodeNetworkConfig = nnc + return nil } func (*MonitorFake) Reconcile() error { diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 9e280ad67d..645a2d8834 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -367,7 +367,7 @@ func GenerateARMID(nc *v1alpha.NetworkContainer) string { // the pool reconcile loop. // If the Monitor has not been Started, this will block until Start() is called, which will // immediately read this passed NNC and start the pool reconcile loop. -func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) { +func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) error { pm.clampScaler(&nnc.Status.Scaler) // if the nnc has converged, observe the pool scaling latency (if any). @@ -377,6 +377,7 @@ func (pm *Monitor) Update(nnc *v1alpha.NodeNetworkConfig) { metric.ObserverPoolScaleLatency() } pm.nncSource <- *nnc + return nil } // clampScaler makes sure that the values stored in the scaler are sane. diff --git a/cns/ipampool/monitor_test.go b/cns/ipampool/monitor_test.go index 34d81581de..fcdbbb00c0 100644 --- a/cns/ipampool/monitor_test.go +++ b/cns/ipampool/monitor_test.go @@ -26,10 +26,11 @@ type directUpdatePoolMonitor struct { cns.IPAMPoolMonitor } -func (d *directUpdatePoolMonitor) Update(nnc *v1alpha.NodeNetworkConfig) { +func (d *directUpdatePoolMonitor) Update(nnc *v1alpha.NodeNetworkConfig) error { scaler := nnc.Status.Scaler d.m.spec = nnc.Spec d.m.metastate.minFreeCount, d.m.metastate.maxFreeCount = CalculateMinFreeIPs(scaler), CalculateMaxFreeIPs(scaler) + return nil } type testState struct { diff --git a/cns/service/main.go b/cns/service/main.go index 5abaf7069f..cecbd7fbad 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1010,7 +1010,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrapf(err, "failed to get node %s", nodeName) } - reconciler := kubecontroller.NewReconciler(nnccli, httpRestServiceImplementation, poolMonitor) + reconciler := kubecontroller.NewReconciler(nnccli, kubecontroller.SwiftNodeNetworkConfigListener(httpRestServiceImplementation), poolMonitor) // pass Node to the Reconciler for Controller xref if err := reconciler.SetupWithManager(manager, node); err != nil { return errors.Wrapf(err, "failed to setup reconciler with manager") diff --git a/cns/singletenantcontroller/conversion.go b/cns/singletenantcontroller/conversion.go index e1dbd5be5d..8cf7dd2de0 100644 --- a/cns/singletenantcontroller/conversion.go +++ b/cns/singletenantcontroller/conversion.go @@ -7,6 +7,9 @@ import ( "strings" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" + cnstypes "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" ) @@ -20,6 +23,42 @@ var ( ErrUnsupportedNCQuantity = errors.New("unsupported number of network containers") ) +type cnsClient interface { + CreateOrUpdateNetworkContainerInternal(*cns.CreateNetworkContainerRequest) cnstypes.ResponseCode +} + +var _ nodeNetworkConfigListener = (NodeNetworkConfigListenerFunc)(nil) //nolint:gocritic // clarity + +type NodeNetworkConfigListenerFunc func(*v1alpha.NodeNetworkConfig) error + +func (f NodeNetworkConfigListenerFunc) Update(nnc *v1alpha.NodeNetworkConfig) error { + return f(nnc) +} + +// SwiftNodeNetworkConfigListener return a function which satisfies the NodeNetworkConfigListener +// interface. It accepts a CreateOrUpdateNetworkContainerInternal implementation, and when Update +// is called, transforms the NNC in to an NC Request and calls the CNS Service implementation with +// that request. +func SwiftNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerFunc { + return func(nnc *v1alpha.NodeNetworkConfig) error { + // Create NC request and hand it off to CNS + ncRequest, err := CRDStatusToNCRequest(&nnc.Status) + if err != nil { + return errors.Wrap(err, "failed to convert NNC status to network container request") + } + responseCode := cnscli.CreateOrUpdateNetworkContainerInternal(&ncRequest) + err = restserver.ResponseCodeToError(responseCode) + if err != nil { + logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err) + return errors.Wrap(err, "failed to create or update network container") + } + + // record assigned IPs metric + allocatedIPs.Set(float64(len(nnc.Status.NetworkContainers[0].IPAssignments))) + return nil + } +} + // CRDStatusToNCRequest translates a crd status to createnetworkcontainer request func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNetworkContainerRequest, error) { // if NNC has no NC, return an empty request diff --git a/cns/singletenantcontroller/reconciler.go b/cns/singletenantcontroller/reconciler.go index f591eb058a..e0b436911b 100644 --- a/cns/singletenantcontroller/reconciler.go +++ b/cns/singletenantcontroller/reconciler.go @@ -4,10 +4,7 @@ import ( "context" "sync" - "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/restserver" - cnstypes "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" @@ -21,12 +18,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -type cnsClient interface { - CreateOrUpdateNetworkContainerInternal(*cns.CreateNetworkContainerRequest) cnstypes.ResponseCode -} - -type ipamPoolMonitorClient interface { - Update(*v1alpha.NodeNetworkConfig) +type nodeNetworkConfigListener interface { + Update(*v1alpha.NodeNetworkConfig) error } type nncGetter interface { @@ -35,19 +28,21 @@ type nncGetter interface { // Reconciler watches for CRD status changes type Reconciler struct { - cnscli cnsClient - ipampoolmonitorcli ipamPoolMonitorClient - nnccli nncGetter - started chan interface{} - once sync.Once + nncListeners []nodeNetworkConfigListener + nnccli nncGetter + once sync.Once + started chan interface{} } -func NewReconciler(nnccli nncGetter, cnscli cnsClient, ipampipampoolmonitorcli ipamPoolMonitorClient) *Reconciler { +// NewReconciler creates a NodeNetworkConfig Reconciler which will get updates from the Kubernetes +// apiserver for NNC events. +// Provided nncListeners are passed the NNC after the Reconcile preprocesses it. Note: order matters! The +// passed Listeners are notified in the order provided. +func NewReconciler(nnccli nncGetter, nncListeners ...nodeNetworkConfigListener) *Reconciler { return &Reconciler{ - cnscli: cnscli, - ipampoolmonitorcli: ipampipampoolmonitorcli, - nnccli: nnccli, - started: make(chan interface{}), + nncListeners: nncListeners, + nnccli: nnccli, + started: make(chan interface{}), } } @@ -65,32 +60,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco logger.Printf("[cns-rc] CRD Spec: %v", nnc.Spec) - // If there are no network containers, don't hand it off to CNS + // If there are no network containers, don't continue to updating Listeners if len(nnc.Status.NetworkContainers) == 0 { logger.Errorf("[cns-rc] Empty NetworkContainers") return reconcile.Result{}, nil } - // Create NC request and hand it off to CNS - ncRequest, err := CRDStatusToNCRequest(&nnc.Status) - if err != nil { - logger.Errorf("[cns-rc] Error translating crd status to nc request %v", err) - // requeue - return reconcile.Result{}, errors.Wrap(err, "failed to convert NNC status to network container request") - } - - responseCode := r.cnscli.CreateOrUpdateNetworkContainerInternal(&ncRequest) - err = restserver.ResponseCodeToError(responseCode) - if err != nil { - logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err) - // requeue - return reconcile.Result{}, errors.Wrap(err, "failed to create or update network container") + // push the NNC to the registered NNC Sinks + for i := range r.nncListeners { + if err := r.nncListeners[i].Update(nnc); err != nil { + return reconcile.Result{}, errors.Wrap(err, "nnc listener return error during update") + } } - r.ipampoolmonitorcli.Update(nnc) - // record assigned IPs metric - allocatedIPs.Set(float64(len(nnc.Status.NetworkContainers[0].IPAssignments))) - // we have received and pushed an NNC update, we are "Started" r.once.Do(func() { close(r.started) }) return reconcile.Result{}, nil diff --git a/cns/singletenantcontroller/reconciler_test.go b/cns/singletenantcontroller/reconciler_test.go index 9e396c3ae2..39c9cd2e4e 100644 --- a/cns/singletenantcontroller/reconciler_test.go +++ b/cns/singletenantcontroller/reconciler_test.go @@ -25,7 +25,7 @@ type cnsClientState struct { type mockCNSClient struct { state cnsClientState createOrUpdateNC func(*cns.CreateNetworkContainerRequest) cnstypes.ResponseCode - update func(*v1alpha.NodeNetworkConfig) + update func(*v1alpha.NodeNetworkConfig) error } //nolint:gocritic // ignore hugeParam pls @@ -34,9 +34,9 @@ func (m *mockCNSClient) CreateOrUpdateNetworkContainerInternal(req *cns.CreateNe return m.createOrUpdateNC(req) } -func (m *mockCNSClient) Update(nnc *v1alpha.NodeNetworkConfig) { +func (m *mockCNSClient) Update(nnc *v1alpha.NodeNetworkConfig) error { m.state.nnc = nnc - m.update(nnc) + return m.update(nnc) } type mockNCGetter struct { @@ -131,7 +131,9 @@ func TestReconcile(t *testing.T) { createOrUpdateNC: func(*cns.CreateNetworkContainerRequest) cnstypes.ResponseCode { return cnstypes.Success }, - update: func(*v1alpha.NodeNetworkConfig) {}, + update: func(*v1alpha.NodeNetworkConfig) error { + return nil + }, }, wantErr: false, wantCNSClientState: cnsClientState{ @@ -148,7 +150,7 @@ func TestReconcile(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - r := NewReconciler(&tt.ncGetter, &tt.cnsClient, &tt.cnsClient) + r := NewReconciler(&tt.ncGetter, SwiftNodeNetworkConfigListener(&tt.cnsClient), &tt.cnsClient) got, err := r.Reconcile(context.Background(), tt.in) if tt.wantErr { require.Error(t, err) From 1c1c980e6eb6acc5cedf603193996d68f1706b5d Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Fri, 8 Apr 2022 23:20:45 +0000 Subject: [PATCH 2/3] add overlay static nnc to nc listener Signed-off-by: Evan Baker --- cns/service/main.go | 2 +- cns/singletenantcontroller/conversion.go | 102 +++++++++++-- cns/singletenantcontroller/conversion_test.go | 140 ++++++++++-------- 3 files changed, 169 insertions(+), 75 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index cecbd7fbad..6178ccced9 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -865,7 +865,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter, } // Convert to CreateNetworkContainerRequest - ncRequest, err := kubecontroller.CRDStatusToNCRequest(&nnc.Status) + ncRequest, err := kubecontroller.CreateNCRequestFromDynamicNNC(nnc) if err != nil { return errors.Wrap(err, "failed to convert NNC status to network container request") } diff --git a/cns/singletenantcontroller/conversion.go b/cns/singletenantcontroller/conversion.go index 8cf7dd2de0..8b3e897d25 100644 --- a/cns/singletenantcontroller/conversion.go +++ b/cns/singletenantcontroller/conversion.go @@ -37,12 +37,12 @@ func (f NodeNetworkConfigListenerFunc) Update(nnc *v1alpha.NodeNetworkConfig) er // SwiftNodeNetworkConfigListener return a function which satisfies the NodeNetworkConfigListener // interface. It accepts a CreateOrUpdateNetworkContainerInternal implementation, and when Update -// is called, transforms the NNC in to an NC Request and calls the CNS Service implementation with +// is called, transforms the dynamic NNC in to an NC Request and calls the CNS Service implementation with // that request. func SwiftNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerFunc { return func(nnc *v1alpha.NodeNetworkConfig) error { // Create NC request and hand it off to CNS - ncRequest, err := CRDStatusToNCRequest(&nnc.Status) + ncRequest, err := CreateNCRequestFromDynamicNNC(nnc) if err != nil { return errors.Wrap(err, "failed to convert NNC status to network container request") } @@ -59,19 +59,19 @@ func SwiftNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerF } } -// CRDStatusToNCRequest translates a crd status to createnetworkcontainer request -func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNetworkContainerRequest, error) { +// CreateNCRequestFromDynamicNNC translates a crd status to createnetworkcontainer request for dynamic NNC (swift) +func CreateNCRequestFromDynamicNNC(nnc *v1alpha.NodeNetworkConfig) (cns.CreateNetworkContainerRequest, error) { // if NNC has no NC, return an empty request - if len(status.NetworkContainers) == 0 { + if len(nnc.Status.NetworkContainers) == 0 { return cns.CreateNetworkContainerRequest{}, nil } // only support a single NC per node, error on more - if len(status.NetworkContainers) > 1 { - return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(status.NetworkContainers)) + if len(nnc.Status.NetworkContainers) > 1 { + return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(nnc.Status.NetworkContainers)) } - nc := status.NetworkContainers[0] + nc := nnc.Status.NetworkContainers[0] primaryIP := nc.PrimaryIP // if the PrimaryIP is not a CIDR, append a /32 @@ -84,14 +84,14 @@ func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNe return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "IP: %s", primaryIP) } - secondaryPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace) + subnetPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace) if err != nil { return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "invalid SubnetAddressSpace %s", nc.SubnetAddressSpace) } subnet := cns.IPSubnet{ IPAddress: primaryPrefix.Addr().String(), - PrefixLength: uint8(secondaryPrefix.Bits()), + PrefixLength: uint8(subnetPrefix.Bits()), } secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} @@ -109,7 +109,87 @@ func CRDStatusToNCRequest(status *v1alpha.NodeNetworkConfigStatus) (cns.CreateNe SecondaryIPConfigs: secondaryIPConfigs, NetworkContainerid: nc.ID, NetworkContainerType: cns.Docker, - Version: strconv.FormatInt(nc.Version, 10), + Version: strconv.FormatInt(nc.Version, 10), //nolint:gomnd // it's decimal + IPConfiguration: cns.IPConfiguration{ + IPSubnet: subnet, + GatewayIPAddress: nc.DefaultGateway, + }, + }, nil +} + +// OverlayNodeNetworkConfigListener returns a function which satisfies the NodeNetworkConfigListener +// interface. It accepts a CreateOrUpdateNetworkContainerInternal implementation, and when Update +// is called, transforms the static NNC in to an NC Request and calls the CNS Service implementation with +// that request. +func OverlayNodeNetworkConfigListener(cnscli cnsClient) NodeNetworkConfigListenerFunc { + return func(nnc *v1alpha.NodeNetworkConfig) error { + // Create NC request and hand it off to CNS + ncRequest, err := CreateNCRequestFromDynamicNNC(nnc) + if err != nil { + return errors.Wrap(err, "failed to convert NNC status to network container request") + } + responseCode := cnscli.CreateOrUpdateNetworkContainerInternal(&ncRequest) + err = restserver.ResponseCodeToError(responseCode) + if err != nil { + logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err) + return errors.Wrap(err, "failed to create or update network container") + } + + // record assigned IPs metric + allocatedIPs.Set(float64(len(nnc.Status.NetworkContainers[0].IPAssignments))) + return nil + } +} + +// CreateNCRequestFromStaticNNC translates a crd status to createnetworkcontainer request for static NNC (overlay) +func CreateNCRequestFromStaticNNC(nnc *v1alpha.NodeNetworkConfig) (cns.CreateNetworkContainerRequest, error) { + // if NNC has no NC, return an empty request + if len(nnc.Status.NetworkContainers) == 0 { + return cns.CreateNetworkContainerRequest{}, nil + } + + // only support a single NC per node, error on more + if len(nnc.Status.NetworkContainers) > 1 { + return cns.CreateNetworkContainerRequest{}, errors.Wrapf(ErrUnsupportedNCQuantity, "count: %d", len(nnc.Status.NetworkContainers)) + } + + nc := nnc.Status.NetworkContainers[0] + + primaryPrefix, err := netip.ParsePrefix(nc.PrimaryIP) + if err != nil { + return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "IP: %s", nc.PrimaryIP) + } + + subnetPrefix, err := netip.ParsePrefix(nc.SubnetAddressSpace) + if err != nil { + return cns.CreateNetworkContainerRequest{}, errors.Wrapf(err, "invalid SubnetAddressSpace %s", nc.SubnetAddressSpace) + } + + subnet := cns.IPSubnet{ + IPAddress: primaryPrefix.Addr().String(), + PrefixLength: uint8(subnetPrefix.Bits()), + } + + secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} + + // iterate through all IP addresses in the subnet described by primaryPrefix and + // add them to the request as secondary IPConfigs. Skip the specific IP of the + // primaryPrefix; that is the gateway. + zeroAddr := primaryPrefix.Masked().Addr() // the masked address is the 0th IP in the subnet + for addr := zeroAddr.Next(); primaryPrefix.Contains(addr); addr = addr.Next() { + if addr == primaryPrefix.Addr() { + continue + } + secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{ + IPAddress: addr.String(), + NCVersion: int(nc.Version), + } + } + return cns.CreateNetworkContainerRequest{ + SecondaryIPConfigs: secondaryIPConfigs, + NetworkContainerid: nc.ID, + NetworkContainerType: cns.Docker, + Version: strconv.FormatInt(nc.Version, 10), //nolint:gomnd // it's decimal IPConfiguration: cns.IPConfiguration{ IPSubnet: subnet, GatewayIPAddress: nc.DefaultGateway, diff --git a/cns/singletenantcontroller/conversion_test.go b/cns/singletenantcontroller/conversion_test.go index 4dd4ea4651..ab8b0b9089 100644 --- a/cns/singletenantcontroller/conversion_test.go +++ b/cns/singletenantcontroller/conversion_test.go @@ -74,41 +74,47 @@ var validRequest = cns.CreateNetworkContainerRequest{ func TestConvertNNCStatusToNCRequest(t *testing.T) { tests := []struct { name string - input v1alpha.NodeNetworkConfigStatus + input v1alpha.NodeNetworkConfig want cns.CreateNetworkContainerRequest wantErr bool }{ { - name: "valid", - input: validStatus, + name: "valid", + input: v1alpha.NodeNetworkConfig{ + Status: validStatus, + }, wantErr: false, want: validRequest, }, { name: "no nc", - input: v1alpha.NodeNetworkConfigStatus{}, + input: v1alpha.NodeNetworkConfig{}, wantErr: false, want: cns.CreateNetworkContainerRequest{}, }, { - name: ">1 nc", - input: invalidStatusMultiNC, + name: ">1 nc", + input: v1alpha.NodeNetworkConfig{ + Status: invalidStatusMultiNC, + }, wantErr: true, }, { name: "malformed primary IP", - input: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - PrimaryIP: ipMalformed, - ID: ncID, - IPAssignments: []v1alpha.IPAssignment{ - { - Name: uuid, - IP: testSecIP, + input: v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + PrimaryIP: ipMalformed, + ID: ncID, + IPAssignments: []v1alpha.IPAssignment{ + { + Name: uuid, + IP: testSecIP, + }, }, + SubnetAddressSpace: subnetAddressSpace, }, - SubnetAddressSpace: subnetAddressSpace, }, }, }, @@ -116,18 +122,20 @@ func TestConvertNNCStatusToNCRequest(t *testing.T) { }, { name: "malformed IP assignment", - input: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - PrimaryIP: primaryIP, - ID: ncID, - IPAssignments: []v1alpha.IPAssignment{ - { - Name: uuid, - IP: ipMalformed, + input: v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + PrimaryIP: primaryIP, + ID: ncID, + IPAssignments: []v1alpha.IPAssignment{ + { + Name: uuid, + IP: ipMalformed, + }, }, + SubnetAddressSpace: subnetAddressSpace, }, - SubnetAddressSpace: subnetAddressSpace, }, }, }, @@ -135,25 +143,27 @@ func TestConvertNNCStatusToNCRequest(t *testing.T) { }, { name: "IP is CIDR", - input: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - PrimaryIP: ipIsCIDR, - ID: ncID, - IPAssignments: []v1alpha.IPAssignment{ - { - Name: uuid, - IP: testSecIP, + input: v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + PrimaryIP: ipIsCIDR, + ID: ncID, + IPAssignments: []v1alpha.IPAssignment{ + { + Name: uuid, + IP: testSecIP, + }, }, + SubnetName: subnetName, + DefaultGateway: defaultGateway, + SubnetAddressSpace: subnetAddressSpace, + Version: version, }, - SubnetName: subnetName, - DefaultGateway: defaultGateway, - SubnetAddressSpace: subnetAddressSpace, - Version: version, }, - }, - Scaler: v1alpha.Scaler{ - BatchSize: 1, + Scaler: v1alpha.Scaler{ + BatchSize: 1, + }, }, }, wantErr: false, @@ -161,18 +171,20 @@ func TestConvertNNCStatusToNCRequest(t *testing.T) { }, { name: "IP assignment is CIDR", - input: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - PrimaryIP: primaryIP, - ID: ncID, - IPAssignments: []v1alpha.IPAssignment{ - { - Name: uuid, - IP: ipIsCIDR, + input: v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + PrimaryIP: primaryIP, + ID: ncID, + IPAssignments: []v1alpha.IPAssignment{ + { + Name: uuid, + IP: ipIsCIDR, + }, }, + SubnetAddressSpace: subnetAddressSpace, }, - SubnetAddressSpace: subnetAddressSpace, }, }, }, @@ -180,18 +192,20 @@ func TestConvertNNCStatusToNCRequest(t *testing.T) { }, { name: "address space is not CIDR", - input: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - PrimaryIP: primaryIP, - ID: ncID, - IPAssignments: []v1alpha.IPAssignment{ - { - Name: uuid, - IP: testSecIP, + input: v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + PrimaryIP: primaryIP, + ID: ncID, + IPAssignments: []v1alpha.IPAssignment{ + { + Name: uuid, + IP: testSecIP, + }, }, + SubnetAddressSpace: "10.0.0.0", // not a cidr range }, - SubnetAddressSpace: "10.0.0.0", // not a cidr range }, }, }, @@ -201,7 +215,7 @@ func TestConvertNNCStatusToNCRequest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := CRDStatusToNCRequest(&tt.input) + got, err := CreateNCRequestFromDynamicNNC(&tt.input) if tt.wantErr { assert.Error(t, err) return From 5f250224709fe056fa906a48b5dc67e8b69a81cd Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Mon, 11 Apr 2022 20:43:07 +0000 Subject: [PATCH 3/3] add overlay config option and update reconciler listener injection Signed-off-by: Evan Baker --- cns/configuration/configuration.go | 4 +++- cns/service/main.go | 13 ++++++++++--- .../api/v1alpha/nodenetworkconfig.go | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index b4547d82f8..15002d7164 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" ) @@ -20,9 +21,10 @@ const ( type CNSConfig struct { ChannelMode string + Debug bool + IPAMMode v1alpha.Mode InitializeFromCNI bool ManagedSettings ManagedSettings - Debug bool SyncHostNCTimeoutMs int SyncHostNCVersionIntervalMs int TLSCertificatePath string diff --git a/cns/service/main.go b/cns/service/main.go index 6178ccced9..27dcf11541 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1010,7 +1010,16 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrapf(err, "failed to get node %s", nodeName) } - reconciler := kubecontroller.NewReconciler(nnccli, kubecontroller.SwiftNodeNetworkConfigListener(httpRestServiceImplementation), poolMonitor) + // set up the reconciler for the IPAM mode that we are running in + var reconciler *kubecontroller.Reconciler + switch cnsconfig.IPAMMode { + case v1alpha.Static: + // don't run the pool monitor in Overlay mode + reconciler = kubecontroller.NewReconciler(nnccli, kubecontroller.OverlayNodeNetworkConfigListener(httpRestServiceImplementation)) + case v1alpha.Dynamic: + // run the pool monitor to autoscale the IPs on the node NNC when in Swift mode + reconciler = kubecontroller.NewReconciler(nnccli, kubecontroller.SwiftNodeNetworkConfigListener(httpRestServiceImplementation), poolMonitor) + } // pass Node to the Reconciler for Controller xref if err := reconciler.SetupWithManager(manager, node); err != nil { return errors.Wrapf(err, "failed to setup reconciler with manager") @@ -1038,8 +1047,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn mux.Handle("/healthz", http.StripPrefix("/healthz", &healthzhandler)) // Start the Manager which starts the reconcile loop. - // The Reconciler will send an initial NodeNetworkConfig update to the PoolMonitor, starting the - // Monitor's internal loop. go func() { logger.Printf("Starting NodeNetworkConfig reconciler.") for { diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 9437505c72..6a655c92fb 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -50,8 +50,8 @@ type NodeNetworkConfigSpec struct { type Mode string const ( - Overlay Mode = "overlay" - Swift Mode = "swift" + Static Mode = "static" // overlay IPAM + Dynamic Mode = "dynamic" // swift IPAM ) // NodeNetworkConfigStatus defines the observed state of NetworkConfig