From ec638128378763e8b952d34ddd98e27ccc846f3c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 16 Jun 2020 16:59:23 -0700 Subject: [PATCH 01/20] NewRequestController and StartRequestController --- cns/examplerequestcontroller/main.go | 27 ++++++ cns/requestcontroller/channels/cnschannel.go | 6 ++ .../nodenetworkconfigreconciler.go | 44 +++++++++ cns/requestcontroller/requestcontroller.go | 93 +++++++++++++++++++ .../api/v1alpha/nodenetworkconfig.go | 2 +- 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 cns/examplerequestcontroller/main.go create mode 100644 cns/requestcontroller/channels/cnschannel.go create mode 100644 cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go create mode 100644 cns/requestcontroller/requestcontroller.go diff --git a/cns/examplerequestcontroller/main.go b/cns/examplerequestcontroller/main.go new file mode 100644 index 0000000000..b593722e4a --- /dev/null +++ b/cns/examplerequestcontroller/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/requestcontroller" + "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" +) + +//Example of using the requestcontroller package +func main() { + //Assuming logger is already setup and stuff + logger.InitLogger("Azure CNS", 3, 3, "") + + cnsChannel := make(chan channels.CNSChannel) + + rc, err := requestcontroller.NewRequestController(cnsChannel) + if err != nil { + logger.Errorf("Error making new RequestController: %v", err) + } + + //Spawn off a goroutine running the reconcile loop + + if err = rc.StartRequestController(); err != nil { + logger.Errorf("Error starting requestController: %v", err) + } + +} diff --git a/cns/requestcontroller/channels/cnschannel.go b/cns/requestcontroller/channels/cnschannel.go new file mode 100644 index 0000000000..d782b60d7b --- /dev/null +++ b/cns/requestcontroller/channels/cnschannel.go @@ -0,0 +1,6 @@ +package channels + +// Channel used to communicate changes in CRD status to CNS +type CNSChannel struct { + Foo string +} diff --git a/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go new file mode 100644 index 0000000000..8fbf831d8c --- /dev/null +++ b/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go @@ -0,0 +1,44 @@ +package reconcilers + +import ( + "context" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects +type NodeNetworkConfigReconciler struct { + K8sClient client.Client + CNSchannel chan channels.CNSChannel +} + +// Reconcile relays changes in NodeNetworkConfig to CNS +// Returning non-nil error causes a requeue +// Returning ctrl.Result{}, nil causes the queue to "forget" the item +// Other return values are possible, see kubebuilder docs for details +func (n *NodeNetworkConfigReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { + var nodeNetConfig nnc.NodeNetworkConfig + + //Get the CRD object + if err := n.K8sClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { + logger.Printf("[cns-rc] CRD not found, ignoring %v", err) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) + + //TODO: Pass the updates to CNS via the CNSChannel + + return ctrl.Result{}, nil +} + +// SetupWithManager Sets up the reconciler with a new manager +func (n *NodeNetworkConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nnc.NodeNetworkConfig{}). + Complete(n) +} diff --git a/cns/requestcontroller/requestcontroller.go b/cns/requestcontroller/requestcontroller.go new file mode 100644 index 0000000000..42f9054e77 --- /dev/null +++ b/cns/requestcontroller/requestcontroller.go @@ -0,0 +1,93 @@ +package requestcontroller + +import ( + "errors" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" + "github.com/Azure/azure-container-networking/cns/requestcontroller/reconcilers" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +//requestController watches CRDs for status updates and publishes CRD spec changes +// cnsChannel acts as the communication between CNS and requestController +// mgr acts as the communication between requestController and API server +type requestController struct { + cnsChannel chan channels.CNSChannel + mgr manager.Manager //mgr has method GetClient() to get k8s client +} + +//NewRequestController given a CNSChannel, returns a requestController struct +func NewRequestController(cnsChannel chan channels.CNSChannel) (*requestController, error) { + const k8sNamespace = "kube-system" + + //Check that logger package has been intialized + if logger.Log == nil { + return nil, errors.New("Must initialize logger before calling") + } + + //Add CRD scheme to runtime sheme so manager can recognize it + var scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = nnc.AddToScheme(scheme) + + // GetConfig precedence + // * --kubeconfig flag pointing at a file at this cmd line + // * KUBECONFIG environment variable pointing at a file + // * In-cluster config if running in cluster + // * $HOME/.kube/config if exists + // We're not using GetConfigOrDie becuase we want to propogate the error if there is one + k8sconfig, err := ctrl.GetConfig() + if err != nil { + logger.Errorf("[cns-rc] Error getting kubeconfig: %v", err) + return nil, err + } + + // Create manager for NodeNetworkConfigReconciler + // MetricsBindAddress is the tcp address that the controller should bind to + // for serving prometheus metrics, set to "0" to disable + mgr, err := ctrl.NewManager(k8sconfig, ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: "0", + Namespace: k8sNamespace, + }) + if err != nil { + logger.Errorf("[cns-rc] Error creating new request controller manager: %v", err) + return nil, err + } + + // Create the requestController struct + requestController := requestController{ + cnsChannel: cnsChannel, + mgr: mgr, + } + + return &requestController, nil +} + +// StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. +// When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. +func (rc *requestController) StartRequestController() error { + nodenetworkconfigreconciler := &reconcilers.NodeNetworkConfigReconciler{ + K8sClient: rc.mgr.GetClient(), + CNSchannel: rc.cnsChannel, + } + + // Setup manager with NodeNetworkConfigReconciler + if err := nodenetworkconfigreconciler.SetupWithManager(rc.mgr); err != nil { + logger.Errorf("[cns-rc] Error creating new NodeNetworkConfigReconciler: %v", err) + return err + } + + // Start manager and consequently, the reconciler + if err := rc.mgr.Start(ctrl.SetupSignalHandler()); err != nil { + logger.Errorf("[cns-rc] Error starting manager: %v", err) + return err + } + + return nil +} diff --git a/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 8b45b50f80..52689cb60b 100644 --- a/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -25,7 +25,7 @@ import ( // +kubebuilder:object:root=true // NodeNetworkConfig is the Schema for the nodenetworkconfigs API -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Namespaced // +kubebuilder:subresource:status type NodeNetworkConfig struct { metav1.TypeMeta `json:",inline"` From e861bcaa49d29b6a71d8fd9471542cc36b05ba57 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 17 Jun 2020 09:46:41 -0700 Subject: [PATCH 02/20] Making Start Manager in go routine --- cns/examplerequestcontroller/main.go | 5 ++--- cns/requestcontroller/requestcontroller.go | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cns/examplerequestcontroller/main.go b/cns/examplerequestcontroller/main.go index b593722e4a..276336b8d5 100644 --- a/cns/examplerequestcontroller/main.go +++ b/cns/examplerequestcontroller/main.go @@ -18,9 +18,8 @@ func main() { logger.Errorf("Error making new RequestController: %v", err) } - //Spawn off a goroutine running the reconcile loop - - if err = rc.StartRequestController(); err != nil { + //Start the RequestController which starts the reconcile loop + if err := rc.StartRequestController(); err != nil { logger.Errorf("Error starting requestController: %v", err) } diff --git a/cns/requestcontroller/requestcontroller.go b/cns/requestcontroller/requestcontroller.go index 42f9054e77..1c1fc819cc 100644 --- a/cns/requestcontroller/requestcontroller.go +++ b/cns/requestcontroller/requestcontroller.go @@ -84,10 +84,13 @@ func (rc *requestController) StartRequestController() error { } // Start manager and consequently, the reconciler - if err := rc.mgr.Start(ctrl.SetupSignalHandler()); err != nil { - logger.Errorf("[cns-rc] Error starting manager: %v", err) - return err - } + // Start() blocks until SIGINT or SIGTERM is received + go func() { + logger.Printf("Starting manager") + if err := rc.mgr.Start(ctrl.SetupSignalHandler()); err != nil { + logger.Errorf("[cns-rc] Error starting manager: %v", err) + } + }() return nil } From 086a64b320d3efb1dedf2cccab8f4a0b0a8b9fa8 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 17 Jun 2020 18:09:42 -0700 Subject: [PATCH 03/20] Lookup HOSTNAME env var --- cns/examplerequestcontroller/main.go | 6 ++++- .../nodenetworkconfigreconciler.go | 22 ++++++++++++++++--- cns/requestcontroller/requestcontroller.go | 17 ++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cns/examplerequestcontroller/main.go b/cns/examplerequestcontroller/main.go index 276336b8d5..d14b569990 100644 --- a/cns/examplerequestcontroller/main.go +++ b/cns/examplerequestcontroller/main.go @@ -11,7 +11,7 @@ func main() { //Assuming logger is already setup and stuff logger.InitLogger("Azure CNS", 3, 3, "") - cnsChannel := make(chan channels.CNSChannel) + cnsChannel := make(chan channels.CNSChannel, 1) rc, err := requestcontroller.NewRequestController(cnsChannel) if err != nil { @@ -23,4 +23,8 @@ func main() { logger.Errorf("Error starting requestController: %v", err) } + x := <-cnsChannel + + logger.Printf("done: %v", x) + } diff --git a/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go index 8fbf831d8c..dbc504f4cf 100644 --- a/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go @@ -2,38 +2,54 @@ package reconcilers import ( "context" + "errors" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects type NodeNetworkConfigReconciler struct { K8sClient client.Client CNSchannel chan channels.CNSChannel + HostName string } // Reconcile relays changes in NodeNetworkConfig to CNS // Returning non-nil error causes a requeue // Returning ctrl.Result{}, nil causes the queue to "forget" the item // Other return values are possible, see kubebuilder docs for details -func (n *NodeNetworkConfigReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) { +func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + // We are only interested in requests coming for the node that this program is running on + // Requeue if it's not for this node + if request.Name != n.HostName { + return reconcile.Result{}, errors.New("Requeing") + } + var nodeNetConfig nnc.NodeNetworkConfig //Get the CRD object if err := n.K8sClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { logger.Printf("[cns-rc] CRD not found, ignoring %v", err) - return ctrl.Result{}, client.IgnoreNotFound(err) + return reconcile.Result{}, client.IgnoreNotFound(err) } logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) //TODO: Pass the updates to CNS via the CNSChannel - return ctrl.Result{}, nil + logger.Printf("Sending channel") + tempChannel := channels.CNSChannel{ + Foo: "hi", + } + + n.CNSchannel <- tempChannel + + return reconcile.Result{}, nil } // SetupWithManager Sets up the reconciler with a new manager diff --git a/cns/requestcontroller/requestcontroller.go b/cns/requestcontroller/requestcontroller.go index 1c1fc819cc..82bc26dd78 100644 --- a/cns/requestcontroller/requestcontroller.go +++ b/cns/requestcontroller/requestcontroller.go @@ -2,6 +2,7 @@ package requestcontroller import ( "errors" + "os" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" @@ -19,6 +20,7 @@ import ( type requestController struct { cnsChannel chan channels.CNSChannel mgr manager.Manager //mgr has method GetClient() to get k8s client + hostName string //name of node running this program } //NewRequestController given a CNSChannel, returns a requestController struct @@ -30,6 +32,12 @@ func NewRequestController(cnsChannel chan channels.CNSChannel) (*requestControll return nil, errors.New("Must initialize logger before calling") } + // Check that HOSTNAME environment variable is set. HOSTNAME is name of node running this program + hostName := os.Getenv("HOSTNAME") + if hostName == "" { + return nil, errors.New("Must declare HOSTNAME environment variable. HOSTNAME is name of node.") + } + //Add CRD scheme to runtime sheme so manager can recognize it var scheme = runtime.NewScheme() _ = clientgoscheme.AddToScheme(scheme) @@ -64,6 +72,7 @@ func NewRequestController(cnsChannel chan channels.CNSChannel) (*requestControll requestController := requestController{ cnsChannel: cnsChannel, mgr: mgr, + hostName: hostName, } return &requestController, nil @@ -75,6 +84,7 @@ func (rc *requestController) StartRequestController() error { nodenetworkconfigreconciler := &reconcilers.NodeNetworkConfigReconciler{ K8sClient: rc.mgr.GetClient(), CNSchannel: rc.cnsChannel, + HostName: rc.hostName, } // Setup manager with NodeNetworkConfigReconciler @@ -94,3 +104,10 @@ func (rc *requestController) StartRequestController() error { return nil } + +// This function will translate a release request from CNS into updating the CRD spec +// It implements cnsipaminterface method ReleaseIPs +func (rc *requestController) ReleaseIpConfigs() error { + + return nil +} From 82fa79d7d6f60f5aa0c8714a6f2d2157adc0d6b6 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 17 Jun 2020 18:10:04 -0700 Subject: [PATCH 04/20] Adding cnsipaminterface.go --- cns/ipaminterface/cnsipaminterface.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cns/ipaminterface/cnsipaminterface.go diff --git a/cns/ipaminterface/cnsipaminterface.go b/cns/ipaminterface/cnsipaminterface.go new file mode 100644 index 0000000000..da9f52da11 --- /dev/null +++ b/cns/ipaminterface/cnsipaminterface.go @@ -0,0 +1,10 @@ +package cnsipaminterface + +// CNSIpamInterface is the interface that CNS calls when it wants to release or allocate ip configs. +// These two methods are responsible to relay these changes to DNC +// I will be implementing these via RequestController +// But say tomorrow we want to do this via privatelink. +type CNSIpamInterface interface { + ReleaseIPAddress() error + AllocateIpAddress() error +} From 6a3edf35c13f350ad6ab7403a587a284a70e3299 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 22 Jun 2020 09:50:48 -0700 Subject: [PATCH 05/20] Created requestController interface and implemented updating CRD --- cns/examplerequestcontroller/main.go | 34 +++++++++--- cns/ipaminterface/cnsipaminterface.go | 10 ---- ...tcontroller.go => k8srequestcontroller.go} | 55 ++++++++++++++----- .../nodenetworkconfigreconciler.go | 2 + .../requestcontrollerintreface.go | 7 +++ 5 files changed, 77 insertions(+), 31 deletions(-) delete mode 100644 cns/ipaminterface/cnsipaminterface.go rename cns/requestcontroller/{requestcontroller.go => k8srequestcontroller.go} (67%) rename cns/requestcontroller/{ => kubernetes}/reconcilers/nodenetworkconfigreconciler.go (96%) create mode 100644 cns/requestcontroller/requestcontrollerintreface.go diff --git a/cns/examplerequestcontroller/main.go b/cns/examplerequestcontroller/main.go index d14b569990..daf397e7f0 100644 --- a/cns/examplerequestcontroller/main.go +++ b/cns/examplerequestcontroller/main.go @@ -1,30 +1,50 @@ package main import ( + "time" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller" "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" ) +func goRequestController(rc requestcontroller.RequestController) { + //Start the RequestController which starts the reconcile loop + if err := rc.StartRequestController(); err != nil { + logger.Errorf("Error starting requestController: %v", err) + } + // After calling StartRequestController, there needs to be some pause or, the calling program + // must keep running because the reconcile loop is spawned off on a different go-routine inside + // rc.StartRequestController() + time.Sleep(5 * time.Second) + + // Create some dummy uuids + uuids := make([]string, 5) + uuids[0] = "uuid0" + uuids[1] = "uuid1" + uuids[2] = "uuid2" + uuids[3] = "uuid3" + uuids[4] = "uuid4" + rc.ReleaseIpsByUUID(uuids, int64(10)) +} + //Example of using the requestcontroller package func main() { + var requestController requestcontroller.RequestController + //Assuming logger is already setup and stuff logger.InitLogger("Azure CNS", 3, 3, "") cnsChannel := make(chan channels.CNSChannel, 1) - rc, err := requestcontroller.NewRequestController(cnsChannel) + requestController, err := requestcontroller.NewK8sRequestController(cnsChannel) if err != nil { logger.Errorf("Error making new RequestController: %v", err) } - //Start the RequestController which starts the reconcile loop - if err := rc.StartRequestController(); err != nil { - logger.Errorf("Error starting requestController: %v", err) - } + //Rely on the interface + goRequestController(requestController) x := <-cnsChannel - logger.Printf("done: %v", x) - } diff --git a/cns/ipaminterface/cnsipaminterface.go b/cns/ipaminterface/cnsipaminterface.go deleted file mode 100644 index da9f52da11..0000000000 --- a/cns/ipaminterface/cnsipaminterface.go +++ /dev/null @@ -1,10 +0,0 @@ -package cnsipaminterface - -// CNSIpamInterface is the interface that CNS calls when it wants to release or allocate ip configs. -// These two methods are responsible to relay these changes to DNC -// I will be implementing these via RequestController -// But say tomorrow we want to do this via privatelink. -type CNSIpamInterface interface { - ReleaseIPAddress() error - AllocateIpAddress() error -} diff --git a/cns/requestcontroller/requestcontroller.go b/cns/requestcontroller/k8srequestcontroller.go similarity index 67% rename from cns/requestcontroller/requestcontroller.go rename to cns/requestcontroller/k8srequestcontroller.go index 82bc26dd78..1ac62b353f 100644 --- a/cns/requestcontroller/requestcontroller.go +++ b/cns/requestcontroller/k8srequestcontroller.go @@ -1,30 +1,35 @@ package requestcontroller import ( + "context" "errors" "os" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" - "github.com/Azure/azure-container-networking/cns/requestcontroller/reconcilers" + "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes/reconcilers" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) +const k8sNamespace = "kube-system" + //requestController watches CRDs for status updates and publishes CRD spec changes // cnsChannel acts as the communication between CNS and requestController // mgr acts as the communication between requestController and API server -type requestController struct { +// implements the +type k8sRequestController struct { cnsChannel chan channels.CNSChannel mgr manager.Manager //mgr has method GetClient() to get k8s client hostName string //name of node running this program } -//NewRequestController given a CNSChannel, returns a requestController struct -func NewRequestController(cnsChannel chan channels.CNSChannel) (*requestController, error) { +//NewK8sRequestController given a CNSChannel, returns a k8sRequestController struct +func NewK8sRequestController(cnsChannel chan channels.CNSChannel) (*k8sRequestController, error) { const k8sNamespace = "kube-system" //Check that logger package has been intialized @@ -69,26 +74,26 @@ func NewRequestController(cnsChannel chan channels.CNSChannel) (*requestControll } // Create the requestController struct - requestController := requestController{ + k8sRequestController := k8sRequestController{ cnsChannel: cnsChannel, mgr: mgr, hostName: hostName, } - return &requestController, nil + return &k8sRequestController, nil } // StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. -func (rc *requestController) StartRequestController() error { +func (k8sRC *k8sRequestController) StartRequestController() error { nodenetworkconfigreconciler := &reconcilers.NodeNetworkConfigReconciler{ - K8sClient: rc.mgr.GetClient(), - CNSchannel: rc.cnsChannel, - HostName: rc.hostName, + K8sClient: k8sRC.mgr.GetClient(), + CNSchannel: k8sRC.cnsChannel, + HostName: k8sRC.hostName, } // Setup manager with NodeNetworkConfigReconciler - if err := nodenetworkconfigreconciler.SetupWithManager(rc.mgr); err != nil { + if err := nodenetworkconfigreconciler.SetupWithManager(k8sRC.mgr); err != nil { logger.Errorf("[cns-rc] Error creating new NodeNetworkConfigReconciler: %v", err) return err } @@ -97,7 +102,7 @@ func (rc *requestController) StartRequestController() error { // Start() blocks until SIGINT or SIGTERM is received go func() { logger.Printf("Starting manager") - if err := rc.mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := k8sRC.mgr.Start(ctrl.SetupSignalHandler()); err != nil { logger.Errorf("[cns-rc] Error starting manager: %v", err) } }() @@ -106,8 +111,30 @@ func (rc *requestController) StartRequestController() error { } // This function will translate a release request from CNS into updating the CRD spec -// It implements cnsipaminterface method ReleaseIPs -func (rc *requestController) ReleaseIpConfigs() error { +func (k8sRC *k8sRequestController) ReleaseIpsByUUID(listOfIPUUIDS []string, newCount int64) error { + k8sClient := k8sRC.mgr.GetClient() + nodeNetworkConfig := &nnc.NodeNetworkConfig{} + + //Get the CRD object + err := k8sClient.Get(context.Background(), client.ObjectKey{ + Namespace: k8sNamespace, + Name: k8sRC.hostName, + }, nodeNetworkConfig) + + if err != nil { + logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) + return err + } + + logger.Printf("Fetched nnc: %v", nodeNetworkConfig) + + nodeNetworkConfig.Spec.IPsNotInUse = append(nodeNetworkConfig.Spec.IPsNotInUse, listOfIPUUIDS...) + nodeNetworkConfig.Spec.RequestedIPCount = newCount + + if err := k8sClient.Update(context.Background(), nodeNetworkConfig); err != nil { + logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) + return err + } return nil } diff --git a/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go similarity index 96% rename from cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go rename to cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go index dbc504f4cf..6d6a43a614 100644 --- a/cns/requestcontroller/reconcilers/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go @@ -24,6 +24,8 @@ type NodeNetworkConfigReconciler struct { // Returning ctrl.Result{}, nil causes the queue to "forget" the item // Other return values are possible, see kubebuilder docs for details func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + // TODO: Implement a cache system so as not to call cns redunantly + // We are only interested in requests coming for the node that this program is running on // Requeue if it's not for this node if request.Name != n.HostName { diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go new file mode 100644 index 0000000000..fdf72badca --- /dev/null +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -0,0 +1,7 @@ +package requestcontroller + +// interface for cns to interact with the request controller +type RequestController interface { + StartRequestController() error + ReleaseIpsByUUID(listOfIPUUIDS []string, newCount int64) error +} From c3321b7a529d6de8e2d157cb2b885494e7d882e2 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 22 Jun 2020 09:42:48 -0700 Subject: [PATCH 06/20] fix windows 1903 test apimodel.json (#585) --- test/e2e/kubernetes/cniWindows1809.json | 52 ++++++++++--------------- test/e2e/kubernetes/cniWindows1903.json | 52 ++++++++++--------------- 2 files changed, 42 insertions(+), 62 deletions(-) diff --git a/test/e2e/kubernetes/cniWindows1809.json b/test/e2e/kubernetes/cniWindows1809.json index 7be581d0b5..d2b7f0a385 100644 --- a/test/e2e/kubernetes/cniWindows1809.json +++ b/test/e2e/kubernetes/cniWindows1809.json @@ -13,18 +13,14 @@ "--tls-min-version": "VersionTLS12", "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" }, - "addons": [ - { + "addons": [{ + "name": "azure-npm-daemonset", + "enabled": true, + "containers": [{ "name": "azure-npm-daemonset", - "enabled": true, - "containers": [ - { - "name": "azure-npm-daemonset", - "image": "" - } - ] - } - ] + "image": "" + }] + }] } }, "masterProfile": { @@ -32,20 +28,16 @@ "dnsPrefix": "cniWindows", "vmSize": "Standard_D2_v2" }, - "agentPoolProfiles": [ - { - "name": "windowspool2", - "count": 2, - "vmSize": "Standard_D2_v2", - "availabilityProfile": "AvailabilitySet", - "osType": "Windows", - "extensions": [ - { - "name": "windows-patches" - } - ] - } - ], + "agentPoolProfiles": [{ + "name": "windowspool2", + "count": 2, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "AvailabilitySet", + "osType": "Windows", + "extensions": [{ + "name": "windows-patches" + }] + }], "windowsProfile": { "adminUsername": "azureuser", "adminPassword": "azureTest@!", @@ -54,16 +46,14 @@ "windowsPublisher": "MicrosoftWindowsServer", "windowsOffer": "WindowsServerSemiAnnual", "windowsSku": "Datacenter-Core-1809-with-Containers-smalldisk", - "imageVersion": "1809.0.20190826" + "imageVersion": "latest" }, "linuxProfile": { "adminUsername": "azureuser", "ssh": { - "publicKeys": [ - { - "keyData": "" - } - ] + "publicKeys": [{ + "keyData": "" + }] } }, "servicePrincipalProfile": { diff --git a/test/e2e/kubernetes/cniWindows1903.json b/test/e2e/kubernetes/cniWindows1903.json index 4d545ecbd8..d65ac102da 100644 --- a/test/e2e/kubernetes/cniWindows1903.json +++ b/test/e2e/kubernetes/cniWindows1903.json @@ -13,18 +13,14 @@ "--tls-min-version": "VersionTLS12", "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" }, - "addons": [ - { + "addons": [{ + "name": "azure-npm-daemonset", + "enabled": true, + "containers": [{ "name": "azure-npm-daemonset", - "enabled": true, - "containers": [ - { - "name": "azure-npm-daemonset", - "image": "" - } - ] - } - ] + "image": "" + }] + }] } }, "masterProfile": { @@ -32,20 +28,16 @@ "dnsPrefix": "cniWindows", "vmSize": "Standard_D2_v2" }, - "agentPoolProfiles": [ - { - "name": "windowspool2", - "count": 2, - "vmSize": "Standard_D2_v2", - "availabilityProfile": "AvailabilitySet", - "osType": "Windows", - "extensions": [ - { - "name": "windows-patches" - } - ] - } - ], + "agentPoolProfiles": [{ + "name": "windowspool2", + "count": 2, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "AvailabilitySet", + "osType": "Windows", + "extensions": [{ + "name": "windows-patches" + }] + }], "windowsProfile": { "adminUsername": "azureuser", "adminPassword": "azureTest@!", @@ -54,16 +46,14 @@ "windowsPublisher": "MicrosoftWindowsServer", "windowsOffer": "WindowsServer", "windowsSku": "Datacenter-Core-1903-with-Containers-smalldisk", - "imageVersion": "1903.0.20190603" + "imageVersion": "latest" }, "linuxProfile": { "adminUsername": "azureuser", "ssh": { - "publicKeys": [ - { - "keyData": "" - } - ] + "publicKeys": [{ + "keyData": "" + }] } }, "servicePrincipalProfile": { From 887ab161ced654edeb00cb8425d52f1262f83deb Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 22 Jun 2020 16:10:56 -0700 Subject: [PATCH 07/20] Avoiding redundant calls into cns by only watching for status updates in reconcile loop, ignoring spec updates in reconcil loop. Also adding ability for updating CRD spec through k8sRequestController methods --- cns/examplerequestcontroller/main.go | 10 ++-- cns/requestcontroller/k8srequestcontroller.go | 57 ++++++++++++++++--- .../nodenetworkconfigreconciler.go | 28 +++++---- .../requestcontrollerintreface.go | 3 +- 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/cns/examplerequestcontroller/main.go b/cns/examplerequestcontroller/main.go index daf397e7f0..bb357f7915 100644 --- a/cns/examplerequestcontroller/main.go +++ b/cns/examplerequestcontroller/main.go @@ -24,8 +24,11 @@ func goRequestController(rc requestcontroller.RequestController) { uuids[1] = "uuid1" uuids[2] = "uuid2" uuids[3] = "uuid3" - uuids[4] = "uuid4" - rc.ReleaseIpsByUUID(uuids, int64(10)) + uuids[4] = "uuid100" + rc.ReleaseIPsByUUIDs(uuids) + + // Update to dummy count + rc.UpdateRequestedIPCount(int64(10)) } //Example of using the requestcontroller package @@ -44,7 +47,4 @@ func main() { //Rely on the interface goRequestController(requestController) - - x := <-cnsChannel - logger.Printf("done: %v", x) } diff --git a/cns/requestcontroller/k8srequestcontroller.go b/cns/requestcontroller/k8srequestcontroller.go index 1ac62b353f..36fe53c67a 100644 --- a/cns/requestcontroller/k8srequestcontroller.go +++ b/cns/requestcontroller/k8srequestcontroller.go @@ -110,29 +110,68 @@ func (k8sRC *k8sRequestController) StartRequestController() error { return nil } -// This function will translate a release request from CNS into updating the CRD spec -func (k8sRC *k8sRequestController) ReleaseIpsByUUID(listOfIPUUIDS []string, newCount int64) error { +// ReleaseIPsByUUIDs sends release ip request to the API server. Provide the UUIDs of the IP allocations +func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(listOfIPUUIDS []string) error { + nodeNetworkConfig, err := k8sRC.getNodeNetConfig(k8sRC.hostName, k8sNamespace) + if err != nil { + logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) + return err + } + + //Update the CRD IpsNotInUse + nodeNetworkConfig.Spec.IPsNotInUse = append(nodeNetworkConfig.Spec.IPsNotInUse, listOfIPUUIDS...) + + //Send update to API server + if err := k8sRC.updateNodeNetConfig(nodeNetworkConfig); err != nil { + logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) + return err + } + + return nil +} + +// UpdateIPCount sends the new requested ip count to the API server +func (k8sRC *k8sRequestController) UpdateRequestedIPCount(newCount int64) error { + nodeNetworkConfig, err := k8sRC.getNodeNetConfig(k8sRC.hostName, k8sNamespace) + if err != nil { + logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) + return err + } + + //Update the CRD IP count + nodeNetworkConfig.Spec.RequestedIPCount = newCount + + //Send update to API server + if err := k8sRC.updateNodeNetConfig(nodeNetworkConfig); err != nil { + logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) + return err + } + + return nil +} + +// getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object +func (k8sRC *k8sRequestController) getNodeNetConfig(name, namespace string) (*nnc.NodeNetworkConfig, error) { k8sClient := k8sRC.mgr.GetClient() nodeNetworkConfig := &nnc.NodeNetworkConfig{} - //Get the CRD object err := k8sClient.Get(context.Background(), client.ObjectKey{ Namespace: k8sNamespace, Name: k8sRC.hostName, }, nodeNetworkConfig) if err != nil { - logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) - return err + return nil, err } - logger.Printf("Fetched nnc: %v", nodeNetworkConfig) + return nodeNetworkConfig, nil +} - nodeNetworkConfig.Spec.IPsNotInUse = append(nodeNetworkConfig.Spec.IPsNotInUse, listOfIPUUIDS...) - nodeNetworkConfig.Spec.RequestedIPCount = newCount +// updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object +func (k8sRC *k8sRequestController) updateNodeNetConfig(nodeNetworkConfig *nnc.NodeNetworkConfig) error { + k8sClient := k8sRC.mgr.GetClient() if err := k8sClient.Update(context.Background(), nodeNetworkConfig); err != nil { - logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) return err } diff --git a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go index 6d6a43a614..6e08abdf5c 100644 --- a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go @@ -9,6 +9,8 @@ import ( nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -19,13 +21,11 @@ type NodeNetworkConfigReconciler struct { HostName string } -// Reconcile relays changes in NodeNetworkConfig to CNS +// Reconcile relays status changes in NodeNetworkConfig to CNS // Returning non-nil error causes a requeue // Returning ctrl.Result{}, nil causes the queue to "forget" the item // Other return values are possible, see kubebuilder docs for details func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { - // TODO: Implement a cache system so as not to call cns redunantly - // We are only interested in requests coming for the node that this program is running on // Requeue if it's not for this node if request.Name != n.HostName { @@ -44,13 +44,6 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco //TODO: Pass the updates to CNS via the CNSChannel - logger.Printf("Sending channel") - tempChannel := channels.CNSChannel{ - Foo: "hi", - } - - n.CNSchannel <- tempChannel - return reconcile.Result{}, nil } @@ -58,5 +51,20 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco func (n *NodeNetworkConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&nnc.NodeNetworkConfig{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: isStatusUpdated, + }). Complete(n) } + +// If the generations are the same, it means it's status change, and we should return true, so that the +// reconcile loop is triggered by it. +// If they're different, it means a spec change, and we should ignore, by returning false, to avoid redundant calls to cns when the +// status hasn't changed +// See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource +// for more details +func isStatusUpdated(e event.UpdateEvent) bool { + oldGeneration := e.MetaOld.GetGeneration() + newGeneration := e.MetaNew.GetGeneration() + return oldGeneration == newGeneration +} diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index fdf72badca..3db178b7ad 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -3,5 +3,6 @@ package requestcontroller // interface for cns to interact with the request controller type RequestController interface { StartRequestController() error - ReleaseIpsByUUID(listOfIPUUIDS []string, newCount int64) error + ReleaseIPsByUUIDs(listOfIPUUIDS []string) error + UpdateRequestedIPCount(newCount int64) error } From c56e1e7e415919c78f3d19cadb0622a16458b967 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 22 Jun 2020 16:13:34 -0700 Subject: [PATCH 08/20] fixing comments --- cns/requestcontroller/k8srequestcontroller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cns/requestcontroller/k8srequestcontroller.go b/cns/requestcontroller/k8srequestcontroller.go index 36fe53c67a..c70f34ae86 100644 --- a/cns/requestcontroller/k8srequestcontroller.go +++ b/cns/requestcontroller/k8srequestcontroller.go @@ -18,7 +18,7 @@ import ( const k8sNamespace = "kube-system" -//requestController watches CRDs for status updates and publishes CRD spec changes +// k8sRequestController watches CRDs for status updates and publishes CRD spec changes // cnsChannel acts as the communication between CNS and requestController // mgr acts as the communication between requestController and API server // implements the @@ -30,7 +30,6 @@ type k8sRequestController struct { //NewK8sRequestController given a CNSChannel, returns a k8sRequestController struct func NewK8sRequestController(cnsChannel chan channels.CNSChannel) (*k8sRequestController, error) { - const k8sNamespace = "kube-system" //Check that logger package has been intialized if logger.Log == nil { From 2eb52799f0802cba702729b1c505836addbc7406 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 23 Jun 2020 12:31:44 -0700 Subject: [PATCH 09/20] Cleaned up code and added more comments --- cns/requestcontroller/channels/cnschannel.go | 6 ---- .../example}/main.go | 13 ++++++--- cns/requestcontroller/k8srequestcontroller.go | 29 +++++++++---------- .../nodenetworkconfigreconciler.go | 10 +++---- 4 files changed, 28 insertions(+), 30 deletions(-) delete mode 100644 cns/requestcontroller/channels/cnschannel.go rename cns/{examplerequestcontroller => requestcontroller/example}/main.go (67%) diff --git a/cns/requestcontroller/channels/cnschannel.go b/cns/requestcontroller/channels/cnschannel.go deleted file mode 100644 index d782b60d7b..0000000000 --- a/cns/requestcontroller/channels/cnschannel.go +++ /dev/null @@ -1,6 +0,0 @@ -package channels - -// Channel used to communicate changes in CRD status to CNS -type CNSChannel struct { - Foo string -} diff --git a/cns/examplerequestcontroller/main.go b/cns/requestcontroller/example/main.go similarity index 67% rename from cns/examplerequestcontroller/main.go rename to cns/requestcontroller/example/main.go index bb357f7915..bb68458ec3 100644 --- a/cns/examplerequestcontroller/main.go +++ b/cns/requestcontroller/example/main.go @@ -5,7 +5,7 @@ import ( "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller" - "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" + "github.com/Azure/azure-container-networking/cns/restserver" ) func goRequestController(rc requestcontroller.RequestController) { @@ -18,15 +18,20 @@ func goRequestController(rc requestcontroller.RequestController) { // rc.StartRequestController() time.Sleep(5 * time.Second) + // Example of releasing ips, give the requestController list of uuids which correspond to the ips you want to release // Create some dummy uuids uuids := make([]string, 5) uuids[0] = "uuid0" uuids[1] = "uuid1" uuids[2] = "uuid2" uuids[3] = "uuid3" - uuids[4] = "uuid100" + uuids[4] = "uuid4" rc.ReleaseIPsByUUIDs(uuids) + // This method is not synchronous, all it does is send the new count to the API server through the CRD spec. + // Dnc would see this new ip count, and in turn, send the requested IPS in the CRD status, which would + // trigger the reconcile loop, and in that reconcile loop is where the ips would be passed to cns. + // So this method only relays the message, it does nothing else // Update to dummy count rc.UpdateRequestedIPCount(int64(10)) } @@ -38,9 +43,9 @@ func main() { //Assuming logger is already setup and stuff logger.InitLogger("Azure CNS", 3, 3, "") - cnsChannel := make(chan channels.CNSChannel, 1) + restService := &restserver.HTTPRestService{} - requestController, err := requestcontroller.NewK8sRequestController(cnsChannel) + requestController, err := requestcontroller.NewK8sRequestController(restService) if err != nil { logger.Errorf("Error making new RequestController: %v", err) } diff --git a/cns/requestcontroller/k8srequestcontroller.go b/cns/requestcontroller/k8srequestcontroller.go index c70f34ae86..77e1416149 100644 --- a/cns/requestcontroller/k8srequestcontroller.go +++ b/cns/requestcontroller/k8srequestcontroller.go @@ -6,8 +6,8 @@ import ( "os" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes/reconcilers" + "github.com/Azure/azure-container-networking/cns/restserver" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -19,17 +19,16 @@ import ( const k8sNamespace = "kube-system" // k8sRequestController watches CRDs for status updates and publishes CRD spec changes -// cnsChannel acts as the communication between CNS and requestController // mgr acts as the communication between requestController and API server -// implements the +// implements the RequestController interface type k8sRequestController struct { - cnsChannel chan channels.CNSChannel - mgr manager.Manager //mgr has method GetClient() to get k8s client - hostName string //name of node running this program + mgr manager.Manager //mgr has method GetClient() to get k8s client + restService *restserver.HTTPRestService //restService is given to nodeNetworkConfigReconciler (the reconcile loop) + hostName string //name of node running this program } -//NewK8sRequestController given a CNSChannel, returns a k8sRequestController struct -func NewK8sRequestController(cnsChannel chan channels.CNSChannel) (*k8sRequestController, error) { +//NewK8sRequestController given a reference to CNS's HTTPRestService state, returns a k8sRequestController struct +func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sRequestController, error) { //Check that logger package has been intialized if logger.Log == nil { @@ -74,9 +73,9 @@ func NewK8sRequestController(cnsChannel chan channels.CNSChannel) (*k8sRequestCo // Create the requestController struct k8sRequestController := k8sRequestController{ - cnsChannel: cnsChannel, - mgr: mgr, - hostName: hostName, + mgr: mgr, + restService: restService, + hostName: hostName, } return &k8sRequestController, nil @@ -86,9 +85,9 @@ func NewK8sRequestController(cnsChannel chan channels.CNSChannel) (*k8sRequestCo // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. func (k8sRC *k8sRequestController) StartRequestController() error { nodenetworkconfigreconciler := &reconcilers.NodeNetworkConfigReconciler{ - K8sClient: k8sRC.mgr.GetClient(), - CNSchannel: k8sRC.cnsChannel, - HostName: k8sRC.hostName, + K8sClient: k8sRC.mgr.GetClient(), + RestService: k8sRC.restService, + HostName: k8sRC.hostName, } // Setup manager with NodeNetworkConfigReconciler @@ -129,7 +128,7 @@ func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(listOfIPUUIDS []string) err return nil } -// UpdateIPCount sends the new requested ip count to the API server +// UpdateIPCount sends the new requested ip count to the API server. func (k8sRC *k8sRequestController) UpdateRequestedIPCount(newCount int64) error { nodeNetworkConfig, err := k8sRC.getNodeNetConfig(k8sRC.hostName, k8sNamespace) if err != nil { diff --git a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go index 6e08abdf5c..ecf4fa29a5 100644 --- a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/requestcontroller/channels" + "github.com/Azure/azure-container-networking/cns/restserver" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -16,9 +16,9 @@ import ( // NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects type NodeNetworkConfigReconciler struct { - K8sClient client.Client - CNSchannel chan channels.CNSChannel - HostName string + K8sClient client.Client + RestService *restserver.HTTPRestService + HostName string } // Reconcile relays status changes in NodeNetworkConfig to CNS @@ -42,7 +42,7 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) - //TODO: Pass the updates to CNS via the CNSChannel + //TODO: Translate CRD status into HTTPRestService state return reconcile.Result{}, nil } From c817700414f99d4a526626672b8784cc55cabc70 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 23 Jun 2020 17:28:18 -0700 Subject: [PATCH 10/20] Made client interface for testing purposes and changed structure of files to be less folder-y --- cns/requestcontroller/example/main.go | 3 ++- .../kubernetes/eventfilter.go | 22 +++++++++++++++++++ .../kubernetes/k8sclientinterface.go | 17 ++++++++++++++ .../{ => kubernetes}/k8srequestcontroller.go | 16 ++++++-------- .../kubernetes/k8srequestcontroller_test.go | 9 ++++++++ .../nodenetworkconfigreconciler.go | 22 +++---------------- go.sum | 10 +++++++++ 7 files changed, 70 insertions(+), 29 deletions(-) create mode 100644 cns/requestcontroller/kubernetes/eventfilter.go create mode 100644 cns/requestcontroller/kubernetes/k8sclientinterface.go rename cns/requestcontroller/{ => kubernetes}/k8srequestcontroller.go (92%) create mode 100644 cns/requestcontroller/kubernetes/k8srequestcontroller_test.go rename cns/requestcontroller/kubernetes/{reconcilers => }/nodenetworkconfigreconciler.go (68%) diff --git a/cns/requestcontroller/example/main.go b/cns/requestcontroller/example/main.go index bb68458ec3..b62239bd51 100644 --- a/cns/requestcontroller/example/main.go +++ b/cns/requestcontroller/example/main.go @@ -5,6 +5,7 @@ import ( "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller" + "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes" "github.com/Azure/azure-container-networking/cns/restserver" ) @@ -45,7 +46,7 @@ func main() { restService := &restserver.HTTPRestService{} - requestController, err := requestcontroller.NewK8sRequestController(restService) + requestController, err := kubernetes.NewK8sRequestController(restService) if err != nil { logger.Errorf("Error making new RequestController: %v", err) } diff --git a/cns/requestcontroller/kubernetes/eventfilter.go b/cns/requestcontroller/kubernetes/eventfilter.go new file mode 100644 index 0000000000..bdbf5ff4bf --- /dev/null +++ b/cns/requestcontroller/kubernetes/eventfilter.go @@ -0,0 +1,22 @@ +package kubernetes + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +type NodeNetworkConfigFilter struct { + predicate.Funcs +} + +// If the generations are the same, it means it's status change, and we should return true, so that the +// reconcile loop is triggered by it. +// If they're different, it means a spec change, and we should ignore, by returning false, to avoid redundant calls to cns when the +// status hasn't changed +// See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource +// for more details +func (NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { + oldGeneration := e.MetaOld.GetGeneration() + newGeneration := e.MetaNew.GetGeneration() + return oldGeneration == newGeneration +} diff --git a/cns/requestcontroller/kubernetes/k8sclientinterface.go b/cns/requestcontroller/kubernetes/k8sclientinterface.go new file mode 100644 index 0000000000..2bd1199285 --- /dev/null +++ b/cns/requestcontroller/kubernetes/k8sclientinterface.go @@ -0,0 +1,17 @@ +package kubernetes + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// K8sClient is an interface used by nodeNetworkConfigReconciler and k8sRequestController +// They rely on this interface in order to be able to make unit tests that overload these methods +type K8sClient interface { + Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error + Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error +} + +//TODO: make a mock client for testing diff --git a/cns/requestcontroller/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go similarity index 92% rename from cns/requestcontroller/k8srequestcontroller.go rename to cns/requestcontroller/kubernetes/k8srequestcontroller.go index 77e1416149..6d8552ef17 100644 --- a/cns/requestcontroller/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -1,4 +1,4 @@ -package requestcontroller +package kubernetes import ( "context" @@ -6,7 +6,6 @@ import ( "os" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes/reconcilers" "github.com/Azure/azure-container-networking/cns/restserver" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" "k8s.io/apimachinery/pkg/runtime" @@ -23,6 +22,7 @@ const k8sNamespace = "kube-system" // implements the RequestController interface type k8sRequestController struct { mgr manager.Manager //mgr has method GetClient() to get k8s client + Client K8sClient //Relying on the K8sClient interface to more easily test restService *restserver.HTTPRestService //restService is given to nodeNetworkConfigReconciler (the reconcile loop) hostName string //name of node running this program } @@ -74,6 +74,7 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque // Create the requestController struct k8sRequestController := k8sRequestController{ mgr: mgr, + Client: mgr.GetClient(), restService: restService, hostName: hostName, } @@ -84,8 +85,8 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque // StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. func (k8sRC *k8sRequestController) StartRequestController() error { - nodenetworkconfigreconciler := &reconcilers.NodeNetworkConfigReconciler{ - K8sClient: k8sRC.mgr.GetClient(), + nodenetworkconfigreconciler := &NodeNetworkConfigReconciler{ + K8sClient: k8sRC.Client, RestService: k8sRC.restService, HostName: k8sRC.hostName, } @@ -150,10 +151,9 @@ func (k8sRC *k8sRequestController) UpdateRequestedIPCount(newCount int64) error // getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object func (k8sRC *k8sRequestController) getNodeNetConfig(name, namespace string) (*nnc.NodeNetworkConfig, error) { - k8sClient := k8sRC.mgr.GetClient() nodeNetworkConfig := &nnc.NodeNetworkConfig{} - err := k8sClient.Get(context.Background(), client.ObjectKey{ + err := k8sRC.Client.Get(context.Background(), client.ObjectKey{ Namespace: k8sNamespace, Name: k8sRC.hostName, }, nodeNetworkConfig) @@ -167,9 +167,7 @@ func (k8sRC *k8sRequestController) getNodeNetConfig(name, namespace string) (*nn // updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object func (k8sRC *k8sRequestController) updateNodeNetConfig(nodeNetworkConfig *nnc.NodeNetworkConfig) error { - k8sClient := k8sRC.mgr.GetClient() - - if err := k8sClient.Update(context.Background(), nodeNetworkConfig); err != nil { + if err := k8sRC.Client.Update(context.Background(), nodeNetworkConfig); err != nil { return err } diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go new file mode 100644 index 0000000000..6d5ffdc22d --- /dev/null +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -0,0 +1,9 @@ +package kubernetes + +import ( + "testing" +) + +func TestNewK8sRequestController(t *testing.T) { + +} diff --git a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go similarity index 68% rename from cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go rename to cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go index ecf4fa29a5..cb7c674834 100644 --- a/cns/requestcontroller/kubernetes/reconcilers/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go @@ -1,4 +1,4 @@ -package reconcilers +package kubernetes import ( "context" @@ -9,14 +9,12 @@ import ( nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects type NodeNetworkConfigReconciler struct { - K8sClient client.Client + K8sClient K8sClient RestService *restserver.HTTPRestService HostName string } @@ -51,20 +49,6 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco func (n *NodeNetworkConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&nnc.NodeNetworkConfig{}). - WithEventFilter(predicate.Funcs{ - UpdateFunc: isStatusUpdated, - }). + WithEventFilter(NodeNetworkConfigFilter{}). Complete(n) } - -// If the generations are the same, it means it's status change, and we should return true, so that the -// reconcile loop is triggered by it. -// If they're different, it means a spec change, and we should ignore, by returning false, to avoid redundant calls to cns when the -// status hasn't changed -// See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource -// for more details -func isStatusUpdated(e event.UpdateEvent) bool { - oldGeneration := e.MetaOld.GetGeneration() - newGeneration := e.MetaNew.GetGeneration() - return oldGeneration == newGeneration -} diff --git a/go.sum b/go.sum index 326b3150df..84e4850ed4 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -92,6 +93,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -101,6 +103,7 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -239,6 +242,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -284,14 +288,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -442,6 +450,7 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -493,6 +502,7 @@ k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= From 52c72681306c4c7df50b17ddabfb8224918972b8 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 25 Jun 2020 16:46:29 -0700 Subject: [PATCH 11/20] Addressed comments from Paul Miller and Wei --- cns/requestcontroller/example/main.go | 31 +++++++-- .../kubernetes/eventfilter.go | 21 ++++++- .../kubernetes/k8sclientinterface.go | 6 +- .../kubernetes/k8srequestcontroller.go | 63 ++++++++----------- .../kubernetes/k8srequestcontroller_test.go | 56 +++++++++++++++++ .../kubernetes/nodenetworkconfigreconciler.go | 9 +-- .../kubernetes/signalhandler.go | 35 +++++++++++ .../requestcontrollerintreface.go | 7 ++- 8 files changed, 172 insertions(+), 56 deletions(-) create mode 100644 cns/requestcontroller/kubernetes/signalhandler.go diff --git a/cns/requestcontroller/example/main.go b/cns/requestcontroller/example/main.go index b62239bd51..19d3385964 100644 --- a/cns/requestcontroller/example/main.go +++ b/cns/requestcontroller/example/main.go @@ -7,19 +7,31 @@ import ( "github.com/Azure/azure-container-networking/cns/requestcontroller" "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes" "github.com/Azure/azure-container-networking/cns/restserver" + "golang.org/x/net/context" ) func goRequestController(rc requestcontroller.RequestController) { + //Exit channel for requestController, this channel is notified when requestController receives + //SIGINT or SIGTERM, requestControllerExitChan is sent 'true' and you can clean up anything then + requestControllerExitChan := make(chan bool, 1) + //Start the RequestController which starts the reconcile loop - if err := rc.StartRequestController(); err != nil { + if err := rc.StartRequestController(requestControllerExitChan); err != nil { logger.Errorf("Error starting requestController: %v", err) + return } + // After calling StartRequestController, there needs to be some pause or, the calling program // must keep running because the reconcile loop is spawned off on a different go-routine inside - // rc.StartRequestController() time.Sleep(5 * time.Second) + logger.Printf("Done sleeping") + + //For all request controller interactions with CRD spec (ReleaseIPsByUUIDs and UpdateRequestedIPCount) provide a context + // in order to be able to cancel if needed in the future + cntxt := context.Background() // Example of releasing ips, give the requestController list of uuids which correspond to the ips you want to release + // as well as the new requestedIpCount // Create some dummy uuids uuids := make([]string, 5) uuids[0] = "uuid0" @@ -27,14 +39,22 @@ func goRequestController(rc requestcontroller.RequestController) { uuids[2] = "uuid2" uuids[3] = "uuid3" uuids[4] = "uuid4" - rc.ReleaseIPsByUUIDs(uuids) + + // newCount = oldCount - #ips releasing + // In this example, say we had 20 allocated to the node, we want to release 5, new count would be 15 + oldCount := 20 + newRequestedIPCount := oldCount - len(uuids) // This method is not synchronous, all it does is send the new count to the API server through the CRD spec. // Dnc would see this new ip count, and in turn, send the requested IPS in the CRD status, which would // trigger the reconcile loop, and in that reconcile loop is where the ips would be passed to cns. // So this method only relays the message, it does nothing else - // Update to dummy count - rc.UpdateRequestedIPCount(int64(10)) + rc.ReleaseIPsByUUIDs(cntxt, uuids, newRequestedIPCount) + time.Sleep(5 * time.Second) + + <-requestControllerExitChan + logger.Printf("Request controller received sigint or sigterm, time to cleanup") + // Clean clean... } //Example of using the requestcontroller package @@ -49,6 +69,7 @@ func main() { requestController, err := kubernetes.NewK8sRequestController(restService) if err != nil { logger.Errorf("Error making new RequestController: %v", err) + return } //Rely on the interface diff --git a/cns/requestcontroller/kubernetes/eventfilter.go b/cns/requestcontroller/kubernetes/eventfilter.go index bdbf5ff4bf..2a41c75e5c 100644 --- a/cns/requestcontroller/kubernetes/eventfilter.go +++ b/cns/requestcontroller/kubernetes/eventfilter.go @@ -7,6 +7,7 @@ import ( type NodeNetworkConfigFilter struct { predicate.Funcs + hostname string } // If the generations are the same, it means it's status change, and we should return true, so that the @@ -15,8 +16,24 @@ type NodeNetworkConfigFilter struct { // status hasn't changed // See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource // for more details -func (NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { +func (n NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { + isHostName := n.isHostName(e.MetaOld.GetName()) oldGeneration := e.MetaOld.GetGeneration() newGeneration := e.MetaNew.GetGeneration() - return oldGeneration == newGeneration + return (oldGeneration == newGeneration) && isHostName +} + +// Only process create events if CRD name equals this host's name +func (n NodeNetworkConfigFilter) Create(e event.CreateEvent) bool { + return n.isHostName(e.Meta.GetName()) +} + +// Only process delete events if CRD name equals this host's name +func (n NodeNetworkConfigFilter) Delete(e event.DeleteEvent) bool { + return n.isHostName(e.Meta.GetName()) +} + +// Given a string, returns if that string equals the hostname running this program +func (n NodeNetworkConfigFilter) isHostName(metaName string) bool { + return metaName == n.hostname } diff --git a/cns/requestcontroller/kubernetes/k8sclientinterface.go b/cns/requestcontroller/kubernetes/k8sclientinterface.go index 2bd1199285..4130ebe28e 100644 --- a/cns/requestcontroller/kubernetes/k8sclientinterface.go +++ b/cns/requestcontroller/kubernetes/k8sclientinterface.go @@ -4,14 +4,16 @@ import ( "context" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ObjectKey identifies a Kubernetes Object. +type ObjectKey = types.NamespacedName + // K8sClient is an interface used by nodeNetworkConfigReconciler and k8sRequestController // They rely on this interface in order to be able to make unit tests that overload these methods type K8sClient interface { Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error } - -//TODO: make a mock client for testing diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go index 6d8552ef17..a1e9b1beec 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -41,10 +41,16 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque return nil, errors.New("Must declare HOSTNAME environment variable. HOSTNAME is name of node.") } - //Add CRD scheme to runtime sheme so manager can recognize it + //Add client-go scheme to runtime sheme so manager can recognize it var scheme = runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = nnc.AddToScheme(scheme) + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding client-go scheme to runtime scheme") + } + + //Add CRD scheme to runtime sheme so manager can recognize it + if err := nnc.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding NodeNetworkConfig scheme to runtime scheme") + } // GetConfig precedence // * --kubeconfig flag pointing at a file at this cmd line @@ -84,7 +90,8 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque // StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. -func (k8sRC *k8sRequestController) StartRequestController() error { +// exitChan will be notified when requestController receives a kill signal +func (k8sRC *k8sRequestController) StartRequestController(exitChan chan bool) error { nodenetworkconfigreconciler := &NodeNetworkConfigReconciler{ K8sClient: k8sRC.Client, RestService: k8sRC.restService, @@ -99,9 +106,10 @@ func (k8sRC *k8sRequestController) StartRequestController() error { // Start manager and consequently, the reconciler // Start() blocks until SIGINT or SIGTERM is received + // When SIGINT or SIGTERm are recived, notifies exitChan before exiting go func() { logger.Printf("Starting manager") - if err := k8sRC.mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := k8sRC.mgr.Start(SetupSignalHandler(exitChan)); err != nil { logger.Errorf("[cns-rc] Error starting manager: %v", err) } }() @@ -109,39 +117,22 @@ func (k8sRC *k8sRequestController) StartRequestController() error { return nil } -// ReleaseIPsByUUIDs sends release ip request to the API server. Provide the UUIDs of the IP allocations -func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(listOfIPUUIDS []string) error { - nodeNetworkConfig, err := k8sRC.getNodeNetConfig(k8sRC.hostName, k8sNamespace) +// ReleaseIPsByUUIDs sends release ip request to the API server and updates requested ip count. +// Provide the UUIDs of the IP allocations and the new requested ip count +func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(cntxt context.Context, listOfIPUUIDS []string, newRequestedIPCount int) error { + nodeNetworkConfig, err := k8sRC.getNodeNetConfig(cntxt, k8sRC.hostName, k8sNamespace) if err != nil { - logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) + logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %v", err) return err } //Update the CRD IpsNotInUse nodeNetworkConfig.Spec.IPsNotInUse = append(nodeNetworkConfig.Spec.IPsNotInUse, listOfIPUUIDS...) + //Update the CRD requestedIPCount + nodeNetworkConfig.Spec.RequestedIPCount = int64(newRequestedIPCount) //Send update to API server - if err := k8sRC.updateNodeNetConfig(nodeNetworkConfig); err != nil { - logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) - return err - } - - return nil -} - -// UpdateIPCount sends the new requested ip count to the API server. -func (k8sRC *k8sRequestController) UpdateRequestedIPCount(newCount int64) error { - nodeNetworkConfig, err := k8sRC.getNodeNetConfig(k8sRC.hostName, k8sNamespace) - if err != nil { - logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %V", err) - return err - } - - //Update the CRD IP count - nodeNetworkConfig.Spec.RequestedIPCount = newCount - - //Send update to API server - if err := k8sRC.updateNodeNetConfig(nodeNetworkConfig); err != nil { + if err := k8sRC.updateNodeNetConfig(cntxt, nodeNetworkConfig); err != nil { logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) return err } @@ -150,12 +141,12 @@ func (k8sRC *k8sRequestController) UpdateRequestedIPCount(newCount int64) error } // getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object -func (k8sRC *k8sRequestController) getNodeNetConfig(name, namespace string) (*nnc.NodeNetworkConfig, error) { +func (k8sRC *k8sRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { nodeNetworkConfig := &nnc.NodeNetworkConfig{} - err := k8sRC.Client.Get(context.Background(), client.ObjectKey{ - Namespace: k8sNamespace, - Name: k8sRC.hostName, + err := k8sRC.Client.Get(cntxt, client.ObjectKey{ + Namespace: namespace, + Name: name, }, nodeNetworkConfig) if err != nil { @@ -166,8 +157,8 @@ func (k8sRC *k8sRequestController) getNodeNetConfig(name, namespace string) (*nn } // updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object -func (k8sRC *k8sRequestController) updateNodeNetConfig(nodeNetworkConfig *nnc.NodeNetworkConfig) error { - if err := k8sRC.Client.Update(context.Background(), nodeNetworkConfig); err != nil { +func (k8sRC *k8sRequestController) updateNodeNetConfig(cntxt context.Context, nodeNetworkConfig *nnc.NodeNetworkConfig) error { + if err := k8sRC.Client.Update(cntxt, nodeNetworkConfig); err != nil { return err } diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go index 6d5ffdc22d..59eeae5f95 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -1,9 +1,65 @@ package kubernetes import ( + "context" + "errors" + "fmt" "testing" + + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + mockClient MockClient ) +type MockClient struct { + mockStore map[string]*nnc.NodeNetworkConfig //Mock store of namespace/name -> nodeNetworkConfig +} + +// Mock implementation of the K8sClientInterface Get method +func (mc *MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + nodeNetConfig, ok := mc.mockStore[key.String()] + if !ok { + return errors.New("CRD not found") + } + obj = nodeNetConfig + return nil +} + +func initMockClient() { + //Make a mock store + mockStore := make(map[string]*nnc.NodeNetworkConfig) + + //Fill it with an example namespace/name -> nodeNetConfig obj + mockStore["namespace/name"] = &nnc.NodeNetworkConfig{} + + // Make mock client initialized with mock store + mockClient = MockClient{mockStore: mockStore} +} + +func TestMain(m *testing.M) { + //Setup mock client + initMockClient() + //run test + m.Run() +} + func TestNewK8sRequestController(t *testing.T) { + // Attempt to retrieve mock obj + key := client.ObjectKey{ + Namespace: "namespace", + Name: "name", + } + toFill := &nnc.NodeNetworkConfig{} + if err := mockClient.Get(context.Background(), key, toFill); err != nil { + fmt.Printf("Some error ocured: %v \n", err) + } + + return } + +// TODO: Finish tests, make interface for reconciler diff --git a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go index cb7c674834..4daa730826 100644 --- a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go @@ -2,7 +2,6 @@ package kubernetes import ( "context" - "errors" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/restserver" @@ -24,12 +23,6 @@ type NodeNetworkConfigReconciler struct { // Returning ctrl.Result{}, nil causes the queue to "forget" the item // Other return values are possible, see kubebuilder docs for details func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { - // We are only interested in requests coming for the node that this program is running on - // Requeue if it's not for this node - if request.Name != n.HostName { - return reconcile.Result{}, errors.New("Requeing") - } - var nodeNetConfig nnc.NodeNetworkConfig //Get the CRD object @@ -49,6 +42,6 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco func (n *NodeNetworkConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&nnc.NodeNetworkConfig{}). - WithEventFilter(NodeNetworkConfigFilter{}). + WithEventFilter(NodeNetworkConfigFilter{hostname: n.HostName}). Complete(n) } diff --git a/cns/requestcontroller/kubernetes/signalhandler.go b/cns/requestcontroller/kubernetes/signalhandler.go new file mode 100644 index 0000000000..af4203da57 --- /dev/null +++ b/cns/requestcontroller/kubernetes/signalhandler.go @@ -0,0 +1,35 @@ +package kubernetes + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +var onlyOneSignalHandler = make(chan struct{}) +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + +// SetupSignalHandler registers for SIGTERM and SIGINT. A stop channel is returned +// which is closed on one of these signals. If a second signal is caught, the program +// is terminated with exit code 1. +// exitChan is notified when a SIGINT or SIGTERM signal is received. +func SetupSignalHandler(exitChan chan<- bool) (stopCh <-chan struct{}) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + // Notify the provided exitChan + // this will allow whoever provided the channel time to cleanup before requestController exits + exitChan <- true + fmt.Println("Sending to exitChan") + close(stop) + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return stop +} diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index 3db178b7ad..0dee9eaa08 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -1,8 +1,9 @@ package requestcontroller +import "context" + // interface for cns to interact with the request controller type RequestController interface { - StartRequestController() error - ReleaseIPsByUUIDs(listOfIPUUIDS []string) error - UpdateRequestedIPCount(newCount int64) error + StartRequestController(exitChan chan bool) error + ReleaseIPsByUUIDs(cntxt context.Context, listOfIPUUIDS []string, newRequestedIPCount int) error } From 109e60564e769949f05db789cc7ab0be81528405 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Fri, 26 Jun 2020 14:04:40 -0700 Subject: [PATCH 12/20] Beginning unit tests --- .../kubernetes/k8scnsinteractor.go | 18 +++++++ .../kubernetes/k8srequestcontroller.go | 48 +++++++++++-------- .../kubernetes/k8srequestcontroller_test.go | 4 +- .../kubernetes/nodenetworkconfigreconciler.go | 9 ++-- .../requestcontrollerintreface.go | 11 ++++- 5 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 cns/requestcontroller/kubernetes/k8scnsinteractor.go diff --git a/cns/requestcontroller/kubernetes/k8scnsinteractor.go b/cns/requestcontroller/kubernetes/k8scnsinteractor.go new file mode 100644 index 0000000000..264bcd21bf --- /dev/null +++ b/cns/requestcontroller/kubernetes/k8scnsinteractor.go @@ -0,0 +1,18 @@ +package kubernetes + +import ( + "github.com/Azure/azure-container-networking/cns/restserver" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" +) + +//This K8sCNSInteractor implements the CNSInteractor interface. It's used by nodenetworkconfigreconciler to translate +// CRD status to HttpRestService changes +type K8sCNSInteractor struct { + RestService *restserver.HTTPRestService +} + +func (interactor *K8sCNSInteractor) UpdateCNSState(status nnc.NodeNetworkConfigStatus) error { + //TODO: translate CNS Status into CNS Rest Service changes. + //Mat will pick up from here + return nil +} diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go index a1e9b1beec..acff15a6d5 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -21,10 +21,10 @@ const k8sNamespace = "kube-system" // mgr acts as the communication between requestController and API server // implements the RequestController interface type k8sRequestController struct { - mgr manager.Manager //mgr has method GetClient() to get k8s client - Client K8sClient //Relying on the K8sClient interface to more easily test - restService *restserver.HTTPRestService //restService is given to nodeNetworkConfigReconciler (the reconcile loop) - hostName string //name of node running this program + mgr manager.Manager //mgr has method GetClient() to get k8s client + Client K8sClient //Relying on the K8sClient interface to more easily test + hostName string //name of node running this program + Reconciler *NodeNetworkConfigReconciler } //NewK8sRequestController given a reference to CNS's HTTPRestService state, returns a k8sRequestController struct @@ -77,12 +77,30 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque return nil, err } - // Create the requestController struct + //Create k8scnsinteractor + k8scnsinteractor := &K8sCNSInteractor{ + RestService: restService, + } + + //Create reconciler + nodenetworkconfigreconciler := &NodeNetworkConfigReconciler{ + K8sClient: mgr.GetClient(), + HostName: hostName, + CNSInteractor: k8scnsinteractor, + } + + // Setup manager with reconciler + if err := nodenetworkconfigreconciler.SetupWithManager(mgr); err != nil { + logger.Errorf("[cns-rc] Error creating new NodeNetworkConfigReconciler: %v", err) + return nil, err + } + + // Create the requestController k8sRequestController := k8sRequestController{ - mgr: mgr, - Client: mgr.GetClient(), - restService: restService, - hostName: hostName, + mgr: mgr, + Client: mgr.GetClient(), + hostName: hostName, + Reconciler: nodenetworkconfigreconciler, } return &k8sRequestController, nil @@ -92,18 +110,6 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. // exitChan will be notified when requestController receives a kill signal func (k8sRC *k8sRequestController) StartRequestController(exitChan chan bool) error { - nodenetworkconfigreconciler := &NodeNetworkConfigReconciler{ - K8sClient: k8sRC.Client, - RestService: k8sRC.restService, - HostName: k8sRC.hostName, - } - - // Setup manager with NodeNetworkConfigReconciler - if err := nodenetworkconfigreconciler.SetupWithManager(k8sRC.mgr); err != nil { - logger.Errorf("[cns-rc] Error creating new NodeNetworkConfigReconciler: %v", err) - return err - } - // Start manager and consequently, the reconciler // Start() blocks until SIGINT or SIGTERM is received // When SIGINT or SIGTERm are recived, notifies exitChan before exiting diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go index 59eeae5f95..422cd00cec 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -16,7 +16,7 @@ var ( ) type MockClient struct { - mockStore map[string]*nnc.NodeNetworkConfig //Mock store of namespace/name -> nodeNetworkConfig + mockStore map[string]*nnc.NodeNetworkConfig //Mock store of "namespace/name" is key } // Mock implementation of the K8sClientInterface Get method @@ -33,7 +33,7 @@ func initMockClient() { //Make a mock store mockStore := make(map[string]*nnc.NodeNetworkConfig) - //Fill it with an example namespace/name -> nodeNetConfig obj + //Fill the mock store mockStore["namespace/name"] = &nnc.NodeNetworkConfig{} // Make mock client initialized with mock store diff --git a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go index 4daa730826..a7bbb6950e 100644 --- a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go +++ b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go @@ -4,7 +4,7 @@ import ( "context" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/restserver" + "github.com/Azure/azure-container-networking/cns/requestcontroller" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -13,9 +13,9 @@ import ( // NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects type NodeNetworkConfigReconciler struct { - K8sClient K8sClient - RestService *restserver.HTTPRestService - HostName string + K8sClient K8sClient + HostName string + CNSInteractor requestcontroller.CNSInteractor } // Reconcile relays status changes in NodeNetworkConfig to CNS @@ -34,6 +34,7 @@ func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reco logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) //TODO: Translate CRD status into HTTPRestService state + n.CNSInteractor.UpdateCNSState(nodeNetConfig.Status) return reconcile.Result{}, nil } diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index 0dee9eaa08..9e09cb6dab 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -1,9 +1,18 @@ package requestcontroller -import "context" +import ( + "context" + + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" +) // interface for cns to interact with the request controller type RequestController interface { StartRequestController(exitChan chan bool) error ReleaseIPsByUUIDs(cntxt context.Context, listOfIPUUIDS []string, newRequestedIPCount int) error } + +// interface for request controller to interact with cns +type CNSInteractor interface { + UpdateCNSState(nnc.NodeNetworkConfigStatus) error +} From 8786e2fdac87979354ab744f6989cb21533786ef Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 30 Jun 2020 07:37:22 -0700 Subject: [PATCH 13/20] Finished unit tests --- Makefile | 1 + cns/requestcontroller/example/main.go | 8 +- .../kubernetes/k8srequestcontroller.go | 38 ++- .../kubernetes/k8srequestcontroller_test.go | 310 ++++++++++++++++-- 4 files changed, 310 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index f8f968f2e5..3780b85548 100644 --- a/Makefile +++ b/Makefile @@ -377,6 +377,7 @@ test-all: ./cnm/network/ \ ./cni/ipam/ \ ./cns/ipamclient/ \ + ./cns/requestcontroller/kubernetes/ \ ./cnms/service/ \ ./npm/iptm/ \ ./npm/ipsm/ diff --git a/cns/requestcontroller/example/main.go b/cns/requestcontroller/example/main.go index 19d3385964..e0a8a6275b 100644 --- a/cns/requestcontroller/example/main.go +++ b/cns/requestcontroller/example/main.go @@ -66,7 +66,13 @@ func main() { restService := &restserver.HTTPRestService{} - requestController, err := kubernetes.NewK8sRequestController(restService) + //Provide kubeconfig, this method was abstracted out for testing + kubeconfig, err := kubernetes.GetKubeConfig() + if err != nil { + logger.Errorf("Error getting kubeconfig: %v", err) + } + + requestController, err = kubernetes.NewK8sRequestController(restService, kubeconfig) if err != nil { logger.Errorf("Error making new RequestController: %v", err) return diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go index acff15a6d5..7349401804 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -10,6 +10,7 @@ import ( nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -22,13 +23,26 @@ const k8sNamespace = "kube-system" // implements the RequestController interface type k8sRequestController struct { mgr manager.Manager //mgr has method GetClient() to get k8s client - Client K8sClient //Relying on the K8sClient interface to more easily test + K8sClient K8sClient //Relying on the K8sClient interface to more easily test hostName string //name of node running this program Reconciler *NodeNetworkConfigReconciler } +// GetKubeConfig precedence +// * --kubeconfig flag pointing at a file at this cmd line +// * KUBECONFIG environment variable pointing at a file +// * In-cluster config if running in cluster +// * $HOME/.kube/config if exists +func GetKubeConfig() (*rest.Config, error) { + k8sconfig, err := ctrl.GetConfig() + if err != nil { + return nil, err + } + return k8sconfig, nil +} + //NewK8sRequestController given a reference to CNS's HTTPRestService state, returns a k8sRequestController struct -func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sRequestController, error) { +func NewK8sRequestController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*k8sRequestController, error) { //Check that logger package has been intialized if logger.Log == nil { @@ -52,22 +66,10 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque return nil, errors.New("Error adding NodeNetworkConfig scheme to runtime scheme") } - // GetConfig precedence - // * --kubeconfig flag pointing at a file at this cmd line - // * KUBECONFIG environment variable pointing at a file - // * In-cluster config if running in cluster - // * $HOME/.kube/config if exists - // We're not using GetConfigOrDie becuase we want to propogate the error if there is one - k8sconfig, err := ctrl.GetConfig() - if err != nil { - logger.Errorf("[cns-rc] Error getting kubeconfig: %v", err) - return nil, err - } - // Create manager for NodeNetworkConfigReconciler // MetricsBindAddress is the tcp address that the controller should bind to // for serving prometheus metrics, set to "0" to disable - mgr, err := ctrl.NewManager(k8sconfig, ctrl.Options{ + mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{ Scheme: scheme, MetricsBindAddress: "0", Namespace: k8sNamespace, @@ -98,7 +100,7 @@ func NewK8sRequestController(restService *restserver.HTTPRestService) (*k8sReque // Create the requestController k8sRequestController := k8sRequestController{ mgr: mgr, - Client: mgr.GetClient(), + K8sClient: mgr.GetClient(), hostName: hostName, Reconciler: nodenetworkconfigreconciler, } @@ -150,7 +152,7 @@ func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(cntxt context.Context, list func (k8sRC *k8sRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { nodeNetworkConfig := &nnc.NodeNetworkConfig{} - err := k8sRC.Client.Get(cntxt, client.ObjectKey{ + err := k8sRC.K8sClient.Get(cntxt, client.ObjectKey{ Namespace: namespace, Name: name, }, nodeNetworkConfig) @@ -164,7 +166,7 @@ func (k8sRC *k8sRequestController) getNodeNetConfig(cntxt context.Context, name, // updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object func (k8sRC *k8sRequestController) updateNodeNetConfig(cntxt context.Context, nodeNetworkConfig *nnc.NodeNetworkConfig) error { - if err := k8sRC.Client.Update(cntxt, nodeNetworkConfig); err != nil { + if err := k8sRC.K8sClient.Update(cntxt, nodeNetworkConfig); err != nil { return err } diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go index 422cd00cec..a03220966e 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -3,63 +3,317 @@ package kubernetes import ( "context" "errors" - "fmt" + "reflect" + "strings" "testing" + "github.com/Azure/azure-container-networking/cns/logger" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +const existingNNCName = "nodenetconfig_1" +const existingNamespace = k8sNamespace +const nonexistingNNCName = "nodenetconfig_nonexisting" +const nonexistingNamespace = "namespace_nonexisting" + var ( - mockClient MockClient + mockStore map[MockKey]*nnc.NodeNetworkConfig + mockCNSUpdated bool ) +//MockKey is the key to the mockStore, namespace+"/"+name like in API server +type MockKey struct { + Namespace string + Name string +} + +func (m MockKey) String() string { + return m.Namespace + "/" + m.Name +} + +// MockClient implements K8SClient interface type MockClient struct { - mockStore map[string]*nnc.NodeNetworkConfig //Mock store of "namespace/name" is key + mockStore map[MockKey]*nnc.NodeNetworkConfig } // Mock implementation of the K8sClientInterface Get method -func (mc *MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { - nodeNetConfig, ok := mc.mockStore[key.String()] +//Mimics that of controller-runtime's client.Client +func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + mockKey := MockKey{ + Namespace: key.Namespace, + Name: key.Name, + } + + nodeNetConfig, ok := mc.mockStore[mockKey] if !ok { - return errors.New("CRD not found") + return errors.New("Node Net Config not found in mock store") } - obj = nodeNetConfig + nodeNetConfig.DeepCopyInto(obj.(*nnc.NodeNetworkConfig)) + return nil } -func initMockClient() { - //Make a mock store - mockStore := make(map[string]*nnc.NodeNetworkConfig) +//Mock implementation of the K8sClientInterface Update method +//Mimics that of controller-runtime's client.Client +func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { + nodeNetConfig := obj.(*nnc.NodeNetworkConfig) - //Fill the mock store - mockStore["namespace/name"] = &nnc.NodeNetworkConfig{} + mockKey := MockKey{ + Namespace: nodeNetConfig.ObjectMeta.Namespace, + Name: nodeNetConfig.ObjectMeta.Name, + } - // Make mock client initialized with mock store - mockClient = MockClient{mockStore: mockStore} + _, ok := mockStore[mockKey] + + if !ok { + return errors.New("Node Net Config not found in mock store") + } + + nodeNetConfig.DeepCopyInto(mockStore[mockKey]) + return nil +} + +// MockCNSInteractor implements CNSInteractor interface +type MockCNSInteractor struct{} + +// we're just testing that reconciler interacts with CNS on Reconcile(). +func (mi MockCNSInteractor) UpdateCNSState(nnc.NodeNetworkConfigStatus) error { + mockCNSUpdated = true + return nil } -func TestMain(m *testing.M) { - //Setup mock client - initMockClient() - //run test - m.Run() +func ResetCNSInteractionFlag() { + mockCNSUpdated = false } func TestNewK8sRequestController(t *testing.T) { + //Test making request controller without logger initialized, should fail + _, err := NewK8sRequestController(nil, nil) + if err == nil { + t.Fatalf("Expected error when making NewK8sRequestController without initializing logger, got nil error") + } else if !strings.Contains(err.Error(), "logger") { + t.Fatalf("Expected logger error when making NewK8sRequestController without initializing logger, got: %+v", err) + } + + //Initialize logger + logger.InitLogger("Azure CNS Request Controller", 3, 3, "") - // Attempt to retrieve mock obj - key := client.ObjectKey{ - Namespace: "namespace", - Name: "name", + //Test making request controller without HOSTNAME env var set, should fail + _, err = NewK8sRequestController(nil, nil) + if err == nil { + t.Fatalf("Expected error when making NewK8sRequestController without setting HOSTNAME env var, got nil error") + } else if !strings.Contains(err.Error(), "HOSTNAME") { + t.Fatalf("Expected error when making NewK8sRequestController without setting HOSTNAME env var, got: %+v", err) } - toFill := &nnc.NodeNetworkConfig{} - if err := mockClient.Get(context.Background(), key, toFill); err != nil { - fmt.Printf("Some error ocured: %v \n", err) + + //Successful creation is tested in integrationt tests because it requires standing up a minikube cluster +} + +func TestGetNonExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + + //Test getting nonexisting NodeNetconfig obj + _, err := rc.getNodeNetConfig(context.Background(), nonexistingNNCName, nonexistingNamespace) + if err == nil { + t.Fatalf("Expected error when getting nonexisting nodenetconfig obj. Got nil error.") } - return } -// TODO: Finish tests, make interface for reconciler +func TestGetExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + + //Test getting existing NodeNetConfig obj + nodeNetConfig, err := rc.getNodeNetConfig(context.Background(), existingNNCName, existingNamespace) + if err != nil { + t.Fatalf("Expected no error when getting existing NodeNetworkConfig: %+v", err) + } + + mockKey := MockKey{ + Namespace: existingNamespace, + Name: existingNNCName, + } + + if !reflect.DeepEqual(nodeNetConfig, mockStore[mockKey]) { + t.Fatalf("Expected fetched node net config to equal one in mock store") + } +} + +func TestUpdateNonExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + + //Test updating non existing NodeNetworkConfig obj + nodeNetConfig := &nnc.NodeNetworkConfig{ObjectMeta: metav1.ObjectMeta{ + Name: nonexistingNNCName, + Namespace: nonexistingNamespace, + }} + + err := rc.updateNodeNetConfig(context.Background(), nodeNetConfig) + + if err == nil { + t.Fatalf("Expected error when updating non existing NodeNetworkConfig. Got nil error") + } +} + +func TestUpdateExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + + mockKey := MockKey{ + Namespace: existingNamespace, + Name: existingNNCName, + } + + //Update an existing NodeNetworkConfig obj from the mock store + nodeNetConfigUpdated := mockStore[mockKey].DeepCopy() + nodeNetConfigUpdated.ObjectMeta.ClusterName = "New cluster name" + + err := rc.updateNodeNetConfig(context.Background(), nodeNetConfigUpdated) + if err != nil { + t.Fatalf("Expected no error when updating existing NodeNetworkConfig, got :%v", err) + } + + //See that NodeNetworkConfig in mock store was updated + if !reflect.DeepEqual(nodeNetConfigUpdated, mockStore[mockKey]) { + t.Fatal("Update of existing NodeNetworkConfig did not get passed along") + } +} + +func TestReleaseIPsByUUIDsOnNonExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + rc.hostName = nonexistingNNCName + + uuids := make([]string, 3) + uuids[0] = "uuid0" + uuids[1] = "uuid1" + uuids[2] = "uuid2" + newCount := 5 + + //Test releasing ips by uuid for existing NodeNetworkConfig (hostname) + err := rc.ReleaseIPsByUUIDs(context.Background(), uuids, newCount) + + if err == nil { + t.Fatalf("Expected error when releasing ips by uuids") + } +} + +func TestReleaseIPsByUUIDsOnExistingNodeNetConfig(t *testing.T) { + rc := createMockRequestController() + + uuids := make([]string, 3) + uuids[0] = "uuid0" + uuids[1] = "uuid1" + uuids[2] = "uuid2" + newCount := 5 + + //Test releasing ips by uuid for existing NodeNetworkConfig (hostname) + err := rc.ReleaseIPsByUUIDs(context.Background(), uuids, newCount) + + if err != nil { + t.Fatalf("Expected no error when releasing ips by uuids, got :%v", err) + } + + mockKey := MockKey{ + Namespace: existingNamespace, + Name: existingNNCName, + } + + if !reflect.DeepEqual(mockStore[mockKey].Spec.IPsNotInUse, uuids) { + t.Fatalf("Expected IpsNotInUse to equal requested ReleaseIpsByUUIDs") + } + + if mockStore[mockKey].Spec.RequestedIPCount != int64(newCount) { + t.Fatalf("Expected requested ip count to equal count passed into ReleaseIPsByUUIds") + } +} + +func TestReconcileNonExistingNNC(t *testing.T) { + rc := createMockRequestController() + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: nonexistingNamespace, + Name: nonexistingNNCName, + }, + } + + _, err := rc.Reconciler.Reconcile(request) + + if err == nil { + t.Fatalf("Expected error when calling Reconcile for non existing NodeNetworkConfig") + } +} + +func TestReconcileExistingNNC(t *testing.T) { + rc := createMockRequestController() + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: existingNamespace, + Name: existingNNCName, + }, + } + + _, err := rc.Reconciler.Reconcile(request) + + //Want to reset flag to false for next test + defer ResetCNSInteractionFlag() + + if err != nil { + t.Fatalf("Expected no error reconciling existing NodeNetworkConfig, got :%v", err) + } + + if !mockCNSUpdated { + t.Fatalf("Expected MockCNSInteractor's UpdateCNSState() method to be called on Reconcile of existing NodeNetworkConfig") + } +} + +func createMockStore() map[MockKey]*nnc.NodeNetworkConfig { + //Create the mock store + mockStore = make(map[MockKey]*nnc.NodeNetworkConfig) + + mockKey := MockKey{ + Namespace: existingNamespace, + Name: existingNNCName, + } + + //Fill the mock store with one valid nodenetconfig obj + mockStore[mockKey] = &nnc.NodeNetworkConfig{ObjectMeta: v1.ObjectMeta{ + Name: existingNNCName, + Namespace: existingNamespace, + }} + + return mockStore +} + +func createMockClient() MockClient { + mockStore := createMockStore() + // Make mock client initialized with mock store + mockClient := MockClient{mockStore: mockStore} + + return mockClient +} + +func createMockInteractor() MockCNSInteractor { + return MockCNSInteractor{} +} + +func createMockRequestController() *k8sRequestController { + mockClient := createMockClient() + mockInteractor := createMockInteractor() + + rc := &k8sRequestController{} + rc.hostName = existingNNCName + rc.K8sClient = mockClient + rc.Reconciler = &NodeNetworkConfigReconciler{} + rc.Reconciler.K8sClient = mockClient + rc.Reconciler.CNSInteractor = mockInteractor + + //Initialize logger + logger.InitLogger("Azure CNS Request Controller", 0, 0, "") + + return rc +} From 76c1b66f9f5d8b1a406ccfb71de01decb7201a5c Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 30 Jun 2020 08:08:51 -0700 Subject: [PATCH 14/20] Fixing pipeline issues --- go.sum | 10 ---------- nodenetworkconfig/api/v1alpha/nodenetworkconfig.go | 1 - 2 files changed, 11 deletions(-) diff --git a/go.sum b/go.sum index 84e4850ed4..326b3150df 100644 --- a/go.sum +++ b/go.sum @@ -38,7 +38,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -93,7 +92,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -103,7 +101,6 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -242,7 +239,6 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -288,18 +284,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -450,7 +442,6 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -502,7 +493,6 @@ k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= -k8s.io/apiextensions-apiserver v0.18.2 h1:I4v3/jAuQC+89L3Z7dDgAiN4EOjN6sbm6iBqQwHTah8= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= diff --git a/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 52a1cb8ff1..c46a645e47 100644 --- a/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -26,7 +26,6 @@ import ( // NodeNetworkConfig is the Schema for the nodenetworkconfigs API // +kubebuilder:resource:scope=Namespaced - // +kubebuilder:resource:shortName=nnc // +kubebuilder:subresource:status type NodeNetworkConfig struct { From 20386ca03bdd832128fc923ebca92b64b76c00e8 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 30 Jun 2020 08:48:16 -0700 Subject: [PATCH 15/20] found issue, fixed HOSTNAME environment variable dependency --- .../kubernetes/k8srequestcontroller_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go index a03220966e..32c8380846 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -3,6 +3,7 @@ package kubernetes import ( "context" "errors" + "os" "reflect" "strings" "testing" @@ -105,6 +106,15 @@ func TestNewK8sRequestController(t *testing.T) { logger.InitLogger("Azure CNS Request Controller", 3, 3, "") //Test making request controller without HOSTNAME env var set, should fail + //Save old value though + hostName, found := os.LookupEnv("HOSTNAME") + os.Unsetenv("HOSTNAME") + defer func() { + if found { + os.Setenv("HOSTNAME", hostName) + } + }() + _, err = NewK8sRequestController(nil, nil) if err == nil { t.Fatalf("Expected error when making NewK8sRequestController without setting HOSTNAME env var, got nil error") From 99916983567b976c26861a44f7818d6b19421d58 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 30 Jun 2020 14:47:17 -0700 Subject: [PATCH 16/20] review changes requested --- cns/requestcontroller/kubernetes/eventfilter.go | 1 + cns/requestcontroller/kubernetes/k8scnsinteractor.go | 1 + cns/requestcontroller/kubernetes/k8srequestcontroller.go | 7 ++++++- cns/requestcontroller/kubernetes/translator.go | 0 cns/requestcontroller/requestcontrollerintreface.go | 5 ++++- 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 cns/requestcontroller/kubernetes/translator.go diff --git a/cns/requestcontroller/kubernetes/eventfilter.go b/cns/requestcontroller/kubernetes/eventfilter.go index 2a41c75e5c..3048aa3eac 100644 --- a/cns/requestcontroller/kubernetes/eventfilter.go +++ b/cns/requestcontroller/kubernetes/eventfilter.go @@ -16,6 +16,7 @@ type NodeNetworkConfigFilter struct { // status hasn't changed // See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource // for more details +// Generation will not change on status change func (n NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { isHostName := n.isHostName(e.MetaOld.GetName()) oldGeneration := e.MetaOld.GetGeneration() diff --git a/cns/requestcontroller/kubernetes/k8scnsinteractor.go b/cns/requestcontroller/kubernetes/k8scnsinteractor.go index 264bcd21bf..ca970784fb 100644 --- a/cns/requestcontroller/kubernetes/k8scnsinteractor.go +++ b/cns/requestcontroller/kubernetes/k8scnsinteractor.go @@ -11,6 +11,7 @@ type K8sCNSInteractor struct { RestService *restserver.HTTPRestService } +//translate nnc status to create nc request func (interactor *K8sCNSInteractor) UpdateCNSState(status nnc.NodeNetworkConfigStatus) error { //TODO: translate CNS Status into CNS Rest Service changes. //Mat will pick up from here diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go index 7349401804..0aa2c00377 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -16,8 +16,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) +//make environment var name constant const k8sNamespace = "kube-system" +//disable prometheus constant +//nodeNetConfigRequestController + // k8sRequestController watches CRDs for status updates and publishes CRD spec changes // mgr acts as the communication between requestController and API server // implements the RequestController interface @@ -42,7 +46,7 @@ func GetKubeConfig() (*rest.Config, error) { } //NewK8sRequestController given a reference to CNS's HTTPRestService state, returns a k8sRequestController struct -func NewK8sRequestController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*k8sRequestController, error) { +func NewRequestController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*k8sRequestController, error) { //Check that logger package has been intialized if logger.Log == nil { @@ -50,6 +54,7 @@ func NewK8sRequestController(restService *restserver.HTTPRestService, kubeconfig } // Check that HOSTNAME environment variable is set. HOSTNAME is name of node running this program + // NODENAME hostName := os.Getenv("HOSTNAME") if hostName == "" { return nil, errors.New("Must declare HOSTNAME environment variable. HOSTNAME is name of node.") diff --git a/cns/requestcontroller/kubernetes/translator.go b/cns/requestcontroller/kubernetes/translator.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index 9e09cb6dab..abef2f6f9a 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -9,10 +9,13 @@ import ( // interface for cns to interact with the request controller type RequestController interface { StartRequestController(exitChan chan bool) error - ReleaseIPsByUUIDs(cntxt context.Context, listOfIPUUIDS []string, newRequestedIPCount int) error + UpdateCRDSpec(cntxt context.Context, crdSpec nnc.NodeNetworkConfigSpec) error + //pass in cns client } // interface for request controller to interact with cns +//CNSClient type CNSInteractor interface { UpdateCNSState(nnc.NodeNetworkConfigStatus) error + //pass in cns type } From 588f39e346ed32d7f07e593fdc6d0c6e435f50d6 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 30 Jun 2020 14:49:10 -0700 Subject: [PATCH 17/20] more review changes --- cns/requestcontroller/kubernetes/eventfilter.go | 4 ++++ cns/requestcontroller/kubernetes/k8srequestcontroller.go | 7 ++++++- .../kubernetes/k8srequestcontroller_test.go | 3 +++ cns/requestcontroller/kubernetes/translator.go | 9 +++++++++ cns/requestcontroller/requestcontrollerintreface.go | 5 ++--- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cns/requestcontroller/kubernetes/eventfilter.go b/cns/requestcontroller/kubernetes/eventfilter.go index 3048aa3eac..e7eada4ad2 100644 --- a/cns/requestcontroller/kubernetes/eventfilter.go +++ b/cns/requestcontroller/kubernetes/eventfilter.go @@ -26,6 +26,7 @@ func (n NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { // Only process create events if CRD name equals this host's name func (n NodeNetworkConfigFilter) Create(e event.CreateEvent) bool { + //when we add, check status return n.isHostName(e.Meta.GetName()) } @@ -34,6 +35,9 @@ func (n NodeNetworkConfigFilter) Delete(e event.DeleteEvent) bool { return n.isHostName(e.Meta.GetName()) } +//Decide on what deleting CRD means, recreate CRD with CNS state? +//Right now ignore, coordinate with DNC + // Given a string, returns if that string equals the hostname running this program func (n NodeNetworkConfigFilter) isHostName(metaName string) bool { return metaName == n.hostname diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go index 0aa2c00377..e7881795a4 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller.go @@ -116,11 +116,12 @@ func NewRequestController(restService *restserver.HTTPRestService, kubeconfig *r // StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. // When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. // exitChan will be notified when requestController receives a kill signal +//This method blocks func (k8sRC *k8sRequestController) StartRequestController(exitChan chan bool) error { // Start manager and consequently, the reconciler // Start() blocks until SIGINT or SIGTERM is received // When SIGINT or SIGTERm are recived, notifies exitChan before exiting - go func() { + go func() {//get rid of go routine logger.Printf("Starting manager") if err := k8sRC.mgr.Start(SetupSignalHandler(exitChan)); err != nil { logger.Errorf("[cns-rc] Error starting manager: %v", err) @@ -153,6 +154,10 @@ func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(cntxt context.Context, list return nil } +ReconcileCNSState + +// seperate pr for that + // getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object func (k8sRC *k8sRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { nodeNetworkConfig := &nnc.NodeNetworkConfig{} diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go index 32c8380846..bebac11865 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go @@ -60,6 +60,9 @@ func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime. return nil } +//rename folder to kube-controller +//remove k8s filename prefix + //Mock implementation of the K8sClientInterface Update method //Mimics that of controller-runtime's client.Client func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { diff --git a/cns/requestcontroller/kubernetes/translator.go b/cns/requestcontroller/kubernetes/translator.go index e69de29bb2..b5c73a3f6a 100644 --- a/cns/requestcontroller/kubernetes/translator.go +++ b/cns/requestcontroller/kubernetes/translator.go @@ -0,0 +1,9 @@ +package kubernetes + +func CRDtoCNS() error { + return nil +} + +func CNStoCRD() { + +} diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index abef2f6f9a..d1e4e8011e 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -10,12 +10,11 @@ import ( type RequestController interface { StartRequestController(exitChan chan bool) error UpdateCRDSpec(cntxt context.Context, crdSpec nnc.NodeNetworkConfigSpec) error - //pass in cns client } // interface for request controller to interact with cns //CNSClient -type CNSInteractor interface { - UpdateCNSState(nnc.NodeNetworkConfigStatus) error +type CNSClient interface { + UpdateCNSState() error //pass in cns type } From 6ef0a7ad7db997ea42b839fcb7b1701a2bf30645 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 1 Jul 2020 12:13:52 -0700 Subject: [PATCH 18/20] Addressed changes from yesterday's review --- cns/requestcontroller/example/main.go | 46 ++--- .../apiclientinterface.go} | 7 +- .../kubecontroller/cnsinteractor.go | 17 ++ .../kubecontroller/crdreconciler.go | 53 +++++ .../kubecontroller/crdrequestcontroller.go | 168 ++++++++++++++++ .../crdrequestcontroller_test.go} | 100 +++++----- .../kubecontroller/crdtranslator.go | 19 ++ .../kubecontroller/eventfilter.go | 36 ++++ .../signalhandler.go | 2 +- .../kubernetes/eventfilter.go | 44 ----- .../kubernetes/k8scnsinteractor.go | 19 -- .../kubernetes/k8srequestcontroller.go | 184 ------------------ .../kubernetes/nodenetworkconfigreconciler.go | 48 ----- .../kubernetes/translator.go | 9 - .../requestcontrollerintreface.go | 11 +- 15 files changed, 377 insertions(+), 386 deletions(-) rename cns/requestcontroller/{kubernetes/k8sclientinterface.go => kubecontroller/apiclientinterface.go} (62%) create mode 100644 cns/requestcontroller/kubecontroller/cnsinteractor.go create mode 100644 cns/requestcontroller/kubecontroller/crdreconciler.go create mode 100644 cns/requestcontroller/kubecontroller/crdrequestcontroller.go rename cns/requestcontroller/{kubernetes/k8srequestcontroller_test.go => kubecontroller/crdrequestcontroller_test.go} (74%) create mode 100644 cns/requestcontroller/kubecontroller/crdtranslator.go create mode 100644 cns/requestcontroller/kubecontroller/eventfilter.go rename cns/requestcontroller/{kubernetes => kubecontroller}/signalhandler.go (97%) delete mode 100644 cns/requestcontroller/kubernetes/eventfilter.go delete mode 100644 cns/requestcontroller/kubernetes/k8scnsinteractor.go delete mode 100644 cns/requestcontroller/kubernetes/k8srequestcontroller.go delete mode 100644 cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go delete mode 100644 cns/requestcontroller/kubernetes/translator.go diff --git a/cns/requestcontroller/example/main.go b/cns/requestcontroller/example/main.go index e0a8a6275b..58f3acaf25 100644 --- a/cns/requestcontroller/example/main.go +++ b/cns/requestcontroller/example/main.go @@ -5,8 +5,9 @@ import ( "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/requestcontroller" - "github.com/Azure/azure-container-networking/cns/requestcontroller/kubernetes" + "github.com/Azure/azure-container-networking/cns/requestcontroller/kubecontroller" "github.com/Azure/azure-container-networking/cns/restserver" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" "golang.org/x/net/context" ) @@ -15,23 +16,20 @@ func goRequestController(rc requestcontroller.RequestController) { //SIGINT or SIGTERM, requestControllerExitChan is sent 'true' and you can clean up anything then requestControllerExitChan := make(chan bool, 1) - //Start the RequestController which starts the reconcile loop - if err := rc.StartRequestController(requestControllerExitChan); err != nil { - logger.Errorf("Error starting requestController: %v", err) - return - } + //Start the RequestController which starts the reconcile loop, blocks + go func() { + if err := rc.StartRequestController(requestControllerExitChan); err != nil { + logger.Errorf("Error starting requestController: %v", err) + return + } + }() - // After calling StartRequestController, there needs to be some pause or, the calling program - // must keep running because the reconcile loop is spawned off on a different go-routine inside + // After calling StartRequestController, there needs to be some pause before updating CRD spec time.Sleep(5 * time.Second) - logger.Printf("Done sleeping") - //For all request controller interactions with CRD spec (ReleaseIPsByUUIDs and UpdateRequestedIPCount) provide a context - // in order to be able to cancel if needed in the future + // We provide a context when making operations on CRD in case we need to cancel operation cntxt := context.Background() - // Example of releasing ips, give the requestController list of uuids which correspond to the ips you want to release - // as well as the new requestedIpCount // Create some dummy uuids uuids := make([]string, 5) uuids[0] = "uuid0" @@ -40,17 +38,19 @@ func goRequestController(rc requestcontroller.RequestController) { uuids[3] = "uuid3" uuids[4] = "uuid4" - // newCount = oldCount - #ips releasing + // newCount = oldCount - ips releasing // In this example, say we had 20 allocated to the node, we want to release 5, new count would be 15 oldCount := 20 - newRequestedIPCount := oldCount - len(uuids) + newRequestedIPCount := int64(oldCount - len(uuids)) - // This method is not synchronous, all it does is send the new count to the API server through the CRD spec. - // Dnc would see this new ip count, and in turn, send the requested IPS in the CRD status, which would - // trigger the reconcile loop, and in that reconcile loop is where the ips would be passed to cns. - // So this method only relays the message, it does nothing else - rc.ReleaseIPsByUUIDs(cntxt, uuids, newRequestedIPCount) - time.Sleep(5 * time.Second) + //Create CRD spec + spec := &nnc.NodeNetworkConfigSpec{ + RequestedIPCount: newRequestedIPCount, + IPsNotInUse: uuids, + } + + //Update CRD spec + rc.UpdateCRDSpec(cntxt, spec) <-requestControllerExitChan logger.Printf("Request controller received sigint or sigterm, time to cleanup") @@ -67,12 +67,12 @@ func main() { restService := &restserver.HTTPRestService{} //Provide kubeconfig, this method was abstracted out for testing - kubeconfig, err := kubernetes.GetKubeConfig() + kubeconfig, err := kubecontroller.GetKubeConfig() if err != nil { logger.Errorf("Error getting kubeconfig: %v", err) } - requestController, err = kubernetes.NewK8sRequestController(restService, kubeconfig) + requestController, err = kubecontroller.NewCrdRequestController(restService, kubeconfig) if err != nil { logger.Errorf("Error making new RequestController: %v", err) return diff --git a/cns/requestcontroller/kubernetes/k8sclientinterface.go b/cns/requestcontroller/kubecontroller/apiclientinterface.go similarity index 62% rename from cns/requestcontroller/kubernetes/k8sclientinterface.go rename to cns/requestcontroller/kubecontroller/apiclientinterface.go index 4130ebe28e..0f5fa81e84 100644 --- a/cns/requestcontroller/kubernetes/k8sclientinterface.go +++ b/cns/requestcontroller/kubecontroller/apiclientinterface.go @@ -1,4 +1,4 @@ -package kubernetes +package kubecontroller import ( "context" @@ -11,9 +11,8 @@ import ( // ObjectKey identifies a Kubernetes Object. type ObjectKey = types.NamespacedName -// K8sClient is an interface used by nodeNetworkConfigReconciler and k8sRequestController -// They rely on this interface in order to be able to make unit tests that overload these methods -type K8sClient interface { +// APIClient is an interface that talks to the API server +type APIClient interface { Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error } diff --git a/cns/requestcontroller/kubecontroller/cnsinteractor.go b/cns/requestcontroller/kubecontroller/cnsinteractor.go new file mode 100644 index 0000000000..56c0fdde4c --- /dev/null +++ b/cns/requestcontroller/kubecontroller/cnsinteractor.go @@ -0,0 +1,17 @@ +package kubecontroller + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/restserver" +) + +// CNSInteractor implements the CNSClient interface. +type CNSInteractor struct { + RestService *restserver.HTTPRestService +} + +// UpdateCNSState updates cns state +func (interactor *CNSInteractor) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { + //Mat will pick up from here + return nil +} diff --git a/cns/requestcontroller/kubecontroller/crdreconciler.go b/cns/requestcontroller/kubecontroller/crdreconciler.go new file mode 100644 index 0000000000..2da2276b3a --- /dev/null +++ b/cns/requestcontroller/kubecontroller/crdreconciler.go @@ -0,0 +1,53 @@ +package kubecontroller + +import ( + "context" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/requestcontroller" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// CrdReconciler watches for CRD status changes +type CrdReconciler struct { + APIClient APIClient + NodeName string + CNSClient requestcontroller.CNSClient +} + +// Reconcile is called on CRD status changes +func (r *CrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + var nodeNetConfig nnc.NodeNetworkConfig + + //Get the CRD object + if err := r.APIClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { + logger.Printf("[cns-rc] CRD not found, ignoring %v", err) + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) + + //TODO: Translate CRD status into NetworkContainer request + ncRequest, err := CRDStatusToNCRequest(&nodeNetConfig.Status) + if err != nil { + logger.Errorf("[cns-rc] Error translating crd status to nc request %v", err) + //requeue + return reconcile.Result{}, err + } + + //TODO: process the nc request on CNS side + r.CNSClient.UpdateCNSState(ncRequest) + + return reconcile.Result{}, nil +} + +// SetupWithManager Sets up the reconciler with a new manager, filtering using NodeNetworkConfigFilter +func (r *CrdReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nnc.NodeNetworkConfig{}). + WithEventFilter(NodeNetworkConfigFilter{nodeName: r.NodeName}). + Complete(r) +} diff --git a/cns/requestcontroller/kubecontroller/crdrequestcontroller.go b/cns/requestcontroller/kubecontroller/crdrequestcontroller.go new file mode 100644 index 0000000000..a109d2ca1e --- /dev/null +++ b/cns/requestcontroller/kubecontroller/crdrequestcontroller.go @@ -0,0 +1,168 @@ +package kubecontroller + +import ( + "context" + "errors" + "os" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const nodeNameEnvVar = "NODENAME" +const k8sNamespace = "kube-system" +const prometheusAddress = "0" + +// crdRequestController +// - watches CRD status changes +// - updates CRD spec +type crdRequestController struct { + mgr manager.Manager //Manager starts the reconcile loop which watches for crd status changes + APIClient APIClient //Relying on the APIClient interface to more easily test + nodeName string //name of node running this program + Reconciler *CrdReconciler +} + +// GetKubeConfig precedence +// * --kubeconfig flag pointing at a file at this cmd line +// * KUBECONFIG environment variable pointing at a file +// * In-cluster config if running in cluster +// * $HOME/.kube/config if exists +func GetKubeConfig() (*rest.Config, error) { + k8sconfig, err := ctrl.GetConfig() + if err != nil { + return nil, err + } + return k8sconfig, nil +} + +//NewCrdRequestController given a reference to CNS's HTTPRestService state, returns a crdRequestController struct +func NewCrdRequestController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*crdRequestController, error) { + + //Check that logger package has been intialized + if logger.Log == nil { + return nil, errors.New("Must initialize logger before calling") + } + + // Check that NODENAME environment variable is set. NODENAME is name of node running this program + nodeName := os.Getenv(nodeNameEnvVar) + if nodeName == "" { + return nil, errors.New("Must declare " + nodeNameEnvVar + " environment variable.") + } + + //Add client-go scheme to runtime sheme so manager can recognize it + var scheme = runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding client-go scheme to runtime scheme") + } + + //Add CRD scheme to runtime sheme so manager can recognize it + if err := nnc.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding NodeNetworkConfig scheme to runtime scheme") + } + + // Create manager for CrdRequestController + // MetricsBindAddress is the tcp address that the controller should bind to + // for serving prometheus metrics, set to "0" to disable + mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: prometheusAddress, + Namespace: k8sNamespace, + }) + if err != nil { + logger.Errorf("[cns-rc] Error creating new request controller manager: %v", err) + return nil, err + } + + //Create cnsinteractor + cnsInteractor := &CNSInteractor{ + RestService: restService, + } + + //Create reconciler + crdreconciler := &CrdReconciler{ + APIClient: mgr.GetClient(), + NodeName: nodeName, + CNSClient: cnsInteractor, + } + + // Setup manager with reconciler + if err := crdreconciler.SetupWithManager(mgr); err != nil { + logger.Errorf("[cns-rc] Error creating new CrdRequestController: %v", err) + return nil, err + } + + // Create the requestController + crdRequestController := crdRequestController{ + mgr: mgr, + APIClient: mgr.GetClient(), + nodeName: nodeName, + Reconciler: crdreconciler, + } + + return &crdRequestController, nil +} + +// StartRequestController starts the Reconciler loop which watches for CRD status updates +// Blocks until SIGINT or SIGTERM is received +// Notifies exitChan when kill signal received +func (crdRC *crdRequestController) StartRequestController(exitChan chan bool) error { + logger.Printf("Starting manager") + if err := crdRC.mgr.Start(SetupSignalHandler(exitChan)); err != nil { + logger.Errorf("[cns-rc] Error starting manager: %v", err) + } + + return nil +} + +// UpdateCRDSpec updates the CRD spec +func (crdRC *crdRequestController) UpdateCRDSpec(cntxt context.Context, crdSpec *nnc.NodeNetworkConfigSpec) error { + nodeNetworkConfig, err := crdRC.getNodeNetConfig(cntxt, crdRC.nodeName, k8sNamespace) + if err != nil { + logger.Errorf("[cns-rc] Error getting CRD when updating spec %v", err) + return err + } + + //Update the CRD spec + nodeNetworkConfig.Spec = *crdSpec + + //Send update to API server + if err := crdRC.updateNodeNetConfig(cntxt, nodeNetworkConfig); err != nil { + logger.Errorf("[cns-rc] Error updating CRD spec %v", err) + return err + } + + return nil +} + +// getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object +func (crdRC *crdRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { + nodeNetworkConfig := &nnc.NodeNetworkConfig{} + + err := crdRC.APIClient.Get(cntxt, client.ObjectKey{ + Namespace: namespace, + Name: name, + }, nodeNetworkConfig) + + if err != nil { + return nil, err + } + + return nodeNetworkConfig, nil +} + +// updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object +func (crdRC *crdRequestController) updateNodeNetConfig(cntxt context.Context, nodeNetworkConfig *nnc.NodeNetworkConfig) error { + if err := crdRC.APIClient.Update(cntxt, nodeNetworkConfig); err != nil { + return err + } + + return nil +} diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go similarity index 74% rename from cns/requestcontroller/kubernetes/k8srequestcontroller_test.go rename to cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go index bebac11865..d95e02c6c1 100644 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller_test.go +++ b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go @@ -1,4 +1,4 @@ -package kubernetes +package kubecontroller import ( "context" @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/logger" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,17 +35,13 @@ type MockKey struct { Name string } -func (m MockKey) String() string { - return m.Namespace + "/" + m.Name -} - -// MockClient implements K8SClient interface +// MockClient implements APIClient interface type MockClient struct { mockStore map[MockKey]*nnc.NodeNetworkConfig } -// Mock implementation of the K8sClientInterface Get method -//Mimics that of controller-runtime's client.Client +// Mock implementation of the APIClient interface Get method +// Mimics that of controller-runtime's client.Client func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { mockKey := MockKey{ Namespace: key.Namespace, @@ -60,10 +57,7 @@ func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime. return nil } -//rename folder to kube-controller -//remove k8s filename prefix - -//Mock implementation of the K8sClientInterface Update method +//Mock implementation of the APIClient interface Update method //Mimics that of controller-runtime's client.Client func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { nodeNetConfig := obj.(*nnc.NodeNetworkConfig) @@ -83,11 +77,11 @@ func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...cli return nil } -// MockCNSInteractor implements CNSInteractor interface +// MockCNSInteractor implements CNSClient interface type MockCNSInteractor struct{} // we're just testing that reconciler interacts with CNS on Reconcile(). -func (mi MockCNSInteractor) UpdateCNSState(nnc.NodeNetworkConfigStatus) error { +func (mi MockCNSInteractor) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { mockCNSUpdated = true return nil } @@ -96,36 +90,36 @@ func ResetCNSInteractionFlag() { mockCNSUpdated = false } -func TestNewK8sRequestController(t *testing.T) { +func TestNewCrdRequestController(t *testing.T) { //Test making request controller without logger initialized, should fail - _, err := NewK8sRequestController(nil, nil) + _, err := NewCrdRequestController(nil, nil) if err == nil { - t.Fatalf("Expected error when making NewK8sRequestController without initializing logger, got nil error") + t.Fatalf("Expected error when making NewCrdRequestController without initializing logger, got nil error") } else if !strings.Contains(err.Error(), "logger") { - t.Fatalf("Expected logger error when making NewK8sRequestController without initializing logger, got: %+v", err) + t.Fatalf("Expected logger error when making NewCrdRequestController without initializing logger, got: %+v", err) } //Initialize logger - logger.InitLogger("Azure CNS Request Controller", 3, 3, "") + logger.InitLogger("Azure CRD Request Controller", 3, 3, "") - //Test making request controller without HOSTNAME env var set, should fail + //Test making request controller without NODENAME env var set, should fail //Save old value though - hostName, found := os.LookupEnv("HOSTNAME") - os.Unsetenv("HOSTNAME") + nodeName, found := os.LookupEnv(nodeNameEnvVar) + os.Unsetenv(nodeNameEnvVar) defer func() { if found { - os.Setenv("HOSTNAME", hostName) + os.Setenv(nodeNameEnvVar, nodeName) } }() - _, err = NewK8sRequestController(nil, nil) + _, err = NewCrdRequestController(nil, nil) if err == nil { - t.Fatalf("Expected error when making NewK8sRequestController without setting HOSTNAME env var, got nil error") - } else if !strings.Contains(err.Error(), "HOSTNAME") { - t.Fatalf("Expected error when making NewK8sRequestController without setting HOSTNAME env var, got: %+v", err) + t.Fatalf("Expected error when making NewCrdRequestController without setting " + nodeNameEnvVar + " env var, got nil error") + } else if !strings.Contains(err.Error(), nodeNameEnvVar) { + t.Fatalf("Expected error when making NewCrdRequestController without setting "+nodeNameEnvVar+" env var, got: %+v", err) } - //Successful creation is tested in integrationt tests because it requires standing up a minikube cluster + //TODO: Create integration tests with minikube } func TestGetNonExistingNodeNetConfig(t *testing.T) { @@ -197,38 +191,48 @@ func TestUpdateExistingNodeNetConfig(t *testing.T) { } } -func TestReleaseIPsByUUIDsOnNonExistingNodeNetConfig(t *testing.T) { +func TestUpdateSpecOnNonExistingNodeNetConfig(t *testing.T) { rc := createMockRequestController() - rc.hostName = nonexistingNNCName + rc.nodeName = nonexistingNNCName uuids := make([]string, 3) uuids[0] = "uuid0" uuids[1] = "uuid1" uuids[2] = "uuid2" - newCount := 5 + newCount := int64(5) - //Test releasing ips by uuid for existing NodeNetworkConfig (hostname) - err := rc.ReleaseIPsByUUIDs(context.Background(), uuids, newCount) + spec := &nnc.NodeNetworkConfigSpec{ + RequestedIPCount: newCount, + IPsNotInUse: uuids, + } + + //Test updating spec for existing NodeNetworkConfig + err := rc.UpdateCRDSpec(context.Background(), spec) if err == nil { - t.Fatalf("Expected error when releasing ips by uuids") + t.Fatalf("Expected error when updating spec on non-existing crd") } } -func TestReleaseIPsByUUIDsOnExistingNodeNetConfig(t *testing.T) { +func TestUpdateSpecOnExistingNodeNetConfig(t *testing.T) { rc := createMockRequestController() uuids := make([]string, 3) uuids[0] = "uuid0" uuids[1] = "uuid1" uuids[2] = "uuid2" - newCount := 5 + newCount := int64(5) + + spec := &nnc.NodeNetworkConfigSpec{ + RequestedIPCount: newCount, + IPsNotInUse: uuids, + } - //Test releasing ips by uuid for existing NodeNetworkConfig (hostname) - err := rc.ReleaseIPsByUUIDs(context.Background(), uuids, newCount) + //Test releasing ips by uuid for existing NodeNetworkConfig + err := rc.UpdateCRDSpec(context.Background(), spec) if err != nil { - t.Fatalf("Expected no error when releasing ips by uuids, got :%v", err) + t.Fatalf("Expected no error when updating spec on existing crd, got :%v", err) } mockKey := MockKey{ @@ -237,11 +241,11 @@ func TestReleaseIPsByUUIDsOnExistingNodeNetConfig(t *testing.T) { } if !reflect.DeepEqual(mockStore[mockKey].Spec.IPsNotInUse, uuids) { - t.Fatalf("Expected IpsNotInUse to equal requested ReleaseIpsByUUIDs") + t.Fatalf("Expected IpsNotInUse to equal requested ips to release") } if mockStore[mockKey].Spec.RequestedIPCount != int64(newCount) { - t.Fatalf("Expected requested ip count to equal count passed into ReleaseIPsByUUIds") + t.Fatalf("Expected requested ip count to equal count passed into requested ip count") } } @@ -314,16 +318,16 @@ func createMockInteractor() MockCNSInteractor { return MockCNSInteractor{} } -func createMockRequestController() *k8sRequestController { +func createMockRequestController() *crdRequestController { mockClient := createMockClient() mockInteractor := createMockInteractor() - rc := &k8sRequestController{} - rc.hostName = existingNNCName - rc.K8sClient = mockClient - rc.Reconciler = &NodeNetworkConfigReconciler{} - rc.Reconciler.K8sClient = mockClient - rc.Reconciler.CNSInteractor = mockInteractor + rc := &crdRequestController{} + rc.nodeName = existingNNCName + rc.APIClient = mockClient + rc.Reconciler = &CrdReconciler{} + rc.Reconciler.APIClient = mockClient + rc.Reconciler.CNSClient = mockInteractor //Initialize logger logger.InitLogger("Azure CNS Request Controller", 0, 0, "") diff --git a/cns/requestcontroller/kubecontroller/crdtranslator.go b/cns/requestcontroller/kubecontroller/crdtranslator.go new file mode 100644 index 0000000000..14b7ea7c54 --- /dev/null +++ b/cns/requestcontroller/kubecontroller/crdtranslator.go @@ -0,0 +1,19 @@ +package kubecontroller + +import ( + "github.com/Azure/azure-container-networking/cns" + nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" +) + +// CRDStatusToNCRequest translates a crd status to network container request +func CRDStatusToNCRequest(crdStatus *nnc.NodeNetworkConfigStatus) (*cns.CreateNetworkContainerRequest, error) { + //TODO: Translate CRD status into network container request + //Mat will pick up from here + return nil, nil +} + +// CNSToCRDSpec translates CNS's list of Ips to be released and requested ip count into a CRD Spec +func CNSToCRDSpec() (*nnc.NodeNetworkConfigSpec, error) { + //TODO: Translate list of ips to be released and requested ip count to CRD spec + return nil, nil +} diff --git a/cns/requestcontroller/kubecontroller/eventfilter.go b/cns/requestcontroller/kubecontroller/eventfilter.go new file mode 100644 index 0000000000..1704913019 --- /dev/null +++ b/cns/requestcontroller/kubecontroller/eventfilter.go @@ -0,0 +1,36 @@ +package kubecontroller + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +type NodeNetworkConfigFilter struct { + predicate.Funcs + nodeName string +} + +// Returns true if request is to be processed by Reconciler +// Checks that old generation equals new generation because status changes don't change generation number +func (n NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { + isNodeName := n.isNodeName(e.MetaOld.GetName()) + oldGeneration := e.MetaOld.GetGeneration() + newGeneration := e.MetaNew.GetGeneration() + return (oldGeneration == newGeneration) && isNodeName +} + +// Only process create events if CRD name equals this host's name +func (n NodeNetworkConfigFilter) Create(e event.CreateEvent) bool { + return n.isNodeName(e.Meta.GetName()) +} + +//TODO: Decide what deleteing crd means with DNC +// Ignore all for now +func (n NodeNetworkConfigFilter) Delete(e event.DeleteEvent) bool { + return false +} + +// Given a string, returns if that string equals the nodename running this program +func (n NodeNetworkConfigFilter) isNodeName(metaName string) bool { + return metaName == n.nodeName +} diff --git a/cns/requestcontroller/kubernetes/signalhandler.go b/cns/requestcontroller/kubecontroller/signalhandler.go similarity index 97% rename from cns/requestcontroller/kubernetes/signalhandler.go rename to cns/requestcontroller/kubecontroller/signalhandler.go index af4203da57..f6b73124ec 100644 --- a/cns/requestcontroller/kubernetes/signalhandler.go +++ b/cns/requestcontroller/kubecontroller/signalhandler.go @@ -1,4 +1,4 @@ -package kubernetes +package kubecontroller import ( "fmt" diff --git a/cns/requestcontroller/kubernetes/eventfilter.go b/cns/requestcontroller/kubernetes/eventfilter.go deleted file mode 100644 index e7eada4ad2..0000000000 --- a/cns/requestcontroller/kubernetes/eventfilter.go +++ /dev/null @@ -1,44 +0,0 @@ -package kubernetes - -import ( - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -type NodeNetworkConfigFilter struct { - predicate.Funcs - hostname string -} - -// If the generations are the same, it means it's status change, and we should return true, so that the -// reconcile loop is triggered by it. -// If they're different, it means a spec change, and we should ignore, by returning false, to avoid redundant calls to cns when the -// status hasn't changed -// See https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource -// for more details -// Generation will not change on status change -func (n NodeNetworkConfigFilter) Update(e event.UpdateEvent) bool { - isHostName := n.isHostName(e.MetaOld.GetName()) - oldGeneration := e.MetaOld.GetGeneration() - newGeneration := e.MetaNew.GetGeneration() - return (oldGeneration == newGeneration) && isHostName -} - -// Only process create events if CRD name equals this host's name -func (n NodeNetworkConfigFilter) Create(e event.CreateEvent) bool { - //when we add, check status - return n.isHostName(e.Meta.GetName()) -} - -// Only process delete events if CRD name equals this host's name -func (n NodeNetworkConfigFilter) Delete(e event.DeleteEvent) bool { - return n.isHostName(e.Meta.GetName()) -} - -//Decide on what deleting CRD means, recreate CRD with CNS state? -//Right now ignore, coordinate with DNC - -// Given a string, returns if that string equals the hostname running this program -func (n NodeNetworkConfigFilter) isHostName(metaName string) bool { - return metaName == n.hostname -} diff --git a/cns/requestcontroller/kubernetes/k8scnsinteractor.go b/cns/requestcontroller/kubernetes/k8scnsinteractor.go deleted file mode 100644 index ca970784fb..0000000000 --- a/cns/requestcontroller/kubernetes/k8scnsinteractor.go +++ /dev/null @@ -1,19 +0,0 @@ -package kubernetes - -import ( - "github.com/Azure/azure-container-networking/cns/restserver" - nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" -) - -//This K8sCNSInteractor implements the CNSInteractor interface. It's used by nodenetworkconfigreconciler to translate -// CRD status to HttpRestService changes -type K8sCNSInteractor struct { - RestService *restserver.HTTPRestService -} - -//translate nnc status to create nc request -func (interactor *K8sCNSInteractor) UpdateCNSState(status nnc.NodeNetworkConfigStatus) error { - //TODO: translate CNS Status into CNS Rest Service changes. - //Mat will pick up from here - return nil -} diff --git a/cns/requestcontroller/kubernetes/k8srequestcontroller.go b/cns/requestcontroller/kubernetes/k8srequestcontroller.go deleted file mode 100644 index e7881795a4..0000000000 --- a/cns/requestcontroller/kubernetes/k8srequestcontroller.go +++ /dev/null @@ -1,184 +0,0 @@ -package kubernetes - -import ( - "context" - "errors" - "os" - - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/restserver" - nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -//make environment var name constant -const k8sNamespace = "kube-system" - -//disable prometheus constant -//nodeNetConfigRequestController - -// k8sRequestController watches CRDs for status updates and publishes CRD spec changes -// mgr acts as the communication between requestController and API server -// implements the RequestController interface -type k8sRequestController struct { - mgr manager.Manager //mgr has method GetClient() to get k8s client - K8sClient K8sClient //Relying on the K8sClient interface to more easily test - hostName string //name of node running this program - Reconciler *NodeNetworkConfigReconciler -} - -// GetKubeConfig precedence -// * --kubeconfig flag pointing at a file at this cmd line -// * KUBECONFIG environment variable pointing at a file -// * In-cluster config if running in cluster -// * $HOME/.kube/config if exists -func GetKubeConfig() (*rest.Config, error) { - k8sconfig, err := ctrl.GetConfig() - if err != nil { - return nil, err - } - return k8sconfig, nil -} - -//NewK8sRequestController given a reference to CNS's HTTPRestService state, returns a k8sRequestController struct -func NewRequestController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*k8sRequestController, error) { - - //Check that logger package has been intialized - if logger.Log == nil { - return nil, errors.New("Must initialize logger before calling") - } - - // Check that HOSTNAME environment variable is set. HOSTNAME is name of node running this program - // NODENAME - hostName := os.Getenv("HOSTNAME") - if hostName == "" { - return nil, errors.New("Must declare HOSTNAME environment variable. HOSTNAME is name of node.") - } - - //Add client-go scheme to runtime sheme so manager can recognize it - var scheme = runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - return nil, errors.New("Error adding client-go scheme to runtime scheme") - } - - //Add CRD scheme to runtime sheme so manager can recognize it - if err := nnc.AddToScheme(scheme); err != nil { - return nil, errors.New("Error adding NodeNetworkConfig scheme to runtime scheme") - } - - // Create manager for NodeNetworkConfigReconciler - // MetricsBindAddress is the tcp address that the controller should bind to - // for serving prometheus metrics, set to "0" to disable - mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: "0", - Namespace: k8sNamespace, - }) - if err != nil { - logger.Errorf("[cns-rc] Error creating new request controller manager: %v", err) - return nil, err - } - - //Create k8scnsinteractor - k8scnsinteractor := &K8sCNSInteractor{ - RestService: restService, - } - - //Create reconciler - nodenetworkconfigreconciler := &NodeNetworkConfigReconciler{ - K8sClient: mgr.GetClient(), - HostName: hostName, - CNSInteractor: k8scnsinteractor, - } - - // Setup manager with reconciler - if err := nodenetworkconfigreconciler.SetupWithManager(mgr); err != nil { - logger.Errorf("[cns-rc] Error creating new NodeNetworkConfigReconciler: %v", err) - return nil, err - } - - // Create the requestController - k8sRequestController := k8sRequestController{ - mgr: mgr, - K8sClient: mgr.GetClient(), - hostName: hostName, - Reconciler: nodenetworkconfigreconciler, - } - - return &k8sRequestController, nil -} - -// StartRequestController starts the reconcile loop. This loop waits for changes to CRD statuses. -// When a CRD status change is made, Reconcile from nodenetworkconfigreconciler is called. -// exitChan will be notified when requestController receives a kill signal -//This method blocks -func (k8sRC *k8sRequestController) StartRequestController(exitChan chan bool) error { - // Start manager and consequently, the reconciler - // Start() blocks until SIGINT or SIGTERM is received - // When SIGINT or SIGTERm are recived, notifies exitChan before exiting - go func() {//get rid of go routine - logger.Printf("Starting manager") - if err := k8sRC.mgr.Start(SetupSignalHandler(exitChan)); err != nil { - logger.Errorf("[cns-rc] Error starting manager: %v", err) - } - }() - - return nil -} - -// ReleaseIPsByUUIDs sends release ip request to the API server and updates requested ip count. -// Provide the UUIDs of the IP allocations and the new requested ip count -func (k8sRC *k8sRequestController) ReleaseIPsByUUIDs(cntxt context.Context, listOfIPUUIDS []string, newRequestedIPCount int) error { - nodeNetworkConfig, err := k8sRC.getNodeNetConfig(cntxt, k8sRC.hostName, k8sNamespace) - if err != nil { - logger.Errorf("[cns-rc] Error getting CRD when releasing IPs by uuid %v", err) - return err - } - - //Update the CRD IpsNotInUse - nodeNetworkConfig.Spec.IPsNotInUse = append(nodeNetworkConfig.Spec.IPsNotInUse, listOfIPUUIDS...) - //Update the CRD requestedIPCount - nodeNetworkConfig.Spec.RequestedIPCount = int64(newRequestedIPCount) - - //Send update to API server - if err := k8sRC.updateNodeNetConfig(cntxt, nodeNetworkConfig); err != nil { - logger.Errorf("[cns-rc] Error updating CRD when releasing IPs by uuid %v", err) - return err - } - - return nil -} - -ReconcileCNSState - -// seperate pr for that - -// getNodeNetConfig gets the nodeNetworkConfig CRD given the name and namespace of the CRD object -func (k8sRC *k8sRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { - nodeNetworkConfig := &nnc.NodeNetworkConfig{} - - err := k8sRC.K8sClient.Get(cntxt, client.ObjectKey{ - Namespace: namespace, - Name: name, - }, nodeNetworkConfig) - - if err != nil { - return nil, err - } - - return nodeNetworkConfig, nil -} - -// updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object -func (k8sRC *k8sRequestController) updateNodeNetConfig(cntxt context.Context, nodeNetworkConfig *nnc.NodeNetworkConfig) error { - if err := k8sRC.K8sClient.Update(cntxt, nodeNetworkConfig); err != nil { - return err - } - - return nil -} diff --git a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go b/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go deleted file mode 100644 index a7bbb6950e..0000000000 --- a/cns/requestcontroller/kubernetes/nodenetworkconfigreconciler.go +++ /dev/null @@ -1,48 +0,0 @@ -package kubernetes - -import ( - "context" - - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/requestcontroller" - nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// NodeNetworkConfigReconciler (aka controller) watches API server for any creation/deletion/updates of NodeNetworkConfig objects -type NodeNetworkConfigReconciler struct { - K8sClient K8sClient - HostName string - CNSInteractor requestcontroller.CNSInteractor -} - -// Reconcile relays status changes in NodeNetworkConfig to CNS -// Returning non-nil error causes a requeue -// Returning ctrl.Result{}, nil causes the queue to "forget" the item -// Other return values are possible, see kubebuilder docs for details -func (n *NodeNetworkConfigReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { - var nodeNetConfig nnc.NodeNetworkConfig - - //Get the CRD object - if err := n.K8sClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { - logger.Printf("[cns-rc] CRD not found, ignoring %v", err) - return reconcile.Result{}, client.IgnoreNotFound(err) - } - - logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) - - //TODO: Translate CRD status into HTTPRestService state - n.CNSInteractor.UpdateCNSState(nodeNetConfig.Status) - - return reconcile.Result{}, nil -} - -// SetupWithManager Sets up the reconciler with a new manager -func (n *NodeNetworkConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&nnc.NodeNetworkConfig{}). - WithEventFilter(NodeNetworkConfigFilter{hostname: n.HostName}). - Complete(n) -} diff --git a/cns/requestcontroller/kubernetes/translator.go b/cns/requestcontroller/kubernetes/translator.go deleted file mode 100644 index b5c73a3f6a..0000000000 --- a/cns/requestcontroller/kubernetes/translator.go +++ /dev/null @@ -1,9 +0,0 @@ -package kubernetes - -func CRDtoCNS() error { - return nil -} - -func CNStoCRD() { - -} diff --git a/cns/requestcontroller/requestcontrollerintreface.go b/cns/requestcontroller/requestcontrollerintreface.go index d1e4e8011e..86687e38a0 100644 --- a/cns/requestcontroller/requestcontrollerintreface.go +++ b/cns/requestcontroller/requestcontrollerintreface.go @@ -3,18 +3,17 @@ package requestcontroller import ( "context" + "github.com/Azure/azure-container-networking/cns" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" ) -// interface for cns to interact with the request controller +// RequestController interface for cns to interact with the request controller type RequestController interface { StartRequestController(exitChan chan bool) error - UpdateCRDSpec(cntxt context.Context, crdSpec nnc.NodeNetworkConfigSpec) error + UpdateCRDSpec(cntxt context.Context, crdSpec *nnc.NodeNetworkConfigSpec) error } -// interface for request controller to interact with cns -//CNSClient +// CNSClient interface for request controller to interact with cns type CNSClient interface { - UpdateCNSState() error - //pass in cns type + UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error } From 9a9f3ceb6cd445af7c34ff59000f50801e11cd7e Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 1 Jul 2020 12:20:38 -0700 Subject: [PATCH 19/20] Changing makefile line to run correct package --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f47aa3a69e..316c547f5d 100644 --- a/Makefile +++ b/Makefile @@ -378,7 +378,7 @@ test-all: ./cnm/network/ \ ./cni/ipam/ \ ./cns/ipamclient/ \ - ./cns/requestcontroller/kubernetes/ \ + ./cns/requestcontroller/kubecontroller/ \ ./cnms/service/ \ ./npm/iptm/ \ ./npm/ipsm/ From 72ad9887eaec305a06561aaf3913b7e1f72c6baf Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 1 Jul 2020 17:10:03 -0700 Subject: [PATCH 20/20] Addressed Matt Long's suggestions --- Makefile | 2 +- cns/cnsclient/apiclient.go | 8 +++++ .../httpapi/client.go} | 8 ++--- .../kubecontroller/apiclientinterface.go | 4 +-- .../kubecontroller/crdreconciler.go | 20 +++++++---- .../kubecontroller/crdrequestcontroller.go | 23 +++++++------ .../crdrequestcontroller_test.go | 34 +++++++++---------- 7 files changed, 57 insertions(+), 42 deletions(-) create mode 100644 cns/cnsclient/apiclient.go rename cns/{requestcontroller/kubecontroller/cnsinteractor.go => cnsclient/httpapi/client.go} (52%) diff --git a/Makefile b/Makefile index 316c547f5d..3648e1d8e7 100644 --- a/Makefile +++ b/Makefile @@ -378,7 +378,7 @@ test-all: ./cnm/network/ \ ./cni/ipam/ \ ./cns/ipamclient/ \ - ./cns/requestcontroller/kubecontroller/ \ + ./cns/requestcontroller/kubecontroller/ \ ./cnms/service/ \ ./npm/iptm/ \ ./npm/ipsm/ diff --git a/cns/cnsclient/apiclient.go b/cns/cnsclient/apiclient.go new file mode 100644 index 0000000000..d3a1032398 --- /dev/null +++ b/cns/cnsclient/apiclient.go @@ -0,0 +1,8 @@ +package cnsclient + +import "github.com/Azure/azure-container-networking/cns" + +// APIClient interface to update cns state +type APIClient interface { + UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error +} diff --git a/cns/requestcontroller/kubecontroller/cnsinteractor.go b/cns/cnsclient/httpapi/client.go similarity index 52% rename from cns/requestcontroller/kubecontroller/cnsinteractor.go rename to cns/cnsclient/httpapi/client.go index 56c0fdde4c..16740ec25f 100644 --- a/cns/requestcontroller/kubecontroller/cnsinteractor.go +++ b/cns/cnsclient/httpapi/client.go @@ -1,17 +1,17 @@ -package kubecontroller +package httpapi import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/restserver" ) -// CNSInteractor implements the CNSClient interface. -type CNSInteractor struct { +// Client implements APIClient interface. Used to update CNS state +type Client struct { RestService *restserver.HTTPRestService } // UpdateCNSState updates cns state -func (interactor *CNSInteractor) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { +func (client *Client) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { //Mat will pick up from here return nil } diff --git a/cns/requestcontroller/kubecontroller/apiclientinterface.go b/cns/requestcontroller/kubecontroller/apiclientinterface.go index 0f5fa81e84..d57a116221 100644 --- a/cns/requestcontroller/kubecontroller/apiclientinterface.go +++ b/cns/requestcontroller/kubecontroller/apiclientinterface.go @@ -11,8 +11,8 @@ import ( // ObjectKey identifies a Kubernetes Object. type ObjectKey = types.NamespacedName -// APIClient is an interface that talks to the API server -type APIClient interface { +// KubeClient is an interface that talks to the API server +type KubeClient interface { Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error } diff --git a/cns/requestcontroller/kubecontroller/crdreconciler.go b/cns/requestcontroller/kubecontroller/crdreconciler.go index 2da2276b3a..e958a70d51 100644 --- a/cns/requestcontroller/kubecontroller/crdreconciler.go +++ b/cns/requestcontroller/kubecontroller/crdreconciler.go @@ -3,9 +3,10 @@ package kubecontroller import ( "context" + "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/requestcontroller" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -13,9 +14,9 @@ import ( // CrdReconciler watches for CRD status changes type CrdReconciler struct { - APIClient APIClient - NodeName string - CNSClient requestcontroller.CNSClient + KubeClient KubeClient + NodeName string + CNSClient cnsclient.APIClient } // Reconcile is called on CRD status changes @@ -23,9 +24,14 @@ func (r *CrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, var nodeNetConfig nnc.NodeNetworkConfig //Get the CRD object - if err := r.APIClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { - logger.Printf("[cns-rc] CRD not found, ignoring %v", err) - return reconcile.Result{}, client.IgnoreNotFound(err) + if err := r.KubeClient.Get(context.TODO(), request.NamespacedName, &nodeNetConfig); err != nil { + if apierrors.IsNotFound(err) { + logger.Printf("[cns-rc] CRD not found, ignoring %v", err) + return reconcile.Result{}, client.IgnoreNotFound(err) + } else { + logger.Errorf("[cns-rc] Error retrieving CRD from cache : %v", err) + return reconcile.Result{}, err + } } logger.Printf("[cns-rc] CRD object: %v", nodeNetConfig) diff --git a/cns/requestcontroller/kubecontroller/crdrequestcontroller.go b/cns/requestcontroller/kubecontroller/crdrequestcontroller.go index a109d2ca1e..41b760b769 100644 --- a/cns/requestcontroller/kubecontroller/crdrequestcontroller.go +++ b/cns/requestcontroller/kubecontroller/crdrequestcontroller.go @@ -5,6 +5,7 @@ import ( "errors" "os" + "github.com/Azure/azure-container-networking/cns/cnsclient/httpapi" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/restserver" nnc "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" @@ -18,14 +19,14 @@ import ( const nodeNameEnvVar = "NODENAME" const k8sNamespace = "kube-system" -const prometheusAddress = "0" +const prometheusAddress = "0" //0 means disabled // crdRequestController // - watches CRD status changes // - updates CRD spec type crdRequestController struct { mgr manager.Manager //Manager starts the reconcile loop which watches for crd status changes - APIClient APIClient //Relying on the APIClient interface to more easily test + KubeClient KubeClient //KubeClient interacts with API server nodeName string //name of node running this program Reconciler *CrdReconciler } @@ -81,16 +82,16 @@ func NewCrdRequestController(restService *restserver.HTTPRestService, kubeconfig return nil, err } - //Create cnsinteractor - cnsInteractor := &CNSInteractor{ + //Create httpClient + httpClient := &httpapi.Client{ RestService: restService, } //Create reconciler crdreconciler := &CrdReconciler{ - APIClient: mgr.GetClient(), - NodeName: nodeName, - CNSClient: cnsInteractor, + KubeClient: mgr.GetClient(), + NodeName: nodeName, + CNSClient: httpClient, } // Setup manager with reconciler @@ -102,7 +103,7 @@ func NewCrdRequestController(restService *restserver.HTTPRestService, kubeconfig // Create the requestController crdRequestController := crdRequestController{ mgr: mgr, - APIClient: mgr.GetClient(), + KubeClient: mgr.GetClient(), nodeName: nodeName, Reconciler: crdreconciler, } @@ -131,7 +132,7 @@ func (crdRC *crdRequestController) UpdateCRDSpec(cntxt context.Context, crdSpec } //Update the CRD spec - nodeNetworkConfig.Spec = *crdSpec + crdSpec.DeepCopyInto(&nodeNetworkConfig.Spec) //Send update to API server if err := crdRC.updateNodeNetConfig(cntxt, nodeNetworkConfig); err != nil { @@ -146,7 +147,7 @@ func (crdRC *crdRequestController) UpdateCRDSpec(cntxt context.Context, crdSpec func (crdRC *crdRequestController) getNodeNetConfig(cntxt context.Context, name, namespace string) (*nnc.NodeNetworkConfig, error) { nodeNetworkConfig := &nnc.NodeNetworkConfig{} - err := crdRC.APIClient.Get(cntxt, client.ObjectKey{ + err := crdRC.KubeClient.Get(cntxt, client.ObjectKey{ Namespace: namespace, Name: name, }, nodeNetworkConfig) @@ -160,7 +161,7 @@ func (crdRC *crdRequestController) getNodeNetConfig(cntxt context.Context, name, // updateNodeNetConfig updates the nodeNetConfig object in the API server with the given nodeNetworkConfig object func (crdRC *crdRequestController) updateNodeNetConfig(cntxt context.Context, nodeNetworkConfig *nnc.NodeNetworkConfig) error { - if err := crdRC.APIClient.Update(cntxt, nodeNetworkConfig); err != nil { + if err := crdRC.KubeClient.Update(cntxt, nodeNetworkConfig); err != nil { return err } diff --git a/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go index d95e02c6c1..06c7b661ce 100644 --- a/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go +++ b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go @@ -35,14 +35,14 @@ type MockKey struct { Name string } -// MockClient implements APIClient interface -type MockClient struct { +// MockKubeClient implements KubeClient interface +type MockKubeClient struct { mockStore map[MockKey]*nnc.NodeNetworkConfig } // Mock implementation of the APIClient interface Get method // Mimics that of controller-runtime's client.Client -func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { +func (mc MockKubeClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { mockKey := MockKey{ Namespace: key.Namespace, Name: key.Name, @@ -59,7 +59,7 @@ func (mc MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime. //Mock implementation of the APIClient interface Update method //Mimics that of controller-runtime's client.Client -func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { +func (mc MockKubeClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { nodeNetConfig := obj.(*nnc.NodeNetworkConfig) mockKey := MockKey{ @@ -77,11 +77,11 @@ func (mc MockClient) Update(ctx context.Context, obj runtime.Object, opts ...cli return nil } -// MockCNSInteractor implements CNSClient interface -type MockCNSInteractor struct{} +// MockCNSClient implements API client interface +type MockCNSClient struct{} // we're just testing that reconciler interacts with CNS on Reconcile(). -func (mi MockCNSInteractor) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { +func (mi *MockCNSClient) UpdateCNSState(createNetworkContainerRequest *cns.CreateNetworkContainerRequest) error { mockCNSUpdated = true return nil } @@ -306,28 +306,28 @@ func createMockStore() map[MockKey]*nnc.NodeNetworkConfig { return mockStore } -func createMockClient() MockClient { +func createMockKubeClient() MockKubeClient { mockStore := createMockStore() // Make mock client initialized with mock store - mockClient := MockClient{mockStore: mockStore} + MockKubeClient := MockKubeClient{mockStore: mockStore} - return mockClient + return MockKubeClient } -func createMockInteractor() MockCNSInteractor { - return MockCNSInteractor{} +func createMockCNSClient() *MockCNSClient { + return &MockCNSClient{} } func createMockRequestController() *crdRequestController { - mockClient := createMockClient() - mockInteractor := createMockInteractor() + MockKubeClient := createMockKubeClient() + MockCNSClient := createMockCNSClient() rc := &crdRequestController{} rc.nodeName = existingNNCName - rc.APIClient = mockClient + rc.KubeClient = MockKubeClient rc.Reconciler = &CrdReconciler{} - rc.Reconciler.APIClient = mockClient - rc.Reconciler.CNSClient = mockInteractor + rc.Reconciler.KubeClient = MockKubeClient + rc.Reconciler.CNSClient = MockCNSClient //Initialize logger logger.InitLogger("Azure CNS Request Controller", 0, 0, "")