From ae54c5f54d53987f56c5423e70dfd1a53233dbec Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Mon, 10 Jan 2022 23:00:58 +0000 Subject: [PATCH] match on Node controller ref in NNC event predicate Signed-off-by: Evan Baker --- cns/azure-cns.yaml | 9 ++++--- cns/service/main.go | 26 +++++++++++++++---- cns/singletenantcontroller/reconciler.go | 8 +++--- crd/nodenetworkconfig/client.go | 4 +-- .../manifests/cns/clusterrole.yaml | 8 ++++-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index ff2064b54e..9bb98b713e 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -10,9 +10,9 @@ metadata: namespace: kube-system name: nodeNetConfigEditor rules: - - apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] +- apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -22,6 +22,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/cns/service/main.go b/cns/service/main.go index 61e1490e51..521580b708 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -46,8 +46,10 @@ import ( "github.com/Azure/azure-container-networking/store" "github.com/avast/retry-go/v3" "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -863,13 +865,12 @@ func initCNS(ctx context.Context, cli nodeNetworkConfigGetter, ncReconciler ncSt } podInfoByIP, err := podInfoByIPProvider.PodInfoByIP() if err != nil { - return errors.Wrap(err, "err in CNS initialization") + return errors.Wrap(err, "provider failed to provide PodInfoByIP") } - // errors.Wrap provides additional context, and return nil if the err input arg is nil // Call cnsclient init cns passing those two things. err = restserver.ResponseCodeToError(ncReconciler.ReconcileNCState(&ncRequest, podInfoByIP, nnc)) - return errors.Wrap(err, "err in CNS reconciliation") + return errors.Wrap(err, "failed to reconcile NC state") } // InitializeCRDState builds and starts the CRD controllers. @@ -945,6 +946,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn }, }, }) + manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{ Scheme: nodenetworkconfig.Scheme, MetricsBindAddress: cnsconfig.MetricsBindAddress, @@ -954,9 +956,23 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { return errors.Wrap(err, "failed to create manager") } + + clientset, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return errors.Wrap(err, "failed to build clientset") + } + + // get our Node so that we can xref it against the NodeNetworkConfig's to make sure that the + // NNC is not stale and represents the Node we're running on. + node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get node %s", nodeName) + } + reconciler := kubecontroller.NewReconciler(nnccli, httpRestServiceImplementation, httpRestServiceImplementation.IPAMPoolMonitor) - if err := reconciler.SetupWithManager(manager, nodeName); err != nil { - return err + // pass Node to the Reconciler for Controller xref + if err := reconciler.SetupWithManager(manager, node); err != nil { + return errors.Wrapf(err, "failed to setup reconciler with manager") } // Start the RequestController which starts the reconcile loop diff --git a/cns/singletenantcontroller/reconciler.go b/cns/singletenantcontroller/reconciler.go index 0932e4924c..fd1f82a603 100644 --- a/cns/singletenantcontroller/reconciler.go +++ b/cns/singletenantcontroller/reconciler.go @@ -9,7 +9,9 @@ import ( cnstypes "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -89,7 +91,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco } // SetupWithManager Sets up the reconciler with a new manager, filtering using NodeNetworkConfigFilter on nodeName. -func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, nodeName string) error { +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, node *v1.Node) error { err := ctrl.NewControllerManagedBy(mgr). For(&v1alpha.NodeNetworkConfig{}). WithEventFilter(predicate.Funcs{ @@ -99,8 +101,8 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, nodeName string) error { }, }). WithEventFilter(predicate.NewPredicateFuncs(func(object client.Object) bool { - // match on node name for all other events. - return nodeName == object.GetName() + // match on node controller ref for all other events. + return metav1.IsControlledBy(object, node) })). WithEventFilter(predicate.Funcs{ // check that the generation is the same - status changes don't update generation.a diff --git a/crd/nodenetworkconfig/client.go b/crd/nodenetworkconfig/client.go index 031919ee26..ce945809dc 100644 --- a/crd/nodenetworkconfig/client.go +++ b/crd/nodenetworkconfig/client.go @@ -42,13 +42,13 @@ func NewClient(c *rest.Config) (*Client, error) { opts := ctrlcli.Options{ Scheme: Scheme, } - nnnCli, err := ctrlcli.New(c, opts) + nncCli, err := ctrlcli.New(c, opts) if err != nil { return nil, errors.Wrap(err, "failed to init nnc client") } return &Client{ crdcli: crdCli, - nnccli: nnnCli, + nnccli: nncCli, }, nil } diff --git a/test/integration/manifests/cns/clusterrole.yaml b/test/integration/manifests/cns/clusterrole.yaml index a2f397c4d0..43707aaf3a 100644 --- a/test/integration/manifests/cns/clusterrole.yaml +++ b/test/integration/manifests/cns/clusterrole.yaml @@ -1,9 +1,13 @@ +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: pod-reader-all-namespaces - namespace: kube-system + namespace: kube-system rules: - apiGroups: [""] resources: ["pods"] - verbs: ["get", "watch", "list"] \ No newline at end of file + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"]