Add MTPNC reconciler for cache population in Swift V2 (#2164)

add mtpnc watcher

Signed-off-by: Evan Baker <rbtr@users.noreply.github.com>

cns/kubecontroller/multitenantpodnetworkconfig/reconciler.go

@@ -0,0 +1,18 @@
+package multitenantpodnetworkconfig
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
+	"github.com/pkg/errors"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// SetupWithManager registers a noop MTPNC reconciler
+func SetupWithManager(mgr ctrl.Manager) error {
+	err := ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.MultitenantPodNetworkConfig{}).
+		Complete(reconcile.Func(func(context.Context, ctrl.Request) (ctrl.Result, error) { return ctrl.Result{}, nil }))
+	return errors.Wrap(err, "failed to set up mtpnc reconciler")
+}

cns/kubecontroller/podwatcher/podwatcher.go → cns/kubecontroller/pod/reconciler.go

@@ -1,4 +1,4 @@
-package podwatcher
+package pod
 
 import (
 	"context"
@@ -85,8 +85,5 @@ func (p *PodWatcher) SetupWithManager(mgr ctrl.Manager) error {
 			},
 		}).
 		Complete(p)
-	if err != nil {
-		return errors.Wrap(err, "failed to set up pod watcher with manager")
-	}
-	return nil
+	return errors.Wrap(err, "failed to set up pod watcher with manager")
 }

cns/service/main.go

@@ -31,8 +31,9 @@ import (
 	"github.com/Azure/azure-container-networking/cns/hnsclient"
 	"github.com/Azure/azure-container-networking/cns/ipampool"
 	cssctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/clustersubnetstate"
+	mtpncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/multitenantpodnetworkconfig"
 	nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig"
-	"github.com/Azure/azure-container-networking/cns/kubecontroller/podwatcher"
+	podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/multitenantcontroller"
 	"github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator"
@@ -41,7 +42,8 @@ import (
 	"github.com/Azure/azure-container-networking/cns/wireserver"
 	acn "github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/crd"
-	"github.com/Azure/azure-container-networking/crd/clustersubnetstate/api/v1alpha1"
+	cssv1alpha1 "github.com/Azure/azure-container-networking/crd/clustersubnetstate/api/v1alpha1"
+	mtv1alpha1 "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	acnfs "github.com/Azure/azure-container-networking/internal/fs"
@@ -1123,6 +1125,19 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		return errors.Wrap(err, "failed to get NodeName")
 	}
 
+	node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrapf(err, "failed to get node %s", nodeName)
+	}
+
+	// check the Node labels for Swift V2
+	if _, ok := node.Labels[configuration.LabelSwiftV2]; ok {
+		cnsconfig.EnableSwiftV2 = true
+		cnsconfig.WatchPods = true
+		// TODO(rbtr): create the NodeInfo for Swift V2
+		// register the noop mtpnc reconciler to populate the cache
+	}
+
 	var podInfoByIPProvider cns.PodInfoByIPProvider
 	switch {
 	case cnsconfig.ManageEndpointState:
@@ -1195,9 +1210,12 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	if err = v1alpha.AddToScheme(scheme); err != nil {
 		return errors.Wrap(err, "failed to add nodenetworkconfig/v1alpha to scheme")
 	}
-	if err = v1alpha1.AddToScheme(scheme); err != nil {
+	if err = cssv1alpha1.AddToScheme(scheme); err != nil {
 		return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme")
 	}
+	if err = mtv1alpha1.AddToScheme(scheme); err != nil {
+		return errors.Wrap(err, "failed to add multitenantpodnetworkconfig/v1alpha1 to scheme")
+	}
 
 	// Set Selector options on the Manager cache which are used
 	// to perform *server-side* filtering of the cached objects. This is very important
@@ -1211,12 +1229,15 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 					"kube-system": {FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName})},
 				},
 			},
-			&corev1.Pod{}: {
-				Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}),
-			},
 		},
 	}
 
+	if cnsconfig.WatchPods {
+		cacheOpts.ByObject[&corev1.Pod{}] = cache.ByObject{
+			Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}),
+		}
+	}
+
 	managerOpts := ctrlmgr.Options{
 		Scheme:  scheme,
 		Metrics: ctrlmetrics.Options{BindAddress: "0"},
@@ -1230,7 +1251,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	}
 
 	// Build the IPAM Pool monitor
-	clusterSubnetStateChan := make(chan v1alpha1.ClusterSubnetState)
+	clusterSubnetStateChan := make(chan cssv1alpha1.ClusterSubnetState)
 
 	// this cachedscopedclient is built using the Manager's cached client, which is
 	// NOT SAFE TO USE UNTIL THE MANAGER IS STARTED!
@@ -1248,19 +1269,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 
 	// Start building the NNC Reconciler
 
-	// get our Node so that we can xref it against the NodeNetworkConfig's to make sure that the
-	// NNC is not stale and represents the Node we're running on.
-	node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
-	if err != nil {
-		return errors.Wrapf(err, "failed to get node %s", nodeName)
-	}
-
-	// check the Node labels for Swift V2
-	if _, ok := node.Labels[configuration.LabelSwiftV2]; ok {
-		cnsconfig.EnableSwiftV2 = true
-		// TODO(rbtr): create the NodeInfo for Swift V2
-	}
-
 	// get CNS Node IP to compare NC Node IP with this Node IP to ensure NCs were created for this node
 	nodeIP := configuration.NodeIP()
 
@@ -1281,12 +1289,18 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 
 	// TODO: add pod listeners based on Swift V1 vs MT/V2 configuration
 	if cnsconfig.WatchPods {
-		pw := podwatcher.New(nodeName)
+		pw := podctrl.New(nodeName)
 		if err := pw.SetupWithManager(manager); err != nil {
 			return errors.Wrapf(err, "failed to setup pod watcher with manager")
 		}
 	}
 
+	if cnsconfig.EnableSwiftV2 {
+		if err := mtpncctrl.SetupWithManager(manager); err != nil {
+			return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager")
+		}
+	}
+
 	// adding some routes to the root service mux
 	mux := httpRestServiceImplementation.Listener.GetMux()
 	mux.Handle("/readyz", http.StripPrefix("/readyz", &healthz.Handler{}))

go.mod

@@ -14,8 +14,11 @@ require (
 	github.com/billgraziano/dpapi v0.4.0
 	github.com/containernetworking/cni v1.1.2
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20210525090646-64b7a4574d14
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.3
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.1
 	github.com/gorilla/mux v1.8.0
@@ -35,6 +38,7 @@ require (
 	go.uber.org/zap v1.25.0
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/sys v0.11.0
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 	google.golang.org/grpc v1.54.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -46,15 +50,6 @@ require (
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/controller-runtime v0.16.0
-	sigs.k8s.io/yaml v1.3.0
-)
-
-require (
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/go-logr/zapr v1.2.4 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 )
 
 require (
@@ -138,6 +133,10 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 )
 
+require sigs.k8s.io/yaml v1.3.0
+
+require github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+
 replace (
 	github.com/Microsoft/go-winio => github.com/microsoft/go-winio v0.4.17
 	github.com/Microsoft/hcsshim => github.com/vakalapa/hcsshim v0.9.1-0.20211203205307-837d4d06df77

go.sum

@@ -280,8 +280,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
-github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
+github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=