fix: Service Account Mitigation for CNS on k8s Windows 2022

cni/network/network_windows.go

@@ -37,8 +37,8 @@ var (
  * Issue link: https://github.com/kubernetes/kubernetes/issues/57253
  */
 func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string,
-	nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
-
+	nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig,
+) (*cniTypesCurr.Result, error) {
 	epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId)
 	if epInfo == nil {
 		return nil, nil
@@ -141,13 +141,13 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) {
 }
 
 func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResult, nwCfg *cni.NetworkConfig) (string, error) {
+	determineWinVer()
 	// For singletenancy, the network name is simply the nwCfg.Name
 	if !nwCfg.MultiTenancy {
 		return nwCfg.Name, nil
 	}
 
 	// in multitenancy case, the network name will be in the state file or can be built from cnsResponse
-	determineWinVer()
 	if len(strings.TrimSpace(netNs)) == 0 {
 		return "", fmt.Errorf("NetNs cannot be empty")
 	}

cns/azure-cns-windows.yaml

@@ -1,3 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: azure-cns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -13,30 +21,31 @@ spec:
     metadata:
       labels:
         k8s-app: azure-cns-win
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+        prometheus.io/port: "10092"
     spec:
       securityContext:
         windowsOptions:
           hostProcess: true
           runAsUserName: "NT AUTHORITY\\SYSTEM"
-      hostNetwork: true
       containers:
         - name: cns-container
-          image: acnpublic.azurecr.io/azure-cns-win:v1.4.15-1-g1b238a3f9
+          image: acnpublic.azurecr.io/azure-cns:v1.4.26-9-gc40fb852
           imagePullPolicy: IfNotPresent
-          command:
-            ["%CONTAINER_SANDBOX_MOUNT_POINT%/usr/local/bin/azure-cns.exe"]
-          args:
-            [
-              "-cns-url",
-              "tcp://$(CNSIpAddress):$(CNSPort)",
-              "-log-target",
-              "$(CNSLogTarget)",
-              "-log-location",
-              "$(CNSLogDirectory)",
-              "-store-file-path",
-              "$(CNSStoreFilePath)",
-              "-config-path",
-              "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)",
+          securityContext:
+            privileged: true
+          command: ["powershell.exe"]
+          args: 
+            [ 
+              '.\setkubeconfigpath.ps1', ";",
+              'powershell.exe', '.\azure-cns.exe',
+              '-c', "tcp://$(CNSIpAddress):$(CNSPort)",
+              '-t', "$(CNSLogTarget)",
+              '-o', "$(CNSLogDir)",
+              '-storefilepath', "$(CNSStoreFilePath)",
+              '-config-path', "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)",
+              '--kubeconfig', '.\kubeconfig',
             ]
           volumeMounts:
             - name: log
@@ -45,26 +54,28 @@ spec:
               mountPath: etc/azure-cns
           ports:
             - containerPort: 10090
+              name: api
+            - containerPort: 10092
+              name: metrics
           env:
             - name: CNSIpAddress
               value: "127.0.0.1"
             - name: CNSPort
               value: "10090"
             - name: CNSLogTarget
               value: "stdoutfile"
-            - name: CNSLogDirectory
-              value: "C:\\k"
+            - name: CNSLogDir
+              value: /k
             - name: CNSStoreFilePath
-              value: "C:\\k\\"
+              value: /k/
             - name: CNS_CONFIGURATION_PATH
-              value: "etc\\azure-cns\\cns_config.json"
+              value: etc/azure-cns/cns_config.json
             - name: NODENAME
               valueFrom:
                 fieldRef:
                   apiVersion: v1
-                  fieldPath: spec.nodeName
-            - name: KUBECONFIG
-              value: "C:\\k\\config"
+                  fieldPath: spec.nodeName 
+      hostNetwork: true
       volumes:
         - name: log
           hostPath:
@@ -75,6 +86,7 @@ spec:
             name: cns-config
       nodeSelector:
         kubernetes.io/os: windows
+      serviceAccountName: azure-cns
 ---
 apiVersion: v1
 kind: ConfigMap

cns/kubeconfigtemplate.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: <ca>
+    <server>
+contexts:
+- name: azure-cns-win@kubernetes
+  context:
+    cluster: kubernetes
+    namespace: kube-system
+    user: azure-cns-win
+current-context: azure-cns-win@kubernetes
+users:
+- name: azure-cns-win
+  user:
+    token: <token>

cns/service/main.go

@@ -910,11 +910,12 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 
 	// build default clientset.
 	kubeConfig, err := ctrl.GetConfig()
-	kubeConfig.UserAgent = fmt.Sprintf("azure-cns-%s", version)
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to get kubeconfig for request controller: %v", err)
-		return err
+		return errors.Wrap(err, "failed to get kubeconfig")
 	}
+	kubeConfig.UserAgent = fmt.Sprintf("azure-cns-%s", version)
+
 	clientset, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to build clientset")

cns/singletenantcontroller/reconciler.go

@@ -88,7 +88,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
 			req, err = CreateNCRequestFromDynamicNC(nnc.Status.NetworkContainers[i])
 			// in dynamic, we will also push this NNC to the IPAM Pool Monitor when we're done.
 			listenersToNotify = append(listenersToNotify, r.ipampoolmonitorcli)
-
 		}
 
 		if err != nil {

cns/windows.Dockerfile

@@ -5,18 +5,20 @@ ARG VERSION
 ARG CNS_AI_PATH
 ARG CNS_AI_ID
 
-WORKDIR /usr/local/src/cns
+WORKDIR /usr/src/cns
+RUN mkdir /usr/bin/
 
 # Copy the source
 COPY . .
 
 # Build cns
-RUN $Env:CGO_ENABLED=0; go build -v -o /usr/local/bin/azure-cns.exe -ldflags """-X main.version=${env:VERSION} -X ${env:CNS_AI_PATH}=${env:CNS_AI_ID}""" -gcflags="-dwarflocationlists=true" ./cns/service
+RUN $Env:CGO_ENABLED=0; go build -v -o /usr/bin/azure-cns.exe -ldflags """-X main.version=${env:VERSION} -X ${env:CNS_AI_PATH}=${env:CNS_AI_ID}""" -gcflags="-dwarflocationlists=true" ./cns/service
 
 # Copy into final image
 FROM mcr.microsoft.com/windows/servercore:ltsc2022
-COPY --from=builder /usr/local/bin/azure-cns.exe \
-    /usr/local/bin/azure-cns.exe
+COPY --from=builder /usr/src/cns/cns/kubeconfigtemplate.yaml kubeconfigtemplate.yaml
+COPY --from=builder /usr/src/cns/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1
+COPY --from=builder /usr/bin/azure-cns.exe azure-cns.exe
 
-ENTRYPOINT ["/usr/local/bin/azure-cns.exe"]
+ENTRYPOINT ["azure-cns.exe"]
 EXPOSE 10090