feat: add image and manifest for windows npm

.pipelines/singletenancy/aks-engine/e2e-step-template.yaml

@@ -102,8 +102,6 @@ steps:
         make test-kubernetes
     name: DeployAKSEngine
     displayName: Run AKS-Engine E2E Tests
-
-
 
   - task: CopyFiles@2
     inputs:
@@ -116,4 +114,3 @@ steps:
       artifactName: ${{ parameters.name }}
       pathtoPublish: "$(Build.ArtifactStagingDirectory)/${{ parameters.name }}"
     condition: always()
-

Makefile

@@ -237,6 +237,23 @@ ifeq ($(GOOS),linux)
 	echo $(AZURE_NPM_IMAGE):$(VERSION) > $(IMAGE_DIR)/$(NPM_IMAGE_INFO_FILE)
 endif
 
+
+# Build the Azure NPM image, because the buildx command breaks other runtimes
+.PHONY: azure-npm-image-classic
+azure-npm-image-classic: azure-npm
+ifeq ($(GOOS),linux)
+	mkdir -p $(IMAGE_DIR)
+	docker build \
+	--no-cache \
+	-f npm/Dockerfile \
+	-t $(AZURE_NPM_IMAGE):$(VERSION) \
+	--build-arg VERSION=$(VERSION) \
+	--build-arg NPM_AI_PATH=$(NPM_AI_PATH) \
+	--build-arg NPM_AI_ID=$(NPM_AI_ID) \
+	--build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \
+	.
+endif
+
 # Build the Azure CNS image
 .PHONY: azure-cns-image
 azure-cns-image:

npm/Dockerfile.windows

@@ -13,7 +13,8 @@ RUN $Env:CGO_ENABLED=0; go build -v -o /usr/bin/npm.exe -ldflags """-X main.vers
 
 # Copy into final image
 FROM mcr.microsoft.com/windows/servercore:ltsc2022
-COPY --from=builder /usr/bin/npm.exe \
-    /usr/bin/npm.exe
+COPY --from=builder /usr/src/npm/npm/examples/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml
+COPY --from=builder /usr/src/npm/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1
+COPY --from=builder /usr/bin/npm.exe npm.exe
 
-ENTRYPOINT ["/usr/bin/npm.exe", "start"]
+CMD ["npm.exe", "start" "--kubeconfig=.\\kubeconfig"]

npm/azure-npm.yaml

@@ -34,7 +34,7 @@ rules:
       - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: ClusterRoleBinding  
 metadata:
   name: azure-npm-binding
   namespace: kube-system

npm/cmd/main.go

@@ -4,12 +4,54 @@ package main
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 )
 
+const (
+	flagVersion        = "version"
+	flagKubeConfigPath = "kubeconfig"
+)
+
+var flagDefaults = map[string]string{
+	flagKubeConfigPath: "",
+}
+
 // Version is populated by make during build.
 var version string
 
 func main() {
 	rootCmd := NewRootCmd()
+
+	if version != "" {
+		viper.Set(flagVersion, version)
+	}
+
+	cobra.OnInitialize(func() {
+		viper.AutomaticEnv()
+		initCommandFlags(rootCmd.Commands())
+	})
+
 	cobra.CheckErr(rootCmd.Execute())
 }
+
+func initCommandFlags(commands []*cobra.Command) {
+	for _, cmd := range commands {
+		// bind vars from env or conf to pflags
+		err := viper.BindPFlags(cmd.Flags())
+		cobra.CheckErr(err)
+
+		c := cmd
+		c.Flags().VisitAll(func(flag *pflag.Flag) {
+			if viper.IsSet(flag.Name) && viper.GetString(flag.Name) != "" {
+				err := c.Flags().Set(flag.Name, viper.GetString(flag.Name))
+				cobra.CheckErr(err)
+			}
+		})
+
+		// call recursively on subcommands
+		if cmd.HasSubCommands() {
+			initCommandFlags(cmd.Commands())
+		}
+	}
+}

npm/cmd/start.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
 	"k8s.io/utils/exec"
 )
@@ -40,8 +41,9 @@ func newStartNPMCmd() *cobra.Command {
 			viper.SetConfigFile(cfgFile)
 
 			// If a config file is found, read it in.
+			// NOTE: there is no config merging with default, if config is loaded, options must be set
 			if err := viper.ReadInConfig(); err == nil {
-				klog.Info("Using config file: ", viper.ConfigFileUsed())
+				klog.Infof("Using config file: %+v", viper.ConfigFileUsed())
 			} else {
 				klog.Infof("Failed to load config from env %s: %v", npmconfig.ConfigEnvPath, err)
 				b, _ := json.Marshal(npmconfig.DefaultConfig)
@@ -58,36 +60,50 @@ func newStartNPMCmd() *cobra.Command {
 			config := &npmconfig.Config{}
 			err := viper.Unmarshal(config)
 			if err != nil {
-				return fmt.Errorf("failed to load config with error %w", err)
+				return fmt.Errorf("failed to load config with error: %w", err)
 			}
 
-			return start(*config)
+			flags := npmconfig.Flags{
+				KubeConfigPath: viper.GetString(flagKubeConfigPath),
+			}
+
+			return start(*config, flags)
 		},
 	}
+
+	startNPMCmd.Flags().String(flagKubeConfigPath, flagDefaults[flagKubeConfigPath], "path to kubeconfig")
+
 	return startNPMCmd
 }
 
-func start(config npmconfig.Config) error {
+func start(config npmconfig.Config, flags npmconfig.Flags) error {
 	klog.Infof("loaded config: %+v", config)
 	klog.Infof("Start NPM version: %s", version)
 
 	var err error
-	defer func() {
-		if r := recover(); r != nil {
-			klog.Infof("recovered from error: %v", err)
-		}
-	}()
 
-	if err = initLogging(); err != nil {
+	err = initLogging()
+	if err != nil {
 		return err
 	}
 
+	klog.Infof("initializing metrics")
 	metrics.InitializeAll()
 
-	// Creates the in-cluster config
-	k8sConfig, err := rest.InClusterConfig()
-	if err != nil {
-		return fmt.Errorf("failed to load in cluster config: %w", err)
+	// Create the kubernetes client
+	var k8sConfig *rest.Config
+	if flags.KubeConfigPath == "" {
+		klog.Infof("loading in cluster kubeconfig")
+		k8sConfig, err = rest.InClusterConfig()
+		if err != nil {
+			return fmt.Errorf("failed to load in cluster config: %w", err)
+		}
+	} else {
+		klog.Infof("loading kubeconfig from flag: %s", flags.KubeConfigPath)
+		k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath)
+		if err != nil {
+			return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err)
+		}
 	}
 
 	// Creates the clientset
@@ -101,7 +117,7 @@ func start(config npmconfig.Config) error {
 	minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute
 
 	// Adding some randomness so all NPM pods will not request for info at once.
-	factor := rand.Float64() + 1
+	factor := rand.Float64() + 1 //nolint
 	resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor)
 	klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute))
 	factory := informers.NewSharedInformerFactory(clientset, resyncPeriod)
@@ -125,8 +141,8 @@ func start(config npmconfig.Config) error {
 	go restserver.NPMRestServerListenAndServe(config, npMgr)
 
 	if err = npMgr.Start(config, wait.NeverStop); err != nil {
-		metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %s", err)
-		panic(err.Error)
+		metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %+v", err)
+		return fmt.Errorf("failed to start with err: %w", err)
 	}
 
 	select {}

npm/config/config.go

@@ -36,3 +36,7 @@ type Toggles struct {
 	EnableV2NPM             bool
 	PlaceAzureChainFirst    bool
 }
+
+type Flags struct {
+	KubeConfigPath string `json:"KubeConfigPath"`
+}

npm/examples/windows/azure-npm.yaml

@@ -0,0 +1,147 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: azure-npm
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: azure-npm
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - pods
+      - nodes
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+    - networking.k8s.io
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: azure-npm-binding
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+subjects:
+  - kind: ServiceAccount
+    name: azure-npm
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azure-npm
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: azure-npm
+  namespace: kube-system
+  labels:
+    app: azure-npm
+    addonmanager.kubernetes.io/mode: EnsureExists
+spec:
+  selector:
+    matchLabels:
+      k8s-app: azure-npm
+  template:
+    metadata:
+      labels:
+        k8s-app: azure-npm
+      annotations:  
+        azure.npm/scrapeable: ''
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: "Exists"
+        effect: NoExecute
+      - operator: "Exists"
+        effect: NoSchedule
+      - key: CriticalAddonsOnly
+        operator: Exists
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+      hostNetwork: true
+      containers:
+        - name: azure-npm
+          image: acnpublic.azurecr.io/azure-npm:v26-windows-amd64
+          command: ["powershell.exe"]
+          args: ['.\setkubeconfigpath.ps1', ';', 'powershell.exe', '.\npm.exe', "start", '--kubeconfig=.\kubeconfig']
+          resources:
+            limits:
+              cpu: 250m
+              memory: 300Mi
+            requests:
+              cpu: 250m
+          env:
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: NPM_CONFIG
+              value: .\\etc\\azure-npm\\azure-npm.json
+          volumeMounts:
+          - name: azure-npm-config
+            mountPath: .\\etc\\azure-npm
+      nodeSelector:
+        kubernetes.io/os: windows
+      volumes:
+      - name: azure-npm-config
+        configMap:
+          name: azure-npm-config
+      serviceAccountName: azure-npm
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: npm-metrics-cluster-service
+  namespace: kube-system
+  labels:
+    app: npm-metrics
+spec:
+  selector:
+    k8s-app: azure-npm
+  ports:
+    - port: 9000
+      targetPort: 10091
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: azure-npm-config
+  namespace: kube-system
+data:
+  azure-npm.json: |
+    {
+        "ResyncPeriodInMinutes": 15,
+        "ListeningPort":         10091,
+        "ListeningAddress":      "0.0.0.0",
+        "Toggles": {
+            "EnablePrometheusMetrics": true,
+            "EnablePprof":             true,
+            "EnableHTTPDebugAPI":      true,
+            "EnableV2Controllers":     true     
+        }
+    }
+
+

npm/examples/windows/kubeconfigtemplate.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: <ca>
+    <server>
+contexts:
+- name: azure-npm-windows@kubernetes
+  context:
+    cluster: kubernetes
+    namespace: kube-system
+    user: azure-npm-windows
+current-context: azure-npm-windows@kubernetes
+users:
+- name: azure-npm-windows
+  user:
+    token: <token>