diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml index 9209fed795..481f4f2418 100644 --- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml @@ -102,8 +102,6 @@ steps: make test-kubernetes name: DeployAKSEngine displayName: Run AKS-Engine E2E Tests - - - task: CopyFiles@2 inputs: @@ -116,4 +114,3 @@ steps: artifactName: ${{ parameters.name }} pathtoPublish: "$(Build.ArtifactStagingDirectory)/${{ parameters.name }}" condition: always() - \ No newline at end of file diff --git a/Makefile b/Makefile index 953ab7c1b6..3e8bf9c0cf 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,23 @@ ifeq ($(GOOS),linux) echo $(AZURE_NPM_IMAGE):$(VERSION) > $(IMAGE_DIR)/$(NPM_IMAGE_INFO_FILE) endif + +# Build the Azure NPM image, because the buildx command breaks other runtimes +.PHONY: azure-npm-image-classic +azure-npm-image-classic: azure-npm +ifeq ($(GOOS),linux) + mkdir -p $(IMAGE_DIR) + docker build \ + --no-cache \ + -f npm/Dockerfile \ + -t $(AZURE_NPM_IMAGE):$(VERSION) \ + --build-arg VERSION=$(VERSION) \ + --build-arg NPM_AI_PATH=$(NPM_AI_PATH) \ + --build-arg NPM_AI_ID=$(NPM_AI_ID) \ + --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ + . +endif + # Build the Azure CNS image .PHONY: azure-cns-image azure-cns-image: diff --git a/npm/Dockerfile.windows b/npm/Dockerfile.windows index e20696a9a7..0ff95fe08a 100644 --- a/npm/Dockerfile.windows +++ b/npm/Dockerfile.windows @@ -13,7 +13,8 @@ RUN $Env:CGO_ENABLED=0; go build -v -o /usr/bin/npm.exe -ldflags """-X main.vers # Copy into final image FROM mcr.microsoft.com/windows/servercore:ltsc2022 -COPY --from=builder /usr/bin/npm.exe \ - /usr/bin/npm.exe +COPY --from=builder /usr/src/npm/npm/examples/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml +COPY --from=builder /usr/src/npm/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1 +COPY --from=builder /usr/bin/npm.exe npm.exe -ENTRYPOINT ["/usr/bin/npm.exe", "start"] +CMD ["npm.exe", "start" "--kubeconfig=.\\kubeconfig"] diff --git a/npm/azure-npm.yaml b/npm/azure-npm.yaml index 289eba7aaf..dfdcac0c8c 100644 --- a/npm/azure-npm.yaml +++ b/npm/azure-npm.yaml @@ -34,7 +34,7 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRoleBinding metadata: name: azure-npm-binding namespace: kube-system diff --git a/npm/cmd/main.go b/npm/cmd/main.go index df198512bf..b855402ac6 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -4,12 +4,54 @@ package main import ( "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" ) +const ( + flagVersion = "version" + flagKubeConfigPath = "kubeconfig" +) + +var flagDefaults = map[string]string{ + flagKubeConfigPath: "", +} + // Version is populated by make during build. var version string func main() { rootCmd := NewRootCmd() + + if version != "" { + viper.Set(flagVersion, version) + } + + cobra.OnInitialize(func() { + viper.AutomaticEnv() + initCommandFlags(rootCmd.Commands()) + }) + cobra.CheckErr(rootCmd.Execute()) } + +func initCommandFlags(commands []*cobra.Command) { + for _, cmd := range commands { + // bind vars from env or conf to pflags + err := viper.BindPFlags(cmd.Flags()) + cobra.CheckErr(err) + + c := cmd + c.Flags().VisitAll(func(flag *pflag.Flag) { + if viper.IsSet(flag.Name) && viper.GetString(flag.Name) != "" { + err := c.Flags().Set(flag.Name, viper.GetString(flag.Name)) + cobra.CheckErr(err) + } + }) + + // call recursively on subcommands + if cmd.HasSubCommands() { + initCommandFlags(cmd.Commands()) + } + } +} diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 9a13d52ffc..2917eaf16e 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" "k8s.io/utils/exec" ) @@ -40,8 +41,9 @@ func newStartNPMCmd() *cobra.Command { viper.SetConfigFile(cfgFile) // If a config file is found, read it in. + // NOTE: there is no config merging with default, if config is loaded, options must be set if err := viper.ReadInConfig(); err == nil { - klog.Info("Using config file: ", viper.ConfigFileUsed()) + klog.Infof("Using config file: %+v", viper.ConfigFileUsed()) } else { klog.Infof("Failed to load config from env %s: %v", npmconfig.ConfigEnvPath, err) b, _ := json.Marshal(npmconfig.DefaultConfig) @@ -58,36 +60,50 @@ func newStartNPMCmd() *cobra.Command { config := &npmconfig.Config{} err := viper.Unmarshal(config) if err != nil { - return fmt.Errorf("failed to load config with error %w", err) + return fmt.Errorf("failed to load config with error: %w", err) } - return start(*config) + flags := npmconfig.Flags{ + KubeConfigPath: viper.GetString(flagKubeConfigPath), + } + + return start(*config, flags) }, } + + startNPMCmd.Flags().String(flagKubeConfigPath, flagDefaults[flagKubeConfigPath], "path to kubeconfig") + return startNPMCmd } -func start(config npmconfig.Config) error { +func start(config npmconfig.Config, flags npmconfig.Flags) error { klog.Infof("loaded config: %+v", config) klog.Infof("Start NPM version: %s", version) var err error - defer func() { - if r := recover(); r != nil { - klog.Infof("recovered from error: %v", err) - } - }() - if err = initLogging(); err != nil { + err = initLogging() + if err != nil { return err } + klog.Infof("initializing metrics") metrics.InitializeAll() - // Creates the in-cluster config - k8sConfig, err := rest.InClusterConfig() - if err != nil { - return fmt.Errorf("failed to load in cluster config: %w", err) + // Create the kubernetes client + var k8sConfig *rest.Config + if flags.KubeConfigPath == "" { + klog.Infof("loading in cluster kubeconfig") + k8sConfig, err = rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to load in cluster config: %w", err) + } + } else { + klog.Infof("loading kubeconfig from flag: %s", flags.KubeConfigPath) + k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath) + if err != nil { + return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err) + } } // Creates the clientset @@ -101,7 +117,7 @@ func start(config npmconfig.Config) error { minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute // Adding some randomness so all NPM pods will not request for info at once. - factor := rand.Float64() + 1 + factor := rand.Float64() + 1 //nolint resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) @@ -125,8 +141,8 @@ func start(config npmconfig.Config) error { go restserver.NPMRestServerListenAndServe(config, npMgr) if err = npMgr.Start(config, wait.NeverStop); err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %s", err) - panic(err.Error) + metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %+v", err) + return fmt.Errorf("failed to start with err: %w", err) } select {} diff --git a/npm/config/config.go b/npm/config/config.go index 32d1b6da09..d343f93da3 100644 --- a/npm/config/config.go +++ b/npm/config/config.go @@ -36,3 +36,7 @@ type Toggles struct { EnableV2NPM bool PlaceAzureChainFirst bool } + +type Flags struct { + KubeConfigPath string `json:"KubeConfigPath"` +} diff --git a/npm/examples/windows/azure-npm.yaml b/npm/examples/windows/azure-npm.yaml new file mode 100644 index 0000000000..68bbf75bc5 --- /dev/null +++ b/npm/examples/windows/azure-npm.yaml @@ -0,0 +1,147 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: azure-npm-binding + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +subjects: + - kind: ServiceAccount + name: azure-npm + namespace: kube-system +roleRef: + kind: ClusterRole + name: azure-npm + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-npm + namespace: kube-system + labels: + app: azure-npm + addonmanager.kubernetes.io/mode: EnsureExists +spec: + selector: + matchLabels: + k8s-app: azure-npm + template: + metadata: + labels: + k8s-app: azure-npm + annotations: + azure.npm/scrapeable: '' + spec: + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: azure-npm + image: acnpublic.azurecr.io/azure-npm:v26-windows-amd64 + command: ["powershell.exe"] + args: ['.\setkubeconfigpath.ps1', ';', 'powershell.exe', '.\npm.exe', "start", '--kubeconfig=.\kubeconfig'] + resources: + limits: + cpu: 250m + memory: 300Mi + requests: + cpu: 250m + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NPM_CONFIG + value: .\\etc\\azure-npm\\azure-npm.json + volumeMounts: + - name: azure-npm-config + mountPath: .\\etc\\azure-npm + nodeSelector: + kubernetes.io/os: windows + volumes: + - name: azure-npm-config + configMap: + name: azure-npm-config + serviceAccountName: azure-npm +--- +apiVersion: v1 +kind: Service +metadata: + name: npm-metrics-cluster-service + namespace: kube-system + labels: + app: npm-metrics +spec: + selector: + k8s-app: azure-npm + ports: + - port: 9000 + targetPort: 10091 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-npm-config + namespace: kube-system +data: + azure-npm.json: | + { + "ResyncPeriodInMinutes": 15, + "ListeningPort": 10091, + "ListeningAddress": "0.0.0.0", + "Toggles": { + "EnablePrometheusMetrics": true, + "EnablePprof": true, + "EnableHTTPDebugAPI": true, + "EnableV2Controllers": true + } + } + + diff --git a/npm/examples/windows/kubeconfigtemplate.yaml b/npm/examples/windows/kubeconfigtemplate.yaml new file mode 100644 index 0000000000..1f1da274c5 --- /dev/null +++ b/npm/examples/windows/kubeconfigtemplate.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Config +clusters: +- name: kubernetes + cluster: + certificate-authority-data: + +contexts: +- name: azure-npm-windows@kubernetes + context: + cluster: kubernetes + namespace: kube-system + user: azure-npm-windows +current-context: azure-npm-windows@kubernetes +users: +- name: azure-npm-windows + user: + token: diff --git a/npm/examples/windows/setkubeconfigpath.ps1 b/npm/examples/windows/setkubeconfigpath.ps1 new file mode 100644 index 0000000000..38ec3440b2 --- /dev/null +++ b/npm/examples/windows/setkubeconfigpath.ps1 @@ -0,0 +1,11 @@ +# pull the server value from the kubeconfig on host to construct our own kubeconfig, but using service principal settings +# this is required to build a kubeconfig using the kubeconfig on disk in c:\k, and the service principle granted in the container mount, to generate clientset +$cpEndpoint = Get-Content C:\k\config | ForEach-Object -Process {if($_.Contains("server:")) {$_.Trim().Split()[1]}} +$token = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\token +$ca = Get-Content -Raw -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\ca.crt +$caBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($ca)) +$server = "server: $cpEndpoint" +(Get-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfigtemplate.yaml). + replace("", $caBase64). + replace("", $server.Trim()). + replace("", $token) | Set-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfig -Force diff --git a/npm/npm.go b/npm/npm.go index eb0dd3da93..70315c7777 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -83,7 +83,7 @@ func NewNetworkPolicyManager(config npmconfig.Config, exec utilexec.Interface, npmVersion string, k8sServerVersion *version.Info) *NetworkPolicyManager { - klog.Infof("API server version: %+v ai meta data %+v", k8sServerVersion, aiMetadata) + klog.Infof("API server version: %+v AI metadata %+v", k8sServerVersion, aiMetadata) npMgr := &NetworkPolicyManager{ config: config, diff --git a/npm/testpolicies/testing/only-ports.yaml b/npm/testpolicies/testing/only-ports.yaml index 31826ce0ea..234cc22fe4 100644 --- a/npm/testpolicies/testing/only-ports.yaml +++ b/npm/testpolicies/testing/only-ports.yaml @@ -15,4 +15,4 @@ spec: matchLabels: app: server policyTypes: - - Ingress \ No newline at end of file + - Ingress diff --git a/scripts/New-ContainerHostVm.ps1 b/scripts/New-ContainerHostVm.ps1 index 5e4531fc49..93c92cea77 100644 --- a/scripts/New-ContainerHostVm.ps1 +++ b/scripts/New-ContainerHostVm.ps1 @@ -1,4 +1,4 @@ -<# +<# .SYNOPSIS Creates an Azure VM with given number of network interfaces and IP addresses. diff --git a/vendor/github.com/hashicorp/hcl/.gitignore b/vendor/github.com/hashicorp/hcl/.gitignore index 15586a2b54..822fa09f52 100644 --- a/vendor/github.com/hashicorp/hcl/.gitignore +++ b/vendor/github.com/hashicorp/hcl/.gitignore @@ -1,9 +1,9 @@ -y.output - -# ignore intellij files -.idea -*.iml -*.ipr -*.iws - -*.test +y.output + +# ignore intellij files +.idea +*.iml +*.ipr +*.iws + +*.test diff --git a/vendor/github.com/hashicorp/hcl/Makefile b/vendor/github.com/hashicorp/hcl/Makefile index 84fd743f5c..9fafd5017c 100644 --- a/vendor/github.com/hashicorp/hcl/Makefile +++ b/vendor/github.com/hashicorp/hcl/Makefile @@ -1,18 +1,18 @@ -TEST?=./... - -default: test - -fmt: generate - go fmt ./... - -test: generate - go get -t ./... - go test $(TEST) $(TESTARGS) - -generate: - go generate ./... - -updatedeps: - go get -u golang.org/x/tools/cmd/stringer - -.PHONY: default generate test updatedeps +TEST?=./... + +default: test + +fmt: generate + go fmt ./... + +test: generate + go get -t ./... + go test $(TEST) $(TESTARGS) + +generate: + go generate ./... + +updatedeps: + go get -u golang.org/x/tools/cmd/stringer + +.PHONY: default generate test updatedeps diff --git a/vendor/github.com/pelletier/go-toml/example-crlf.toml b/vendor/github.com/pelletier/go-toml/example-crlf.toml index 780d9c68f2..f45bf88b8f 100644 --- a/vendor/github.com/pelletier/go-toml/example-crlf.toml +++ b/vendor/github.com/pelletier/go-toml/example-crlf.toml @@ -1,30 +1,30 @@ -# This is a TOML document. Boom. - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it score = 4e-08 # to make sure leading zeroes in exponent parts of floats are supported \ No newline at end of file diff --git a/windows.ps1 b/windows.ps1 index adb0be6b8c..39c051a902 100644 --- a/windows.ps1 +++ b/windows.ps1 @@ -1,3 +1,5 @@ +# example usage: +# powershell.exe -command "& { . .\windows.ps1; azure-npm-image }" # Retry({azure-npm-image $(tag)-windows-amd64}) function Retry([Action]$action) {