From 78f5698d7a52dbec1ce96150467d886f366a5908 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 10 Nov 2021 18:15:17 -0800 Subject: [PATCH 01/11] add image and manifest for windows npm --- npm/Dockerfile.windows | 9 +- npm/cmd/main.go | 40 +++++ npm/cmd/start.go | 59 ++++--- npm/config/config.go | 4 + npm/examples/windows/azure-npm.yaml | 147 ++++++++++++++++++ npm/examples/windows/kubeconfigtemplate.yaml | 18 +++ npm/examples/windows/setkubeconfigpath.ps1 | 9 ++ npm/npm.go | 14 +- npm/testpolicies/testing/only-ports.yaml | 2 +- scripts/New-ContainerHostVm.ps1 | 2 +- vendor/github.com/hashicorp/hcl/.gitignore | 18 +-- vendor/github.com/hashicorp/hcl/Makefile | 36 ++--- .../pelletier/go-toml/example-crlf.toml | 58 +++---- 13 files changed, 330 insertions(+), 86 deletions(-) create mode 100644 npm/examples/windows/azure-npm.yaml create mode 100644 npm/examples/windows/kubeconfigtemplate.yaml create mode 100644 npm/examples/windows/setkubeconfigpath.ps1 diff --git a/npm/Dockerfile.windows b/npm/Dockerfile.windows index 910954d2c0..4c5e8f4ab0 100644 --- a/npm/Dockerfile.windows +++ b/npm/Dockerfile.windows @@ -12,9 +12,10 @@ COPY . . RUN $Env:CGO_ENABLED=0; go build -v -o /usr/bin/npm.exe -ldflags """-X main.version=${env:VERSION} -X ${env:NPM_AI_PATH}=${env:NPM_AI_ID}""" -gcflags="-dwarflocationlists=true" ./npm/cmd/ # Copy into final image -FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 +FROM mcr.microsoft.com/windows/servercore:ltsc2022 COPY --from=builder /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll -COPY --from=builder /usr/bin/npm.exe \ - /usr/bin/npm.exe +COPY --from=builder /usr/src/npm/npm/examples/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml +COPY --from=builder /usr/src/npm/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1 +COPY --from=builder /usr/bin/npm.exe npm.exe -ENTRYPOINT ["/usr/bin/npm.exe", "start"] +CMD ["npm.exe", "start" "--kubeconfig=.\\kubeconfig"] diff --git a/npm/cmd/main.go b/npm/cmd/main.go index df198512bf..f39c60aa80 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -4,6 +4,19 @@ package main import ( "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + FlagVersion = "version" + FlagKubeConfigPath = "kubeconfig" +) + +var ( + FlagDefaults = map[string]string{ + FlagKubeConfigPath: "", + } ) // Version is populated by make during build. @@ -11,5 +24,32 @@ var version string func main() { rootCmd := NewRootCmd() + + if version != "" { + viper.Set(FlagVersion, version) + } + + cobra.OnInitialize(func() { + viper.AutomaticEnv() + initCommandFlags(rootCmd.Commands()) + }) + cobra.CheckErr(rootCmd.Execute()) } + +func initCommandFlags(commands []*cobra.Command) { + for _, cmd := range commands { + // bind vars from env or conf to pflags + viper.BindPFlags(cmd.Flags()) + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + if viper.IsSet(flag.Name) && viper.GetString(flag.Name) != "" { + cmd.Flags().Set(flag.Name, viper.GetString(flag.Name)) + } + }) + + // call recursively on subcommands + if cmd.HasSubCommands() { + initCommandFlags(cmd.Commands()) + } + } +} diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 7812738efd..8df54019dc 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" "k8s.io/utils/exec" ) @@ -37,11 +38,12 @@ func newStartNPMCmd() *cobra.Command { viper.AutomaticEnv() // read in environment variables that match viper.SetDefault(npmconfig.ConfigEnvPath, npmconfig.GetConfigPath()) cfgFile := viper.GetString(npmconfig.ConfigEnvPath) + viper.SetConfigFile(cfgFile) // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { - klog.Info("Using config file: ", viper.ConfigFileUsed()) + klog.Info("Using config file: %+v", viper.ConfigFileUsed()) } else { klog.Infof("Failed to load config from env %s: %v", npmconfig.ConfigEnvPath, err) b, _ := json.Marshal(npmconfig.DefaultConfig) @@ -58,25 +60,27 @@ func newStartNPMCmd() *cobra.Command { config := &npmconfig.Config{} err := viper.Unmarshal(config) if err != nil { - return fmt.Errorf("failed to load config with error %w", err) + return fmt.Errorf("failed to load config with error: %w", err) + } + + flags := npmconfig.Flags{ + KubeConfigPath: viper.GetString(FlagKubeConfigPath), } - return start(*config) + return start(*config, flags) }, } + + startNPMCmd.Flags().String(FlagKubeConfigPath, FlagDefaults[FlagKubeConfigPath], "path to kubeconfig") + return startNPMCmd } -func start(config npmconfig.Config) error { +func start(config npmconfig.Config, flags npmconfig.Flags) error { klog.Infof("loaded config: %+v", config) klog.Infof("Start NPM version: %s", version) var err error - defer func() { - if r := recover(); r != nil { - klog.Infof("recovered from error: %v", err) - } - }() if err = initLogging(); err != nil { return err @@ -84,10 +88,20 @@ func start(config npmconfig.Config) error { metrics.InitializeAll() - // Creates the in-cluster config - k8sConfig, err := rest.InClusterConfig() - if err != nil { - return fmt.Errorf("failed to load in cluster config: %w", err) + // Create the kubernetes client + var k8sConfig *rest.Config + if flags.KubeConfigPath == "" { + var err error + k8sConfig, err = rest.InClusterConfig() + if err != nil { + return fmt.Errorf("failed to load in cluster config: %w", err) + } + } else { + var err error + k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath) + if err != nil { + return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err) + } } // Creates the clientset @@ -106,7 +120,11 @@ func start(config npmconfig.Config) error { klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) - k8sServerVersion := k8sServerVersion(clientset) + k8sServerVersion, err := k8sServerVersion(clientset) + if err != nil { + return fmt.Errorf("failed to retrieve kubernetes server version %w", err) + + } var dp dataplane.GenericDataplane if config.Toggles.EnableV2Controllers { @@ -125,8 +143,8 @@ func start(config npmconfig.Config) error { go restserver.NPMRestServerListenAndServe(config, npMgr) if err = npMgr.Start(config, wait.NeverStop); err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %s", err) - panic(err.Error) + metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %w", err) + return err } select {} @@ -143,7 +161,7 @@ func initLogging() error { return nil } -func k8sServerVersion(kubeclientset kubernetes.Interface) *k8sversion.Info { +func k8sServerVersion(kubeclientset kubernetes.Interface) (*k8sversion.Info, error) { var err error var serverVersion *k8sversion.Info for ticker, start := time.NewTicker(1*time.Second).C, time.Now(); time.Since(start) < time.Minute*1; { @@ -156,12 +174,13 @@ func k8sServerVersion(kubeclientset kubernetes.Interface) *k8sversion.Info { if err != nil { metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to retrieving kubernetes version") - panic(err.Error) + return nil, fmt.Errorf("failed to discover kuberntes server version with err %w", err) } if err = util.SetIsNewNwPolicyVerFlag(serverVersion); err != nil { metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to set IsNewNwPolicyVerFlag") - panic(err.Error) + return nil, fmt.Errorf("failed to check if new netowrk policy version is set with err %w", err) } - return serverVersion + + return serverVersion, err } diff --git a/npm/config/config.go b/npm/config/config.go index b7266f3235..3ba09a0a1a 100644 --- a/npm/config/config.go +++ b/npm/config/config.go @@ -34,3 +34,7 @@ type Toggles struct { EnableHTTPDebugAPI bool EnableV2Controllers bool } + +type Flags struct { + KubeConfigPath string `json:"KubeConfigPath"` +} diff --git a/npm/examples/windows/azure-npm.yaml b/npm/examples/windows/azure-npm.yaml new file mode 100644 index 0000000000..68bbf75bc5 --- /dev/null +++ b/npm/examples/windows/azure-npm.yaml @@ -0,0 +1,147 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: azure-npm-binding + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +subjects: + - kind: ServiceAccount + name: azure-npm + namespace: kube-system +roleRef: + kind: ClusterRole + name: azure-npm + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-npm + namespace: kube-system + labels: + app: azure-npm + addonmanager.kubernetes.io/mode: EnsureExists +spec: + selector: + matchLabels: + k8s-app: azure-npm + template: + metadata: + labels: + k8s-app: azure-npm + annotations: + azure.npm/scrapeable: '' + spec: + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: azure-npm + image: acnpublic.azurecr.io/azure-npm:v26-windows-amd64 + command: ["powershell.exe"] + args: ['.\setkubeconfigpath.ps1', ';', 'powershell.exe', '.\npm.exe', "start", '--kubeconfig=.\kubeconfig'] + resources: + limits: + cpu: 250m + memory: 300Mi + requests: + cpu: 250m + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NPM_CONFIG + value: .\\etc\\azure-npm\\azure-npm.json + volumeMounts: + - name: azure-npm-config + mountPath: .\\etc\\azure-npm + nodeSelector: + kubernetes.io/os: windows + volumes: + - name: azure-npm-config + configMap: + name: azure-npm-config + serviceAccountName: azure-npm +--- +apiVersion: v1 +kind: Service +metadata: + name: npm-metrics-cluster-service + namespace: kube-system + labels: + app: npm-metrics +spec: + selector: + k8s-app: azure-npm + ports: + - port: 9000 + targetPort: 10091 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-npm-config + namespace: kube-system +data: + azure-npm.json: | + { + "ResyncPeriodInMinutes": 15, + "ListeningPort": 10091, + "ListeningAddress": "0.0.0.0", + "Toggles": { + "EnablePrometheusMetrics": true, + "EnablePprof": true, + "EnableHTTPDebugAPI": true, + "EnableV2Controllers": true + } + } + + diff --git a/npm/examples/windows/kubeconfigtemplate.yaml b/npm/examples/windows/kubeconfigtemplate.yaml new file mode 100644 index 0000000000..1f1da274c5 --- /dev/null +++ b/npm/examples/windows/kubeconfigtemplate.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Config +clusters: +- name: kubernetes + cluster: + certificate-authority-data: + +contexts: +- name: azure-npm-windows@kubernetes + context: + cluster: kubernetes + namespace: kube-system + user: azure-npm-windows +current-context: azure-npm-windows@kubernetes +users: +- name: azure-npm-windows + user: + token: diff --git a/npm/examples/windows/setkubeconfigpath.ps1 b/npm/examples/windows/setkubeconfigpath.ps1 new file mode 100644 index 0000000000..ef599874db --- /dev/null +++ b/npm/examples/windows/setkubeconfigpath.ps1 @@ -0,0 +1,9 @@ +$cpEndpoint = Get-Content C:\k\config | ForEach-Object -Process {if($_.Contains("server:")) {$_.Trim().Split()[1]}} +$token = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\token +$ca = Get-Content -Raw -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\ca.crt +$caBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($ca)) +$server = "server: $cpEndpoint" +(Get-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfigtemplate.yaml). + replace("", $caBase64). + replace("", $server.Trim()). + replace("", $token) | Set-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfig -Force diff --git a/npm/npm.go b/npm/npm.go index 7404be5bd5..21bc60d9a3 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -65,6 +65,7 @@ type NetworkPolicyManager struct { // V2 controllers podControllerV2 *controllersv2.PodController namespaceControllerV2 *controllersv2.NamespaceController + netPolControllerV2 *controllersv2.NetworkPolicyController npmNamespaceCacheV2 *controllersv2.NpmNamespaceCache npInformer networkinginformers.NetworkPolicyInformer @@ -87,7 +88,7 @@ func NewNetworkPolicyManager(config npmconfig.Config, exec utilexec.Interface, npmVersion string, k8sServerVersion *version.Info) *NetworkPolicyManager { - klog.Infof("API server version: %+v ai meta data %+v", k8sServerVersion, aiMetadata) + klog.Infof("API server version: %+v AI metadata %+v", k8sServerVersion, aiMetadata) npMgr := &NetworkPolicyManager{ config: config, @@ -97,6 +98,7 @@ func NewNetworkPolicyManager(config npmconfig.Config, npInformer: informerFactory.Networking().V1().NetworkPolicies(), ipsMgr: ipsm.NewIpsetManager(exec), npmNamespaceCacheV1: &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*controllersv1.Namespace)}, + npmNamespaceCacheV2: &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}, k8sServerVersion: k8sServerVersion, NodeName: GetNodeName(), version: npmVersion, @@ -108,6 +110,8 @@ func NewNetworkPolicyManager(config npmconfig.Config, npMgr.podControllerV2 = controllersv2.NewPodController(npMgr.podInformer, dp, npMgr.npmNamespaceCacheV2) // create NameSpace controller npMgr.namespaceControllerV2 = controllersv2.NewNamespaceController(npMgr.nsInformer, dp, npMgr.npmNamespaceCacheV2) + // create Network Policy controller + npMgr.netPolControllerV2 = controllersv2.NewNetworkPolicyController(npMgr.npInformer, dp) return npMgr } @@ -219,8 +223,10 @@ func (npMgr *NetworkPolicyManager) SendClusterMetrics() { // Start starts shared informers and waits for the shared informer cache to sync. func (npMgr *NetworkPolicyManager) Start(config npmconfig.Config, stopCh <-chan struct{}) error { // Do initialization of data plane before starting syncup of each controller to avoid heavy call to api-server - if err := npMgr.netPolControllerV1.ResetDataPlane(); err != nil { - return fmt.Errorf("Failed to initialized data plane") + if !config.Toggles.EnableV2Controllers { + if err := npMgr.netPolControllerV1.ResetDataPlane(); err != nil { + return fmt.Errorf("Failed to initialized data plane") + } } // Starts all informers manufactured by npMgr's informerFactory. @@ -243,7 +249,7 @@ func (npMgr *NetworkPolicyManager) Start(config npmconfig.Config, stopCh <-chan go npMgr.podControllerV2.Run(stopCh) go npMgr.namespaceControllerV2.Run(stopCh) // TODO add in netpol controller v2 - // go npMgr.netPolControllerV1.Run(stopCh) + go npMgr.netPolControllerV2.Run(stopCh) // go npMgr.netPolControllerV1.RunPeriodicTasks(stopCh) return nil } diff --git a/npm/testpolicies/testing/only-ports.yaml b/npm/testpolicies/testing/only-ports.yaml index 31826ce0ea..234cc22fe4 100644 --- a/npm/testpolicies/testing/only-ports.yaml +++ b/npm/testpolicies/testing/only-ports.yaml @@ -15,4 +15,4 @@ spec: matchLabels: app: server policyTypes: - - Ingress \ No newline at end of file + - Ingress diff --git a/scripts/New-ContainerHostVm.ps1 b/scripts/New-ContainerHostVm.ps1 index 5e4531fc49..93c92cea77 100644 --- a/scripts/New-ContainerHostVm.ps1 +++ b/scripts/New-ContainerHostVm.ps1 @@ -1,4 +1,4 @@ -<# +<# .SYNOPSIS Creates an Azure VM with given number of network interfaces and IP addresses. diff --git a/vendor/github.com/hashicorp/hcl/.gitignore b/vendor/github.com/hashicorp/hcl/.gitignore index 15586a2b54..822fa09f52 100644 --- a/vendor/github.com/hashicorp/hcl/.gitignore +++ b/vendor/github.com/hashicorp/hcl/.gitignore @@ -1,9 +1,9 @@ -y.output - -# ignore intellij files -.idea -*.iml -*.ipr -*.iws - -*.test +y.output + +# ignore intellij files +.idea +*.iml +*.ipr +*.iws + +*.test diff --git a/vendor/github.com/hashicorp/hcl/Makefile b/vendor/github.com/hashicorp/hcl/Makefile index 84fd743f5c..9fafd5017c 100644 --- a/vendor/github.com/hashicorp/hcl/Makefile +++ b/vendor/github.com/hashicorp/hcl/Makefile @@ -1,18 +1,18 @@ -TEST?=./... - -default: test - -fmt: generate - go fmt ./... - -test: generate - go get -t ./... - go test $(TEST) $(TESTARGS) - -generate: - go generate ./... - -updatedeps: - go get -u golang.org/x/tools/cmd/stringer - -.PHONY: default generate test updatedeps +TEST?=./... + +default: test + +fmt: generate + go fmt ./... + +test: generate + go get -t ./... + go test $(TEST) $(TESTARGS) + +generate: + go generate ./... + +updatedeps: + go get -u golang.org/x/tools/cmd/stringer + +.PHONY: default generate test updatedeps diff --git a/vendor/github.com/pelletier/go-toml/example-crlf.toml b/vendor/github.com/pelletier/go-toml/example-crlf.toml index 780d9c68f2..f45bf88b8f 100644 --- a/vendor/github.com/pelletier/go-toml/example-crlf.toml +++ b/vendor/github.com/pelletier/go-toml/example-crlf.toml @@ -1,30 +1,30 @@ -# This is a TOML document. Boom. - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it score = 4e-08 # to make sure leading zeroes in exponent parts of floats are supported \ No newline at end of file From 50f0dc5b938615a20d8a96c713d99f5080c688ed Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 16 Nov 2021 12:01:25 -0800 Subject: [PATCH 02/11] address feedback --- npm/cmd/main.go | 8 +++---- npm/cmd/start.go | 9 ++++---- npm/examples/windows/setkubeconfigpath.ps1 | 1 + npm/npm.go | 26 ++++++++++++---------- windows.ps1 | 2 ++ 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/npm/cmd/main.go b/npm/cmd/main.go index f39c60aa80..c751ec8ea6 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -9,13 +9,13 @@ import ( ) const ( - FlagVersion = "version" - FlagKubeConfigPath = "kubeconfig" + flagVersion = "version" + flagKubeConfigPath = "kubeconfig" ) var ( FlagDefaults = map[string]string{ - FlagKubeConfigPath: "", + flagKubeConfigPath: "", } ) @@ -26,7 +26,7 @@ func main() { rootCmd := NewRootCmd() if version != "" { - viper.Set(FlagVersion, version) + viper.Set(flagVersion, version) } cobra.OnInitialize(func() { diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 8df54019dc..6d20769107 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -38,10 +38,10 @@ func newStartNPMCmd() *cobra.Command { viper.AutomaticEnv() // read in environment variables that match viper.SetDefault(npmconfig.ConfigEnvPath, npmconfig.GetConfigPath()) cfgFile := viper.GetString(npmconfig.ConfigEnvPath) - viper.SetConfigFile(cfgFile) // If a config file is found, read it in. + // NOTE: there is no config merging with default, if config is loaded, options must be set if err := viper.ReadInConfig(); err == nil { klog.Info("Using config file: %+v", viper.ConfigFileUsed()) } else { @@ -64,14 +64,14 @@ func newStartNPMCmd() *cobra.Command { } flags := npmconfig.Flags{ - KubeConfigPath: viper.GetString(FlagKubeConfigPath), + KubeConfigPath: viper.GetString(flagKubeConfigPath), } return start(*config, flags) }, } - startNPMCmd.Flags().String(FlagKubeConfigPath, FlagDefaults[FlagKubeConfigPath], "path to kubeconfig") + startNPMCmd.Flags().String(flagKubeConfigPath, FlagDefaults[flagKubeConfigPath], "path to kubeconfig") return startNPMCmd } @@ -89,15 +89,14 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { metrics.InitializeAll() // Create the kubernetes client + var k8sConfig *rest.Config if flags.KubeConfigPath == "" { - var err error k8sConfig, err = rest.InClusterConfig() if err != nil { return fmt.Errorf("failed to load in cluster config: %w", err) } } else { - var err error k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath) if err != nil { return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err) diff --git a/npm/examples/windows/setkubeconfigpath.ps1 b/npm/examples/windows/setkubeconfigpath.ps1 index ef599874db..8a15fdbccd 100644 --- a/npm/examples/windows/setkubeconfigpath.ps1 +++ b/npm/examples/windows/setkubeconfigpath.ps1 @@ -1,3 +1,4 @@ +# pull the server value from the kubeconfig on host to construct our own kubeconfig, but using service principal settings $cpEndpoint = Get-Content C:\k\config | ForEach-Object -Process {if($_.Contains("server:")) {$_.Trim().Split()[1]}} $token = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\token $ca = Get-Content -Raw -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\ca.crt diff --git a/npm/npm.go b/npm/npm.go index 21bc60d9a3..ab7911accb 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -91,21 +91,21 @@ func NewNetworkPolicyManager(config npmconfig.Config, klog.Infof("API server version: %+v AI metadata %+v", k8sServerVersion, aiMetadata) npMgr := &NetworkPolicyManager{ - config: config, - informerFactory: informerFactory, - podInformer: informerFactory.Core().V1().Pods(), - nsInformer: informerFactory.Core().V1().Namespaces(), - npInformer: informerFactory.Networking().V1().NetworkPolicies(), - ipsMgr: ipsm.NewIpsetManager(exec), - npmNamespaceCacheV1: &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*controllersv1.Namespace)}, - npmNamespaceCacheV2: &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}, - k8sServerVersion: k8sServerVersion, - NodeName: GetNodeName(), - version: npmVersion, - TelemetryEnabled: true, + config: config, + informerFactory: informerFactory, + podInformer: informerFactory.Core().V1().Pods(), + nsInformer: informerFactory.Core().V1().Namespaces(), + npInformer: informerFactory.Networking().V1().NetworkPolicies(), + ipsMgr: ipsm.NewIpsetManager(exec), + k8sServerVersion: k8sServerVersion, + NodeName: GetNodeName(), + version: npmVersion, + TelemetryEnabled: true, } if npMgr.config.Toggles.EnableV2Controllers { + // initialize v2 cache + npMgr.npmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)} // create pod controller npMgr.podControllerV2 = controllersv2.NewPodController(npMgr.podInformer, dp, npMgr.npmNamespaceCacheV2) // create NameSpace controller @@ -115,6 +115,8 @@ func NewNetworkPolicyManager(config npmconfig.Config, return npMgr } + // initialize V1 cache + npMgr.npmNamespaceCacheV1 = &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*controllersv1.Namespace)} // create pod controller npMgr.podControllerV1 = controllersv1.NewPodController(npMgr.podInformer, npMgr.ipsMgr, npMgr.npmNamespaceCacheV1) // create NameSpace controller diff --git a/windows.ps1 b/windows.ps1 index 32cd475d65..5d2638cce1 100644 --- a/windows.ps1 +++ b/windows.ps1 @@ -1,3 +1,5 @@ +# example usage: +# powershell.exe -command "& { . .\windows.ps1; azure-npm-image }" function azure-npm-image { $env:ACN_PACKAGE_PATH = "github.com/Azure/azure-container-networking" From 49e971bd014f0089ff198d7e2a0df36c1045f27f Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 16 Nov 2021 15:03:35 -0800 Subject: [PATCH 03/11] lint --- npm/cmd/start.go | 4 ++-- npm/cmd/start_test.go | 3 ++- npm/npm.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 6d20769107..4806613775 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -43,7 +43,7 @@ func newStartNPMCmd() *cobra.Command { // If a config file is found, read it in. // NOTE: there is no config merging with default, if config is loaded, options must be set if err := viper.ReadInConfig(); err == nil { - klog.Info("Using config file: %+v", viper.ConfigFileUsed()) + klog.Infof("Using config file: %+v", viper.ConfigFileUsed()) } else { klog.Infof("Failed to load config from env %s: %v", npmconfig.ConfigEnvPath, err) b, _ := json.Marshal(npmconfig.DefaultConfig) @@ -142,7 +142,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { go restserver.NPMRestServerListenAndServe(config, npMgr) if err = npMgr.Start(config, wait.NeverStop); err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %w", err) + metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %+v", err) return err } diff --git a/npm/cmd/start_test.go b/npm/cmd/start_test.go index dbee5aaf31..41952ff925 100644 --- a/npm/cmd/start_test.go +++ b/npm/cmd/start_test.go @@ -107,7 +107,8 @@ func TestK8sServerVersion(t *testing.T) { }) } else { require.NotPanics(t, func() { - got := k8sServerVersion(fc) + got, err := k8sServerVersion(fc) + require.NoError(t, err) require.Equal(t, got, tt.info) require.Equal(t, util.IsNewNwPolicyVerFlag, tt.isNewNwPolicyVer) }) diff --git a/npm/npm.go b/npm/npm.go index ab7911accb..2d3a00a08d 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -227,7 +227,7 @@ func (npMgr *NetworkPolicyManager) Start(config npmconfig.Config, stopCh <-chan // Do initialization of data plane before starting syncup of each controller to avoid heavy call to api-server if !config.Toggles.EnableV2Controllers { if err := npMgr.netPolControllerV1.ResetDataPlane(); err != nil { - return fmt.Errorf("Failed to initialized data plane") + return fmt.Errorf("Failed to initialized data plane with err %w", err) } } From b0717d0f5bf351736c11c7d461dbaabcd734ba8b Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 17 Nov 2021 10:43:47 -0800 Subject: [PATCH 04/11] server version --- npm/cmd/debug_test.go | 4 ++-- npm/cmd/main.go | 18 ++++++++++-------- npm/cmd/start.go | 37 ++++++++++++++++++++----------------- npm/cmd/start_test.go | 33 +++++++++++++++------------------ 4 files changed, 47 insertions(+), 45 deletions(-) diff --git a/npm/cmd/debug_test.go b/npm/cmd/debug_test.go index 32d0482e3e..87ee7c7246 100644 --- a/npm/cmd/debug_test.go +++ b/npm/cmd/debug_test.go @@ -2,7 +2,7 @@ package main import ( "bytes" - "io/ioutil" + "io" "testing" "github.com/stretchr/testify/require" @@ -51,7 +51,7 @@ func testCommand(t *testing.T, tests []*testCases) { require.NoError(t, err) - out, err := ioutil.ReadAll(b) + out, err := io.ReadAll(b) require.NoError(t, err) if tt.wantErr { require.NotEmpty(t, out) diff --git a/npm/cmd/main.go b/npm/cmd/main.go index c751ec8ea6..b855402ac6 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -13,11 +13,9 @@ const ( flagKubeConfigPath = "kubeconfig" ) -var ( - FlagDefaults = map[string]string{ - flagKubeConfigPath: "", - } -) +var flagDefaults = map[string]string{ + flagKubeConfigPath: "", +} // Version is populated by make during build. var version string @@ -40,10 +38,14 @@ func main() { func initCommandFlags(commands []*cobra.Command) { for _, cmd := range commands { // bind vars from env or conf to pflags - viper.BindPFlags(cmd.Flags()) - cmd.Flags().VisitAll(func(flag *pflag.Flag) { + err := viper.BindPFlags(cmd.Flags()) + cobra.CheckErr(err) + + c := cmd + c.Flags().VisitAll(func(flag *pflag.Flag) { if viper.IsSet(flag.Name) && viper.GetString(flag.Name) != "" { - cmd.Flags().Set(flag.Name, viper.GetString(flag.Name)) + err := c.Flags().Set(flag.Name, viper.GetString(flag.Name)) + cobra.CheckErr(err) } }) diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 4806613775..c9c3b4afd8 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -4,9 +4,11 @@ package main import ( "bytes" + "crypto/rand" "encoding/json" "fmt" - "math/rand" + "math" + "math/big" "time" "github.com/Azure/azure-container-networking/common" @@ -71,7 +73,7 @@ func newStartNPMCmd() *cobra.Command { }, } - startNPMCmd.Flags().String(flagKubeConfigPath, FlagDefaults[flagKubeConfigPath], "path to kubeconfig") + startNPMCmd.Flags().String(flagKubeConfigPath, flagDefaults[flagKubeConfigPath], "path to kubeconfig") return startNPMCmd } @@ -82,7 +84,8 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { var err error - if err = initLogging(); err != nil { + err = initLogging() + if err != nil { return err } @@ -113,8 +116,12 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { // Setting reSyncPeriod minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute + random, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) + if err != nil { + return fmt.Errorf("failed to generate random resyncPeriod with err %w", err) + } // Adding some randomness so all NPM pods will not request for info at once. - factor := rand.Float64() + 1 + factor := float64(random.Int64() + 1) resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) @@ -122,7 +129,6 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { k8sServerVersion, err := k8sServerVersion(clientset) if err != nil { return fmt.Errorf("failed to retrieve kubernetes server version %w", err) - } var dp dataplane.GenericDataplane @@ -143,7 +149,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { if err = npMgr.Start(config, wait.NeverStop); err != nil { metrics.SendErrorLogAndMetric(util.NpmID, "Failed to start NPM due to %+v", err) - return err + return fmt.Errorf("failed to start with err: %w", err) } select {} @@ -161,25 +167,22 @@ func initLogging() error { } func k8sServerVersion(kubeclientset kubernetes.Interface) (*k8sversion.Info, error) { - var err error var serverVersion *k8sversion.Info for ticker, start := time.NewTicker(1*time.Second).C, time.Now(); time.Since(start) < time.Minute*1; { <-ticker + var err error serverVersion, err = kubeclientset.Discovery().ServerVersion() - if err == nil { + if err != nil { + metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to retrieving kubernetes version with err: %v", err) + } else { break } } - if err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to retrieving kubernetes version") - return nil, fmt.Errorf("failed to discover kuberntes server version with err %w", err) - } - - if err = util.SetIsNewNwPolicyVerFlag(serverVersion); err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to set IsNewNwPolicyVerFlag") - return nil, fmt.Errorf("failed to check if new netowrk policy version is set with err %w", err) + if err := util.SetIsNewNwPolicyVerFlag(serverVersion); err != nil { + metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to set IsNewNwPolicyVerFlag with err: %v", err) + return nil, fmt.Errorf("failed to check if new network policy version is set with err %w", err) } - return serverVersion, err + return serverVersion, nil } diff --git a/npm/cmd/start_test.go b/npm/cmd/start_test.go index 41952ff925..063c8c7474 100644 --- a/npm/cmd/start_test.go +++ b/npm/cmd/start_test.go @@ -24,7 +24,7 @@ func TestK8sServerVersion(t *testing.T) { tests := []struct { name string info *k8sversion.Info - wantPanic bool + wantErr bool isNewNwPolicyVer bool }{ { @@ -34,7 +34,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "2", GitVersion: "v1.20.2", }, - wantPanic: false, + wantErr: false, isNewNwPolicyVer: true, }, { @@ -44,7 +44,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v1.11", }, - wantPanic: false, + wantErr: false, isNewNwPolicyVer: true, }, { @@ -54,7 +54,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "1", GitVersion: "v1.10.1", }, - wantPanic: false, + wantErr: false, isNewNwPolicyVer: false, }, { @@ -64,7 +64,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v0.0", }, - wantPanic: false, + wantErr: false, isNewNwPolicyVer: false, }, { @@ -74,7 +74,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v-1.11", }, - wantPanic: true, + wantErr: true, }, { name: "Test wrong alphabet version", @@ -83,7 +83,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "cc", GitVersion: "vab.cc", }, - wantPanic: true, + wantErr: true, }, { name: "Test wrong alphabet version", @@ -92,7 +92,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "cc", GitVersion: "v1.1.cc", }, - wantPanic: true, + wantErr: true, }, } @@ -101,17 +101,14 @@ func TestK8sServerVersion(t *testing.T) { tt := tt fc.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = tt.info t.Run(tt.name, func(t *testing.T) { - if tt.wantPanic { - require.Panics(t, func() { - k8sServerVersion(fc) - }) + if tt.wantErr { + _, err := k8sServerVersion(fc) + require.Error(t, err) } else { - require.NotPanics(t, func() { - got, err := k8sServerVersion(fc) - require.NoError(t, err) - require.Equal(t, got, tt.info) - require.Equal(t, util.IsNewNwPolicyVerFlag, tt.isNewNwPolicyVer) - }) + got, err := k8sServerVersion(fc) + require.NoError(t, err) + require.Equal(t, got, tt.info) + require.Equal(t, util.IsNewNwPolicyVerFlag, tt.isNewNwPolicyVer) } }) } From 7c55fde86e80a0efe0581036175dec45a4472038 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 18 Nov 2021 11:03:07 -0800 Subject: [PATCH 05/11] logs --- npm/cmd/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/npm/cmd/main.go b/npm/cmd/main.go index b855402ac6..0dd38b0da5 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -3,6 +3,8 @@ package main import ( + "fmt" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -21,16 +23,20 @@ var flagDefaults = map[string]string{ var version string func main() { + fmt.Println("start here") + rootCmd := NewRootCmd() if version != "" { viper.Set(flagVersion, version) } + fmt.Println("made it here") cobra.OnInitialize(func() { viper.AutomaticEnv() initCommandFlags(rootCmd.Commands()) }) + fmt.Println("made it here2") cobra.CheckErr(rootCmd.Execute()) } From 3a5af5610f0e571d4995f9b987bac3a67c0a8faf Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 18 Nov 2021 12:58:23 -0800 Subject: [PATCH 06/11] revert server version --- Makefile | 15 ++++----------- npm/cmd/start.go | 25 ++++++++++++------------- npm/cmd/start_test.go | 32 +++++++++++++++++--------------- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index a55619e40b..b6f8b40d8f 100644 --- a/Makefile +++ b/Makefile @@ -219,22 +219,15 @@ azure-cnm-plugin-image: azure-cnm-plugin # Build the Azure NPM image. .PHONY: azure-npm-image -azure-npm-image: +azure-npm-image: azure-npm ifeq ($(GOOS),linux) - $(MKDIR) $(IMAGE_DIR) - docker buildx create --use - docker buildx build \ + docker build \ --no-cache \ -f npm/Dockerfile \ -t $(AZURE_NPM_IMAGE):$(VERSION) \ - --build-arg VERSION=$(VERSION) \ - --build-arg NPM_AI_PATH=$(NPM_AI_PATH) \ - --build-arg NPM_AI_ID=$(NPM_AI_ID) \ - --platform=$(IMAGE_PLATFORM_ARCHES) \ - --$(IMAGE_ACTION) \ + --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ . - - echo $(AZURE_NPM_IMAGE):$(VERSION) > $(IMAGE_DIR)/$(NPM_IMAGE_INFO_FILE) + docker save $(AZURE_NPM_IMAGE):$(VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_IMAGE_ARCHIVE_NAME) endif # Build the Azure CNS image diff --git a/npm/cmd/start.go b/npm/cmd/start.go index c9c3b4afd8..69f1b71a4e 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -126,10 +126,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) - k8sServerVersion, err := k8sServerVersion(clientset) - if err != nil { - return fmt.Errorf("failed to retrieve kubernetes server version %w", err) - } + k8sServerVersion := k8sServerVersion(clientset) var dp dataplane.GenericDataplane if config.Toggles.EnableV2Controllers { @@ -166,23 +163,25 @@ func initLogging() error { return nil } -func k8sServerVersion(kubeclientset kubernetes.Interface) (*k8sversion.Info, error) { +func k8sServerVersion(kubeclientset kubernetes.Interface) *k8sversion.Info { + var err error var serverVersion *k8sversion.Info for ticker, start := time.NewTicker(1*time.Second).C, time.Now(); time.Since(start) < time.Minute*1; { <-ticker - var err error serverVersion, err = kubeclientset.Discovery().ServerVersion() - if err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to retrieving kubernetes version with err: %v", err) - } else { + if err == nil { break } } - if err := util.SetIsNewNwPolicyVerFlag(serverVersion); err != nil { - metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to set IsNewNwPolicyVerFlag with err: %v", err) - return nil, fmt.Errorf("failed to check if new network policy version is set with err %w", err) + if err != nil { + metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to retrieving kubernetes version") + panic(err.Error) } - return serverVersion, nil + if err = util.SetIsNewNwPolicyVerFlag(serverVersion); err != nil { + metrics.SendErrorLogAndMetric(util.NpmID, "Error: failed to set IsNewNwPolicyVerFlag") + panic(err.Error) + } + return serverVersion } diff --git a/npm/cmd/start_test.go b/npm/cmd/start_test.go index 063c8c7474..dbee5aaf31 100644 --- a/npm/cmd/start_test.go +++ b/npm/cmd/start_test.go @@ -24,7 +24,7 @@ func TestK8sServerVersion(t *testing.T) { tests := []struct { name string info *k8sversion.Info - wantErr bool + wantPanic bool isNewNwPolicyVer bool }{ { @@ -34,7 +34,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "2", GitVersion: "v1.20.2", }, - wantErr: false, + wantPanic: false, isNewNwPolicyVer: true, }, { @@ -44,7 +44,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v1.11", }, - wantErr: false, + wantPanic: false, isNewNwPolicyVer: true, }, { @@ -54,7 +54,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "1", GitVersion: "v1.10.1", }, - wantErr: false, + wantPanic: false, isNewNwPolicyVer: false, }, { @@ -64,7 +64,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v0.0", }, - wantErr: false, + wantPanic: false, isNewNwPolicyVer: false, }, { @@ -74,7 +74,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "0", GitVersion: "v-1.11", }, - wantErr: true, + wantPanic: true, }, { name: "Test wrong alphabet version", @@ -83,7 +83,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "cc", GitVersion: "vab.cc", }, - wantErr: true, + wantPanic: true, }, { name: "Test wrong alphabet version", @@ -92,7 +92,7 @@ func TestK8sServerVersion(t *testing.T) { Minor: "cc", GitVersion: "v1.1.cc", }, - wantErr: true, + wantPanic: true, }, } @@ -101,14 +101,16 @@ func TestK8sServerVersion(t *testing.T) { tt := tt fc.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = tt.info t.Run(tt.name, func(t *testing.T) { - if tt.wantErr { - _, err := k8sServerVersion(fc) - require.Error(t, err) + if tt.wantPanic { + require.Panics(t, func() { + k8sServerVersion(fc) + }) } else { - got, err := k8sServerVersion(fc) - require.NoError(t, err) - require.Equal(t, got, tt.info) - require.Equal(t, util.IsNewNwPolicyVerFlag, tt.isNewNwPolicyVer) + require.NotPanics(t, func() { + got := k8sServerVersion(fc) + require.Equal(t, got, tt.info) + require.Equal(t, util.IsNewNwPolicyVerFlag, tt.isNewNwPolicyVer) + }) } }) } From 54341df10f3bf990e2d1d3f4d89c4b81b6ee675b Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 18 Nov 2021 13:05:21 -0800 Subject: [PATCH 07/11] makefile --- Makefile | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b6f8b40d8f..a55619e40b 100644 --- a/Makefile +++ b/Makefile @@ -219,15 +219,22 @@ azure-cnm-plugin-image: azure-cnm-plugin # Build the Azure NPM image. .PHONY: azure-npm-image -azure-npm-image: azure-npm +azure-npm-image: ifeq ($(GOOS),linux) - docker build \ + $(MKDIR) $(IMAGE_DIR) + docker buildx create --use + docker buildx build \ --no-cache \ -f npm/Dockerfile \ -t $(AZURE_NPM_IMAGE):$(VERSION) \ - --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ + --build-arg VERSION=$(VERSION) \ + --build-arg NPM_AI_PATH=$(NPM_AI_PATH) \ + --build-arg NPM_AI_ID=$(NPM_AI_ID) \ + --platform=$(IMAGE_PLATFORM_ARCHES) \ + --$(IMAGE_ACTION) \ . - docker save $(AZURE_NPM_IMAGE):$(VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_IMAGE_ARCHIVE_NAME) + + echo $(AZURE_NPM_IMAGE):$(VERSION) > $(IMAGE_DIR)/$(NPM_IMAGE_INFO_FILE) endif # Build the Azure CNS image From e940baeaaba2e5baf6f07d425326bc9d07776fde Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 19 Nov 2021 16:40:29 -0800 Subject: [PATCH 08/11] update --- Makefile | 17 +++++++++++++ npm.txt | 59 ++++++++++++++++++++++++++++++++++++++++++++ npm/azure-npm.yaml | 2 +- npm/clusterrole.yaml | 42 +++++++++++++++++++++++++++++++ npm/cmd/start.go | 7 +++++- 5 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 npm.txt create mode 100644 npm/clusterrole.yaml diff --git a/Makefile b/Makefile index a55619e40b..bc8e4e68a3 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,23 @@ ifeq ($(GOOS),linux) echo $(AZURE_NPM_IMAGE):$(VERSION) > $(IMAGE_DIR)/$(NPM_IMAGE_INFO_FILE) endif + +# Build the Azure NPM image, because the buildx command breaks other runtimes +.PHONY: azure-npm-image-classic +azure-npm-image-classic: azure-npm +ifeq ($(GOOS),linux) + mkdir -p $(IMAGE_DIR) + docker build \ + --no-cache \ + -f npm/Dockerfile \ + -t $(AZURE_NPM_IMAGE):$(VERSION) \ + --build-arg VERSION=$(VERSION) \ + --build-arg NPM_AI_PATH=$(NPM_AI_PATH) \ + --build-arg NPM_AI_ID=$(NPM_AI_ID) \ + --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ + . +endif + # Build the Azure CNS image .PHONY: azure-cns-image azure-cns-image: diff --git a/npm.txt b/npm.txt new file mode 100644 index 0000000000..ba5bafad79 --- /dev/null +++ b/npm.txt @@ -0,0 +1,59 @@ +start here +made it here +made it here2 +2021/11/19 22:47:04 [1] Finished initializing all Prometheus metrics +I1119 22:47:04.572518 1 start.go:50] Failed to load config from env NPM_CONFIG: open /etc/azure/azure-vnet/azure-npm.json: no such file or directory +I1119 22:47:04.572812 1 start.go:82] loaded config: {ResyncPeriodInMinutes:15 ListeningPort:10091 ListeningAddress:0.0.0.0 Toggles:{EnablePrometheusMetrics:true EnablePprof:true EnableHTTPDebugAPI:true EnableV2Controllers:false}} +I1119 22:47:04.572826 1 start.go:83] Start NPM version: v1.4.15-10-g54341df1-dirty +I1119 22:47:04.574496 1 start.go:126] Resync period for NPM pod is set to -153722867. +2021/11/19 22:47:05 [1] GetAzureCloud querying url: http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text +2021/11/19 22:47:05 [1] [Utils] Initializing HTTP client with connection timeout: 7, response header timeout: 7 +I1119 22:47:05.592800 1 npm.go:91] API server version: v1.22.3 AI metadata 014c22bd-4107-459e-8475-67909e96edcb +2021/11/19 22:47:05 [1] [AppInsights] CloudName: AzurePublicCloud +2021/11/19 22:47:05 [1] Initialized AppInsights handle +I1119 22:47:05.598433 1 networkPolicyController.go:70] Initiailize data plane. Clean up Azure-NPM chains and start reconcile iptables +2021/11/19 22:47:05 [1] Executing iptables command iptables [-w 60 -D FORWARD -j AZURE-NPM] +2021/11/19 22:47:05 [1] [Telemetry] Request metadata from wireserver +I1119 22:47:05.599151 1 server.go:59] Starting NPM HTTP API on 0.0.0.0:10091... +2021/11/19 22:47:05 [1] Error: There was an error running command: [iptables -w 60 -D FORWARD -j AZURE-NPM] Stderr: [exit status 2, iptables v1.8.4 (legacy): Couldn't load target `AZURE-NPM':No such file or directory + +Try `iptables -h' or 'iptables --help' for more information.] +2021/11/19 22:47:05 [1] Error: failed to delete AZURE-NPM from Forward chain +2021/11/19 22:47:05 [1] Azure-NPM creating, cleaning existing Azure NPM IPSets +2021/11/19 22:47:05 [1] Acquiring process lock +2021/11/19 22:47:05 [1] Acquired process lock +E1119 22:47:05.605266 1 networkPolicyController.go:77] Failed to UninitNpmChains with err: exit status 2 +2021/11/19 22:47:05 [1] Released process lock +2021/11/19 22:47:05 [1] {DestroyNpmIpsets} Received empty string from ipset list while destroying azure-npm ipsets +E1119 22:47:05.611015 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:47:05.611082 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:47:05.611146 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:06.899585 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:47:07.001923 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:07.222449 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:47:09.205860 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:47:09.284164 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:09.925324 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:47:12.618537 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:47:12.988198 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:13.438476 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:47:20.953326 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:47:22.690244 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:25.049791 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +2021/11/19 22:47:35 [1] [AppInsights] [Fri Nov 19 22:47:35 UTC 2021] --------- Transmitting 6 items --------- +E1119 22:47:36.503673 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Telemetry transmitted in 1.916782656s +2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Response: 200 +2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Items accepted/received: 6/6 +E1119 22:47:40.372713 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:47:41.923033 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:48:13.224112 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:48:14.111110 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:48:15.031252 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:48:50.682171 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:48:51.130953 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:49:03.603203 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:49:31.514361 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope +E1119 22:49:38.260726 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope +E1119 22:49:59.488782 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope +E1119 22:50:10.313803 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope diff --git a/npm/azure-npm.yaml b/npm/azure-npm.yaml index fb5574f486..44d357ea86 100644 --- a/npm/azure-npm.yaml +++ b/npm/azure-npm.yaml @@ -34,7 +34,7 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRoleBinding metadata: name: azure-npm-binding namespace: kube-system diff --git a/npm/clusterrole.yaml b/npm/clusterrole.yaml new file mode 100644 index 0000000000..0699212301 --- /dev/null +++ b/npm/clusterrole.yaml @@ -0,0 +1,42 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: azure-npm-binding + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +subjects: + - kind: ServiceAccount + name: azure-npm + namespace: kube-system +roleRef: + kind: ClusterRole + name: azure-npm + apiGroup: rbac.authorization.k8s.io diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 69f1b71a4e..f6b2dfde26 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -89,17 +89,20 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { return err } + klog.Infof("initializing metrics") metrics.InitializeAll() // Create the kubernetes client - + klog.Infof("loading kubeconfig") var k8sConfig *rest.Config if flags.KubeConfigPath == "" { + klog.Infof("loading in cluster kubeconfig") k8sConfig, err = rest.InClusterConfig() if err != nil { return fmt.Errorf("failed to load in cluster config: %w", err) } } else { + klog.Infof("loading kubeconfig from flags") k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath) if err != nil { return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err) @@ -113,6 +116,8 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { return fmt.Errorf("failed to generate clientset with cluster config: %w", err) } + klog.Infof("received clientset %+v", clientset) + // Setting reSyncPeriod minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute From eb6f90d82b966f0230a0cb2e7013c28dfd8d89fc Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 29 Nov 2021 13:18:33 -0800 Subject: [PATCH 09/11] rand --- npm/cmd/start.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 7e67f41a89..ef0bda7b76 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -4,11 +4,9 @@ package main import ( "bytes" - "crypto/rand" "encoding/json" "fmt" - "math" - "math/big" + "math/rand" "time" "github.com/Azure/azure-container-networking/common" @@ -121,12 +119,8 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { // Setting reSyncPeriod minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute - random, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - return fmt.Errorf("failed to generate random resyncPeriod with err %w", err) - } // Adding some randomness so all NPM pods will not request for info at once. - factor := float64(random.Int64() + 1) + factor := rand.Float64() + 1 resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) From a5f9f67601f340f2a977cb7f75ab91312124a940 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 30 Nov 2021 13:54:56 -0800 Subject: [PATCH 10/11] cleanup --- .../aks-engine/e2e-step-template.yaml | 4 +- npm.txt | 59 ------------------- npm/clusterrole.yaml | 42 ------------- npm/cmd/main.go | 6 -- npm/cmd/start.go | 7 +-- npm/examples/windows/setkubeconfigpath.ps1 | 1 + 6 files changed, 4 insertions(+), 115 deletions(-) delete mode 100644 npm.txt delete mode 100644 npm/clusterrole.yaml diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml index 9209fed795..bd532289b7 100644 --- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml @@ -97,13 +97,12 @@ steps: export IS_JENKINS=false export DEBUG_CRASHING_PODS=true export AZURE_CORE_ONLY_SHOW_ERRORS=True + export GINKGO_FOCUS="should apply various network policies and enforce access to nginx" echo Cluster Def $CLUSTER_DEFINITION cat $CLUSTER_DEFINITION make test-kubernetes name: DeployAKSEngine displayName: Run AKS-Engine E2E Tests - - - task: CopyFiles@2 inputs: @@ -116,4 +115,3 @@ steps: artifactName: ${{ parameters.name }} pathtoPublish: "$(Build.ArtifactStagingDirectory)/${{ parameters.name }}" condition: always() - \ No newline at end of file diff --git a/npm.txt b/npm.txt deleted file mode 100644 index ba5bafad79..0000000000 --- a/npm.txt +++ /dev/null @@ -1,59 +0,0 @@ -start here -made it here -made it here2 -2021/11/19 22:47:04 [1] Finished initializing all Prometheus metrics -I1119 22:47:04.572518 1 start.go:50] Failed to load config from env NPM_CONFIG: open /etc/azure/azure-vnet/azure-npm.json: no such file or directory -I1119 22:47:04.572812 1 start.go:82] loaded config: {ResyncPeriodInMinutes:15 ListeningPort:10091 ListeningAddress:0.0.0.0 Toggles:{EnablePrometheusMetrics:true EnablePprof:true EnableHTTPDebugAPI:true EnableV2Controllers:false}} -I1119 22:47:04.572826 1 start.go:83] Start NPM version: v1.4.15-10-g54341df1-dirty -I1119 22:47:04.574496 1 start.go:126] Resync period for NPM pod is set to -153722867. -2021/11/19 22:47:05 [1] GetAzureCloud querying url: http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text -2021/11/19 22:47:05 [1] [Utils] Initializing HTTP client with connection timeout: 7, response header timeout: 7 -I1119 22:47:05.592800 1 npm.go:91] API server version: v1.22.3 AI metadata 014c22bd-4107-459e-8475-67909e96edcb -2021/11/19 22:47:05 [1] [AppInsights] CloudName: AzurePublicCloud -2021/11/19 22:47:05 [1] Initialized AppInsights handle -I1119 22:47:05.598433 1 networkPolicyController.go:70] Initiailize data plane. Clean up Azure-NPM chains and start reconcile iptables -2021/11/19 22:47:05 [1] Executing iptables command iptables [-w 60 -D FORWARD -j AZURE-NPM] -2021/11/19 22:47:05 [1] [Telemetry] Request metadata from wireserver -I1119 22:47:05.599151 1 server.go:59] Starting NPM HTTP API on 0.0.0.0:10091... -2021/11/19 22:47:05 [1] Error: There was an error running command: [iptables -w 60 -D FORWARD -j AZURE-NPM] Stderr: [exit status 2, iptables v1.8.4 (legacy): Couldn't load target `AZURE-NPM':No such file or directory - -Try `iptables -h' or 'iptables --help' for more information.] -2021/11/19 22:47:05 [1] Error: failed to delete AZURE-NPM from Forward chain -2021/11/19 22:47:05 [1] Azure-NPM creating, cleaning existing Azure NPM IPSets -2021/11/19 22:47:05 [1] Acquiring process lock -2021/11/19 22:47:05 [1] Acquired process lock -E1119 22:47:05.605266 1 networkPolicyController.go:77] Failed to UninitNpmChains with err: exit status 2 -2021/11/19 22:47:05 [1] Released process lock -2021/11/19 22:47:05 [1] {DestroyNpmIpsets} Received empty string from ipset list while destroying azure-npm ipsets -E1119 22:47:05.611015 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:47:05.611082 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:47:05.611146 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:06.899585 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:47:07.001923 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:07.222449 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:47:09.205860 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:47:09.284164 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:09.925324 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:47:12.618537 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:47:12.988198 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:13.438476 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:47:20.953326 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:47:22.690244 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:25.049791 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -2021/11/19 22:47:35 [1] [AppInsights] [Fri Nov 19 22:47:35 UTC 2021] --------- Transmitting 6 items --------- -E1119 22:47:36.503673 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Telemetry transmitted in 1.916782656s -2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Response: 200 -2021/11/19 22:47:37 [1] [AppInsights] [Fri Nov 19 22:47:37 UTC 2021] Items accepted/received: 6/6 -E1119 22:47:40.372713 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:47:41.923033 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:48:13.224112 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:48:14.111110 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:48:15.031252 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:48:50.682171 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:48:51.130953 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:49:03.603203 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:49:31.514361 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope -E1119 22:49:38.260726 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Namespace: failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "namespaces" in API group "" at the cluster scope -E1119 22:49:59.488782 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "pods" in API group "" at the cluster scope -E1119 22:50:10.313803 1 reflector.go:138] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.NetworkPolicy: failed to list *v1.NetworkPolicy: networkpolicies.networking.k8s.io is forbidden: User "system:serviceaccount:kube-system:azure-npm" cannot list resource "networkpolicies" in API group "networking.k8s.io" at the cluster scope diff --git a/npm/clusterrole.yaml b/npm/clusterrole.yaml deleted file mode 100644 index 0699212301..0000000000 --- a/npm/clusterrole.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: azure-npm - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists -rules: - - apiGroups: - - "" - resources: - - pods - - nodes - - namespaces - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: azure-npm-binding - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists -subjects: - - kind: ServiceAccount - name: azure-npm - namespace: kube-system -roleRef: - kind: ClusterRole - name: azure-npm - apiGroup: rbac.authorization.k8s.io diff --git a/npm/cmd/main.go b/npm/cmd/main.go index 0dd38b0da5..b855402ac6 100644 --- a/npm/cmd/main.go +++ b/npm/cmd/main.go @@ -3,8 +3,6 @@ package main import ( - "fmt" - "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -23,20 +21,16 @@ var flagDefaults = map[string]string{ var version string func main() { - fmt.Println("start here") - rootCmd := NewRootCmd() if version != "" { viper.Set(flagVersion, version) } - fmt.Println("made it here") cobra.OnInitialize(func() { viper.AutomaticEnv() initCommandFlags(rootCmd.Commands()) }) - fmt.Println("made it here2") cobra.CheckErr(rootCmd.Execute()) } diff --git a/npm/cmd/start.go b/npm/cmd/start.go index ef0bda7b76..2917eaf16e 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -91,7 +91,6 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { metrics.InitializeAll() // Create the kubernetes client - klog.Infof("loading kubeconfig") var k8sConfig *rest.Config if flags.KubeConfigPath == "" { klog.Infof("loading in cluster kubeconfig") @@ -100,7 +99,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { return fmt.Errorf("failed to load in cluster config: %w", err) } } else { - klog.Infof("loading kubeconfig from flags") + klog.Infof("loading kubeconfig from flag: %s", flags.KubeConfigPath) k8sConfig, err = clientcmd.BuildConfigFromFlags("", flags.KubeConfigPath) if err != nil { return fmt.Errorf("failed to load kubeconfig [%s] with err config: %w", flags.KubeConfigPath, err) @@ -114,13 +113,11 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { return fmt.Errorf("failed to generate clientset with cluster config: %w", err) } - klog.Infof("received clientset %+v", clientset) - // Setting reSyncPeriod minResyncPeriod := time.Duration(config.ResyncPeriodInMinutes) * time.Minute // Adding some randomness so all NPM pods will not request for info at once. - factor := rand.Float64() + 1 + factor := rand.Float64() + 1 //nolint resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) klog.Infof("Resync period for NPM pod is set to %d.", int(resyncPeriod/time.Minute)) factory := informers.NewSharedInformerFactory(clientset, resyncPeriod) diff --git a/npm/examples/windows/setkubeconfigpath.ps1 b/npm/examples/windows/setkubeconfigpath.ps1 index 8a15fdbccd..38ec3440b2 100644 --- a/npm/examples/windows/setkubeconfigpath.ps1 +++ b/npm/examples/windows/setkubeconfigpath.ps1 @@ -1,4 +1,5 @@ # pull the server value from the kubeconfig on host to construct our own kubeconfig, but using service principal settings +# this is required to build a kubeconfig using the kubeconfig on disk in c:\k, and the service principle granted in the container mount, to generate clientset $cpEndpoint = Get-Content C:\k\config | ForEach-Object -Process {if($_.Contains("server:")) {$_.Trim().Split()[1]}} $token = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\token $ca = Get-Content -Raw -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\ca.crt From 8a9b306cda55e65fb1a242f3a7c38409d3449409 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Wed, 1 Dec 2021 13:02:58 -0800 Subject: [PATCH 11/11] remove ginkgo focus --- .pipelines/singletenancy/aks-engine/e2e-step-template.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml index bd532289b7..481f4f2418 100644 --- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml @@ -97,7 +97,6 @@ steps: export IS_JENKINS=false export DEBUG_CRASHING_PODS=true export AZURE_CORE_ONLY_SHOW_ERRORS=True - export GINKGO_FOCUS="should apply various network policies and enforce access to nginx" echo Cluster Def $CLUSTER_DEFINITION cat $CLUSTER_DEFINITION make test-kubernetes