From 3a522d4c5e9ed54937262de2563a9b36ac468a76 Mon Sep 17 00:00:00 2001 From: John Schnake Date: Mon, 16 Aug 2021 10:17:53 -0500 Subject: [PATCH] Fix regression regarding certified-conformance mode When the mode is set as certified conformance mode, the skip value does not get properly reset and as a result skips the disruptive tests. Fixes: #1388 Signed-off-by: John Schnake --- cmd/sonobuoy/app/args.go | 6 + test/integration/sonobuoy_integration_test.go | 14 +- .../testdata/gen-config-no-flags.golden | 6 +- .../testdata/gen-config-then-flags.golden | 6 +- .../testdata/gen-issue-1375.golden | 6 +- .../testdata/gen-issue-1388.golden | 203 ++++++++++++++++++ test/integration/testdata/gen-no-uuid.golden | 6 +- .../testdata/gen-subfield-flags.golden | 6 +- 8 files changed, 233 insertions(+), 20 deletions(-) create mode 100644 test/integration/testdata/gen-issue-1388.golden diff --git a/cmd/sonobuoy/app/args.go b/cmd/sonobuoy/app/args.go index fa3f705d1..2e719bf3e 100644 --- a/cmd/sonobuoy/app/args.go +++ b/cmd/sonobuoy/app/args.go @@ -466,10 +466,16 @@ func (m *Mode) Set(str string) error { if err := m.env.Set(fmt.Sprintf("e2e.E2E_FOCUS=%v", quickFocus)); err != nil { return fmt.Errorf("failed to set flag with value %v", str) } + if err := m.env.Set("e2e.E2E_SKIP"); err != nil { + return fmt.Errorf("failed to set flag with value %v", str) + } case CertifiedConformance: if err := m.env.Set(fmt.Sprintf("e2e.E2E_FOCUS=%v", conformanceFocus)); err != nil { return fmt.Errorf("failed to set flag with value %v", str) } + if err := m.env.Set("e2e.E2E_SKIP"); err != nil { + return fmt.Errorf("failed to set flag with value %v", str) + } default: return fmt.Errorf("unknown mode %v", lcase) } diff --git a/test/integration/sonobuoy_integration_test.go b/test/integration/sonobuoy_integration_test.go index 1e6bd8c57..56c25d09c 100644 --- a/test/integration/sonobuoy_integration_test.go +++ b/test/integration/sonobuoy_integration_test.go @@ -495,7 +495,7 @@ func TestExactOutput_LocalGolden(t *testing.T) { expectFile: "testdata/gen-variable-image.golden", }, { desc: "gen doesnt provide UUID", - cmdLine: "gen", + cmdLine: "gen --kubernetes-version=ignore", expectFile: "testdata/gen-no-uuid.golden", }, { desc: "gen config doesnt provide UUID", @@ -503,15 +503,15 @@ func TestExactOutput_LocalGolden(t *testing.T) { expectFile: "testdata/gen-config-no-uuid.golden", }, { desc: "gen with config testing fields that also have flags", - cmdLine: "gen --config=testdata/subfieldTest.json", + cmdLine: "gen --config=testdata/subfieldTest.json --kubernetes-version=ignore", expectFile: "testdata/gen-config-no-flags.golden", }, { desc: "gen with flags targeting nested config fields", - cmdLine: "gen -n=cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99", + cmdLine: "gen -n=cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99 --kubernetes-version=ignore", expectFile: "testdata/gen-subfield-flags.golden", }, { desc: "gen with config then flags targeting subfields", - cmdLine: "gen --config=testdata/subfieldTest.json -n cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99", + cmdLine: "gen --config=testdata/subfieldTest.json -n cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99 --kubernetes-version=ignore", expectFile: "testdata/gen-config-then-flags.golden", }, { desc: "gen respects kube-conformance-image for both plugin and config issue 1376", @@ -519,8 +519,12 @@ func TestExactOutput_LocalGolden(t *testing.T) { expectFile: "testdata/gen-issue-1376.golden", }, { desc: "e2e-repo-config should cause KUBE_TEST_REPO_LIST env var to match location used for mount", - cmdLine: "gen --e2e-repo-config=./testdata/tiny-configmap.yaml", + cmdLine: "gen --e2e-repo-config=./testdata/tiny-configmap.yaml --kubernetes-version=ignore", expectFile: "testdata/gen-issue-1375.golden", + }, { + desc: "certified conformance should have no skip value", + cmdLine: "gen --mode=certified-conformance --kubernetes-version=ignore", + expectFile: "testdata/gen-issue-1388.golden", }, } for _, tc := range testCases { diff --git a/test/integration/testdata/gen-config-no-flags.golden b/test/integration/testdata/gen-config-no-flags.golden index 19fbeb9dd..3a3909c7e 100644 --- a/test/integration/testdata/gen-config-no-flags.golden +++ b/test/integration/testdata/gen-config-no-flags.golden @@ -96,8 +96,8 @@ data: - name: E2E_USE_GO_RUNNER value: "true" - name: SONOBUOY_K8S_VERSION - value: v1.20.0 - image: k8s.gcr.io/conformance:v1.20.0 + value: ignore + image: k8s.gcr.io/conformance:ignore imagePullPolicy: Never name: e2e resources: {} @@ -125,7 +125,7 @@ data: - name: RESULTS_DIR value: /tmp/results - name: SONOBUOY_K8S_VERSION - value: v1.20.0 + value: ignore image: sonobuoy/systemd-logs:v0.3 imagePullPolicy: Never name: systemd-logs diff --git a/test/integration/testdata/gen-config-then-flags.golden b/test/integration/testdata/gen-config-then-flags.golden index 894b9d369..f914fc3ce 100644 --- a/test/integration/testdata/gen-config-then-flags.golden +++ b/test/integration/testdata/gen-config-then-flags.golden @@ -96,8 +96,8 @@ data: - name: E2E_USE_GO_RUNNER value: "true" - name: SONOBUOY_K8S_VERSION - value: v1.20.0 - image: k8s.gcr.io/conformance:v1.20.0 + value: ignore + image: k8s.gcr.io/conformance:ignore imagePullPolicy: Always name: e2e resources: {} @@ -125,7 +125,7 @@ data: - name: RESULTS_DIR value: /tmp/results - name: SONOBUOY_K8S_VERSION - value: v1.20.0 + value: ignore image: sonobuoy/systemd-logs:v0.3 imagePullPolicy: Always name: systemd-logs diff --git a/test/integration/testdata/gen-issue-1375.golden b/test/integration/testdata/gen-issue-1375.golden index 968e7c820..af8007df2 100644 --- a/test/integration/testdata/gen-issue-1375.golden +++ b/test/integration/testdata/gen-issue-1375.golden @@ -106,8 +106,8 @@ data: - name: KUBE_TEST_REPO_LIST value: /tmp/sonobuoy/config/tiny-configmap.yaml - name: SONOBUOY_K8S_VERSION - value: v1.20.0 - image: k8s.gcr.io/conformance:v1.20.0 + value: ignore + image: k8s.gcr.io/conformance:ignore imagePullPolicy: IfNotPresent name: e2e resources: {} @@ -137,7 +137,7 @@ data: - name: RESULTS_DIR value: /tmp/results - name: SONOBUOY_K8S_VERSION - value: v1.20.0 + value: ignore image: sonobuoy/systemd-logs:v0.3 imagePullPolicy: IfNotPresent name: systemd-logs diff --git a/test/integration/testdata/gen-issue-1388.golden b/test/integration/testdata/gen-issue-1388.golden new file mode 100644 index 000000000..f732e5f7f --- /dev/null +++ b/test/integration/testdata/gen-issue-1388.golden @@ -0,0 +1,203 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sonobuoy +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount + namespace: sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + component: sonobuoy + namespace: sonobuoy + name: sonobuoy-serviceaccount-sonobuoy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sonobuoy-serviceaccount-sonobuoy +subjects: +- kind: ServiceAccount + name: sonobuoy-serviceaccount + namespace: sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + component: sonobuoy + namespace: sonobuoy + name: sonobuoy-serviceaccount-sonobuoy +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - '/metrics' + - '/logs' + - '/logs/*' + verbs: + - 'get' +--- +apiVersion: v1 +data: + config.json: | + {"Description":"DEFAULT","UUID":"","Version":"*STATIC_FOR_TESTING*","ResultsDir":"/tmp/sonobuoy","Resources":["apiservices","certificatesigningrequests","clusterrolebindings","clusterroles","componentstatuses","configmaps","controllerrevisions","cronjobs","customresourcedefinitions","daemonsets","deployments","endpoints","ingresses","jobs","leases","limitranges","mutatingwebhookconfigurations","namespaces","networkpolicies","nodes","persistentvolumeclaims","persistentvolumes","poddisruptionbudgets","pods","podlogs","podsecuritypolicies","podtemplates","priorityclasses","replicasets","replicationcontrollers","resourcequotas","rolebindings","roles","servergroups","serverversion","serviceaccounts","services","statefulsets","storageclasses","validatingwebhookconfigurations","volumeattachments"],"Filters":{"Namespaces":".*","LabelSelector":""},"Limits":{"PodLogs":{"Namespaces":"","SonobuoyNamespace":true,"FieldSelectors":[],"LabelSelector":"","Previous":false,"SinceSeconds":null,"SinceTime":null,"Timestamps":false,"TailLines":null,"LimitBytes":null,"LimitSize":"","LimitTime":""}},"QPS":30,"Burst":50,"Server":{"bindaddress":"0.0.0.0","bindport":8080,"advertiseaddress":"","timeoutseconds":21600},"Plugins":null,"PluginSearchPath":["./plugins.d","/etc/sonobuoy/plugins.d","~/sonobuoy/plugins.d"],"Namespace":"sonobuoy","WorkerImage":"sonobuoy/sonobuoy:*STATIC_FOR_TESTING*","ImagePullPolicy":"IfNotPresent","ImagePullSecrets":"","ProgressUpdatesPort":"8099"} +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-config-cm + namespace: sonobuoy +--- +apiVersion: v1 +data: + plugin-0.yaml: | + podSpec: + containers: [] + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - key: kubernetes.io/e2e-evict-taint-key + operator: Exists + sonobuoy-config: + driver: Job + plugin-name: e2e + result-format: junit + spec: + command: + - /run_e2e.sh + env: + - name: E2E_EXTRA_ARGS + value: --progress-report-url=http://localhost:8099/progress + - name: E2E_FOCUS + value: \[Conformance\] + - name: E2E_PARALLEL + value: "false" + - name: E2E_USE_GO_RUNNER + value: "true" + - name: SONOBUOY_K8S_VERSION + value: ignore + image: k8s.gcr.io/conformance:ignore + imagePullPolicy: IfNotPresent + name: e2e + resources: {} + volumeMounts: + - mountPath: /tmp/results + name: results + plugin-1.yaml: | + sonobuoy-config: + driver: DaemonSet + plugin-name: systemd-logs + result-format: raw + spec: + command: + - /bin/sh + - -c + - /get_systemd_logs.sh; while true; do echo "Plugin is complete. Sleeping indefinitely + to avoid container exit and automatic restarts from Kubernetes"; sleep 3600; done + env: + - name: CHROOT_DIR + value: /node + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: RESULTS_DIR + value: /tmp/results + - name: SONOBUOY_K8S_VERSION + value: ignore + image: sonobuoy/systemd-logs:v0.3 + imagePullPolicy: IfNotPresent + name: systemd-logs + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp/results + name: results + - mountPath: /node + name: root +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-plugins-cm + namespace: sonobuoy +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + component: sonobuoy + run: sonobuoy-master + sonobuoy-component: aggregator + tier: analysis + name: sonobuoy + namespace: sonobuoy +spec: + containers: + - env: + - name: SONOBUOY_ADVERTISE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: sonobuoy/sonobuoy:*STATIC_FOR_TESTING* + imagePullPolicy: IfNotPresent + name: kube-sonobuoy + volumeMounts: + - mountPath: /etc/sonobuoy + name: sonobuoy-config-volume + - mountPath: /plugins.d + name: sonobuoy-plugins-volume + - mountPath: /tmp/sonobuoy + name: output-volume + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + tolerations: + - key: "kubernetes.io/e2e-evict-taint-key" + operator: "Exists" + volumes: + - configMap: + name: sonobuoy-config-cm + name: sonobuoy-config-volume + - configMap: + name: sonobuoy-plugins-cm + name: sonobuoy-plugins-volume + - emptyDir: {} + name: output-volume +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: sonobuoy + sonobuoy-component: aggregator + name: sonobuoy-aggregator + namespace: sonobuoy +spec: + ports: + - port: 8080 + protocol: TCP + targetPort: 8080 + selector: + sonobuoy-component: aggregator + type: ClusterIP + diff --git a/test/integration/testdata/gen-no-uuid.golden b/test/integration/testdata/gen-no-uuid.golden index e49e39c8d..fa50e13b9 100644 --- a/test/integration/testdata/gen-no-uuid.golden +++ b/test/integration/testdata/gen-no-uuid.golden @@ -96,8 +96,8 @@ data: - name: E2E_USE_GO_RUNNER value: "true" - name: SONOBUOY_K8S_VERSION - value: v1.20.0 - image: k8s.gcr.io/conformance:v1.20.0 + value: ignore + image: k8s.gcr.io/conformance:ignore imagePullPolicy: IfNotPresent name: e2e resources: {} @@ -125,7 +125,7 @@ data: - name: RESULTS_DIR value: /tmp/results - name: SONOBUOY_K8S_VERSION - value: v1.20.0 + value: ignore image: sonobuoy/systemd-logs:v0.3 imagePullPolicy: IfNotPresent name: systemd-logs diff --git a/test/integration/testdata/gen-subfield-flags.golden b/test/integration/testdata/gen-subfield-flags.golden index 11731ec33..402d539eb 100644 --- a/test/integration/testdata/gen-subfield-flags.golden +++ b/test/integration/testdata/gen-subfield-flags.golden @@ -96,8 +96,8 @@ data: - name: E2E_USE_GO_RUNNER value: "true" - name: SONOBUOY_K8S_VERSION - value: v1.20.0 - image: k8s.gcr.io/conformance:v1.20.0 + value: ignore + image: k8s.gcr.io/conformance:ignore imagePullPolicy: Always name: e2e resources: {} @@ -125,7 +125,7 @@ data: - name: RESULTS_DIR value: /tmp/results - name: SONOBUOY_K8S_VERSION - value: v1.20.0 + value: ignore image: sonobuoy/systemd-logs:v0.3 imagePullPolicy: Always name: systemd-logs