Skip to content

Commit

Permalink
Fix regression regarding certified-conformance mode
Browse files Browse the repository at this point in the history
When the mode is set as certified conformance mode, the skip value
does not get properly reset and as a result skips the disruptive
tests.

Fixes: #1388

Signed-off-by: John Schnake <jschnake@vmware.com>
  • Loading branch information
johnSchnake committed Aug 16, 2021
1 parent 0f7c4a2 commit 3a522d4
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 20 deletions.
6 changes: 6 additions & 0 deletions cmd/sonobuoy/app/args.go
Expand Up @@ -466,10 +466,16 @@ func (m *Mode) Set(str string) error {
if err := m.env.Set(fmt.Sprintf("e2e.E2E_FOCUS=%v", quickFocus)); err != nil {
return fmt.Errorf("failed to set flag with value %v", str)
}
if err := m.env.Set("e2e.E2E_SKIP"); err != nil {
return fmt.Errorf("failed to set flag with value %v", str)
}
case CertifiedConformance:
if err := m.env.Set(fmt.Sprintf("e2e.E2E_FOCUS=%v", conformanceFocus)); err != nil {
return fmt.Errorf("failed to set flag with value %v", str)
}
if err := m.env.Set("e2e.E2E_SKIP"); err != nil {
return fmt.Errorf("failed to set flag with value %v", str)
}
default:
return fmt.Errorf("unknown mode %v", lcase)
}
Expand Down
14 changes: 9 additions & 5 deletions test/integration/sonobuoy_integration_test.go
Expand Up @@ -495,32 +495,36 @@ func TestExactOutput_LocalGolden(t *testing.T) {
expectFile: "testdata/gen-variable-image.golden",
}, {
desc: "gen doesnt provide UUID",
cmdLine: "gen",
cmdLine: "gen --kubernetes-version=ignore",
expectFile: "testdata/gen-no-uuid.golden",
}, {
desc: "gen config doesnt provide UUID",
cmdLine: "gen config",
expectFile: "testdata/gen-config-no-uuid.golden",
}, {
desc: "gen with config testing fields that also have flags",
cmdLine: "gen --config=testdata/subfieldTest.json",
cmdLine: "gen --config=testdata/subfieldTest.json --kubernetes-version=ignore",
expectFile: "testdata/gen-config-no-flags.golden",
}, {
desc: "gen with flags targeting nested config fields",
cmdLine: "gen -n=cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99",
cmdLine: "gen -n=cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99 --kubernetes-version=ignore",
expectFile: "testdata/gen-subfield-flags.golden",
}, {
desc: "gen with config then flags targeting subfields",
cmdLine: "gen --config=testdata/subfieldTest.json -n cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99",
cmdLine: "gen --config=testdata/subfieldTest.json -n cmdlineNS --image-pull-policy=Always --sonobuoy-image=cmdlineimg --timeout=99 --kubernetes-version=ignore",
expectFile: "testdata/gen-config-then-flags.golden",
}, {
desc: "gen respects kube-conformance-image for both plugin and config issue 1376",
cmdLine: "gen --kube-conformance-image=custom-image --kubernetes-version=v9.8.7",
expectFile: "testdata/gen-issue-1376.golden",
}, {
desc: "e2e-repo-config should cause KUBE_TEST_REPO_LIST env var to match location used for mount",
cmdLine: "gen --e2e-repo-config=./testdata/tiny-configmap.yaml",
cmdLine: "gen --e2e-repo-config=./testdata/tiny-configmap.yaml --kubernetes-version=ignore",
expectFile: "testdata/gen-issue-1375.golden",
}, {
desc: "certified conformance should have no skip value",
cmdLine: "gen --mode=certified-conformance --kubernetes-version=ignore",
expectFile: "testdata/gen-issue-1388.golden",
},
}
for _, tc := range testCases {
Expand Down
6 changes: 3 additions & 3 deletions test/integration/testdata/gen-config-no-flags.golden
Expand Up @@ -96,8 +96,8 @@ data:
- name: E2E_USE_GO_RUNNER
value: "true"
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
image: k8s.gcr.io/conformance:v1.20.0
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: Never
name: e2e
resources: {}
Expand Down Expand Up @@ -125,7 +125,7 @@ data:
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: Never
name: systemd-logs
Expand Down
6 changes: 3 additions & 3 deletions test/integration/testdata/gen-config-then-flags.golden
Expand Up @@ -96,8 +96,8 @@ data:
- name: E2E_USE_GO_RUNNER
value: "true"
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
image: k8s.gcr.io/conformance:v1.20.0
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: Always
name: e2e
resources: {}
Expand Down Expand Up @@ -125,7 +125,7 @@ data:
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: Always
name: systemd-logs
Expand Down
6 changes: 3 additions & 3 deletions test/integration/testdata/gen-issue-1375.golden
Expand Up @@ -106,8 +106,8 @@ data:
- name: KUBE_TEST_REPO_LIST
value: /tmp/sonobuoy/config/tiny-configmap.yaml
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
image: k8s.gcr.io/conformance:v1.20.0
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: IfNotPresent
name: e2e
resources: {}
Expand Down Expand Up @@ -137,7 +137,7 @@ data:
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: IfNotPresent
name: systemd-logs
Expand Down
203 changes: 203 additions & 0 deletions test/integration/testdata/gen-issue-1388.golden
@@ -0,0 +1,203 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: sonobuoy
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
component: sonobuoy
name: sonobuoy-serviceaccount
namespace: sonobuoy
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
component: sonobuoy
namespace: sonobuoy
name: sonobuoy-serviceaccount-sonobuoy
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: sonobuoy-serviceaccount-sonobuoy
subjects:
- kind: ServiceAccount
name: sonobuoy-serviceaccount
namespace: sonobuoy
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
component: sonobuoy
namespace: sonobuoy
name: sonobuoy-serviceaccount-sonobuoy
rules:
- apiGroups:
- '*'
resources:
- '*'
verbs:
- '*'
- nonResourceURLs:
- '/metrics'
- '/logs'
- '/logs/*'
verbs:
- 'get'
---
apiVersion: v1
data:
config.json: |
{"Description":"DEFAULT","UUID":"","Version":"*STATIC_FOR_TESTING*","ResultsDir":"/tmp/sonobuoy","Resources":["apiservices","certificatesigningrequests","clusterrolebindings","clusterroles","componentstatuses","configmaps","controllerrevisions","cronjobs","customresourcedefinitions","daemonsets","deployments","endpoints","ingresses","jobs","leases","limitranges","mutatingwebhookconfigurations","namespaces","networkpolicies","nodes","persistentvolumeclaims","persistentvolumes","poddisruptionbudgets","pods","podlogs","podsecuritypolicies","podtemplates","priorityclasses","replicasets","replicationcontrollers","resourcequotas","rolebindings","roles","servergroups","serverversion","serviceaccounts","services","statefulsets","storageclasses","validatingwebhookconfigurations","volumeattachments"],"Filters":{"Namespaces":".*","LabelSelector":""},"Limits":{"PodLogs":{"Namespaces":"","SonobuoyNamespace":true,"FieldSelectors":[],"LabelSelector":"","Previous":false,"SinceSeconds":null,"SinceTime":null,"Timestamps":false,"TailLines":null,"LimitBytes":null,"LimitSize":"","LimitTime":""}},"QPS":30,"Burst":50,"Server":{"bindaddress":"0.0.0.0","bindport":8080,"advertiseaddress":"","timeoutseconds":21600},"Plugins":null,"PluginSearchPath":["./plugins.d","/etc/sonobuoy/plugins.d","~/sonobuoy/plugins.d"],"Namespace":"sonobuoy","WorkerImage":"sonobuoy/sonobuoy:*STATIC_FOR_TESTING*","ImagePullPolicy":"IfNotPresent","ImagePullSecrets":"","ProgressUpdatesPort":"8099"}
kind: ConfigMap
metadata:
labels:
component: sonobuoy
name: sonobuoy-config-cm
namespace: sonobuoy
---
apiVersion: v1
data:
plugin-0.yaml: |
podSpec:
containers: []
nodeSelector:
kubernetes.io/os: linux
restartPolicy: Never
serviceAccountName: sonobuoy-serviceaccount
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
- key: kubernetes.io/e2e-evict-taint-key
operator: Exists
sonobuoy-config:
driver: Job
plugin-name: e2e
result-format: junit
spec:
command:
- /run_e2e.sh
env:
- name: E2E_EXTRA_ARGS
value: --progress-report-url=http://localhost:8099/progress
- name: E2E_FOCUS
value: \[Conformance\]
- name: E2E_PARALLEL
value: "false"
- name: E2E_USE_GO_RUNNER
value: "true"
- name: SONOBUOY_K8S_VERSION
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: IfNotPresent
name: e2e
resources: {}
volumeMounts:
- mountPath: /tmp/results
name: results
plugin-1.yaml: |
sonobuoy-config:
driver: DaemonSet
plugin-name: systemd-logs
result-format: raw
spec:
command:
- /bin/sh
- -c
- /get_systemd_logs.sh; while true; do echo "Plugin is complete. Sleeping indefinitely
to avoid container exit and automatic restarts from Kubernetes"; sleep 3600; done
env:
- name: CHROOT_DIR
value: /node
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: IfNotPresent
name: systemd-logs
resources: {}
securityContext:
privileged: true
volumeMounts:
- mountPath: /tmp/results
name: results
- mountPath: /node
name: root
kind: ConfigMap
metadata:
labels:
component: sonobuoy
name: sonobuoy-plugins-cm
namespace: sonobuoy
---
apiVersion: v1
kind: Pod
metadata:
labels:
component: sonobuoy
run: sonobuoy-master
sonobuoy-component: aggregator
tier: analysis
name: sonobuoy
namespace: sonobuoy
spec:
containers:
- env:
- name: SONOBUOY_ADVERTISE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
image: sonobuoy/sonobuoy:*STATIC_FOR_TESTING*
imagePullPolicy: IfNotPresent
name: kube-sonobuoy
volumeMounts:
- mountPath: /etc/sonobuoy
name: sonobuoy-config-volume
- mountPath: /plugins.d
name: sonobuoy-plugins-volume
- mountPath: /tmp/sonobuoy
name: output-volume
restartPolicy: Never
serviceAccountName: sonobuoy-serviceaccount
tolerations:
- key: "kubernetes.io/e2e-evict-taint-key"
operator: "Exists"
volumes:
- configMap:
name: sonobuoy-config-cm
name: sonobuoy-config-volume
- configMap:
name: sonobuoy-plugins-cm
name: sonobuoy-plugins-volume
- emptyDir: {}
name: output-volume
---
apiVersion: v1
kind: Service
metadata:
labels:
component: sonobuoy
sonobuoy-component: aggregator
name: sonobuoy-aggregator
namespace: sonobuoy
spec:
ports:
- port: 8080
protocol: TCP
targetPort: 8080
selector:
sonobuoy-component: aggregator
type: ClusterIP

6 changes: 3 additions & 3 deletions test/integration/testdata/gen-no-uuid.golden
Expand Up @@ -96,8 +96,8 @@ data:
- name: E2E_USE_GO_RUNNER
value: "true"
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
image: k8s.gcr.io/conformance:v1.20.0
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: IfNotPresent
name: e2e
resources: {}
Expand Down Expand Up @@ -125,7 +125,7 @@ data:
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: IfNotPresent
name: systemd-logs
Expand Down
6 changes: 3 additions & 3 deletions test/integration/testdata/gen-subfield-flags.golden
Expand Up @@ -96,8 +96,8 @@ data:
- name: E2E_USE_GO_RUNNER
value: "true"
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
image: k8s.gcr.io/conformance:v1.20.0
value: ignore
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: Always
name: e2e
resources: {}
Expand Down Expand Up @@ -125,7 +125,7 @@ data:
- name: RESULTS_DIR
value: /tmp/results
- name: SONOBUOY_K8S_VERSION
value: v1.20.0
value: ignore
image: sonobuoy/systemd-logs:v0.3
imagePullPolicy: Always
name: systemd-logs
Expand Down

0 comments on commit 3a522d4

Please sign in to comment.