Skip to content

Commit

Permalink
Allow setting aggregator env vars via CLI
Browse files Browse the repository at this point in the history
In a few situations we may want to set env vars on the
aggregator via the CLI. This is just a convenience change
but makes the plugin name `sonobuoy` reserved to avoid
an issue.

Signed-off-by: John Schnake <jschnake@vmware.com>
  • Loading branch information
johnSchnake committed Jan 9, 2022
1 parent 928b7b4 commit 94b6f18
Show file tree
Hide file tree
Showing 3 changed files with 281 additions and 12 deletions.
41 changes: 29 additions & 12 deletions pkg/client/gen.go
Expand Up @@ -45,6 +45,8 @@ const (
e2ePluginName = "e2e"
systemdLogsName = "systemd-logs"

aggregatorEnvOverrideKey = `sonobuoy`

envVarKeyExtraArgs = "E2E_EXTRA_ARGS"

// sonobuoyKey is just a true/false env to indicate that the container was launched/tagged by Sonobuoy.
Expand Down Expand Up @@ -175,7 +177,7 @@ func (*SonobuoyClient) GenerateManifestAndPlugins(cfg *GenConfig) ([]byte, []*ma
)
}

err := checkPluginsUnique(plugins)
err := checkPluginNames(plugins)
if err != nil {
return nil, nil, errors.Wrap(err, "plugin YAML generation")
}
Expand Down Expand Up @@ -379,6 +381,10 @@ func generateAggregatorAndService(w io.Writer, cfg *GenConfig) error {
FSGroup: &defaultFSGroup,
}
}
if len(cfg.PluginEnvOverrides) > 0 && cfg.PluginEnvOverrides[aggregatorEnvOverrideKey] != nil {
newEnv, removeVals := processEnvVals(cfg.PluginEnvOverrides[aggregatorEnvOverrideKey])
p.Spec.Containers[0].Env = mergeEnv(newEnv, p.Spec.Containers[0].Env, removeVals)
}
ser := &corev1.Service{
Spec: corev1.ServiceSpec{
Selector: map[string]string{"sonobuoy-component": "aggregator"},
Expand Down Expand Up @@ -608,15 +614,7 @@ func applyEnvOverrides(pluginEnvOverrides map[string]map[string]string, plugins
for _, p := range plugins {
if p.SonobuoyConfig.PluginName == pluginName {
found = true
newEnv := []corev1.EnvVar{}
removeVals := map[string]struct{}{}
for k, v := range envVars {
if v != "" {
newEnv = append(newEnv, corev1.EnvVar{Name: k, Value: v})
} else {
removeVals[k] = struct{}{}
}
}
newEnv, removeVals := processEnvVals(envVars)
p.Spec.Env = mergeEnv(newEnv, p.Spec.Env, removeVals)
if p.PodSpec != nil {
for i := range p.PodSpec.Containers {
Expand All @@ -629,7 +627,7 @@ func applyEnvOverrides(pluginEnvOverrides map[string]map[string]string, plugins
// Require overrides to target existing plugins and provide a helpful message if there is a mismatch.
// Dont error if the plugin in question is "e2e" since we default to setting those values regardless of
// if they choose that plugin or not.
if !found && pluginName != e2ePluginName {
if !found && pluginName != e2ePluginName && pluginName != aggregatorEnvOverrideKey {
pluginNames := []string{}
for _, p := range plugins {
pluginNames = append(pluginNames, p.SonobuoyConfig.PluginName)
Expand All @@ -641,6 +639,22 @@ func applyEnvOverrides(pluginEnvOverrides map[string]map[string]string, plugins
return nil
}

// processEnvVals takes a map of key/value pairs representing env vars (from the --plugin-env flag)
// and processes them to represent which envs should be added vs removed. These can then be
// used with the mergeEnv function as needed to assign them into a pod.
func processEnvVals(envVars map[string]string) ([]corev1.EnvVar, map[string]struct{}) {
newEnv := []corev1.EnvVar{}
removeVals := map[string]struct{}{}
for k, v := range envVars {
if v != "" {
newEnv = append(newEnv, corev1.EnvVar{Name: k, Value: v})
} else {
removeVals[k] = struct{}{}
}
}
return newEnv, removeVals
}

// autoAttachResultsDir will either add the volumemount for the results dir or modify the existing
// one to have the right path set.
func autoAttachResultsDir(plugins []*manifest.Manifest, resultsDir string) {
Expand Down Expand Up @@ -694,9 +708,12 @@ func applyAutoEnvVars(imageVersion, resultsDir, progressPort string, env map[str
return env, plugins
}

func checkPluginsUnique(plugins []*manifest.Manifest) error {
func checkPluginNames(plugins []*manifest.Manifest) error {
names := map[string]struct{}{}
for _, v := range plugins {
if v.SonobuoyConfig.PluginName == aggregatorEnvOverrideKey {
return fmt.Errorf("plugin name %q is a reserved value", aggregatorEnvOverrideKey)
}
if _, exists := names[v.SonobuoyConfig.PluginName]; exists {
return fmt.Errorf("plugin names must be unique, got duplicated plugin name '%v'", v.SonobuoyConfig.PluginName)
}
Expand Down
4 changes: 4 additions & 0 deletions test/integration/sonobuoy_integration_test.go
Expand Up @@ -729,6 +729,10 @@ func TestExactOutput_LocalGolden(t *testing.T) {
desc: "sonobuoy modes command",
cmdLine: "modes",
expectFile: "testdata/modes.golden",
}, {
desc: "sonobuoy plugin-env supports aggregator",
cmdLine: "gen --plugin-env=sonobuoy.FOO=bar --kubernetes-version=ignore",
expectFile: "testdata/gen-plugin-env-sonobuoy.golden",
},
}
for _, tc := range testCases {
Expand Down
248 changes: 248 additions & 0 deletions test/integration/testdata/gen-plugin-env-sonobuoy.golden
@@ -0,0 +1,248 @@
apiVersion: v1
kind: Namespace
metadata:
name: sonobuoy
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
component: sonobuoy
name: sonobuoy-serviceaccount
namespace: sonobuoy
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
component: sonobuoy
namespace: sonobuoy
name: sonobuoy-serviceaccount-sonobuoy
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: sonobuoy-serviceaccount-sonobuoy
subjects:
- kind: ServiceAccount
name: sonobuoy-serviceaccount
namespace: sonobuoy
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
component: sonobuoy
namespace: sonobuoy
name: sonobuoy-serviceaccount-sonobuoy
rules:
- apiGroups:
- '*'
resources:
- '*'
verbs:
- '*'
- nonResourceURLs:
- /metrics
- /logs
- /logs/*
verbs:
- get
---
apiVersion: v1
data:
config.json: '{"Description":"DEFAULT","UUID":"","Version":"*STATIC_FOR_TESTING*","ResultsDir":"/tmp/sonobuoy/results","Resources":null,"Filters":{"Namespaces":".*","LabelSelector":""},"Limits":{"PodLogs":{"Namespaces":"kube-system","SonobuoyNamespace":true,"FieldSelectors":[],"LabelSelector":"","Previous":false,"SinceSeconds":null,"SinceTime":null,"Timestamps":false,"TailLines":null,"LimitBytes":null}},"QPS":30,"Burst":50,"Server":{"bindaddress":"0.0.0.0","bindport":8080,"advertiseaddress":"","timeoutseconds":21600},"Plugins":null,"PluginSearchPath":["./plugins.d","/etc/sonobuoy/plugins.d","~/sonobuoy/plugins.d"],"Namespace":"sonobuoy","WorkerImage":"sonobuoy/sonobuoy:*STATIC_FOR_TESTING*","ImagePullPolicy":"IfNotPresent","ImagePullSecrets":"","AggregatorPermissions":"clusterAdmin","ProgressUpdatesPort":"8099","SecurityContextMode":"nonroot"}'
kind: ConfigMap
metadata:
labels:
component: sonobuoy
name: sonobuoy-config-cm
namespace: sonobuoy
---
apiVersion: v1
data:
plugin-0.yaml: |-
podSpec:
containers: []
nodeSelector:
kubernetes.io/os: linux
restartPolicy: Never
serviceAccountName: sonobuoy-serviceaccount
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
- key: kubernetes.io/e2e-evict-taint-key
operator: Exists
sonobuoy-config:
driver: Job
plugin-name: e2e
result-format: junit
spec:
command:
- /run_e2e.sh
env:
- name: E2E_EXTRA_ARGS
value: --progress-report-url=http://localhost:8099/progress
- name: E2E_FOCUS
value: \[Conformance\]
- name: E2E_PARALLEL
value: "false"
- name: E2E_SKIP
value: \[Disruptive\]|NoExecuteTaintManager
- name: E2E_USE_GO_RUNNER
value: "true"
- name: RESULTS_DIR
value: /tmp/sonobuoy/results
- name: SONOBUOY
value: "true"
- name: SONOBUOY_CONFIG_DIR
value: /tmp/sonobuoy/config
- name: SONOBUOY_K8S_VERSION
value: ignore
- name: SONOBUOY_PROGRESS_PORT
value: "8099"
- name: SONOBUOY_RESULTS_DIR
value: /tmp/sonobuoy/results
image: k8s.gcr.io/conformance:ignore
imagePullPolicy: IfNotPresent
name: e2e
volumeMounts:
- mountPath: /tmp/sonobuoy/results
name: results
plugin-1.yaml: |-
podSpec:
containers: []
dnsPolicy: ClusterFirstWithHostNet
hostIPC: true
hostNetwork: true
hostPID: true
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: sonobuoy-serviceaccount
tolerations:
- operator: Exists
volumes:
- hostPath:
path: /
name: root
sonobuoy-config:
driver: DaemonSet
plugin-name: systemd-logs
result-format: raw
spec:
command:
- /bin/sh
- -c
- /get_systemd_logs.sh; while true; do echo "Plugin is complete. Sleeping indefinitely
to avoid container exit and automatic restarts from Kubernetes"; sleep 3600; done
env:
- name: CHROOT_DIR
value: /node
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: RESULTS_DIR
value: /tmp/sonobuoy/results
- name: SONOBUOY
value: "true"
- name: SONOBUOY_CONFIG_DIR
value: /tmp/sonobuoy/config
- name: SONOBUOY_K8S_VERSION
value: ignore
- name: SONOBUOY_PROGRESS_PORT
value: "8099"
- name: SONOBUOY_RESULTS_DIR
value: /tmp/sonobuoy/results
image: sonobuoy/systemd-logs:v0.4
imagePullPolicy: IfNotPresent
name: systemd-logs
securityContext:
privileged: true
volumeMounts:
- mountPath: /node
name: root
- mountPath: /tmp/sonobuoy/results
name: results
kind: ConfigMap
metadata:
labels:
component: sonobuoy
name: sonobuoy-plugins-cm
namespace: sonobuoy
---
apiVersion: v1
kind: Pod
metadata:
labels:
component: sonobuoy
sonobuoy-component: aggregator
tier: analysis
name: sonobuoy
namespace: sonobuoy
spec:
containers:
- args:
- aggregator
- --no-exit
- --level=info
- -v=4
- --alsologtostderr
command:
- /sonobuoy
env:
- name: FOO
value: bar
- name: SONOBUOY_ADVERTISE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
image: sonobuoy/sonobuoy:*STATIC_FOR_TESTING*
imagePullPolicy: IfNotPresent
name: kube-sonobuoy
volumeMounts:
- mountPath: /etc/sonobuoy
name: sonobuoy-config-volume
- mountPath: /plugins.d
name: sonobuoy-plugins-volume
- mountPath: /tmp/sonobuoy
name: output-volume
restartPolicy: Never
securityContext:
fsGroup: 2000
runAsGroup: 3000
runAsUser: 1000
serviceAccountName: sonobuoy-serviceaccount
tolerations:
- key: kubernetes.io/e2e-evict-taint-key
operator: Exists
volumes:
- configMap:
name: sonobuoy-config-cm
name: sonobuoy-config-volume
- configMap:
name: sonobuoy-plugins-cm
name: sonobuoy-plugins-volume
- emptyDir: {}
name: output-volume
---
apiVersion: v1
kind: Service
metadata:
labels:
component: sonobuoy
sonobuoy-component: aggregator
name: sonobuoy-aggregator
namespace: sonobuoy
spec:
ports:
- port: 8080
protocol: TCP
targetPort: 8080
selector:
sonobuoy-component: aggregator
type: ClusterIP
---

0 comments on commit 94b6f18

Please sign in to comment.