Allow DNS pod selectors to be configured (#987)

The preflight checks for Sonobuoy assume the namespace and labels of the
DNS pods. Although the preflight checks can be skipped, some users may
want to still run these checks and ensure that they pass before testing.
To allow this, this change introduces two new flags that are applied to
the `PreflightConfig` so that the namespace and labels to use for
finding the DNS pods can be configured.

The two new flags are: `--dns-namespace` and `--dns-pod-labels`.
The default values are the values that were previously being used so
this does not impact existing users.

Signed-off-by: Bridget McErlean <bmcerlean@vmware.com>

cmd/sonobuoy/app/args.go

@@ -50,6 +50,26 @@ func AddNamespaceFlag(str *string, flags *pflag.FlagSet) {
 	)
 }
 
+// AddDNSNamespaceFlag initialises the dns-namespace flag.
+// The value of this flag is used during preflight checks to determine which namespace to use
+// when looking for the DNS pods.
+func AddDNSNamespaceFlag(str *string, flags *pflag.FlagSet) {
+	flags.StringVar(
+		str, "dns-namespace", config.DefaultDNSNamespace,
+		"The namespace to check for DNS pods during preflight checks.",
+	)
+}
+
+// AddDNSPodLabelsFlag initialises the dns-pod-labels flag.
+// The value of this flag is used during preflight checks to determine which labels to use
+// when looking for the DNS pods.
+func AddDNSPodLabelsFlag(str *[]string, flags *pflag.FlagSet) {
+	flags.StringSliceVar(
+		str, "dns-pod-labels", config.DefaultDNSPodLabels,
+		"The label selectors to use for locating DNS pods during preflight checks. Can be specified multiple times or as a comma-separated list.",
+	)
+}
+
 // AddModeFlag initialises a mode flag.
 // The mode is a preset configuration of sonobuoy configuration and e2e configuration variables.
 // Mode can be partially or fully overridden by specifying config, e2e-focus, and e2e-skip.

cmd/sonobuoy/app/e2e.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/vmware-tanzu/sonobuoy/pkg/client"
 	"github.com/vmware-tanzu/sonobuoy/pkg/errlog"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -114,9 +115,12 @@ func e2es(cmd *cobra.Command, args []string) {
 	}
 
 	if !e2eflags.skipPreflight {
-		errs := sonobuoy.PreflightChecks(&client.PreflightConfig{
-			Namespace: e2eflags.namespace,
-		})
+		pcfg := &client.PreflightConfig{
+			Namespace:    runflags.namespace,
+			DNSNamespace: runflags.dnsNamespace,
+			DNSPodLabels: runflags.dnsPodLabels,
+		}
+		errs := sonobuoy.PreflightChecks(pcfg)
 		if len(errs) > 0 {
 			errlog.LogError(errors.New("Preflight checks failed"))
 			for _, err := range errs {

cmd/sonobuoy/app/gen.go

@@ -24,6 +24,7 @@ import (
 	"github.com/vmware-tanzu/sonobuoy/pkg/config"
 	"github.com/vmware-tanzu/sonobuoy/pkg/errlog"
 	imagepkg "github.com/vmware-tanzu/sonobuoy/pkg/image"
+
 	"github.com/imdario/mergo"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -38,6 +39,8 @@ type genFlags struct {
 	rbacMode                    RBACMode
 	kubecfg                     Kubeconfig
 	namespace                   string
+	dnsNamespace                string
+	dnsPodLabels                []string
 	sonobuoyImage               string
 	kubeConformanceImage        string
 	sshKeyPath                  string
@@ -78,6 +81,8 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode) *pflag.FlagSet {
 	AddShowDefaultPodSpecFlag(&cfg.showDefaultPodSpec, genset)
 
 	AddNamespaceFlag(&cfg.namespace, genset)
+	AddDNSNamespaceFlag(&cfg.dnsNamespace, genset)
+	AddDNSPodLabelsFlag(&cfg.dnsPodLabels, genset)
 	AddSonobuoyImage(&cfg.sonobuoyImage, genset)
 	AddKubeConformanceImage(&cfg.kubeConformanceImage, genset)
 	AddKubeConformanceImageVersion(&cfg.kubeConformanceImageVersion, genset)

cmd/sonobuoy/app/run.go

@@ -97,7 +97,12 @@ func submitSonobuoyRun(cmd *cobra.Command, args []string) {
 	}
 
 	if !runflags.skipPreflight {
-		if errs := sbc.PreflightChecks(&client.PreflightConfig{Namespace: runflags.namespace}); len(errs) > 0 {
+		pcfg := &client.PreflightConfig{
+			Namespace:    runflags.namespace,
+			DNSNamespace: runflags.dnsNamespace,
+			DNSPodLabels: runflags.dnsPodLabels,
+		}
+		if errs := sbc.PreflightChecks(pcfg); len(errs) > 0 {
 			errlog.LogError(errors.New("Preflight checks failed"))
 			for _, err := range errs {
 				errlog.LogError(err)

pkg/client/interfaces.go

@@ -176,7 +176,9 @@ func (sc *StatusConfig) Validate() error {
 
 // PreflightConfig are the options passed to PreflightChecks.
 type PreflightConfig struct {
-	Namespace string
+	Namespace    string
+	DNSNamespace string
+	DNSPodLabels []string
 }
 
 // Validate checks the config to determine if it is valid.

pkg/client/preflight.go

@@ -20,8 +20,9 @@ import (
 	"fmt"
 	"strings"
 
-	version "github.com/hashicorp/go-version"
 	"github.com/vmware-tanzu/sonobuoy/pkg/buildinfo"
+
+	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apicorev1 "k8s.io/api/core/v1"
@@ -84,23 +85,21 @@ func (c *SonobuoyClient) PreflightChecks(cfg *PreflightConfig) []error {
 
 func preflightDNSCheck(client kubernetes.Interface, cfg *PreflightConfig) error {
 	return dnsCheck(
-		client.CoreV1().Pods(kubeSystemNamespace).List,
-		expectedDNSLabels...,
+		client.CoreV1().Pods(cfg.DNSNamespace).List,
+		cfg.DNSNamespace,
+		cfg.DNSPodLabels...,
 	)
 }
 
-func dnsCheck(listPods listFunc, dnsLabels ...string) error {
+func dnsCheck(listPods listFunc, dnsNamespace string, dnsLabels ...string) error {
 	if len(dnsLabels) == 0 {
 		return nil
 	}
 
 	var nPods = 0
-	for _, labelValue := range dnsLabels {
-		selector := metav1.AddLabelToSelector(&metav1.LabelSelector{}, kubeDNSLabelKey, labelValue)
+	for _, label := range dnsLabels {
 
-		obj, err := listPods(
-			metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)},
-		)
+		obj, err := listPods(metav1.ListOptions{LabelSelector: label})
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve list of pods")
 		}
@@ -112,7 +111,7 @@ func dnsCheck(listPods listFunc, dnsLabels ...string) error {
 	}
 
 	if nPods == 0 {
-		return fmt.Errorf("no dns pods found with the labels [%s] in namespace kube-system", strings.Join(dnsLabels, ", "))
+		return fmt.Errorf("no dns pods found with the labels [%s] in namespace %s", strings.Join(dnsLabels, ", "), dnsNamespace)
 	}
 
 	return nil

pkg/client/preflight_test.go

@@ -133,10 +133,11 @@ func (f *fakeServerVersionInterface) ServerVersion() (*k8sversion.Info, error) {
 
 func TestDNSCheck(t *testing.T) {
 	testCases := []struct {
-		desc      string
-		lister    listFunc
-		dnsLabels []string
-		expectErr string
+		desc         string
+		lister       listFunc
+		dnsNamespace string
+		dnsLabels    []string
+		expectErr    string
 	}{
 		{
 			desc: "Needs only a single pod",
@@ -147,7 +148,8 @@ func TestDNSCheck(t *testing.T) {
 					},
 				}, nil
 			},
-			dnsLabels: []string{"foo"},
+			dnsNamespace: "dns-namespace",
+			dnsLabels:    []string{"foo"},
 		}, {
 			desc: "Multiple pods OK",
 			lister: func(metav1.ListOptions) (*apicorev1.PodList, error) {
@@ -158,14 +160,16 @@ func TestDNSCheck(t *testing.T) {
 					},
 				}, nil
 			},
-			dnsLabels: []string{"foo"},
+			dnsNamespace: "dns-namespace",
+			dnsLabels:    []string{"foo"},
 		}, {
 			desc: "Requires at least one pod",
 			lister: func(metav1.ListOptions) (*apicorev1.PodList, error) {
 				return &apicorev1.PodList{}, nil
 			},
-			dnsLabels: []string{"foo"},
-			expectErr: "no dns pods found with the labels [foo] in namespace kube-system",
+			dnsNamespace: "dns-namespace",
+			dnsLabels:    []string{"foo"},
+			expectErr:    "no dns pods found with the labels [foo] in namespace dns-namespace",
 		}, {
 			desc: "Skipped if no labels required",
 			lister: func(metav1.ListOptions) (*apicorev1.PodList, error) {
@@ -175,7 +179,7 @@ func TestDNSCheck(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			err := dnsCheck(tc.lister, tc.dnsLabels...)
+			err := dnsCheck(tc.lister, tc.dnsNamespace, tc.dnsLabels...)
 			if err != nil && len(tc.expectErr) == 0 {
 				t.Fatalf("Expected nil error but got %q", err)
 			}

pkg/config/config.go

@@ -58,6 +58,9 @@ const (
 	DefaultQueryBurst = 50
 	// DefaultProgressUpdatesPort is the port on which the Sonobuoy worker will listen for status updates from its plugin.
 	DefaultProgressUpdatesPort = "8099"
+
+	// DefaultDNSNamespace is the namespace where the DNS pods for the cluster are found.
+	DefaultDNSNamespace = "kube-system"
 )
 
 var (
@@ -112,6 +115,12 @@ var (
 		"validatingwebhookconfigurations",
 		"volumeattachments",
 	}
+
+	// DefaultDNSPodLabels are the label selectors that are used to locate the DNS pods in the cluster.
+	DefaultDNSPodLabels = []string{
+		"k8s-app=kube-dns",
+		"k8s-app=coredns",
+	}
 )
 
 // FilterOptions allow operators to select sets to include in a report