multus: add host checking to validation tool

In order to help users check that they have implemented the newly-added
Multus host configuration prerequisites, add a check to the validation
tool to verify connectivity.

Because users who are already running clusters with Multus enabled, add
a flag that allows users to only check for host configuration
prerequisites. This mode will not start the large number of clients that
would normally be started because those clients could disrupt a running
Rook cluster negatively.

Host checking pods require host network access. Many Kubernetes
distributions have pod security features enabled. In order to allow
non-Vanilla distros to run this tool, allow specifying a service account
that pods will run as, which can be configured by the admin to allow
test pods.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>

.github/workflows/multus.yaml

@@ -55,6 +55,9 @@ jobs:
       - name: Setup multus
         run: ./tests/scripts/multus/setup-multus.sh
 
+      - name: Set up multus prerequisite host routing
+        run: kubectl create -f tests/scripts/multus/host-cfg-ds.yaml
+
       - name: Install public and cluster NADs in default namespace
         run: kubectl create -f tests/scripts/multus/default-public-cluster-nads.yaml
 

cmd/rook/userfacing/multus/validation/validation.go

@@ -38,6 +38,9 @@ var (
 
 	// keep special var for `--daemons-per-node` that needs put into node config for validation run
 	flagDaemonsPerNode = -1
+
+	// keep special var for --host-check-only flag that can override what is from config file
+	flagHostCheckOnly = false
 )
 
 // commands
@@ -131,6 +134,9 @@ func init() {
 			"The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, "+
 			"and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. "+
 			"If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3.")
+	runCmd.Flags().BoolVar(&flagHostCheckOnly, "host-check-only", defaultConfig.HostCheckOnly,
+		"Only check that hosts can connect to the server via the public network. Do not start clients. "+
+			"This mode is recommended when a Rook cluster is already running and consuming the public network specified.")
 	runCmd.Flags().StringVar(&validationConfig.NginxImage, "nginx-image", defaultConfig.NginxImage,
 		"The Nginx image used for the validation server and clients.")
 
@@ -147,7 +153,8 @@ func init() {
 			"clients to start, and it therefore may take longer for all clients to become 'Ready'; in that case, this value can be set slightly higher.")
 
 	runCmd.Flags().StringVarP(&validationConfigFile, "config", "c", "",
-		"The validation test config file to use. This cannot be used with other flags.")
+		"The validation test config file to use. This cannot be used with other flags except --host-check-only.")
+	// allow using --host-check-only in combo with --config so the same config can be used with that flag if desired
 	runCmd.MarkFlagsMutuallyExclusive("config", "timeout-minutes")
 	runCmd.MarkFlagsMutuallyExclusive("config", "namespace")
 	runCmd.MarkFlagsMutuallyExclusive("config", "public-network")
@@ -184,6 +191,11 @@ func runValidation(ctx context.Context) {
 		}
 	}
 
+	// allow --host-check-only(=true) flag to override default/configfile settings
+	if flagHostCheckOnly {
+		validationConfig.HostCheckOnly = true
+	}
+
 	if err := validationConfig.ValidationTestConfig.Validate(); err != nil {
 		fmt.Print(err.Error() + "\n")
 		os.Exit(22 /* EINVAL */)

deploy/examples/multus-validation.yaml

@@ -26,10 +26,11 @@
 #
 # Flags:
 #       --cluster-network string                   The name of the Network Attachment Definition (NAD) that will be used for Ceph's cluster network. This should be a namespaced name in the form <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.
-#   -c, --config string                            The validation test config file to use. This cannot be used with other flags.
+#   -c, --config string                            The validation test config file to use. This cannot be used with other flags except --host-check-only.
 #       --daemons-per-node int                     The number of validation test daemons to run per node. It is recommended to set this to the maximum number of Ceph daemons that can run on any node in the worst case of node failure(s). The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3. (default 19)
 #       --flaky-threshold-seconds timeoutSeconds   This is the time window in which validation clients are all expected to become 'Ready' together. Validation clients are all started at approximately the same time, and they should all stabilize at approximately the same time. Once the first validation client becomes 'Ready', the tool checks that all of the remaining clients become 'Ready' before this threshold duration elapses. In networks that have connectivity issues, limited bandwidth, or high latency, clients will contend for network traffic with each other, causing some clients to randomly fail and become 'Ready' later than others. These randomly-failing clients are considered 'flaky.' Adjust this value to reflect expectations for the underlying network. For fast and reliable networks, this can be set to a smaller value. For networks that are intended to be slow, this can be set to a larger value. Additionally, for very large Kubernetes clusters, it may take longer for all clients to start, and it therefore may take longer for all clients to become 'Ready'; in that case, this value can be set slightly higher. (default 30s)
 #   -h, --help                                     help for run
+#       --host-check-only                          Only check that hosts can connect to the server via the public network. Do not start clients. This mode is recommended when a Rook cluster is already running and consuming the public network specified.
 #   -n, --namespace string                         The namespace for validation test resources. It is recommended to set this to the namespace in which Rook's Ceph cluster will be installed. (default "rook-ceph")
 #       --nginx-image string                       The Nginx image used for the validation server and clients. (default "quay.io/nginx/nginx-unprivileged:stable-alpine")
 #       --public-network string                    The name of the Network Attachment Definition (NAD) that will be used for Ceph's public network. This should be a namespaced name in the form <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.

pkg/daemon/multus/config.go

@@ -38,6 +38,8 @@ var (
 var (
 	DefaultValidationNamespace = "rook-ceph"
 
+	DefaultServiceAccountName = ""
+
 	DefaultValidationOSDsPerNode = 3
 
 	DefaultValidationOtherDaemonsPerNode = 16
@@ -70,13 +72,15 @@ func init() {
 // for this struct from getting out of date, see the output of ValidationTestConfig.ToYAML() for
 // usage text for each field.
 type ValidationTestConfig struct {
-	Namespace       string                `yaml:"namespace"`
-	PublicNetwork   string                `yaml:"publicNetwork"`
-	ClusterNetwork  string                `yaml:"clusterNetwork"`
-	ResourceTimeout time.Duration         `yaml:"resourceTimeout"`
-	FlakyThreshold  time.Duration         `yaml:"flakyThreshold"`
-	NginxImage      string                `yaml:"nginxImage"`
-	NodeTypes       map[string]NodeConfig `yaml:"nodeTypes"`
+	Namespace          string                `yaml:"namespace"`
+	ServiceAccountName string                `yaml:"serviceAccountName"`
+	PublicNetwork      string                `yaml:"publicNetwork"`
+	ClusterNetwork     string                `yaml:"clusterNetwork"`
+	ResourceTimeout    time.Duration         `yaml:"resourceTimeout"`
+	FlakyThreshold     time.Duration         `yaml:"flakyThreshold"`
+	HostCheckOnly      bool                  `yaml:"hostCheckOnly"`
+	NginxImage         string                `yaml:"nginxImage"`
+	NodeTypes          map[string]NodeConfig `yaml:"nodeTypes"`
 }
 
 type NodeConfig struct {
@@ -114,10 +118,11 @@ func (t *TolerationType) ToJSON() (string, error) {
 // The default test is a converged-node test with no placement.
 func NewDefaultValidationTestConfig() *ValidationTestConfig {
 	return &ValidationTestConfig{
-		Namespace:       DefaultValidationNamespace,
-		ResourceTimeout: DefaultValidationResourceTimeout,
-		FlakyThreshold:  DefaultValidationFlakyThreshold,
-		NginxImage:      DefaultValidationNginxImage,
+		Namespace:          DefaultValidationNamespace,
+		ServiceAccountName: DefaultServiceAccountName,
+		ResourceTimeout:    DefaultValidationResourceTimeout,
+		FlakyThreshold:     DefaultValidationFlakyThreshold,
+		NginxImage:         DefaultValidationNginxImage,
 		NodeTypes: map[string]NodeConfig{
 			DefaultValidationNodeType: {
 				OSDsPerNode:         DefaultValidationOSDsPerNode,
@@ -266,10 +271,11 @@ var dedicatedWorkerNodeConfig = NodeConfig{
 
 func NewDedicatedStorageNodesValidationTestConfig() *ValidationTestConfig {
 	return &ValidationTestConfig{
-		Namespace:       DefaultValidationNamespace,
-		ResourceTimeout: DefaultValidationResourceTimeout,
-		FlakyThreshold:  DefaultValidationFlakyThreshold,
-		NginxImage:      DefaultValidationNginxImage,
+		Namespace:          DefaultValidationNamespace,
+		ServiceAccountName: DefaultServiceAccountName,
+		ResourceTimeout:    DefaultValidationResourceTimeout,
+		FlakyThreshold:     DefaultValidationFlakyThreshold,
+		NginxImage:         DefaultValidationNginxImage,
 		NodeTypes: map[string]NodeConfig{
 			DedicatedStorageNodeType: dedicatedStorageNodeConfig,
 			DedicatedWorkerNodeType:  dedicatedWorkerNodeConfig,
@@ -283,10 +289,11 @@ const (
 
 func NewArbiterValidationTestConfig() *ValidationTestConfig {
 	return &ValidationTestConfig{
-		Namespace:       DefaultValidationNamespace,
-		ResourceTimeout: DefaultValidationResourceTimeout,
-		FlakyThreshold:  DefaultValidationFlakyThreshold,
-		NginxImage:      DefaultValidationNginxImage,
+		Namespace:          DefaultValidationNamespace,
+		ServiceAccountName: DefaultServiceAccountName,
+		ResourceTimeout:    DefaultValidationResourceTimeout,
+		FlakyThreshold:     DefaultValidationFlakyThreshold,
+		NginxImage:         DefaultValidationNginxImage,
 		NodeTypes: map[string]NodeConfig{
 			DedicatedStorageNodeType: dedicatedStorageNodeConfig,
 			DedicatedWorkerNodeType:  dedicatedWorkerNodeConfig,

pkg/daemon/multus/config.yaml

@@ -2,6 +2,11 @@
 # Rook-Ceph cluster will be installed.
 namespace: "{{ .Namespace }}"
 
+# The service account to run validation test pods as. Empty means pods won't use a service account.
+# This is useful in Kubernetes environments where security rules are in place.
+# Notably, host checker pods require host network access.
+serviceAccountName: "{{ .ServiceAccountName }}"
+
 # These fields should be set to the name of the Network Attachment Definition (NAD) which will be
 # used for the Ceph cluster's public or cluster network, respectively. This should be a namespaced
 # name in the form <namespace>/<name> if the NAD is defined in a different namespace from the
@@ -29,6 +34,13 @@ resourceTimeout: "{{ .ResourceTimeout }}"
 # longer for all clients to become "Ready"; in that case, this value can be set slightly higher.
 flakyThreshold: "{{ .FlakyThreshold }}"
 
+# Enable host-check-only mode. This will instruct the validation test routine to only check host
+# connectivity to the server via the public network. It won't start clients and cannot check for
+# network flakiness. This mode is recommended when a Rook cluster is already running and consuming
+# the public network specified. This mode avoids disrupting to the running Rook cluster that could
+# be impacted by a large number of running clients that might run without this mode enabled.
+hostCheckOnly: {{ .HostCheckOnly }}
+
 # The Nginx image which will be used for the web server and clients.
 nginxImage: "{{ .NginxImage }}"
 

pkg/daemon/multus/config_test.go

@@ -36,12 +36,14 @@ func TestValidationTestConfig_YAML(t *testing.T) {
 		{"empty config", emptyValidationTestConfig},
 		{"default config", NewDefaultValidationTestConfig()},
 		{"full config", &ValidationTestConfig{
-			Namespace:       "my-rook",
-			PublicNetwork:   "my-pub",
-			ClusterNetwork:  "my-priv",
-			ResourceTimeout: 2 * time.Minute,
-			FlakyThreshold:  30 * time.Second,
-			NginxImage:      "myorg/nginx:latest",
+			Namespace:          "my-rook",
+			ServiceAccountName: "my-svc-acct",
+			PublicNetwork:      "my-pub",
+			ClusterNetwork:     "my-priv",
+			ResourceTimeout:    2 * time.Minute,
+			FlakyThreshold:     30 * time.Second,
+			HostCheckOnly:      true,
+			NginxImage:         "myorg/nginx:latest",
 			NodeTypes: map[string]NodeConfig{
 				"osdOnlyNodes": {
 					OSDsPerNode:         9,

pkg/daemon/multus/host-daemonset.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: multus-validation-test-host-checker-{{ .NodeType }}
+  labels:
+    app: multus-validation-test-host-checker
+    nodeType: "{{ .NodeType }}"
+    app.kubernetes.io/name: "host-checker"
+    app.kubernetes.io/instance: "host-checker-{{ .NodeType }}"
+    app.kubernetes.io/component: "host-checker"
+    app.kubernetes.io/part-of: "multus-validation-test"
+    app.kubernetes.io/managed-by: "rook-cli"
+spec:
+  selector:
+    matchLabels:
+      app: multus-validation-test-host-checker
+      nodeType: "{{ .NodeType }}"
+  template:
+    metadata:
+      labels:
+        app: multus-validation-test-host-checker
+        nodeType: "{{ .NodeType }}"
+    spec:
+      nodeSelector:
+      {{- range $k, $v := .Placement.NodeSelector }}
+        {{ $k }}: {{ $v }}
+      {{- end }}
+      tolerations:
+      {{- range $idx, $toleration := .Placement.Tolerations }}
+        - {{ $toleration.ToJSON }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      hostNetwork: true
+      containers:
+        - name: readiness-check-web-server-public-addr
+          # use nginx image because it's already used for the web server pod and has a non-root user
+          image: "{{ .NginxImage }}"
+          command:
+            - sleep
+            - infinity
+          resources: {}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          # A readiness probe makes validation testing easier than investigate container logs.
+          # Additionally, readiness probe failures don't result in CrashLoopBackoff -- ideal here,
+          # where ever-longer back-offs would cause tests to run for much longer than necessary.
+          readinessProbe:
+            # Low failure threshold and high success threshold. Intended to be very sensitive to
+            # failures. If probe fails with any regularity, Ceph OSDs likely won't be stable.
+            failureThreshold: 1
+            successThreshold: 12
+            periodSeconds: 5
+            # Assumption: a network with a latency more than 4 seconds for this validation test's
+            # simple client-server response likely won't support acceptable performance for any
+            # production Ceph cluster.
+            timeoutSeconds: 4
+            # TODO: exec:curl works but httpGet fails. Why? need custom header?
+            exec:
+              command:
+                - "curl"
+                - "--insecure"
+                - "{{ .PublicNetworkAddress }}:8080"

pkg/daemon/multus/resources.go

@@ -157,13 +157,13 @@ func (vt *ValidationTest) startImagePullers(ctx context.Context, owners []meta.O
 	return nil
 }
 
-func (vt *ValidationTest) deleteImagePullers(ctx context.Context) error {
+func (vt *ValidationTest) deleteDaemonsetsWithLabel(ctx context.Context, label string) error {
 	noGracePeriod := int64(0)
 	delOpts := meta.DeleteOptions{
 		GracePeriodSeconds: &noGracePeriod,
 	}
 	listOpts := meta.ListOptions{
-		LabelSelector: imagePullAppLabel(),
+		LabelSelector: label,
 	}
 	err := vt.Clientset.AppsV1().DaemonSets(vt.Namespace).DeleteCollection(ctx, delOpts, listOpts)
 	if err != nil {
@@ -176,6 +176,27 @@ func (vt *ValidationTest) deleteImagePullers(ctx context.Context) error {
 	return nil
 }
 
+func (vt *ValidationTest) startHostCheckers(
+	ctx context.Context,
+	owners []meta.OwnerReference,
+	serverPublicAddr string,
+) error {
+	for typeName, nodeType := range vt.NodeTypes {
+		ds, err := vt.generateHostCheckerDaemonSet(serverPublicAddr, typeName, nodeType.Placement)
+		if err != nil {
+			return fmt.Errorf("failed to generate host checker daemonset: %w", err)
+		}
+		ds.SetOwnerReferences(owners) // set owner so cleanup is easier
+
+		_, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to create host checker daemonset: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func (vt *ValidationTest) startClients(
 	ctx context.Context,
 	owners []meta.OwnerReference,
@@ -335,10 +356,10 @@ func (vt *ValidationTest) getNumRunningPods(
 	return numRunning, nil
 }
 
-func (vt *ValidationTest) numClientsReady(ctx context.Context, expectedNumPods int) (int, error) {
-	pods, err := vt.getClientPods(ctx, expectedNumPods)
+func (vt *ValidationTest) numPodsReadyWithLabel(ctx context.Context, label string) (int, error) {
+	pods, err := vt.getPodsWithLabel(ctx, label)
 	if err != nil {
-		return 0, fmt.Errorf("unexpected error getting client pods: %w", err)
+		return 0, fmt.Errorf("unexpected error getting pods with label %q: %w", label, err)
 	}
 	numReady := 0
 	for _, p := range pods.Items {
@@ -349,16 +370,13 @@ func (vt *ValidationTest) numClientsReady(ctx context.Context, expectedNumPods i
 	return numReady, nil
 }
 
-func (vt *ValidationTest) getClientPods(ctx context.Context, expectedNumPods int) (*core.PodList, error) {
+func (vt *ValidationTest) getPodsWithLabel(ctx context.Context, label string) (*core.PodList, error) {
 	listOpts := meta.ListOptions{
-		LabelSelector: clientAppLabel(),
+		LabelSelector: label,
 	}
 	pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to list client pods: %w", err)
-	}
-	if len(pods.Items) != expectedNumPods {
-		return nil, fmt.Errorf("the number of pods listed [%d] does not match the number expected [%d]", len(pods.Items), expectedNumPods)
+		return nil, fmt.Errorf("failed to list pods with label %q: %w", label, err)
 	}
 	return pods, err
 }