Tweaks/fix fio args (#32)

* fio is missing parsing the arguments
* add fio back to CI testing with smaller storage ask!
* add pre and post commands to storage

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/main.yaml

@@ -60,8 +60,8 @@ jobs:
       matrix:
         test: [["perf-hello-world", "ghcr.io/converged-computing/metric-sysstat:latest", 60],               # performance test
                ["io-host-volume", "ghcr.io/converged-computing/metric-sysstat:latest", 60],                 # storage test
-               ["network-osu-benchmark", "ghcr.io/converged-computing/metric-osu-benchmark:latest", 120]]   # standalone test
-               # ["io-fio", "ghcr.io/converged-computing/metric-fio:latest", 120]]                             # storage test
+               ["network-osu-benchmark", "ghcr.io/converged-computing/metric-osu-benchmark:latest", 120],   # standalone test
+               ["io-fio", "ghcr.io/converged-computing/metric-fio:latest", 120]]                            # storage test
 
     steps:
     - name: Clone the code

api/v1alpha1/metric_types.go

@@ -81,6 +81,10 @@ type Storage struct {
 
 	// Volume type to test
 	Volume Volume `json:"volume"`
+
+	// Commands to run (pre is supported to make bind)
+	// +optional
+	Commands Commands `json:"commands"`
 }
 
 // Application that will be monitored
@@ -121,6 +125,17 @@ type ContainerResources struct {
 	Requests ContainerResource `json:"requests"`
 }
 
+type Commands struct {
+
+	// pre command happens at start (before anything else)
+	// +optional
+	Pre string `json:"pre"`
+
+	// post happens at end (after collection end)
+	// +optional
+	Post string `json:"post"`
+}
+
 type ContainerResource map[string]intstr.IntOrString
 
 // A Volume should correspond with an existing volume, either:

config/crd/bases/flux-framework.org_metricsets.yaml

@@ -207,6 +207,17 @@ spec:
                 description: A storage setup that we want to measure performance for.
                   and binding to storage metrics
                 properties:
+                  commands:
+                    description: Commands to run (pre is supported to make bind)
+                    properties:
+                      post:
+                        description: post happens at end (after collection end)
+                        type: string
+                      pre:
+                        description: pre command happens at start (before anything
+                          else)
+                        type: string
+                    type: object
                   volume:
                     description: Volume type to test
                     properties:

docs/_static/data/metrics.json

@@ -1,4 +1,11 @@
 [
+ {
+  "name": "io-fio",
+  "description": "Flexible IO Tester (FIO)",
+  "type": "storage",
+  "image": "ghcr.io/converged-computing/metric-fio:latest",
+  "url": "https://fio.readthedocs.io/en/latest/fio_doc.html"
+ },
  {
   "name": "io-sysstat",
   "description": "statistics for Linux tasks (processes) : I/O, CPU, memory, etc.",
@@ -26,12 +33,5 @@
   "type": "application",
   "image": "ghcr.io/converged-computing/metric-sysstat:latest",
   "url": "https://github.com/sysstat/sysstat"
- },
- {
-  "name": "io-fio",
-  "description": "Flexible IO Tester (FIO)",
-  "type": "storage",
-  "image": "ghcr.io/converged-computing/metric-fio:latest",
-  "url": "https://fio.readthedocs.io/en/latest/fio_doc.html"
  }
 ]

docs/getting_started/custom-resource-definition.md

@@ -105,8 +105,25 @@ pod:
 
 ### storage
 
-When you want to measure some storage performance, you'll want to add a "storage" section to your MetricSet. This will typically just be a reference to some existing storage (see [existing volumes](#existing-volumes)) that we want to measure, and can
-also be done for some number of completions and metrics for storage.
+When you want to measure some storage performance, you'll want to add a "storage" section to your MetricSet. This will typically just be a reference to some existing storage (see [existing volumes](#existing-volumes)) that we want to measure, and can also be done for some number of completions and metrics for storage.
+
+#### commands
+
+If you need to add some special logic to create or cleanup for a storage volume, you are free to define them for storage in each of pre and post sections, which will happen before and after the metric runs, respectively.
+
+```yaml
+storage:
+  volume:
+    claimName: data 
+    path: /data
+  commands:
+    pre: |
+      apt-get update && apt-get install -y mymounter-tool
+      mymounter-tool mount /data
+    post: mymounter-tool unmount /data
+```
+
+Both of the above are strings. The pipe allows for multiple lines, if appropriate.
 
 ### metrics
 

docs/getting_started/metrics.md

@@ -7,7 +7,7 @@ The following metrics are under development (or being planned).
  - [Application Metrics](https://converged-computing.github.io/metrics-operator/getting_started/metrics.html#application)
  - [Standalone Metrics](https://converged-computing.github.io/metrics-operator/getting_started/metrics.html#standalone)
 
-<iframe src="../_static/data/table.html" style="width:100%; height:400px;" frameBorder="0"></iframe>
+<iframe src="../_static/data/table.html" style="width:100%; height:500px;" frameBorder="0"></iframe>
 
 All metrics can be customized with the following variables
 

examples/dist/metrics-operator-arm.yaml

@@ -197,6 +197,16 @@ spec:
               storage:
                 description: A storage setup that we want to measure performance for. and binding to storage metrics
                 properties:
+                  commands:
+                    description: Commands to run (pre is supported to make bind)
+                    properties:
+                      post:
+                        description: post happens at end (after collection end)
+                        type: string
+                      pre:
+                        description: pre command happens at start (before anything else)
+                        type: string
+                    type: object
                   volume:
                     description: Volume type to test
                     properties:

examples/dist/metrics-operator.yaml

@@ -197,6 +197,16 @@ spec:
               storage:
                 description: A storage setup that we want to measure performance for. and binding to storage metrics
                 properties:
+                  commands:
+                    description: Commands to run (pre is supported to make bind)
+                    properties:
+                      post:
+                        description: post happens at end (after collection end)
+                        type: string
+                      pre:
+                        description: pre command happens at start (before anything else)
+                        type: string
+                    type: object
                   volume:
                     description: Volume type to test
                     properties:

examples/python/io-fio/metrics.yaml

@@ -0,0 +1,22 @@
+apiVersion: flux-framework.org/v1alpha1
+kind: MetricSet
+metadata:
+  labels:
+    app.kubernetes.io/name: metricset
+    app.kubernetes.io/instance: metricset-sample
+  name: metricset-sample
+spec:
+  storage:
+    volume:
+      # This is the path on the host (e.g., inside kind container)
+      hostPath: /tmp/workflow
+
+      # This is the path in the container
+      path: /workflow
+
+  metrics:
+     # Fio just runs once - no concept of completions / rate
+    - name: io-fio
+      options:
+        directory: /workflow
+

examples/python/io-fio/run-metric.py

@@ -8,9 +8,7 @@
 import metricsoperator.utils as utils
 
 here = os.path.abspath(os.path.dirname(__file__))
-examples = os.path.dirname(os.path.dirname(here))
-tests = os.path.join(examples, "tests")
-metrics_yaml = os.path.join(tests, "io-fio", "metrics.yaml")
+metrics_yaml = os.path.join(here, "metrics.yaml")
 
 def get_parser():
     parser = argparse.ArgumentParser(

examples/tests/io-fio/metrics.yaml

@@ -12,11 +12,13 @@ spec:
       hostPath: /tmp/workflow
 
       # This is the path in the container
-      path: /workflow
+      path: /tmp/workflow
 
   metrics:
      # Fio just runs once - no concept of completions / rate
     - name: io-fio
       options:
-        directory: /workflow
+        size: 1M
+        blocksize: 1K
+        directory: /tmp/workflow
 

examples/tests/io-fio/post-run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Cleaning up /tmp/workflow in minikube"
+minikube ssh -- sudo rm -rf /tmp/workflow

examples/tests/io-fio/pre-run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+TESTS=$(dirname ${HERE})
+
+echo "Creating local volume in minikube"
+
+# The "data" volume will be mounted at /mnt/data
+minikube ssh -- mkdir -p /tmp/workflow

pkg/metrics/io/fio.go

@@ -74,6 +74,27 @@ func (m *Fio) SetOptions(metric *api.Metric) {
 	m.iodepth = 64
 	m.size = "4G"
 	m.directory = "/tmp"
+
+	v, ok := metric.Options["testname"]
+	if ok {
+		m.testname = v.StrVal
+	}
+	v, ok = metric.Options["blocksize"]
+	if ok {
+		m.blocksize = v.StrVal
+	}
+	v, ok = metric.Options["size"]
+	if ok {
+		m.size = v.StrVal
+	}
+	v, ok = metric.Options["directory"]
+	if ok {
+		m.directory = v.StrVal
+	}
+	v, ok = metric.Options["iodepth"]
+	if ok {
+		m.iodepth = int(v.IntVal)
+	}
 }
 
 // Generate the entrypoint for measuring the storage
@@ -89,6 +110,8 @@ func (m Fio) EntrypointScripts(
 echo "%s"
 # Directory (and filename) for test assuming other storage mounts
 filename=%s/test-$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 32)
+# Run the pre-command here so it has access to the filename.
+%s
 command="fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=%s --bs=%s --iodepth=%d --readwrite=randrw --rwmixread=75 --size=%s --filename=$filename --output-format=json"
 echo "FIO COMMAND START"
 echo $command
@@ -98,19 +121,23 @@ echo "%s"
 echo "%s"
 $command
 echo "%s"
+# Run command here so it's after collection finish, but before removing the filename
+%s 
 rm $filename
 `
 	script := fmt.Sprintf(
 		template,
 		metadata,
 		m.directory,
+		spec.Spec.Storage.Commands.Pre,
 		m.testname,
 		m.blocksize,
 		m.iodepth,
 		m.size,
 		metrics.CollectionStart,
 		metrics.Separator,
 		metrics.CollectionEnd,
+		spec.Spec.Storage.Commands.Post,
 	)
 	// The entrypoint is the entrypoint for the container, while
 	// the command is expected to be what we are monitoring. Often

pkg/metrics/io/sysstat.go

@@ -85,6 +85,8 @@ func (m IOStat) EntrypointScripts(
 		command = "iostat -dxm"
 	}
 	template := `#!/bin/bash
+# Custom pre command
+%s
 i=0
 echo "%s"
 completions=%d
@@ -96,21 +98,27 @@ while true
 	# Note we can do iostat -o JSON
 	if [[ $completions -ne 0 ]] && [[ $i -eq $completions ]]; then
     	echo "%s"
-    	exit 0
+        %s
+		exit 0
     fi
 	sleep %d
 	let i=i+1
 done
+# Custom post command after done, if we get here
+%s
 `
 	script := fmt.Sprintf(
 		template,
+		spec.Spec.Storage.Commands.Pre,
 		metadata,
 		m.completions,
 		metrics.CollectionStart,
 		metrics.Separator,
 		command,
 		metrics.CollectionEnd,
+		spec.Spec.Storage.Commands.Post,
 		m.rate,
+		spec.Spec.Storage.Commands.Post,
 	)
 	// The entrypoint is the entrypoint for the container, while
 	// the command is expected to be what we are monitoring. Often

script/test.sh

@@ -23,6 +23,7 @@ make run > ${out} 2> ${err} &
 pid=$!
 echo "PID for running cluster is ${pid}"
 
+/bin/bash examples/tests/${name}/pre-run.sh || true
 kubectl apply -f examples/tests/${name}/metrics.yaml
 echo "Sleeping for ${jobtime} seconds to allow job to complete 😴️."
 sleep ${jobtime}
@@ -34,8 +35,10 @@ status=$(kubectl get jobset -o json | jq -r .items[0].status.conditions[0].statu
 
 if [[ "${status}" != "True" ]] || [[ "${type}" != "Completed" ]]; then
     echo "Issue with running job ${name}"
+    /bin/bash examples/tests/${name}/post-run.sh || true
     exit 1
 fi
 
 kill ${pid} || true
 kill $(lsof -t -i:8080) || true
+/bin/bash examples/tests/${name}/post-run.sh || true