Draft IOR automation script (#51)

* Update IOR tests

* add IOR config files

* add IOR config files

* rename sc and pvc for IOR

* ior test

* update IOR scripts

* Add script read-ior-output.sh

* remove unnecessary ml lables

deploy/csi-azurelustre-node.yaml

@@ -130,6 +130,8 @@ spec:
               name: azure-cred
             - mountPath: /mnt
               name: azurelustre-cache
+            - mountPath: /dev
+              name: host-dev
           resources:
             limits:
               cpu: 1
@@ -158,3 +160,7 @@ spec:
             path: /mnt
             type: DirectoryOrCreate
           name: azurelustre-cache
+        - hostPath:
+            path: /dev
+            type: Directory
+          name: host-dev

test/ior/pod.yaml

@@ -1,4 +1,30 @@
 ---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ior.sc.azurelustre.csi.azure.com
+provisioner: azurelustre.csi.azure.com
+parameters:
+  mgs-ip-address: "172.18.16.8"      
+  fs-name: "lustrefs"
+mountOptions:
+  - noatime
+  - flock
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: azurelustre-ior-claim
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteMany
+  volumeMode: Filesystem
+  storageClassName: ior.sc.azurelustre.csi.azure.com
+  resources:
+    requests:
+      storage: 100Mi
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -17,6 +43,7 @@ spec:
     spec:
       nodeSelector:
         kubernetes.io/os: linux
+        agentpool: ior
       tolerations:
       # this toleration is to have the daemonset runnable on master nodes
       # remove it if your masters can't run pods
@@ -26,13 +53,6 @@ spec:
       containers:
       - name: ior
         image: azurelustre.azurecr.io/ior:latest
-        resources:
-          limits:
-            cpu: 1000m
-            memory: 2000Mi
-          requests:
-            cpu: 100m
-            memory: 200Mi
         volumeMounts:
         - name: azurelustre
           mountPath: /azurelustre
@@ -51,4 +71,4 @@ spec:
           path: /var/lib/docker/containers
       - name: azurelustre
         persistentVolumeClaim:
-          claimName: azurelustre-claim
+          claimName: azurelustre-ior-claim

test/ior/read-ior-output.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Shell script to run MPI/IOR cluster
+#
+repo="$(git rev-parse --show-toplevel)/test/ior"
+
+resultsPath="$repo"
+iorConfigArray=$(ls $repo/test-config-files)
+clientPerNodeArray=(64 128)
+
+for config in $iorConfigArray
+do
+    for clientPerNode in ${clientPerNodeArray[@]}
+    do 
+        fileName=${config}_${clientPerNode}
+        echo $fileName
+
+        files=$(ls $resultsPath/results*/$fileName)
+        for f in $files
+        do
+            str=$(cat $f | tail -n 3 | head -n 1)
+            strArray=($str)
+            write=${strArray[3]}
+
+            str=$(cat $f | tail -n 2 | head -n 1)
+            strArray=($str)
+            read=${strArray[3]}
+
+            echo -e  $write ' \t ' $read
+        done
+    done    
+done

test/ior/run_ior_aks.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Shell script to run MPI/IOR cluster
+#
+set -o errexit
+set -o pipefail
+set -o nounset
+
+repo="$(git rev-parse --show-toplevel)/test/ior"
+
+skipSetup=${1:-"false"}
+
+if [[ "$skipSetup" == "false" ]]; 
+then 
+    echo "Removing IOR pods"
+    kubectl delete -f $repo/pod.yaml --ignore-not-found
+    sleep 15
+
+    echo "Creating IOR pods"
+    kubectl apply -f $repo/pod.yaml
+    sleep 15
+
+    echo "Setup IOR pods"
+    $repo/setup_ior_aks.sh
+    sleep 15
+fi
+
+pod1=$(kubectl get po --no-headers | grep "ior" | awk '{print $1}' | head -n 1)
+pod2=$(kubectl get po --no-headers | grep "ior" | awk '{print $1}' | tail -n 1)
+
+resultsDirectory="results$(date +%s)"
+mkdir $repo/$resultsDirectory
+
+echo "$(date -u) Starting IOR execution, find test results in directory $repo/$resultsDirectory"
+
+clientPerNode=64
+echo "$(date -u) executing IOR cases for $clientPerNode client per node"
+
+sleep 60
+testcase="bw_n_to_n_rnd_buffered"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod1 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="bw_n_to_n_rnd_direct"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod2 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="bw_n_to_n_seq_buffered"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod1 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="bw_n_to_n_seq_direct"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod2 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="iops_n_to_1_rnd_buffered"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod1 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="iops_n_to_1_rnd_direct"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod2 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="iops_n_to_n_rnd_buffered"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod1 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+sleep 60
+testcase="iops_n_to_n_rnd_direct"
+echo "$(date -u) executing $testcase"
+kubectl exec -it $pod2 -- mpirun --hostfile /app/host_file --map-by node -np $clientPerNode ior -o /azurelustre/test_file -f /app/$testcase 2>&1 >> "$repo/$resultsDirectory/${testcase}_${clientPerNode}"
+
+echo "$(date -u) Finished IOR execution"

test/ior/setup.sh → test/ior/setup_ior_aks.sh

@@ -17,45 +17,61 @@
 #
 # Shell script to setup MPI/IOR cluster
 #
-
-set -o xtrace
 set -o errexit
 set -o pipefail
 set -o nounset
 
+testCaseName=${1:-""}
+
+repo="$(git rev-parse --show-toplevel)/test/ior/"
+
 echo "$(date -u) Start to setup MPI/IOR cluster"
 
 pods=$(kubectl get pods | grep ior | awk '{print $1}')
-
 ips=$(kubectl get pods -o wide | grep ior | awk '{print $6}')
 
-rm --force ./host_file
+rm --force $repo/host_file
 
 for pod in $pods
 do
-
   for ip in $ips
   do
     echo "Pod $pod sshing to ip $ip"
     kubectl exec $pod -- ssh -o StrictHostKeyChecking=no $ip rm --force /app/host_file
   done
-
 done
 
 for ip in $ips
 do
   echo "Adding ip $ip to host_file"
-  echo $ip >> ./host_file
+  echo $ip >> $repo/host_file
 done
 
-cat ./host_file | sort -u | tee ./host_file
+cat $repo/host_file | sort -u | tee $repo/host_file
+
+echo "show host_file content"
+cat $repo/host_file
+
+if [[ $(cat $repo/host_file | wc -l) == 1 ]]; then
+  echo "only one IOR pod in host_file"
+  exit 1
+fi
 
 for pod in $pods
 do
   echo "Copying host_file to Pod $pod"
-  kubectl cp ./host_file $pod:/app/host_file
+  kubectl cp $repo/host_file $pod:host_file
 done
 
-rm --force ./host_file
+rm --force $repo/host_file
+
+for pod in $pods
+do
+  ls $repo/test-config-files/$testCaseName | while read conf
+  do
+    kubectl cp $repo/test-config-files/$conf $pod:/app
+    echo "$conf copied to $pod"
+  done
+done
 
 echo "$(date -u) Finish MPI/IOR cluster setup"

test/ior/test-config-files/bw_n_to_n_rnd_buffered

@@ -0,0 +1,15 @@
+IOR START
+    api=POSIX
+    blockSize=4g
+    filePerProc=1
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    randomOffset=1
+    readFile=1
+    repetitions=3
+    transferSize=4m
+    useO_DIRECT=0
+    writeFile=1
+RUN
+IOR STOP

test/ior/test-config-files/bw_n_to_n_rnd_direct

@@ -0,0 +1,15 @@
+IOR START
+    api=POSIX
+    blockSize=4g
+    filePerProc=1
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    randomOffset=1
+    readFile=1
+    repetitions=3
+    transferSize=4m
+    useO_DIRECT=1
+    writeFile=1
+RUN
+IOR STOP

test/ior/test-config-files/bw_n_to_n_seq_buffered

@@ -0,0 +1,14 @@
+IOR START
+    api=POSIX
+    blockSize=4g
+    filePerProc=1
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    readFile=1
+    repetitions=3
+    transferSize=4m
+    useO_DIRECT=0
+    writeFile=1
+RUN
+IOR STOP

test/ior/test-config-files/bw_n_to_n_seq_direct

@@ -0,0 +1,14 @@
+IOR START
+    api=POSIX
+    blockSize=4g
+    filePerProc=1
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    readFile=1
+    repetitions=3
+    transferSize=4m
+    useO_DIRECT=1
+    writeFile=1
+RUN
+IOR STOP

test/ior/test-config-files/iops_n_to_1_rnd_buffered

@@ -0,0 +1,15 @@
+IOR START
+    api=POSIX
+    blockSize=4m
+    filePerProc=0
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    randomOffset=1
+    readFile=1
+    repetitions=3
+    transferSize=4k
+    useO_DIRECT=0
+    writeFile=1
+RUN
+IOR STOP

test/ior/test-config-files/iops_n_to_1_rnd_direct

@@ -0,0 +1,15 @@
+IOR START
+    api=POSIX
+    blockSize=4m
+    filePerProc=0
+    fsync=1
+    interTestDelay=5
+    multiFile=1
+    randomOffset=1
+    readFile=1
+    repetitions=3
+    transferSize=4k
+    useO_DIRECT=1
+    writeFile=1
+RUN
+IOR STOP