feat(ci,cijoe): add configs, scripts and workflows for perf. eval.

This adds scripts to:

* Prepare bdev-configurations on the target-system

  - This is bench_confs_generate.py
  - It utilizes the list of [['duts']] aka "device-under-test"

* bdevperf.py

  - Wrap the execution of the SPDK benchmark tool bdevperf

* bench_plotter.py

  - Produce plots from the output of 'bdevperf.py'
  - Auxiliary files 'plot-attributes-*.yaml' to style, label and limit
    graphs

* bench_reporter.py

  - Produce a .pdf report using the graphs produced by bench_plotter.py

* {fio,liburing,spdk}_build.py

  - Scripts to build and install fio, liburing, and SPDK from source

todo... describe this...

Signed-off-by: Simon A. F. Lund <simon.lund@samsung.com>

.github/workflows/verify.yml

@@ -6,7 +6,7 @@ on:
     branches: [main, next]
     tags: ['v*']
   pull_request:
-    types: [opened, reopened, synchronize]
+    types: [opened, reopened, synchronize, labeled]
     branches: [next]
   workflow_dispatch:
     inputs:
@@ -18,6 +18,7 @@ on:
         options:
         - analyze
         - build-and-test
+        - build-and-bench
         - docgen
 
 defaults:
@@ -592,6 +593,76 @@ jobs:
         path: test-results-${{ matrix.guest.os }}-${{ matrix.guest.ver }}.tar.gz
         if-no-files-found: error
 
+  #
+  # Build and run performance test of xNVMe
+  #
+  build-and-bench:
+    needs: [source-archive, source-format-check, build-python]
+    runs-on: [self-hosted, linux, ARM64, bench, intel]
+
+    if: (contains('push', github.event_name) || ((github.event_name == 'workflow_dispatch') && (github.event.inputs.job ==
+      'build-and-bench')) || ((github.event_name == 'pull_request') && contains(github.event.pull_request.labels.*.name, 'bench')))
+
+    strategy:
+      fail-fast: false
+
+    container:
+      image: debian:bookworm
+      options: --privileged
+
+    steps:
+    - name: Runnner-prep, clean up self-hosted left-overs
+      run: |
+        pkill -f qemu || true
+        rm -r $HOME/guests || true
+        rm -r /tmp/artifacts || true
+        mkdir -p /tmp/artifacts || true
+        rm -rf *
+        ls -lh
+
+    - name: Retrieve the xNVMe source archive
+      uses: actions/download-artifact@v3.0.1
+      with:
+        name: xnvme-src-archive
+    - name: Extract, and then move, the xNVMe source archive
+      run: |
+        tar xzf xnvme-src.tar.gz --strip 1
+
+    - name: Prep, environment GITHUB_PATH
+      run: |
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - name: Packages, Debian Bookworm
+      run: |
+        apt-get -qy update
+        apt-get -qy upgrade
+        apt-get -qy install python3 python3-dev python3-venv pipx git make gcc fontconfig
+        pipx ensurepath
+
+    - name: CIJOE, setup pipx environment
+      run: |
+        pushd cijoe && make all && popd
+
+    - name: CIJOE, run bench
+      run: |
+        rm -rf cijoe/bench-results
+        cd cijoe && cijoe \
+        --config "configs/bench-intel.toml" \
+        --workflow "workflows/bench.yaml" \
+        --output "bench-results"
+
+    - name: CIJOE, compress workflow-report-bench
+      if: always()
+      run: |
+        tar czf bench-results.tar.gz cijoe/bench-results/* 
+
+    - name: CIJOE, upload workflow-report-test
+      uses: actions/upload-artifact@v3.1.1
+      if: always()
+      with:
+        name: bench-results
+        path: bench-results.tar.gz
+        if-no-files-found: error
 
   #
   # Documentation: generate documentation and deploy it to githubpages (xnvme.io)

cijoe/Makefile

@@ -41,6 +41,9 @@ install:
 	@${PIPX} inject cijoe cijoe-pkg-qemu==v6.1.15
 	@${PIPX} inject cijoe cijoe-pkg-linux==v0.9.7
 	@${PIPX} inject cijoe cijoe-pkg-fio==v0.9.7
+	@${PIPX} inject cijoe matplotlib
+	@${PIPX} inject cijoe numpy
+	@${PIPX} install rst2pdf
 	@echo "## ${PROJECT_NAME}: make install [DONE]"
 
 define uninstall-help

cijoe/auxiliary/plot-legends.yaml

@@ -0,0 +1,116 @@
+---
+# Reference implementations
+libaio-reference:
+  legend: 'libaio ($\bf{reference}$)'
+  color: "#084594"
+  marker: "s" # square
+  hatch: "."
+
+io_uring-reference:
+  legend: 'io_uring ($\bf{reference}$)'
+  color: "#2171b5"
+  marker: "v" # triangle_down
+  hatch: "."
+
+io_uring_cmd-reference:
+  legend: 'io_uring_cmd ($\bf{reference}$)'
+  color: "#2171b5"
+  marker: "P" # plus
+  hatch: "."
+
+# xNVMe implementations
+libaio-xnvme:
+  legend: 'libaio ($\bf{xnvme}$)'
+  color: "#005a32"
+  marker: "p" # pentagon
+  hatch: "."
+
+libaio-bdev_xnvme:
+  legend: 'libaio ($\bf{bdev\_xnvme}$)'
+  color: "#297373"
+  marker: "X" # cross
+  hatch: "O"
+
+libaio-bdev_xnvme_conserve_cpu:
+  legend: 'libaio ($\bf{bdev\_xnvme}$) +conserve'
+  color: "#E9D758"
+  marker: "X" # cross
+  hatch: "O"
+
+io_uring-xnvme:
+  legend: 'io_uring ($\bf{xnvme}$)'
+  color: "#41ab5d"
+  marker: "*" # star
+  hatch: "."
+
+io_uring-bdev_xnvme:
+  legend: 'uring $\bf{bdev\_xnvme}$'
+  color: "#297373"
+  marker: "h" # hexagon1
+  hatch: "O"
+
+io_uring-bdev_xnvme_conserve_cpu:
+  legend: 'uring $\bf{bdev\_xnvme}$ +conserve'
+  color: "#E9D758"
+  marker: "h" # hexagon1
+  hatch: "O"
+
+io_uring-bdev_xnvme_BATCH:
+  legend: 'uring $\bf{bdev\_xnvme}$ +batch'
+  color: "#FF8552"
+  marker: "D" # diamond
+  hatch: "*"
+
+io_uring-bdev_xnvme_conserve_cpu_BATCH:
+  legend: 'uring $\bf{bdev\_xnvme}$ +conserve +batch'
+  color: "#39393A"
+  marker: "D" # diamond
+  hatch: "*"
+
+io_uring_cmd-xnvme:
+  legend: 'io_uring_cmd ($\bf{xnvme}$)'
+  color: "#74c476"
+  marker: "H" # hexagon2
+  hatch: "."
+
+io_uring_cmd-bdev_xnvme:
+  legend: 'ucmd $\bf{bdev\_xnvme}$'
+  color: "#297373"
+  marker: "o" # circle
+  hatch: "O"
+io_uring_cmd-bdev_xnvme_conserve_cpu:
+  legend: 'ucmd $\bf{bdev\_xnvme}$ +conserve'
+  color: "#E9D758"
+  marker: "o" # circle
+  hatch: "O"
+
+io_uring_cmd-bdev_xnvme_BATCH:
+  legend: 'ucmd $\bf{bdev\_xnvme}$ +batch'
+  color: "#FF8552"
+  marker: "d" # thin_diamond
+  hatch: "*"
+
+io_uring_cmd-bdev_xnvme_conserve_cpu_BATCH:
+  legend: 'ucmd $\bf{bdev\_xnvme}$ +conserve +batch'
+  color: "#39393A"
+  marker: "d" # thin_diamond
+  hatch: "*"
+
+# SPDK bdev implementations
+libaio-bdev_aio:
+  legend: 'libaio ($\bf{bdev\_aio}$)'
+  color: "#4a1486"
+  marker: "D" # diamond
+  hatch: "O"
+
+io_uring-bdev_uring:
+  legend: 'uring $\bf{bdev\_uring}$'
+  color: "#807dba"
+  marker: "d" # thin_diamond
+  hatch: "O"
+
+ioctl-xnvme:
+  legend: 'ioctl ($\bf{xnvme}$)'
+  color: "#807dba"
+  marker: "d" # thin_diamond
+  hatch: "O"

cijoe/auxiliary/plot-limits-4k-aio.yaml

@@ -0,0 +1,6 @@
+---
+# Limits on plots, this is to ensure that different plots are visually comparable
+barplot:
+  y_lim: [0, 140000]
+lineplot:
+  y_lim: [0, 1200000]

cijoe/auxiliary/plot-limits-4k.yaml

@@ -0,0 +1,6 @@
+---
+# Limits on plots, this is to ensure that different plots are visually comparable
+barplot:
+  y_lim: [0, 140000]
+lineplot:
+  y_lim: [0, 2750000]

cijoe/auxiliary/plot-limits-512.yaml

@@ -0,0 +1,6 @@
+---
+# Limits on plots, this is to ensure that different plots are visually comparable
+barplot:
+  y_lim: [0, 195000]
+lineplot:
+  y_lim: [0, 2750000]