feat(cijoe): add configs, scripts and workflows for perf. eval.

This adds scripts to: * Prepare bdev-configurations on the target-system - This is bench_prep_bdevconfs.py - It utilizes the list of [['duts']] aka "device-under-test" todo... describe this... Signed-off-by: Simon A. F. Lund <simon.lund@samsung.com>
xnvme · Oct 19, 2023 · a33f0de · a33f0de
1 parent f54c0db
commit a33f0de
Show file tree

Hide file tree

Showing 18 changed files with 1,953 additions and 12 deletions.
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -6,7 +6,7 @@ on:
     branches: [main, next]
     tags: ['v*']
   pull_request:
-    types: [opened, reopened, synchronize]
+    types: [opened, reopened, synchronize, labeled]
     branches: [next]
   workflow_dispatch:
     inputs:
@@ -18,6 +18,7 @@ on:
         options:
         - analyze
         - build-and-test
+        - build-and-bench
         - docgen
 
 defaults:
@@ -597,10 +598,10 @@ jobs:
   #
   build-and-bench:
     needs: [source-archive, source-format-check, build-python]
-    runs-on: [self-hosted, linux, ARM64, perf]
+    runs-on: [self-hosted, linux, ARM64, bench, intel]
 
     if: (contains('push pull_request', github.event_name) || ((github.event_name == 'workflow_dispatch') && (github.event.inputs.job
-      == 'build-and-bench')))
+      == 'build-and-bench')) || (contains( github.event.pull_request.labels.*.name, 'bench')))
 
     strategy:
       fail-fast: false
@@ -626,12 +627,10 @@ jobs:
     - name: Extract, and then move, the xNVMe source archive
       run: |
         tar xzf xnvme-src.tar.gz --strip 1
-        mv xnvme-src.tar.gz /tmp/artifacts/.
 
     - name: Prep, environment GITHUB_PATH
       run: |
         echo "$HOME/.local/bin" >> $GITHUB_PATH
-        find /tmp/artifacts -name "*.tar.gz"
 
     - name: Packages, Debian Bookworm
       run: |
@@ -646,7 +645,7 @@ jobs:
 
     - name: CIJOE, check that it is available
       run: |
-        cijoe -r
+        pushd cijoe && cijoe -c configs/bench-intel.toml -w workflows/bench-run.yaml
 
   #
   # Documentation: generate documentation and deploy it to githubpages (xnvme.io)

diff --git a/cijoe/auxiliary/plot-attributes-4k-aio.yaml b/cijoe/auxiliary/plot-attributes-4k-aio.yaml
@@ -0,0 +1,125 @@
+---
+# Limits on plots, this is to ensure that different plots are visually comparable
+limits:
+  barplot:
+    y_lim: [0, 140000]
+  lineplot:
+    y_lim: [0, 1200000]
+
+# Identifying colors and markers by dataset-label
+datasets:
+  # Reference implementations
+  libaio-reference:
+    legend: 'libaio ($\bf{reference}$)'
+    color: "#084594"
+    marker: "s" # square
+    hatch: "."
+
+  io_uring-reference:
+    legend: 'io_uring ($\bf{reference}$)'
+    color: "#2171b5"
+    marker: "v" # triangle_down
+    hatch: "."
+
+  io_uring_cmd-reference:
+    legend: 'io_uring_cmd ($\bf{reference}$)'
+    color: "#2171b5"
+    marker: "P" # plus
+    hatch: "."
+
+  # xNVMe implementations
+  libaio-xnvme:
+    legend: 'libaio ($\bf{xnvme}$)'
+    color: "#005a32"
+    marker: "p" # pentagon
+    hatch: "."
+
+  libaio-bdev_xnvme:
+    legend: 'libaio ($\bf{bdev\_xnvme}$)'
+    color: "#297373"
+    marker: "X" # cross
+    hatch: "O"
+
+  libaio-bdev_xnvme_conserve_cpu:
+    legend: 'libaio ($\bf{bdev\_xnvme}$) CCPU'
+    color: "#E9D758"
+    marker: "X" # cross
+    hatch: "O"
+
+  io_uring-xnvme:
+    legend: 'io_uring ($\bf{xnvme}$)'
+    color: "#41ab5d"
+    marker: "*" # star
+    hatch: "."
+
+  io_uring-bdev_xnvme:
+    legend: 'uring $\bf{bdev\_xnvme}$'
+    color: "#297373"
+    marker: "h" # hexagon1
+    hatch: "O"
+
+  io_uring-bdev_xnvme_conserve_cpu:
+    legend: 'uring $\bf{bdev\_xnvme}$ CCPU'
+    color: "#E9D758"
+    marker: "h" # hexagon1
+    hatch: "O"
+
+  io_uring-bdev_xnvme_BATCH:
+    legend: 'uring $\bf{bdev\_xnvme}$ BATCH'
+    color: "#FF8552"
+    marker: "D" # diamond
+    hatch: "*"
+
+  io_uring-bdev_xnvme_conserve_cpu_BATCH:
+    legend: 'uring $\bf{bdev\_xnvme}$ CCPU BATCH'
+    color: "#39393A"
+    marker: "D" # diamond
+    hatch: "*"
+
+  io_uring_cmd-xnvme:
+    legend: 'io_uring_cmd ($\bf{xnvme}$)'
+    color: "#74c476"
+    marker: "H" # hexagon2
+    hatch: "."
+
+  io_uring_cmd-bdev_xnvme:
+    legend: 'ucmd $\bf{bdev\_xnvme}$'
+    color: "#297373"
+    marker: "o" # circle
+    hatch: "O"
+  io_uring_cmd-bdev_xnvme_conserve_cpu:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ CCPU'
+    color: "#E9D758"
+    marker: "o" # circle
+    hatch: "O"
+
+  io_uring_cmd-bdev_xnvme_BATCH:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ BATCH'
+    color: "#FF8552"
+    marker: "d" # thin_diamond
+    hatch: "*"
+
+  io_uring_cmd-bdev_xnvme_conserve_cpu_BATCH:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ CCPU BATCH'
+    color: "#39393A"
+    marker: "d" # thin_diamond
+    hatch: "*"
+
+  # SPDK bdev implementations
+  libaio-bdev_aio:
+    legend: 'libaio ($\bf{bdev\_aio}$)'
+    color: "#4a1486"
+    marker: "D" # diamond
+    hatch: "O"
+
+  io_uring-bdev_uring:
+    legend: 'uring $\bf{bdev\_uring}$'
+    color: "#807dba"
+    marker: "d" # thin_diamond
+    hatch: "O"
+
+  ioctl-xnvme:
+    legend: 'ioctl ($\bf{xnvme}$)'
+    color: "#807dba"
+    marker: "d" # thin_diamond
+    hatch: "O"
diff --git a/cijoe/auxiliary/plot-attributes-4k.yaml b/cijoe/auxiliary/plot-attributes-4k.yaml
@@ -0,0 +1,125 @@
+---
+# Limits on plots, this is to ensure that different plots are visually comparable
+limits:
+  barplot:
+    y_lim: [0, 140000]
+  lineplot:
+    y_lim: [0, 2750000]
+
+# Identifying colors and markers by dataset-label
+datasets:
+  # Reference implementations
+  libaio-reference:
+    legend: 'libaio ($\bf{reference}$)'
+    color: "#084594"
+    marker: "s" # square
+    hatch: "."
+
+  io_uring-reference:
+    legend: 'io_uring ($\bf{reference}$)'
+    color: "#2171b5"
+    marker: "v" # triangle_down
+    hatch: "."
+
+  io_uring_cmd-reference:
+    legend: 'io_uring_cmd ($\bf{reference}$)'
+    color: "#2171b5"
+    marker: "P" # plus
+    hatch: "."
+
+  # xNVMe implementations
+  libaio-xnvme:
+    legend: 'libaio ($\bf{xnvme}$)'
+    color: "#005a32"
+    marker: "p" # pentagon
+    hatch: "."
+
+  libaio-bdev_xnvme:
+    legend: 'libaio ($\bf{bdev\_xnvme}$)'
+    color: "#297373"
+    marker: "X" # cross
+    hatch: "O"
+
+  libaio-bdev_xnvme_conserve_cpu:
+    legend: 'libaio ($\bf{bdev\_xnvme}$) CCPU'
+    color: "#E9D758"
+    marker: "X" # cross
+    hatch: "O"
+
+  io_uring-xnvme:
+    legend: 'io_uring ($\bf{xnvme}$)'
+    color: "#41ab5d"
+    marker: "*" # star
+    hatch: "."
+
+  io_uring-bdev_xnvme:
+    legend: 'uring $\bf{bdev\_xnvme}$'
+    color: "#297373"
+    marker: "h" # hexagon1
+    hatch: "O"
+
+  io_uring-bdev_xnvme_conserve_cpu:
+    legend: 'uring $\bf{bdev\_xnvme}$ CCPU'
+    color: "#E9D758"
+    marker: "h" # hexagon1
+    hatch: "O"
+
+  io_uring-bdev_xnvme_BATCH:
+    legend: 'uring $\bf{bdev\_xnvme}$ BATCH'
+    color: "#FF8552"
+    marker: "D" # diamond
+    hatch: "*"
+
+  io_uring-bdev_xnvme_conserve_cpu_BATCH:
+    legend: 'uring $\bf{bdev\_xnvme}$ CCPU BATCH'
+    color: "#39393A"
+    marker: "D" # diamond
+    hatch: "*"
+
+  io_uring_cmd-xnvme:
+    legend: 'io_uring_cmd ($\bf{xnvme}$)'
+    color: "#74c476"
+    marker: "H" # hexagon2
+    hatch: "."
+
+  io_uring_cmd-bdev_xnvme:
+    legend: 'ucmd $\bf{bdev\_xnvme}$'
+    color: "#297373"
+    marker: "o" # circle
+    hatch: "O"
+  io_uring_cmd-bdev_xnvme_conserve_cpu:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ CCPU'
+    color: "#E9D758"
+    marker: "o" # circle
+    hatch: "O"
+
+  io_uring_cmd-bdev_xnvme_BATCH:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ BATCH'
+    color: "#FF8552"
+    marker: "d" # thin_diamond
+    hatch: "*"
+
+  io_uring_cmd-bdev_xnvme_conserve_cpu_BATCH:
+    legend: 'ucmd $\bf{bdev\_xnvme}$ CCPU BATCH'
+    color: "#39393A"
+    marker: "d" # thin_diamond
+    hatch: "*"
+
+  # SPDK bdev implementations
+  libaio-bdev_aio:
+    legend: 'libaio ($\bf{bdev\_aio}$)'
+    color: "#4a1486"
+    marker: "D" # diamond
+    hatch: "O"
+
+  io_uring-bdev_uring:
+    legend: 'uring $\bf{bdev\_uring}$'
+    color: "#807dba"
+    marker: "d" # thin_diamond
+    hatch: "O"
+
+  ioctl-xnvme:
+    legend: 'ioctl ($\bf{xnvme}$)'
+    color: "#807dba"
+    marker: "d" # thin_diamond
+    hatch: "O"