From 1c6848fd97cdeb4512a4b81a4aad67d03a1c31e7 Mon Sep 17 00:00:00 2001 From: "Simon A. F. Lund" Date: Tue, 10 Oct 2023 14:52:26 +0200 Subject: [PATCH] feat(ci,cijoe): add configs, scripts and workflows for perf. eval. This adds scripts to: * Prepare bdev-configurations on the target-system - This is bench_confs_generate.py - It utilizes the list of [['duts']] aka "device-under-test" * bdevperf.py - Wrap the execution of the SPDK benchmark tool bdevperf * bench_plotter.py - Produce plots from the output of 'bdevperf.py' - Auxiliary files 'plot-attributes-*.yaml' to style, label and limit graphs * bench_reporter.py - Produce a .pdf report using the graphs produced by bench_plotter.py * {fio,liburing,spdk}_build.py - Scripts to build and install fio, liburing, and SPDK from source todo... describe this... Signed-off-by: Simon A. F. Lund --- .github/workflows/verify.yml | 73 ++- cijoe/Makefile | 3 + cijoe/auxiliary/plot-legends.yaml | 116 +++++ cijoe/auxiliary/plot-limits-4k-aio.yaml | 6 + cijoe/auxiliary/plot-limits-4k.yaml | 6 + cijoe/auxiliary/plot-limits-512.yaml | 6 + cijoe/configs/bench-amd.toml | 258 ++++++++++ cijoe/configs/bench-intel.toml | 241 +++++++++ cijoe/scripts/bdevperf.py | 133 +++++ cijoe/scripts/bench_plotter.py | 463 ++++++++++++++++++ cijoe/scripts/bench_reporter.py | 82 ++++ cijoe/scripts/fio_build.py | 37 ++ cijoe/scripts/liburing_build.py | 54 ++ ...weaked.py => linux_build_kdebs_tweaked.py} | 0 cijoe/scripts/linux_prepare_nvme.py | 27 + cijoe/scripts/spdk_bdev_confs_generator.py | 169 +++++++ cijoe/scripts/spdk_build_modded.py | 74 +++ cijoe/scripts/xnvme_build.py | 2 + cijoe/scripts/xnvme_install.py | 6 +- cijoe/templates/perf_report/Makefile | 4 + cijoe/templates/perf_report/README.rst | 22 + cijoe/templates/perf_report/bench.jinja2.rst | 232 +++++++++ cijoe/templates/perf_report/cover.jinja2.tmpl | 48 ++ cijoe/templates/perf_report/style.yaml | 37 ++ cijoe/templates/perf_report/xnvme.png | Bin 0 -> 49223 bytes cijoe/workflows/bench-report.yaml | 14 + cijoe/workflows/bench.yaml | 82 ++++ docs/ci/bench/index.rst | 15 +- 28 files changed, 2202 insertions(+), 8 deletions(-) create mode 100644 cijoe/auxiliary/plot-legends.yaml create mode 100644 cijoe/auxiliary/plot-limits-4k-aio.yaml create mode 100644 cijoe/auxiliary/plot-limits-4k.yaml create mode 100644 cijoe/auxiliary/plot-limits-512.yaml create mode 100644 cijoe/configs/bench-amd.toml create mode 100644 cijoe/configs/bench-intel.toml create mode 100644 cijoe/scripts/bdevperf.py create mode 100644 cijoe/scripts/bench_plotter.py create mode 100644 cijoe/scripts/bench_reporter.py create mode 100644 cijoe/scripts/fio_build.py create mode 100644 cijoe/scripts/liburing_build.py rename cijoe/scripts/{build_kdebs_tweaked.py => linux_build_kdebs_tweaked.py} (100%) create mode 100644 cijoe/scripts/linux_prepare_nvme.py create mode 100755 cijoe/scripts/spdk_bdev_confs_generator.py create mode 100644 cijoe/scripts/spdk_build_modded.py create mode 100644 cijoe/templates/perf_report/Makefile create mode 100644 cijoe/templates/perf_report/README.rst create mode 100644 cijoe/templates/perf_report/bench.jinja2.rst create mode 100644 cijoe/templates/perf_report/cover.jinja2.tmpl create mode 100644 cijoe/templates/perf_report/style.yaml create mode 100644 cijoe/templates/perf_report/xnvme.png create mode 100644 cijoe/workflows/bench-report.yaml create mode 100644 cijoe/workflows/bench.yaml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 6d74ccaa8..76cd492d2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -6,7 +6,7 @@ on: branches: [main, next] tags: ['v*'] pull_request: - types: [opened, reopened, synchronize] + types: [opened, reopened, synchronize, labeled] branches: [next] workflow_dispatch: inputs: @@ -18,6 +18,7 @@ on: options: - analyze - build-and-test + - build-and-bench - docgen defaults: @@ -592,6 +593,76 @@ jobs: path: test-results-${{ matrix.guest.os }}-${{ matrix.guest.ver }}.tar.gz if-no-files-found: error + # + # Build and run performance test of xNVMe + # + build-and-bench: + needs: [source-archive, source-format-check, build-python] + runs-on: [self-hosted, linux, ARM64, bench, intel] + + if: (contains('push', github.event_name) || ((github.event_name == 'workflow_dispatch') && (github.event.inputs.job == + 'build-and-bench')) || ((github.event_name == 'pull_request') && contains(github.event.pull_request.labels.*.name, 'bench'))) + + strategy: + fail-fast: false + + container: + image: debian:bookworm + options: --privileged + + steps: + - name: Runnner-prep, clean up self-hosted left-overs + run: | + pkill -f qemu || true + rm -r $HOME/guests || true + rm -r /tmp/artifacts || true + mkdir -p /tmp/artifacts || true + rm -rf * + ls -lh + + - name: Retrieve the xNVMe source archive + uses: actions/download-artifact@v3.0.1 + with: + name: xnvme-src-archive + - name: Extract, and then move, the xNVMe source archive + run: | + tar xzf xnvme-src.tar.gz --strip 1 + + - name: Prep, environment GITHUB_PATH + run: | + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Packages, Debian Bookworm + run: | + apt-get -qy update + apt-get -qy upgrade + apt-get -qy install python3 python3-dev python3-venv pipx git make gcc fontconfig + pipx ensurepath + + - name: CIJOE, setup pipx environment + run: | + pushd cijoe && make all && popd + + - name: CIJOE, run bench + run: | + rm -rf cijoe/bench-results + cd cijoe && cijoe \ + --config "configs/bench-intel.toml" \ + --workflow "workflows/bench.yaml" \ + --output "bench-results" + + - name: CIJOE, compress workflow-report-bench + if: always() + run: | + tar czf bench-results.tar.gz cijoe/bench-results/* + + - name: CIJOE, upload workflow-report-test + uses: actions/upload-artifact@v3.1.1 + if: always() + with: + name: bench-results + path: bench-results.tar.gz + if-no-files-found: error # # Documentation: generate documentation and deploy it to githubpages (xnvme.io) diff --git a/cijoe/Makefile b/cijoe/Makefile index 91f19e28b..cbc89ba87 100644 --- a/cijoe/Makefile +++ b/cijoe/Makefile @@ -41,6 +41,9 @@ install: @${PIPX} inject cijoe cijoe-pkg-qemu==v6.1.15 @${PIPX} inject cijoe cijoe-pkg-linux==v0.9.7 @${PIPX} inject cijoe cijoe-pkg-fio==v0.9.7 + @${PIPX} inject cijoe matplotlib + @${PIPX} inject cijoe numpy + @${PIPX} install rst2pdf @echo "## ${PROJECT_NAME}: make install [DONE]" define uninstall-help diff --git a/cijoe/auxiliary/plot-legends.yaml b/cijoe/auxiliary/plot-legends.yaml new file mode 100644 index 000000000..24b696cd2 --- /dev/null +++ b/cijoe/auxiliary/plot-legends.yaml @@ -0,0 +1,116 @@ +--- +# Reference implementations +libaio-reference: + legend: 'libaio ($\bf{reference}$)' + color: "#084594" + marker: "s" # square + hatch: "." + +io_uring-reference: + legend: 'io_uring ($\bf{reference}$)' + color: "#2171b5" + marker: "v" # triangle_down + hatch: "." + +io_uring_cmd-reference: + legend: 'io_uring_cmd ($\bf{reference}$)' + color: "#2171b5" + marker: "P" # plus + hatch: "." + +# xNVMe implementations +libaio-xnvme: + legend: 'libaio ($\bf{xnvme}$)' + color: "#005a32" + marker: "p" # pentagon + hatch: "." + +libaio-bdev_xnvme: + legend: 'libaio ($\bf{bdev\_xnvme}$)' + color: "#297373" + marker: "X" # cross + hatch: "O" + +libaio-bdev_xnvme_conserve_cpu: + legend: 'libaio ($\bf{bdev\_xnvme}$) +conserve' + color: "#E9D758" + marker: "X" # cross + hatch: "O" + +io_uring-xnvme: + legend: 'io_uring ($\bf{xnvme}$)' + color: "#41ab5d" + marker: "*" # star + hatch: "." + +io_uring-bdev_xnvme: + legend: 'uring $\bf{bdev\_xnvme}$' + color: "#297373" + marker: "h" # hexagon1 + hatch: "O" + +io_uring-bdev_xnvme_conserve_cpu: + legend: 'uring $\bf{bdev\_xnvme}$ +conserve' + color: "#E9D758" + marker: "h" # hexagon1 + hatch: "O" + +io_uring-bdev_xnvme_BATCH: + legend: 'uring $\bf{bdev\_xnvme}$ +batch' + color: "#FF8552" + marker: "D" # diamond + hatch: "*" + +io_uring-bdev_xnvme_conserve_cpu_BATCH: + legend: 'uring $\bf{bdev\_xnvme}$ +conserve +batch' + color: "#39393A" + marker: "D" # diamond + hatch: "*" + +io_uring_cmd-xnvme: + legend: 'io_uring_cmd ($\bf{xnvme}$)' + color: "#74c476" + marker: "H" # hexagon2 + hatch: "." + +io_uring_cmd-bdev_xnvme: + legend: 'ucmd $\bf{bdev\_xnvme}$' + color: "#297373" + marker: "o" # circle + hatch: "O" +io_uring_cmd-bdev_xnvme_conserve_cpu: + legend: 'ucmd $\bf{bdev\_xnvme}$ +conserve' + color: "#E9D758" + marker: "o" # circle + hatch: "O" + +io_uring_cmd-bdev_xnvme_BATCH: + legend: 'ucmd $\bf{bdev\_xnvme}$ +batch' + color: "#FF8552" + marker: "d" # thin_diamond + hatch: "*" + +io_uring_cmd-bdev_xnvme_conserve_cpu_BATCH: + legend: 'ucmd $\bf{bdev\_xnvme}$ +conserve +batch' + color: "#39393A" + marker: "d" # thin_diamond + hatch: "*" + +# SPDK bdev implementations +libaio-bdev_aio: + legend: 'libaio ($\bf{bdev\_aio}$)' + color: "#4a1486" + marker: "D" # diamond + hatch: "O" + +io_uring-bdev_uring: + legend: 'uring $\bf{bdev\_uring}$' + color: "#807dba" + marker: "d" # thin_diamond + hatch: "O" + +ioctl-xnvme: + legend: 'ioctl ($\bf{xnvme}$)' + color: "#807dba" + marker: "d" # thin_diamond + hatch: "O" diff --git a/cijoe/auxiliary/plot-limits-4k-aio.yaml b/cijoe/auxiliary/plot-limits-4k-aio.yaml new file mode 100644 index 000000000..6464d5ec4 --- /dev/null +++ b/cijoe/auxiliary/plot-limits-4k-aio.yaml @@ -0,0 +1,6 @@ +--- +# Limits on plots, this is to ensure that different plots are visually comparable +barplot: + y_lim: [0, 140000] +lineplot: + y_lim: [0, 1200000] diff --git a/cijoe/auxiliary/plot-limits-4k.yaml b/cijoe/auxiliary/plot-limits-4k.yaml new file mode 100644 index 000000000..84d413a38 --- /dev/null +++ b/cijoe/auxiliary/plot-limits-4k.yaml @@ -0,0 +1,6 @@ +--- +# Limits on plots, this is to ensure that different plots are visually comparable +barplot: + y_lim: [0, 140000] +lineplot: + y_lim: [0, 2750000] diff --git a/cijoe/auxiliary/plot-limits-512.yaml b/cijoe/auxiliary/plot-limits-512.yaml new file mode 100644 index 000000000..4ff88a8fe --- /dev/null +++ b/cijoe/auxiliary/plot-limits-512.yaml @@ -0,0 +1,6 @@ +--- +# Limits on plots, this is to ensure that different plots are visually comparable +barplot: + y_lim: [0, 195000] +lineplot: + y_lim: [0, 2750000] diff --git a/cijoe/configs/bench-amd.toml b/cijoe/configs/bench-amd.toml new file mode 100644 index 000000000..6a5deb509 --- /dev/null +++ b/cijoe/configs/bench-amd.toml @@ -0,0 +1,258 @@ +[transport.ssh] +hostname = "bench-amd.home.arpa" +port = 22 +username = "root" + +[os] +name = "debian" +version = "bookworm" + +[liburing.repository] +remote = "https://github.com/axboe/liburing.git" +path = "/root/git/liburing" +tag = "liburing-2.4" + +[xnvme.repository] +remote = "https://github.com/OpenMPDK/xNVMe.git" +path = "/root/git/xnvme" +branch = "next" + +[xnvme.build] +type = "debug" + +[spdk.repository] +remote = "https://github.com/xnvme/spdk-for-ci.git" +path = "/root/git/spdk" +tag = "xnvme-next" + +[spdk.build] +prefix = "/opt/spdk" + +[fio] +bin = "/opt/fio/bin/fio" + +[fio.repository] +remote = "https://github.com/axboe/fio.git" +path = "/root/git/fio" +tag = "fio-3.34" + +[fio.build] +prefix = "/opt/fio" + +[fio.engines] +libaio = {type = "builtin"} +io_uring = {type = "builtin"} +io_uring_cmd = {type = "builtin"} +xnvme = {type = "builtin"} +spdk_nvme = {type = "external_preload", path = "/root/git/spdk/build/fio/spdk_nvme"} +spdk_bdev = {type = "external_preload", path = "/root/git/spdk/build/fio/spdk_bdev"} + +[[devices]] +uri = "/dev/nvme0n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme1n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme2n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme3n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme4n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme5n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme6n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme7n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng0n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng1n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng2n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng3n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng4n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng5n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng6n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng7n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "0000:41:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:42:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:43:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:44:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:45:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:46:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:47:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:48:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[duts]] +pcie = "0000:08:00.0" +os = "0n1" +sn = "PHBT718003QR016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:09:00.0" +os = "1n1" +sn = "BTBT84921Y3W016N" +label = "M10_16GB" + +[[duts]] +pcie = "0000:0a:00.0" +os = "2n1" +sn = "PHBT813407Z8016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:41:00.0" +os = "3n1" +sn = "PHBT81350074016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:42:00.0" +os = "4n1" +sn = "PHBT813407T8016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:43:00.0" +os = "5n1" +sn = "PHBT813407ZW016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:44:00.0" +os = "6n1" +sn = "PHBT81350076016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:45:00.0" +os = "7n1" +sn = "PHBT813407U6016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:46:00.0" +os = "8n1" +sn = "PHBT718003QW016D" +label = "M10_16GB" + +[[duts]] +pcie = "0000:47:00.0" +os = "9n1" +sn = "BTBT84921FM7016N" +label = "M10_16GB" + +[[duts]] +pcie = "0000:48:00.0" +os = "10n1" +sn = "PHBT8134068W016D" +label = "M10_16GB" diff --git a/cijoe/configs/bench-intel.toml b/cijoe/configs/bench-intel.toml new file mode 100644 index 000000000..62d43824b --- /dev/null +++ b/cijoe/configs/bench-intel.toml @@ -0,0 +1,241 @@ +[transport.ssh] +hostname = "bench-intel.home.arpa" +port = 22 +username = "root" +password = "root" + +[os] +name = "debian" +version = "bookworm" + +[liburing.repository] +remote = "https://github.com/axboe/liburing.git" +path = "/root/git/liburing" +tag = "liburing-2.4" + +[xnvme.repository] +remote = "https://github.com/OpenMPDK/xNVMe.git" +path = "/root/git/xnvme" +branch = "next" + +[xnvme.build] +type = "debug" + +[spdk.repository] +remote = "https://github.com/xnvme/spdk-for-ci.git" +path = "/root/git/spdk" +tag = "xnvme-next" + +[spdk.build] +prefix = "/opt/spdk" + +[fio] +bin = "/opt/fio/bin/fio" + +[fio.repository] +remote = "https://github.com/axboe/fio.git" +path = "/root/git/fio" +tag = "fio-3.34" + +[fio.build] +prefix = "/opt/fio" + +[fio.engines] +libaio = {type = "builtin"} +io_uring = {type = "builtin"} +io_uring_cmd = {type = "builtin"} +xnvme = {type = "builtin"} +spdk_nvme = {type = "external_preload", path = "/root/git/spdk/build/fio/spdk_nvme"} +spdk_bdev = {type = "external_preload", path = "/root/git/spdk/build/fio/spdk_bdev"} + +[[devices]] +uri = "/dev/nvme0n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme1n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme2n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme3n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme4n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme5n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme6n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/nvme7n1" +nsid = 1 +labels = [ "dev", "bdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng0n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng1n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng2n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng3n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng4n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng5n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng6n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "/dev/ng7n1" +nsid = 1 +labels = [ "dev", "cdev", "nvm", "scc" ] +driver_attachment = "kernel" + +[[devices]] +uri = "0000:41:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:42:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:43:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:44:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:45:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:46:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:47:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[devices]] +uri = "0000:48:00.0" +nsid = 1 +labels = [ "dev", "pcie", "nvm", "scc" ] +driver_attachment = "userspace" + +[[duts]] +pcie = "0000:01:00.0" +os = "0n1" +sn = "S69ENX0T810892F" +label = "980PRO_2TB" + +[[duts]] +pcie = "0000:02:00.0" +os = "1n1" +sn = "S5GXNX0T875424D" +label = "980PRO_1TB" + +[[duts]] +pcie = "0000:03:00.0" +os = "2n1" +sn = "S6WSNS0T207605K" +label = "980PRO_1TB" + +[[duts]] +pcie = "0000:05:00.0" +os = "3n1" +sn = "S5GXNX0T811825D" +label = "980PRO_1TB" + +[[duts]] +pcie = "0000:06:00.0" +os = "4n1" +sn = "S69ENX0T805102B" +label = "980PRO_2TB" + +[[duts]] +pcie = "0000:07:00.0" +os = "5n1" +sn = "S69ENX0T805226J" +label = "980PRO_2TB" + +[[duts]] +pcie = "0000:09:00.0" +os = "6n1" +sn = "S69ENX0T805189F" +label = "980PRO_2TB" + +[[duts]] +pcie = "0000:0a:00.0" +os = "7n1" +sn = "S5GXNX0T875423K" +label = "980PRO_1TB" diff --git a/cijoe/scripts/bdevperf.py b/cijoe/scripts/bdevperf.py new file mode 100644 index 000000000..b2c6f11a7 --- /dev/null +++ b/cijoe/scripts/bdevperf.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +""" + Run fio... +""" +import json +import logging as log +import os +import shutil +import traceback +from itertools import product +from pathlib import Path + + +def main(args, cijoe, step): + repetitions = step.get("with", {}).get("repetitions", 3) + iosizes = step.get("with", {}).get("iosizes", ["4K"]) + iodepths = step.get("with", {}).get("iodepths", [1, 2, 4, 8]) + + ndevices = str(step.get("with", {}).get("ndevices", "1")) + bdev_confs = step.get("with", {}).get("bdev_confs", "/tmp") + + iopaths = { + "io_uring_cmd-bdev_xnvme": { + "bdev_name": "bdev_xnvme", + "io_mechanism": "io_uring_cmd", + }, + # "io_uring_cmd-bdev_xnvme_BATCH": { + # "bdev_name": "bdev_xnvme", + # "io_mechanism": "io_uring_cmd", + # }, + # "io_uring_cmd-bdev_xnvme_conserve_cpu": { + # "bdev_name": "bdev_xnvme", + # "io_mechanism": "io_uring_cmd", + # }, + "io_uring_cmd-bdev_xnvme_conserve_cpu_BATCH": { + "bdev_name": "bdev_xnvme", + "io_mechanism": "io_uring_cmd", + }, + "io_uring-bdev_uring": { + "bdev_name": "bdev_uring", + "io_mechanism": "io_uring", + }, + "io_uring-bdev_xnvme": { + "bdev_name": "bdev_xnvme", + "io_mechanism": "io_uring", + }, + # "io_uring-bdev_xnvme_BATCH": { + # "bdev_name": "bdev_xnvme", + # "io_mechanism": "io_uring", + # }, + # "io_uring-bdev_xnvme_conserve_cpu": { + # "bdev_name": "bdev_xnvme", + # "io_mechanism": "io_uring", + # }, + "io_uring-bdev_xnvme_conserve_cpu_BATCH": { + "bdev_name": "bdev_xnvme", + "io_mechanism": "io_uring", + }, + "libaio-bdev_aio": { + "bdev_name": "bdev_aio", + "io_mechanism": "libaio", + }, + "libaio-bdev_xnvme": { + "bdev_name": "bdev_xnvme", + "io_mechanism": "libaio", + }, + # "libaio-bdev_xnvme_conserve_cpu": { + # "bdev_name": "bdev_xnvme", + # "io_mechanism": "libaio", + # }, + } + + artifacts = Path(args.output) / "artifacts" + os.makedirs(str(artifacts), exist_ok=True) + + err = 0 + try: + for bs, iodepth, (label, params), rep in product( + iosizes, iodepths, iopaths.items(), range(repetitions) + ): + bdevperf_output_path = ( + artifacts + / f"bdevperf-output_BS={bs}_IODEPTH={iodepth}_LABEL={label}_{rep}.txt" + ) + + # Create a spdk-configuration file and transfer it + spdk_conf_path = Path(bdev_confs) / "_".join( + [ + params["bdev_name"], + params["io_mechanism"], + "conserve_cpu" if "conserve_cpu" in label else "", + f"{ndevices}.conf", + ] + ) + + env = {} + env["XNVME_QUEUE_BATCHING"] = "1" + cores = "[0,1]" + + # if "io_uring_cmd" in label: + # env["XNVME_QUEUE_SQPOLL_AWQ"] = "1" + # env["XNVME_QUEUE_SQPOLL_CPU"] = "2" + # cores = "[0]" + # else: + # cores = "[0,1]" + + # Run bdevperf + command = [ + "/root/git/spdk/build/examples/bdevperf", + f"--json {spdk_conf_path}", + f"-q {iodepth}", + f"-o {bs}", + "-w randread", + "-t 10", + f"-m '{cores}'", + ] + err, state = cijoe.run(" ".join(command), env=env) + if err: + log.error(f"failed: {state}") + + # Save the bdevperf output to a file in artifacts directory + shutil.copyfile(state.output_fpath, bdevperf_output_path) + + except Exception as exc: + log.error(f"Something failed({exc})") + log.error("".join(traceback.format_exception(None, exc, exc.__traceback__))) + print( + type(exc).__name__, # TypeError + __file__, # /tmp/example.py + exc.__traceback__.tb_lineno, # 2 + ) + + return err diff --git a/cijoe/scripts/bench_plotter.py b/cijoe/scripts/bench_plotter.py new file mode 100644 index 000000000..c2c97e33a --- /dev/null +++ b/cijoe/scripts/bench_plotter.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python +""" +Extract and transform fio and bdevperf output +============================================= + +fio has: + +* bandwidth in KiB / second +* latency is mean and in nano-seconds +* IOPS are 'raw' / base-unit + +bdevperf has: + +* bandwidth in MiB / second +* IOPS are 'raw' / base-unit + +.. note:: + The metric-context need addition of backend-options, the options are + semi-encoded by ioengine and 'label', however, that is not very precise. + +.. note:: + This uses matplotlib and numpy for plotting +""" +import errno +import hashlib +import json +import logging as log +import os +import pprint +import re +import traceback +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +from cijoe.core.analyser import to_base_unit +from cijoe.core.resources import dict_from_yamlfile +from cijoe.fio.wrapper import dict_from_fio_output_file + +JSON_DUMP = {"indent": 4} + +FIO_COMPOUND_FILENAME = "fio-output-compound.json" +FIO_OUTPUT_NORMALIZED_FILENAME = "fio-output-normalize.json" + +BDEVPERF_OUTPUT_PREFIX = "bdevperf-output_" +BDEVPERF_OUTPUT_NORMALIZED_FILENAME = "bdevperf-output-normalized.json" +OUTPUT_REGEX = r".*BS=(?P.*)_IODEPTH=(?P\d+)_LABEL=(?P