From 48ed1a6b35b6dbcbc9c23f9095297e3daea5f8cb Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Nov 2025 22:37:21 -0700 Subject: [PATCH 1/3] Add quoting from shellcheck --- util/files-changed.sh | 8 ++++---- util/full-clang-lint.sh | 6 +++--- util/iterate-fio-jobs.sh | 16 ++++++++-------- util/vfio.sh | 20 ++++++++++---------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/util/files-changed.sh b/util/files-changed.sh index 5115fac1..56cfa0e1 100755 --- a/util/files-changed.sh +++ b/util/files-changed.sh @@ -16,17 +16,17 @@ CHECK_PATTERN=$3 FILES_CHANGED=0 changed_files=() -if ! diff_output="$(git diff --name-only ${BASE_REF} ${HEAD_REF})"; then - echo ${diff_output} +if ! diff_output="$(git diff --name-only "${BASE_REF}" "${HEAD_REF}")"; then + echo "${diff_output}" exit 1 fi mapfile -t changed_files <<< "${diff_output}" for changed_file in "${changed_files[@]}"; do - if [[ "${changed_file}" == ${CHECK_PATTERN} ]]; then + if [[ "${changed_file}" == "${CHECK_PATTERN}" ]]; then FILES_CHANGED=1 fi done printf '%s' ${FILES_CHANGED} -exit 0 \ No newline at end of file +exit 0 diff --git a/util/full-clang-lint.sh b/util/full-clang-lint.sh index a86c082e..4aadd397 100755 --- a/util/full-clang-lint.sh +++ b/util/full-clang-lint.sh @@ -16,13 +16,13 @@ fi WORKING_DIRECTORY=$1 ERROR_FOUND=0 -for file_to_lint in $(find ${WORKING_DIRECTORY} -type f \( -name "*.c" -or -name "*.h" \)); +for file_to_lint in $(find "${WORKING_DIRECTORY}" -type f \( -name "*.c" -or -name "*.h" \)); do - clang-format --Werror --dry-run --style=file ${file_to_lint} + clang-format --Werror --dry-run --style=file "${file_to_lint}" if [[ $? -ne 0 ]]; then ERROR_FOUND=1 - clang-format --Werror --style=file ${file_to_lint} | diff -u -p --color ${file_to_lint} - + clang-format --Werror --style=file "${file_to_lint}" | diff -u -p --color "${file_to_lint}" - echo -e "\n" fi done diff --git a/util/iterate-fio-jobs.sh b/util/iterate-fio-jobs.sh index f734a364..f770ab52 100755 --- a/util/iterate-fio-jobs.sh +++ b/util/iterate-fio-jobs.sh @@ -76,10 +76,10 @@ for ioengine in "${fio_ioengines[@]}"; do fio_cmd+="--ioengine=${ioengine} --${_gpu_io_param}=${_gpu_io_value} --gpu_dev_ids=0 " fio_cmd+="--bs=${size} > ${fio_single_job_results_path} &" #echo "fio invocation: ${fio_cmd}" - eval $fio_cmd + eval "$fio_cmd" FIO_PID=$! echo "FIO_PID: ${FIO_PID}" - pidstat_output=$(pidstat -u -p ${FIO_PID} 1 $((${fio_runtime}+${fio_ramptime}))) + pidstat_output=$(pidstat -u -p ${FIO_PID} 1 $((fio_runtime + fio_ramptime))) pidstat_output=$(echo "${pidstat_output}" | awk 'NR > 3 && !/Average:/ {print}') pidstat_copy_results+="${pidstat_output}\n\n" wait ${FIO_PID} @@ -90,12 +90,12 @@ for ioengine in "${fio_ioengines[@]}"; do continue fi fio_output=$(<${fio_single_job_results_path}) - read_bw=$(echo ${fio_output} | jq '.jobs[0].read.bw_bytes') - read_lat=$(echo ${fio_output} | jq '.jobs[0].read.lat_ns.mean') - write_bw=$(echo ${fio_output} | jq '.jobs[0].write.bw_bytes') - write_lat=$(echo ${fio_output} | jq '.jobs[0].write.lat_ns.mean') - usr_cpu=$(echo ${fio_output} | jq '.jobs[0].usr_cpu') - sys_cpu=$(echo ${fio_output} | jq '.jobs[0].sys_cpu') + read_bw=$(echo "${fio_output}" | jq '.jobs[0].read.bw_bytes') + read_lat=$(echo "${fio_output}" | jq '.jobs[0].read.lat_ns.mean') + write_bw=$(echo "${fio_output}" | jq '.jobs[0].write.bw_bytes') + write_lat=$(echo "${fio_output}" | jq '.jobs[0].write.lat_ns.mean') + usr_cpu=$(echo "${fio_output}" | jq '.jobs[0].usr_cpu') + sys_cpu=$(echo "${fio_output}" | jq '.jobs[0].sys_cpu') total_cpu=$(echo "scale=2; ${usr_cpu} + ${sys_cpu}" | bc) # Add two FP numbers together, precision of 2. if [ "${read_bw}" -eq 0 ]; then write_bw_gib=$(echo "scale=8; ${write_bw} / 2^30" | bc) diff --git a/util/vfio.sh b/util/vfio.sh index dcb79c02..889d3319 100755 --- a/util/vfio.sh +++ b/util/vfio.sh @@ -11,22 +11,22 @@ for bdf in "$@"; do # Unbind the current driver - if [ -e /sys/bus/pci/devices/$bdf/driver/unbind ]; then - echo $bdf > /sys/bus/pci/devices/$bdf/driver/unbind + if [ -e /sys/bus/pci/devices/"$bdf"/driver/unbind ]; then + echo "$bdf" > /sys/bus/pci/devices/"$bdf"/driver/unbind fi # Bind the vfio-pci driver - vendor_id=$(cat /sys/bus/pci/devices/$bdf/vendor) - device_id=$(cat /sys/bus/pci/devices/$bdf/device) - echo $vendor_id $device_id > /sys/bus/pci/drivers/vfio-pci/new_id + vendor_id=$(cat /sys/bus/pci/devices/"$bdf"/vendor) + device_id=$(cat /sys/bus/pci/devices/"$bdf"/device) + echo "$vendor_id" "$device_id" > /sys/bus/pci/drivers/vfio-pci/new_id # Add RW permissions for the vfio group to the device's iommu group - iommu_group=$(basename $(readlink -f /sys/bus/pci/devices/$bdf/iommu_group/)) - chgrp vfio /dev/vfio/$iommu_group - chmod g+rw /dev/vfio/$iommu_group + iommu_group=$(basename "$(readlink -f /sys/bus/pci/devices/"$bdf"/iommu_group/)") + chgrp vfio /dev/vfio/"$iommu_group" + chmod g+rw /dev/vfio/"$iommu_group" # Show which driver is now bound to the PCI device - driver=$(basename $(readlink -f /sys/bus/pci/devices/$bdf/driver)) - echo $bdf $driver + driver=$(basename "$(readlink -f /sys/bus/pci/devices/"$bdf"/driver)") + echo "$bdf" "$driver" done From 243369c47b68fd3638e10a5ebe7014f6ef2784f5 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 17 Nov 2025 22:47:27 -0700 Subject: [PATCH 2/3] Tidy README.md * Remove [INTERNAL] tag and rename H1 to hipFile * Add H2 entries for each script --- util/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/util/README.md b/util/README.md index 9344439c..dfe91478 100644 --- a/util/README.md +++ b/util/README.md @@ -1,5 +1,19 @@ -# [INTERNAL] AIS Development Utilities +# hipFile Development Utilities ## clear-journal.sh Clears systemd's journal. + +## files-changed.sh + +## format-source.sh + +## full-clang-lint.sh + +## iterate-fio-jobs.sh + +## llvm-coverage.sh + +## setup-nvmeof.py + +## vfio.sh From 4c7cfda4a57af5a896fcc43ba5f9e71cf6b5b25f Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 18 Nov 2025 13:28:31 -0700 Subject: [PATCH 3/3] Clean up unused scripts in `util` --- util/README.md | 15 +- util/clear-journal.sh | 12 - util/full-clang-lint.sh | 30 --- util/iterate-fio-jobs.sh | 116 ---------- util/setup-nvmeof.py | 458 --------------------------------------- util/vfio.sh | 32 --- 6 files changed, 5 insertions(+), 658 deletions(-) delete mode 100755 util/clear-journal.sh delete mode 100755 util/full-clang-lint.sh delete mode 100755 util/iterate-fio-jobs.sh delete mode 100755 util/setup-nvmeof.py delete mode 100755 util/vfio.sh diff --git a/util/README.md b/util/README.md index dfe91478..6f3cab76 100644 --- a/util/README.md +++ b/util/README.md @@ -1,19 +1,14 @@ # hipFile Development Utilities -## clear-journal.sh - -Clears systemd's journal. - ## files-changed.sh -## format-source.sh +Determines if important files changed. Used by CI. -## full-clang-lint.sh +## format-source.sh -## iterate-fio-jobs.sh +Formats applicable source in the repo using a specific version +of `clang-format`. Run it from the root. ## llvm-coverage.sh -## setup-nvmeof.py - -## vfio.sh +Code coverage script used by CI. diff --git a/util/clear-journal.sh b/util/clear-journal.sh deleted file mode 100755 index a63513ea..00000000 --- a/util/clear-journal.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - - -set -x - -sudo journalctl --rotate -sudo journalctl --vacuum-time=1s -sudo systemctl restart systemd-journald -sudo journalctl --disk-usage diff --git a/util/full-clang-lint.sh b/util/full-clang-lint.sh deleted file mode 100755 index 4aadd397..00000000 --- a/util/full-clang-lint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - -# CI script that will highlight any issues clang-format complains about -# as well as the diff of what needs to change. -# Requires a .clang-format file in the directory selected to lint. - -if [[ $# -ne 1 ]]; -then - echo "Requires one argument of the directory to lint." - exit 1 -fi - -WORKING_DIRECTORY=$1 -ERROR_FOUND=0 - -for file_to_lint in $(find "${WORKING_DIRECTORY}" -type f \( -name "*.c" -or -name "*.h" \)); -do - clang-format --Werror --dry-run --style=file "${file_to_lint}" - if [[ $? -ne 0 ]]; - then - ERROR_FOUND=1 - clang-format --Werror --style=file "${file_to_lint}" | diff -u -p --color "${file_to_lint}" - - echo -e "\n" - fi -done - -exit ${ERROR_FOUND} diff --git a/util/iterate-fio-jobs.sh b/util/iterate-fio-jobs.sh deleted file mode 100755 index f770ab52..00000000 --- a/util/iterate-fio-jobs.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - -cleanup() { - # Attempt to cleanup a leftover FIO results file - # at the end of this process. - local rc=$? - if [ -f ${fio_single_job_results_path} ]; then - rm ${fio_single_job_results_path} - echo "Cleaned up FIO log file." - fi - exit ${rc} -} -trap cleanup EXIT INT TERM - -quick_copy_results="IO Mode,RW Operation,Block Size,Bandwidth (bytes/s),Bandwidth (GiB/s),Avg Latency (ns),User CPU%,Sys CPU%,Total CPU%\n" -pidstat_copy_results="" - -fio_single_job_results_path="/tmp/fio_job_results.json" -fio_runtime=30 -fio_ramptime=5 -fio_filesize="1GiB" -#fio_data_directory="/mnt/ais/ext4/fio_jobs" # NVMe drive locally on Rowlet -fio_data_directory="/mnt/ais/pancham_ais_ext4/fio_data" # NVMeoF drive on Pancham -fio_ioengines=("libcufile" "librocfile") -fio_gpu_io_methods=("posix" "_gpu_io") -fio_rw_modes=("read" "write") -block_sizes=("4KiB" "16KiB" "64KiB" "256KiB" "1MiB" "4MiB" "16MiB" "64MiB" "256MiB" "1GiB") - -for ioengine in "${fio_ioengines[@]}"; do - for gpu_io_method in "${fio_gpu_io_methods[@]}"; do - for rw_mode in "${fio_rw_modes[@]}"; do - for size in "${block_sizes[@]}"; do - # Ideally any ioengine specific args somehow get processed separately. - # But for purposes of this helper script where our benchmarks rely on - # these specific ioengines, we probably are okay. - _gpu_io_param="" - _gpu_io_value="" - _results_gpu_io_mode="" - if [ "${ioengine}" == "libcufile" ]; then - _gpu_io_param="cuda_io" - _results_gpu_io_mode="cuFile" - elif [ "${ioengine}" == "libhipfile" ]; then - _gpu_io_param="rocm_io" - _results_gpu_io_mode="hipFile" - else - echo "Unknown IOEngine for GPU IO." - exit 1 - fi - if [ "${gpu_io_method}" == "_gpu_io" ] && [ "${ioengine}" == "libcufile" ]; then - _gpu_io_value="cufile" - _results_gpu_io_mode+="_GDS" - elif [ "${gpu_io_method}" == "_gpu_io" ] && [ "${ioengine}" == "libhipfile" ]; then - _gpu_io_value="hipFile" - _results_gpu_io_mode+="_AIS" - elif [ "${gpu_io_method}" == "posix" ]; then - _gpu_io_value="posix" - _results_gpu_io_mode+="_POSIX" - else - echo "Unknown GPU IO Method." - exit 1 - fi - - echo "FIO Job Config: ${_results_gpu_io_mode} ${rw_mode} ${size}" - #fio_job_name="${ioengine}_${gpu_io_method}_${rw_mode}_${size}" - # Keeping job name constant in order to minimize the number of data files - # FIO generates/writes to. Not as big of an issue if FIO cleaned these up - # at the end of each job. Do we need to worry about drive cache? - fio_job_name="NVMeoF_Benchmark" - fio_cmd="/home/rildixon/rocfile/fio/fio --warnings-fatal --output-format=json " - fio_cmd+="--name=${fio_job_name} --kb_base=1000 --directory=${fio_data_directory} " - fio_cmd+="--numjobs=1 --time_based --runtime=${fio_runtime} --ramp_time=${fio_ramptime} --thread " - fio_cmd+="--cpus_allowed=15 --direct=1 --rw=${rw_mode} --size=${fio_filesize} " - fio_cmd+="--ioengine=${ioengine} --${_gpu_io_param}=${_gpu_io_value} --gpu_dev_ids=0 " - fio_cmd+="--bs=${size} > ${fio_single_job_results_path} &" - #echo "fio invocation: ${fio_cmd}" - eval "$fio_cmd" - FIO_PID=$! - echo "FIO_PID: ${FIO_PID}" - pidstat_output=$(pidstat -u -p ${FIO_PID} 1 $((fio_runtime + fio_ramptime))) - pidstat_output=$(echo "${pidstat_output}" | awk 'NR > 3 && !/Average:/ {print}') - pidstat_copy_results+="${pidstat_output}\n\n" - wait ${FIO_PID} - fio_exit_code=$? - if [ ${fio_exit_code} -ne 0 ]; then - echo "FIO Job Failed. Logging failure and continuing." - quick_copy_results+="${_results_gpu_io_mode},${rw_mode},${size},0,0,0,0,0,0\n" - continue - fi - fio_output=$(<${fio_single_job_results_path}) - read_bw=$(echo "${fio_output}" | jq '.jobs[0].read.bw_bytes') - read_lat=$(echo "${fio_output}" | jq '.jobs[0].read.lat_ns.mean') - write_bw=$(echo "${fio_output}" | jq '.jobs[0].write.bw_bytes') - write_lat=$(echo "${fio_output}" | jq '.jobs[0].write.lat_ns.mean') - usr_cpu=$(echo "${fio_output}" | jq '.jobs[0].usr_cpu') - sys_cpu=$(echo "${fio_output}" | jq '.jobs[0].sys_cpu') - total_cpu=$(echo "scale=2; ${usr_cpu} + ${sys_cpu}" | bc) # Add two FP numbers together, precision of 2. - if [ "${read_bw}" -eq 0 ]; then - write_bw_gib=$(echo "scale=8; ${write_bw} / 2^30" | bc) - quick_copy_results+="${_results_gpu_io_mode},${rw_mode},${size},${write_bw},${write_bw_gib}," - quick_copy_results+="${write_lat},${usr_cpu},${sys_cpu},${total_cpu}\n" - else - read_bw_gib=$(echo "scale=8; ${read_bw} / 2^30" | bc) - quick_copy_results+="${_results_gpu_io_mode},${rw_mode},${size},${read_bw},${read_bw_gib}," - quick_copy_results+="${read_lat},${usr_cpu},${sys_cpu},${total_cpu}\n" - fi - done - done - done -done - -echo -e "Quick results:\n${quick_copy_results}" -echo -e "Pidstat Data:\n${pidstat_copy_results}" - diff --git a/util/setup-nvmeof.py b/util/setup-nvmeof.py deleted file mode 100755 index b9f9426a..00000000 --- a/util/setup-nvmeof.py +++ /dev/null @@ -1,458 +0,0 @@ -#! /usr/bin/env python3 -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - -""" -SystemD approach if wanting to have this execute on startup: - -/etc/systemd/system/setup-nvmeof.service -************************************************** -[Unit] -Description=Configure an NVMeoF Target device. - -[Service] -ExecStart=/usr/local/bin/setup-nvmeof.py -Type=oneshot -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target -************************************************** -Modify ExecStart as needed. - -Then enable the service file in SystemD: -$ sudo systemctl daemon-reload && sudo systemctl enable setup-nvmeof.service - -Check the status with -$ sudo systemctl status setup-nvmeof.service - -Please note all networking configuration (including RXE if used) -needs to be done prior to running this utility. Otherwise it will -fail to start. - -Nice to have enhancements: -* Automatically grab an IP Address to bind to. -* Automatically determine namespace-num if one is not provided. - (If namespace N exists, generate namespace N+1.) -* Automatically determine port-num if one is not provided. - (If port-num N exists, generate port-num N+1.) -""" -import argparse -import os -import sys -import subprocess - -from logging import Formatter, StreamHandler, FileHandler, INFO, getLogger -from pathlib import Path - -logger = getLogger(Path(__file__).stem) - -# Types must be expressed in lowercase. -NVMeoF_TRANSPORT_TYPES = [ - "tcp", - "rdma" -] - -NVMET_BASE_PATH = Path("/sys/kernel/config/nvmet") -NVMET_SUBSYSTEM_PATH = NVMET_BASE_PATH / "subsystems" -NVMET_PORT_PATH = NVMET_BASE_PATH / "ports" - - -def setup_logging(): - """ - Configure a basic logger that prints to STDOUT & a log file. - - The log file used is kept in a standard location: /var/log/setup-nvmeof.log - If the script is executed directly, the log messages will also be displayed - to the terminal. - """ - logger.setLevel(INFO) - - console_format = Formatter("%(levelname)s - %(message)s") - console_handler = StreamHandler() - console_handler.setFormatter(console_format) - logger.addHandler(console_handler) - - log_file_path = Path("/var/log/setup-nvmeof.log") - log_file_format = Formatter("%(asctime)s - %(levelname)s - %(message)s") - try: - if not log_file_path.exists(): - log_file_path.touch(mode=644) - log_file_handler = FileHandler(log_file_path) - log_file_handler.setFormatter(log_file_format) - logger.addHandler(log_file_handler) - except PermissionError: - print("Unable to write to the NVMeoF-Setup log file.") - except OSError: - print("Unable to create NVMeoF-Setup log file.") - - -def load_kernel_module(kmod_name: str) -> None: - """ - Loads a kernel module. - - Assumes that we cannot do anything more if we are unable to load - the specified kernel module. - - If modprobe has already loaded in the module, modprobe - becomes a no-op. Thus this is safe to perform without - further checks in case the modules were loaded elsewhere. - """ - logger.info(f"Loading kernel module '{kmod_name}'.") - cmd = ["modprobe", kmod_name] - result = subprocess.run( - cmd, - check=True - ) - - -def create_NVMet_subsystem(subsystem_name: str, force: bool) -> None: - """ - Create an NVMet subsystem to encapsulate the target NVMe drives. - """ - # Check if subsystem already exists. - # If it already exists, emit a warning, but continue normally. - subsystem_dir = NVMET_SUBSYSTEM_PATH / subsystem_name - if subsystem_dir.exists(): - logger.warning(f"NVMet subsystem '{subsystem_name}' already exists.") - if force: - logger.warning("Modifying the existing NVMe subsystem.") - else: - logger.error("Aborting further NVMeoF setup.") - sys.exit(1) - else: - logger.info(f"Making new subsystem '{subsystem_name}'.") - os.mkdir(subsystem_dir, 755) - - # Set access to all hosts. - with open(subsystem_dir / "attr_allow_any_host", "w") as dev_attr: - logger.info("Enabling any host to connect to this NVMe Subsystem.") - dev_attr.write("1") - - -def create_NVMet_namespace( - subsystem_name: str, - namespace_num: int, - nvme_device: Path, - force: bool -) -> None: - """ - Create & Configure a NVMet namespace. - - If the namespace already exists within the subsystem, emit a warning but - continue as normal. - - subsystem_name - Name for a new NVMe Subsystem. - namespace_num - Namespace ID # associated under the NVMe Subsystem. - nvme_device - Path to the NVMe device that will be exposed to the network. - (e.g. /dev/nvme0n1, /dev/disk/by-id/nvme-*) - Reminder that the /dev/nvme#n# path does not reliably point - to the same physical disk. This is generated at boot time - during device discovery. Consider using a difference - reference that is fixed (PCIe BDF, UUID, Disk ID/SN). - force - Reconfigure an existing NVMe Nsamespace if it already exists. - """ - namespace_dir = ( - NVMET_SUBSYSTEM_PATH / subsystem_name / "namespaces" / - str(namespace_num) - ) - if namespace_dir.exists(): - logger.warning( - f"NVMet namespace '{subsystem_name}':'{namespace_num}' already " - "exists. " - ) - if force: - logger.warning("Overwriting the existing NVMe Namespace.") - with open(namespace_dir / "enable", "w") as dev_attr: - logger.info( - "Disabling the old NVMet Namespace " - f"'{subsystem_name}':'{namespace_num}'." - ) - dev_attr.write("0") - else: - logger.error( - "Aborting further NVMeoF setup. Please be aware previous " - "setup steps will not be reverted." - ) - sys.exit(1) - else: - logger.info( - f"Making new namespace '{subsystem_name}':'{namespace_num}'." - ) - os.mkdir(namespace_dir, 755) - - with open(namespace_dir / "device_path", "w") as dev_attr: - logger.info(f"Setting NVMe Target drive: {nvme_device}") - dev_attr.write(str(nvme_device)) - with open(namespace_dir / "enable", "w") as dev_attr: - logger.info( - "Enabling the NVMet Namespace " - f"'{subsystem_name}':'{namespace_num}'." - ) - dev_attr.write("1") - - -def create_NVMet_port( - nvme_port: int, - net_port: int, - address: str, - transport_type: str, - force: bool -) -> None: - """ - Create & Configure a new NVMeoF Port. - - If the port already exists, emit a warning and modify the existing port. - NOTE: That this is different behaviour from creating the NVMe subsystem. - The port is easier to modify than the subsystem for **reasons**. - - nvme_port - Port number associated with the local NVMe controller. - net_port - Network port the NVMet drive will listen on for connections. - address - Address to broadcast the NVMet drive on. - (Currently only supports IPv4) - transport_type - Transport type used to establish NVMeoF communication. - force - Reconfigure an existing NVMe Port if it already exists. - """ - - nvme_port_dir = NVMET_PORT_PATH / str(nvme_port) - if nvme_port_dir.exists(): - logger.warning( - f"NVMet Port # {nvme_port} already exists. " - "Modifying the existing port." - ) - if force: - logger.warning("Overwriting the NVMe Port configuration.") - logger.warning( - "Disabling the NVMe Port by removing ALL NVMe subsystem " - "symlinks." - ) - _delete_NVMet_links(nvme_port_dir / "subsystems") - else: - logger.error( - "Aborting further NVMeoF setup. Please be aware previous " - "setup steps will not be reverted." - ) - sys.exit(1) - else: - logger.info(f"Creating new NVMet port '{nvme_port}'.") - os.mkdir(nvme_port_dir, 755) - - with open(nvme_port_dir / "addr_traddr", "w") as dev_attr: - logger.info(f"Exposing the NVMe device on address '{address}'.") - dev_attr.write(address) - with open(nvme_port_dir / "addr_adrfam", "w") as dev_attr: - # Currently fixed to IPv4. If we need to support other address types - # we should redesign this script to be more extendible. - logger.info(f"Setting Address Family as 'ipv4'.") - dev_attr.write("ipv4") - with open(nvme_port_dir / "addr_trtype", "w") as dev_attr: - logger.info(f"Setting NVMeoF Transport method to '{transport_type}'.") - dev_attr.write(transport_type) - with open(nvme_port_dir / "addr_trsvcid", "w") as dev_attr: - logger.info( - f"Binding NVMe device to '{transport_type}' Port '{net_port}'." - ) - dev_attr.write(str(net_port)) - - -def _delete_NVMet_links(port_ns_links: Path) -> None: - """ - Deletes all symlinks in a given directory. - """ - ns_symlinks = [ - ns_symlink for ns_symlink in port_ns_links.iterdir() - if ns_symlink.is_symlink() - ] - for symlink in ns_symlinks: - logger.warning( - f"Removing symlink {symlink} -> {symlink.resolve()}" - ) - os.remove(symlink) - - -def create_NVMet_link(subsystem_name: str, nvme_port: int) -> None: - """ - Add a symlink to connect the NVMeoF Port to the NVMe Subsystem. - """ - subsystem_path = NVMET_SUBSYSTEM_PATH / subsystem_name - port_subsystem_path = ( - NVMET_PORT_PATH / str(nvme_port) / "subsystems" / subsystem_name - ) - logger.info( - f"Creating symlink '{port_subsystem_path}' -> " - f"'{subsystem_path}'." - ) - try: - os.symlink( - src=subsystem_path, - dst=port_subsystem_path - ) - except OSError as exc: - if "No such device" in exc.strerror: - # E501 - line too long (80 char limit) - logger.exception( - f"\n\n{'*' * 40}\n" - f"This error typically indicates that the setup script ran \n" - f"successfully. However, a device that NVMeoF depends on is either \n" # noqa: E501 - f"missing or mis-configured. For example: \n" - f" * The network device as specified by address is not ready.\n" # noqa: E501 - f" * If RDMA is specified, the network device is not RDMA capable.\n" # noqa: E501 - f" * If RDMA is specified, RXE may not be loaded & configured.\n" # noqa: E501 - f"Start by verifying the devices NVMeoF depends on are active and \n" # noqa: E501 - f"correctly configured.\n" - f"{'*' * 40}\n\n" - ) - raise - - -def check_syslog(nvme_port: int, address: str, net_port: int) -> None: - """ - Checks the syslog for the kernel to emit a message. - - This is not a perfect check as an old message may be read that was not - attributed to this exact script invocation. - """ - result = subprocess.run( - ["dmesg"], - capture_output=True, - encoding="utf-8" - ) - # nvmet_rdma: enabling port 1 (192.168.50.67:4420) - check_str = f"nvmet_rdma: enabling port {nvme_port} ({address}:{net_port})" - if check_str in result.stdout: - logger.info(f"NVMeoF Device Enabled message present in kernel logs.") - else: - logger.warning( - "Unable to find NVMeoF Device enabled message in kernel logs.\n" - "Please manually confirm that the NVMeoF device is enabled." - ) - - -parser = argparse.ArgumentParser( - prog="NVMeoF_Target_Setup", - formatter_class=argparse.RawDescriptionHelpFormatter, - description=( - "A helper script to automatically setup NVMeoF on a system.\n" - "Example: # ./setup-nvmeof.py \\\n" - " -s ais-nvmeof -n 1 -d /dev/nvme0n1 \n" - " -P 1 -a 192.168.50.67 -p 4420 -t rdma \n\n\n" - ), - epilog=( - "This script must be run as root." - ) -) -# Can we be intelligent enough to automatically grab a given IP address? -parser.add_argument( - "-a", - "--address", - type=str, - help=( - "Address to broadcast this NVMeoF Target." - ), - required=True -) -parser.add_argument( - "-d", - "--device-path", - type=Path, - help=( - "Path to the NVMe disk to use as the NVMeoF Target. " - "(ex. /dev/nvme0n1)" - ), - required=True -) -parser.add_argument( - "-n", - "--namespace-num", - type=int, - default=1, - help=( - "Number to identify this namespace under the Subsystem. " - "(Similar to LUN.) Defaults to 1." - ) -) -parser.add_argument( - "-p", - "--net-port", - type=int, - default=4420, - help=( - "Network port to bind this NVMeoF Target to. Defaults to 4420." - ), -) -parser.add_argument( - "-P", - "--nvme-port", - type=int, - default=1, - help=( - "NVMe Controller port to bind this NVMeoF Target to. Defaults to 1." - ) -) -parser.add_argument( - "-s", - "--subsystem-name", - type=str, - help="NVMe Subsystem Name.", - required=True -) -parser.add_argument( - "-t", - "--transport", - type=lambda _str: _str.lower(), - choices=NVMeoF_TRANSPORT_TYPES, - help="Transport Type for NVMeoF communication.", - required=True -) -parser.add_argument( - "--force", - action="store_true", - help=( - "Overwrite existing NVMe configurations, if possible. " - "Configurations that cannot be edited or removed may cause the " - "script to fail." - ) -) - -if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) -if os.getuid() != 0: - logger.error("Script must be run as root.") - sys.exit(1) - -args = parser.parse_args() -setup_logging() - -device_path: Path = args.device_path -if not device_path.exists(): - raise FileNotFoundError( - f"'{device_path}' does not exist on this system." - ) -if not device_path.is_block_device(): - raise ValueError( - f"'{device_path}' does not point to a Block Device or a symlink to a " - "Block Device." - ) -# Could also perform other validation checks... - -load_kernel_module("nvmet") -load_kernel_module("nvmet-rdma") -create_NVMet_subsystem( - args.subsystem_name, args.force -) -create_NVMet_namespace( - args.subsystem_name, args.namespace_num, device_path, args.force -) -create_NVMet_port( - args.nvme_port, args.net_port, args.address, args.transport, args.force -) -create_NVMet_link( - args.subsystem_name, args.nvme_port -) -check_syslog( - args.nvme_port, args.address, args.net_port -) - -logger.info(f"NVMeoF Setup complete.") diff --git a/util/vfio.sh b/util/vfio.sh deleted file mode 100755 index 889d3319..00000000 --- a/util/vfio.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -e -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT - -# Binds devices to the vfio-pci driver. Requires root permissions. -# -# usage: vfio DBDF [DBDF ...] -# -# DBDF: PCI Domain Bus Device Function identifier - -for bdf in "$@"; do - # Unbind the current driver - if [ -e /sys/bus/pci/devices/"$bdf"/driver/unbind ]; then - echo "$bdf" > /sys/bus/pci/devices/"$bdf"/driver/unbind - fi - - # Bind the vfio-pci driver - vendor_id=$(cat /sys/bus/pci/devices/"$bdf"/vendor) - device_id=$(cat /sys/bus/pci/devices/"$bdf"/device) - echo "$vendor_id" "$device_id" > /sys/bus/pci/drivers/vfio-pci/new_id - - # Add RW permissions for the vfio group to the device's iommu group - iommu_group=$(basename "$(readlink -f /sys/bus/pci/devices/"$bdf"/iommu_group/)") - chgrp vfio /dev/vfio/"$iommu_group" - chmod g+rw /dev/vfio/"$iommu_group" - - # Show which driver is now bound to the PCI device - driver=$(basename "$(readlink -f /sys/bus/pci/devices/"$bdf"/driver)") - echo "$bdf" "$driver" -done -