From daa1878868e2aff21a9ddbc573141f4bc67781c4 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Thu, 21 Apr 2022 17:17:38 +0000 Subject: [PATCH 1/2] A few fixes to scripts to simulate a Gimlet on commodity machines - Renames VNICs from `vioifN` to `netN` to avoid collision when running the script inside a VM - Remove the loopback address from the virtual hardware scripts - Better handling of existing names - Remove manual handling of xde kernel driver, delegating to the xde ONU consolidation for that - Cleans up the `omicron-package uninstall` command to avoid removing the files installed by the `helios-netdev` publisher for OPTE. This is mostly for the `opteadm` tool. --- package/src/bin/omicron-package.rs | 21 ++++++--- tools/create_virtual_hardware.sh | 47 +++++++++++--------- tools/destroy_virtual_hardware.sh | 19 +------- tools/install_opte.sh | 71 +++++++++++++++++------------- 4 files changed, 83 insertions(+), 75 deletions(-) diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 560b5a7c7c9..6a27a290990 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -422,8 +422,7 @@ fn remove_all_unless_already_removed>(path: P) -> Result<()> { Ok(()) } -fn remove_all_except_databases>(path: P) -> Result<()> { - const TO_KEEP: [&str; 2] = ["clickhouse", "cockroachdb"]; +fn remove_all_except>(path: P, to_keep: &[&str]) -> Result<()> { let dir = match path.as_ref().read_dir() { Ok(dir) => dir, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), @@ -431,7 +430,10 @@ fn remove_all_except_databases>(path: P) -> Result<()> { }; for entry in dir { let entry = entry?; - if !TO_KEEP.contains(&&*(entry.file_name().to_string_lossy())) { + if to_keep.contains(&&*(entry.file_name().to_string_lossy())) { + println!(" Keeping: '{}'", entry.path().to_string_lossy()); + } else { + println!(" Removing: '{}'", entry.path().to_string_lossy()); if entry.metadata()?.is_dir() { remove_all_unless_already_removed(entry.path())?; } else { @@ -452,9 +454,16 @@ fn do_uninstall( println!("Uninstalling all packages"); uninstall_all_packages(config); println!("Removing artifacts in: {}", artifact_dir.to_string_lossy()); - remove_all_except_databases(artifact_dir)?; - println!("Removing: {}", install_dir.to_string_lossy()); - remove_all_unless_already_removed(install_dir)?; + + const ARTIFACTS_TO_KEEP: &[&str] = &["clickhouse", "cockroachdb", "xde"]; + remove_all_except(artifact_dir, ARTIFACTS_TO_KEEP)?; + + println!( + "Removing installed objects in: {}", + install_dir.to_string_lossy() + ); + const INSTALLED_OBJECTS_TO_KEEP: &[&str] = &["opte"]; + remove_all_except(install_dir, INSTALLED_OBJECTS_TO_KEEP)?; Ok(()) } diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index eac2be4fd87..92fa0233ba4 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -50,6 +50,19 @@ function ensure_zpools { done } +# Return the name of a VNIC link if it exists, or the empty string if not. +# +# Arguments: +# $1: The name of the VNIC to look for +function get_vnic_name_if_exists { + NAME="$(dladm show-vnic -p -o LINK "$1")" + if [[ "$?" -eq 0 ]]; then + echo "$NAME" + else + echo "" + fi +} + # Create VNICs to represent the Chelsio physical links # # Arguments: @@ -57,33 +70,26 @@ function ensure_zpools { # first physical link available on the machine. function ensure_simulated_chelsios { local PHYSICAL_LINK="$1" - VNIC_NAMES=("vioif0" "vioif1") + VNIC_NAMES=("net0" "net1") for VNIC in "${VNIC_NAMES[@]}"; do - if [[ -z "$(dladm show-vnic -p -o LINK "$VNIC")" ]]; then + if [[ -z "$(get_vnic_name_if_exists "$VNIC")" ]]; then dladm create-vnic -t -l "$PHYSICAL_LINK" "$VNIC" fi success "VNIC $VNIC exists" - if [[ -z "$(ipadm show-addr -p -o ADDR "$VNIC/v6")" ]]; then - ipadm create-addr -t -T addrconf "$VNIC/v6" - fi - success "IP address $VNIC/v6 exists" done - - # Create an address on the underlay network - UNDERLAY_ADDR="lo0/underlay" - if [[ -z "$(ipadm show-addr -p -o ADDR "$UNDERLAY_ADDR")" ]]; then - ipadm create-addr -t -T static -a fd00:1::1/64 lo0/underlay - fi - success "IP address $UNDERLAY_ADDR exists" } -function ensure_xde_driver { - # Always remove the driver first. There seems to be a bug in the driver, - # preventing it from showing up in `modinfo` on boot, even if it's actually - # installed. - if [[ -z "$(modinfo | grep xde)" ]]; then - rem_drv xde - add_drv xde +# Return the IP address for the provided addrobj name, or the empty string if it +# does not exist. +# +# Arguments: +# $1: The name of the addrobj +function get_ip_addr_if_exists { + ADDR="$(ipadm show-addr -p -o ADDR "$1")" + if [[ "$?" -eq 0 ]]; then + echo "$ADDR" + else + echo "" fi } @@ -97,4 +103,3 @@ function ensure_run_as_root { ensure_run_as_root ensure_zpools ensure_simulated_chelsios "$PHYSICAL_LINK" -ensure_xde_driver diff --git a/tools/destroy_virtual_hardware.sh b/tools/destroy_virtual_hardware.sh index 13e903a43c2..ab8e475a16a 100755 --- a/tools/destroy_virtual_hardware.sh +++ b/tools/destroy_virtual_hardware.sh @@ -27,20 +27,6 @@ function success { echo -e "\e[1;36m$1\e[0m" } -function try_uninstall_xde { - RC=0 - if ! [[ -z "$(modinfo | grep xde)" ]]; then - rem_drv xde - RC=$? - fi - - if [[ $RC -eq 0 ]]; then - success "XDE kernel module uninstalled" - else - warn "Failed to uninstall XDE kernel module" - fi -} - function try_remove_address { local ADDRESS="$1" RC=0 @@ -71,9 +57,8 @@ function try_remove_vnic { function try_remove_vnics { try_remove_address "lo0/underlay" - VNIC_LINKS=("vioif0" "vioif1") + VNIC_LINKS=("net0" "net1") for LINK in "${VNIC_LINKS[@]}"; do - try_remove_address "$LINK/v6" try_remove_vnic "$LINK" done } @@ -94,5 +79,5 @@ function try_destroy_zpools { done } -try_uninstall_xde && try_remove_vnics +try_remove_vnics try_destroy_zpools diff --git a/tools/install_opte.sh b/tools/install_opte.sh index 842eab947a2..c286d99143c 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -19,7 +19,8 @@ SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" cd "${SOURCE_DIR}/.." OMICRON_TOP="$PWD" OUT_DIR="$OMICRON_TOP/out" -mkdir -p "$OUT_DIR" +XDE_DIR="$OUT_DIR/xde" +mkdir -p "$XDE_DIR" # Compute the SHA256 of the path in $1, returning just the sum function file_sha { @@ -30,7 +31,7 @@ function file_sha { function download_and_check_sha { local URL="$1" local FILENAME="$(basename "$URL")" - local OUT_PATH="$OUT_DIR/$FILENAME" + local OUT_PATH="$XDE_DIR/$FILENAME" local SHA="$2" # Check if the file already exists, with the expected SHA @@ -52,36 +53,44 @@ function sha_from_url { curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1 } -OPTE_P5P_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSFZZWPFEQBW7JRS7ST99G/opte-0.1.58.p5p" -OPTE_P5P_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSG01CGP6TH9THNY39G88Z/opte-0.1.58.p5p.sha256" -OPTE_P5P_REPO_PATH="$OUT_DIR/$(basename "$OPTE_P5P_URL")" -XDE_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHNYQ1NS7DBX8VG3JPAP0/xde" -XDE_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHP47353961S3ETXBSD2T/xde.sha256" +# Add a pkg repo by path. This will also run `pkg update`, handling the case +# gracefully where there are no updates required +function add_pkg_repo_and_update { + local REPO_PATH="$1" + pkg set-publisher -p "$REPO_PATH" --search-first + RC=0 + pkg update || RC=$?; + if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then + return 0 + else + return "$RC" + fi +} -download_and_check_sha "$OPTE_P5P_URL" "$(sha_from_url "$OPTE_P5P_SHA_URL")" -XDE_SHA="$(sha_from_url "$XDE_SHA_URL")" -download_and_check_sha "$XDE_URL" "$XDE_SHA" +# The `helios-netdev` provides the XDE kernel driver and the `opteadm` userland +# tool for interacting with it. +HELIOS_NETDEV_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MPQRBX3Q5EF45YDAW6Q/opte-0.1.60.p5p" +HELIOS_NETDEV_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MR60H4N13NJKGWEEA69/opte-0.1.60.p5p.sha256" +HELIOS_NETDEV_REPO_PATH="$XDE_DIR/$(basename "$HELIOS_NETDEV_REPO_URL")" -# Move the XDE driver into it the expected location to allow operating on it -# with `add_drv` and `rem_drv` -DRIVER_DIR="/kernel/drv/amd64" -XDE_FILENAME="$(basename "$XDE_URL")" -XDE_PATH="$DRIVER_DIR/$XDE_FILENAME" -if ! [[ -f "$XDE_PATH" ]] || [[ "$XDE_SHA" != "$(file_sha "$XDE_PATH")" ]]; then - echo "Replacing XDE driver" - mv -f "$OUT_DIR/$XDE_FILENAME" "$XDE_PATH" -else - echo "XDE driver already exists with correct SHA" -fi +# The XDE repo provides a full OS/Net incorporation, with updated kernel bits +# that the `xde` kernel module and OPTE rely on. +XDE_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDDRXQ3Y4E5SG9QX8N9FK/repo.p5p" +XDE_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDJDMJAYHFV9Z6XVE30X5/repo.p5p.sha256" +XDE_REPO_PATH="$XDE_DIR/$(basename "$XDE_REPO_URL")" -# Add the OPTE P5P package repository (at the top of the search order) and -# update the OS packages. This may require a reboot. -pkg set-publisher -p "$OPTE_P5P_REPO_PATH" --search-first +# Download and verify the package repositorieies +download_and_check_sha "$HELIOS_NETDEV_REPO_URL" "$(sha_from_url "$HELIOS_NETDEV_REPO_SHA_URL")" +download_and_check_sha "$XDE_REPO_URL" "$(sha_from_url "$XDE_REPO_SHA_URL")" + +# Set the `helios-dev` repo as non-sticky, meaning that packages that were +# originally provided by it may be updated by another repository, if that repo +# provides newer versions of the packages. pkg set-publisher --non-sticky helios-dev -RC=0 -pkg update || RC=$?; -if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then - exit 0 -else - exit "$RC" -fi + +# Add the OPTE and XDE repositories and update packages. +add_pkg_repo_and_update "$HELIOS_NETDEV_REPO_PATH" +add_pkg_repo_and_update "$XDE_REPO_PATH" + +# Actually install the xde kernel module and opteadm tool +pkg install driver/network/opte From a355a8132710c4b013ebb69330470300c0049c3b Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Thu, 21 Apr 2022 17:33:54 +0000 Subject: [PATCH 2/2] Address review comments --- tools/create_virtual_hardware.sh | 14 -------------- tools/install_opte.sh | 27 +++++++++++---------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 92fa0233ba4..b33c5a991b0 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -79,20 +79,6 @@ function ensure_simulated_chelsios { done } -# Return the IP address for the provided addrobj name, or the empty string if it -# does not exist. -# -# Arguments: -# $1: The name of the addrobj -function get_ip_addr_if_exists { - ADDR="$(ipadm show-addr -p -o ADDR "$1")" - if [[ "$?" -eq 0 ]]; then - echo "$ADDR" - else - echo "" - fi -} - function ensure_run_as_root { if [[ "$(id -u)" -ne 0 ]]; then echo "This script must be run as root" diff --git a/tools/install_opte.sh b/tools/install_opte.sh index c286d99143c..2098a48a927 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -53,20 +53,6 @@ function sha_from_url { curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1 } -# Add a pkg repo by path. This will also run `pkg update`, handling the case -# gracefully where there are no updates required -function add_pkg_repo_and_update { - local REPO_PATH="$1" - pkg set-publisher -p "$REPO_PATH" --search-first - RC=0 - pkg update || RC=$?; - if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then - return 0 - else - return "$RC" - fi -} - # The `helios-netdev` provides the XDE kernel driver and the `opteadm` userland # tool for interacting with it. HELIOS_NETDEV_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MPQRBX3Q5EF45YDAW6Q/opte-0.1.60.p5p" @@ -89,8 +75,17 @@ download_and_check_sha "$XDE_REPO_URL" "$(sha_from_url "$XDE_REPO_SHA_URL")" pkg set-publisher --non-sticky helios-dev # Add the OPTE and XDE repositories and update packages. -add_pkg_repo_and_update "$HELIOS_NETDEV_REPO_PATH" -add_pkg_repo_and_update "$XDE_REPO_PATH" +pkg set-publisher -p "$HELIOS_NETDEV_REPO_PATH" --search-first +pkg set-publisher -p "$XDE_REPO_PATH" --search-first + +# Actually update packages, handling case where no updates are needed +RC=0 +pkg update || RC=$?; +if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then + return 0 +else + return "$RC" +fi # Actually install the xde kernel module and opteadm tool pkg install driver/network/opte