From 83040df118416ece6213f3c74ff1d43217db2b13 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 10:54:57 -0700 Subject: [PATCH 1/5] test(vm): add composable QEMU test guests Signed-off-by: Drew Newberry --- flake.nix | 5 + nix/vm/README.md | 155 +++++++++++ nix/vm/configuration/docker.yml | 67 +++++ nix/vm/configuration/podman.yml | 34 +++ nix/vm/configuration/selinux.yml | 62 +++++ nix/vm/default.nix | 109 ++++++++ nix/vm/distros/centos.nix | 19 ++ nix/vm/distros/rocky.nix | 19 ++ nix/vm/distros/ubuntu.nix | 22 ++ nix/vm/run.sh | 430 +++++++++++++++++++++++++++++++ tasks/scripts/package-deb.sh | 18 -- 11 files changed, 922 insertions(+), 18 deletions(-) create mode 100644 nix/vm/README.md create mode 100644 nix/vm/configuration/docker.yml create mode 100644 nix/vm/configuration/podman.yml create mode 100644 nix/vm/configuration/selinux.yml create mode 100644 nix/vm/default.nix create mode 100644 nix/vm/distros/centos.nix create mode 100644 nix/vm/distros/rocky.nix create mode 100644 nix/vm/distros/ubuntu.nix create mode 100644 nix/vm/run.sh diff --git a/flake.nix b/flake.nix index 13c4857bc6..aa4ce8829a 100644 --- a/flake.nix +++ b/flake.nix @@ -37,11 +37,16 @@ projectRootFile = "flake.nix"; programs.nixfmt.enable = true; }; + testVm = import ./nix/vm { inherit pkgs; }; in { + apps.test-vm = testVm.app; + devShells.default = pkgs.mkShell { packages = with pkgs; [ rustToolchain + # Assemble Debian artifacts on macOS and Linux. + dpkg # Required to find packages pkg-config # Required for bindgen generation. diff --git a/nix/vm/README.md b/nix/vm/README.md new file mode 100644 index 0000000000..862b2ba0e2 --- /dev/null +++ b/nix/vm/README.md @@ -0,0 +1,155 @@ + + +# Test VMs + +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS and KVM on native-architecture Linux hosts. + +## Requirements + +- Nix with flakes enabled. +- Apple Silicon macOS with HVF, or Linux with readable and writable `/dev/kvm`. +- Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay. +- Native-architecture artifacts. The runner does not use software emulation. + +The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. + +## Supported configurations + +| Distro | Docker | Podman | SELinux | Package format | +| --- | --- | --- | --- | --- | +| Ubuntu 24.04 | Yes | Yes | No | `.deb` | +| CentOS Stream 10 | No | Yes | Yes | `.rpm` | +| Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | + +List the available distros and configurations: + +```shell +nix run .#test-vm -- --list +``` + +## Open an interactive VM + +Boot a base Ubuntu VM: + +```shell +nix run .#test-vm -- --distro ubuntu +``` + +Apply the Docker configuration before opening the SSH session: + +```shell +nix run .#test-vm -- --distro ubuntu --with docker +``` + +Other combinations use the same interface: + +```shell +nix run .#test-vm -- --distro rocky --with docker +nix run .#test-vm -- --distro centos --with podman +``` + +Configurations are repeatable: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --with docker \ + --with podman +``` + +Ensure SELinux is enforcing on CentOS or Rocky: + +```shell +nix run .#test-vm -- \ + --distro rocky \ + --with docker \ + --with selinux \ + -- getenforce +``` + +`--with selinux` installs the required tooling, persists `SELINUX=enforcing`, applies enforcing mode live, and verifies the result. It fails on Ubuntu and on guests where SELinux is fully disabled and would require a reboot to enable. + +## Ansible configurations + +Configurations are Ansible playbooks stored under `nix/vm/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible. + +Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed. + +## Install an OpenShell package + +Package existing ARM64 Linux binaries with the repository's `package:deb:arm64` mise task: + +```shell +OPENSHELL_CLI_BINARY="$PWD/target/aarch64-unknown-linux-musl/release/openshell" \ +OPENSHELL_GATEWAY_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-gateway" \ +OPENSHELL_DRIVER_VM_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-driver-vm" \ +OPENSHELL_DEB_VERSION=0.0.0-local \ +OPENSHELL_OUTPUT_DIR="$PWD/artifacts" \ +nix develop --command mise run package:deb:arm64 +``` + +Install the package in an Ubuntu VM and run a command: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --with docker \ + --install artifacts/openshell_0.0.0-local_arm64.deb \ + -- openshell --version +``` + +For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture. + +`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS and Rocky Linux. This prototype can install an existing RPM but does not build one. + +## Copy binaries directly + +Use `--copy SOURCE:DEST` to install an executable without creating a package: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --copy ./openshell:/usr/local/bin/openshell \ + -- openshell --version +``` + +The destination must be an absolute guest path. Copied files are installed with mode `0755`. + +## Runner options + +```text +--distro NAME Base distro: ubuntu, centos, or rocky +--with NAME Apply docker, podman, or selinux; repeatable +--install PATH Install a .deb or .rpm package; repeatable +--copy SRC:DEST Copy an executable into the guest; repeatable +--ssh-port PORT Use a specific loopback SSH forwarding port +--keep Preserve the disk overlay and logs after shutdown +--list List distros and configurations +``` + +Arguments after `--` are executed inside the guest. Without a command, the runner opens an interactive SSH session. + +## Lifecycle + +Nix downloads and caches the selected, hash-pinned cloud image. Each invocation then: + +1. Creates a temporary QCOW2 overlay. +2. Boots QEMU with HVF or KVM acceleration. +3. Creates an ephemeral `openshell` user and SSH key through cloud-init. +4. Applies the selected Ansible configurations. +5. Installs or copies the supplied artifacts. +6. Opens SSH or executes the requested guest command. +7. Powers off QEMU and deletes the overlay. + +Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log for debugging. The retained directory is printed when the runner exits. + +## Current limitations + +- Host and guest architectures must match. +- Software emulation without HVF or KVM is not supported. +- Configurations are applied fresh on every run; prepared VM caching is not implemented. +- Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. +- The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/vm/configuration/docker.yml b/nix/vm/configuration/docker.yml new file mode 100644 index 0000000000..1c158d3653 --- /dev/null +++ b/nix/vm/configuration/docker.yml @@ -0,0 +1,67 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure Docker in a disposable test VM. + +- name: Configure Docker + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate Docker support + ansible.builtin.assert: + that: + - ansible_facts.distribution in ["Ubuntu", "Rocky"] + fail_msg: >- + Docker is supported on Ubuntu and Rocky in this prototype, + not {{ ansible_facts.distribution }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + when: ansible_facts.distribution == "Ubuntu" + + - name: Install Docker on Ubuntu + ansible.builtin.apt: + name: docker.io + state: present + install_recommends: false + when: ansible_facts.distribution == "Ubuntu" + + - name: Configure the Docker CE repository on Rocky + ansible.builtin.get_url: + url: https://download.docker.com/linux/centos/docker-ce.repo + dest: /etc/yum.repos.d/docker-ce.repo + mode: "0644" + when: ansible_facts.distribution == "Rocky" + + - name: Install Docker on Rocky + ansible.builtin.dnf: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + when: ansible_facts.distribution == "Rocky" + + - name: Enable Docker + ansible.builtin.service: + name: docker + enabled: true + state: started + + - name: Add the test user to the Docker group + ansible.builtin.user: + name: openshell + groups: + - docker + append: true + + - name: Verify Docker + ansible.builtin.command: + cmd: docker info + changed_when: false diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml new file mode 100644 index 0000000000..37ef04ea58 --- /dev/null +++ b/nix/vm/configuration/podman.yml @@ -0,0 +1,34 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure Podman in a disposable test VM. + +- name: Configure Podman + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate Podman support + ansible.builtin.assert: + that: + - ansible_facts.distribution in ["Ubuntu", "CentOS", "Rocky"] + fail_msg: >- + Podman is unsupported on {{ ansible_facts.distribution }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + when: ansible_facts.distribution == "Ubuntu" + + - name: Install Podman + ansible.builtin.package: + name: podman + state: present + + - name: Verify rootless Podman + ansible.builtin.command: + cmd: podman info + become: false + changed_when: false diff --git a/nix/vm/configuration/selinux.yml b/nix/vm/configuration/selinux.yml new file mode 100644 index 0000000000..3b4e32dd7b --- /dev/null +++ b/nix/vm/configuration/selinux.yml @@ -0,0 +1,62 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Ensure SELinux is enforcing in a disposable test VM. + +- name: Configure SELinux + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate SELinux support + ansible.builtin.assert: + that: + - ansible_facts.os_family == "RedHat" + fail_msg: >- + SELinux configuration is supported only on CentOS and Rocky, + not {{ ansible_facts.distribution }}. + + - name: Install SELinux tools + ansible.builtin.package: + name: policycoreutils + state: present + + - name: Read the current SELinux mode + ansible.builtin.command: + cmd: getenforce + register: selinux_current + changed_when: false + + - name: Reject a fully disabled SELinux system + ansible.builtin.assert: + that: + - selinux_current.stdout != "Disabled" + fail_msg: >- + SELinux is disabled and cannot be enabled live. Set SELINUX=enforcing + and reboot the guest before applying this configuration. + + - name: Persist enforcing mode + ansible.builtin.lineinfile: + path: /etc/selinux/config + regexp: ^SELINUX= + line: SELINUX=enforcing + + - name: Enable enforcing mode now + ansible.builtin.command: + cmd: setenforce 1 + when: selinux_current.stdout != "Enforcing" + changed_when: true + + - name: Read the resulting SELinux mode + ansible.builtin.command: + cmd: getenforce + register: selinux_result + changed_when: false + + - name: Verify enforcing mode + ansible.builtin.assert: + that: + - selinux_result.stdout == "Enforcing" + fail_msg: SELinux did not enter enforcing mode. diff --git a/nix/vm/default.nix b/nix/vm/default.nix new file mode 100644 index 0000000000..18fb2f0c2b --- /dev/null +++ b/nix/vm/default.nix @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Composable distro VMs for installing and exercising artifacts. + +{ pkgs }: + +let + isAarch64 = pkgs.stdenv.hostPlatform.isAarch64; + isDarwin = pkgs.stdenv.hostPlatform.isDarwin; + architecture = if isAarch64 then "aarch64" else "x86_64"; + qemu = pkgs.qemu.override { hostCpuOnly = true; }; + qemuBinary = + if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; + + ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + centos = import ./distros/centos.nix { inherit pkgs architecture; }; + rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; + + catalog = pkgs.writeText "openshell-test-vm-catalog.sh" '' + test_vm_list_distros() { + printf '%s\n' ubuntu centos rocky + } + + test_vm_load_distro() { + case "$1" in + ubuntu) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext ubuntu.image.drvPath} + TEST_VM_OS_ID=${ubuntu.osId} + TEST_VM_OS_VERSION=${ubuntu.osVersion} + TEST_VM_PACKAGE_FAMILY=${ubuntu.packageFamily} + ;; + centos) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext centos.image.drvPath} + TEST_VM_OS_ID=${centos.osId} + TEST_VM_OS_VERSION=${centos.osVersion} + TEST_VM_PACKAGE_FAMILY=${centos.packageFamily} + ;; + rocky) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext rocky.image.drvPath} + TEST_VM_OS_ID=${rocky.osId} + TEST_VM_OS_VERSION=${rocky.osVersion} + TEST_VM_PACKAGE_FAMILY=${rocky.packageFamily} + ;; + *) + return 1 + ;; + esac + + TEST_VM_QEMU=${qemuBinary} + TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} + TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} + TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} + TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + TEST_VM_ARCHITECTURE=${architecture} + export TEST_VM_IMAGE_DRV TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY + export TEST_VM_QEMU TEST_VM_FIRMWARE_CODE TEST_VM_FIRMWARE_VARS + export TEST_VM_MACHINE TEST_VM_ACCELERATOR TEST_VM_ARCHITECTURE + } + + test_vm_list_configurations() { + printf '%s\n' docker podman selinux + } + + test_vm_load_configuration() { + case "$1" in + docker) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/docker.yml} + ;; + podman) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/podman.yml} + ;; + selinux) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/selinux.yml} + ;; + *) + return 1 + ;; + esac + export TEST_VM_CONFIGURATION_PLAYBOOK + } + ''; + + runner = pkgs.writeShellApplication { + name = "openshell-test-vm"; + runtimeInputs = [ + qemu + pkgs.python3Packages.ansible-core + pkgs.coreutils + pkgs.gnugrep + pkgs.nix + pkgs.openssh + pkgs.python3 + pkgs.xorriso + ]; + text = '' + export OPENSHELL_TEST_VM_RUNTIME=1 + export OPENSHELL_TEST_VM_CATALOG=${catalog} + exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" + ''; + }; +in +{ + app = { + type = "app"; + program = "${runner}/bin/openshell-test-vm"; + meta.description = "Boot and configure a disposable distro VM"; + }; +} diff --git a/nix/vm/distros/centos.nix b/nix/vm/distros/centos.nix new file mode 100644 index 0000000000..f38ad68533 --- /dev/null +++ b/nix/vm/distros/centos.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "centos"; + osVersion = "10"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + url = "https://cloud.centos.org/centos/10-stream/${architecture}/images/CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag=" + else + "sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc="; + }; +} diff --git a/nix/vm/distros/rocky.nix b/nix/vm/distros/rocky.nix new file mode 100644 index 0000000000..48fc162e89 --- /dev/null +++ b/nix/vm/distros/rocky.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "rocky"; + osVersion = "9"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + url = "https://download.rockylinux.org/pub/rocky/9/images/${architecture}/Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4=" + else + "sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg="; + }; +} diff --git a/nix/vm/distros/ubuntu.nix b/nix/vm/distros/ubuntu.nix new file mode 100644 index 0000000000..36d4c75c64 --- /dev/null +++ b/nix/vm/distros/ubuntu.nix @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; +in +{ + osId = "ubuntu"; + osVersion = "24.04"; + packageFamily = "deb"; + image = pkgs.fetchurl { + name = "ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; + url = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; + hash = + if architecture == "aarch64" then + "sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0=" + else + "sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE="; + }; +} diff --git a/nix/vm/run.sh b/nix/vm/run.sh new file mode 100644 index 0000000000..4a8136a55e --- /dev/null +++ b/nix/vm/run.sh @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Boot and configure a disposable cloud image, then install artifacts. + +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: + nix run .#test-vm -- --distro DISTRO [OPTIONS] [-- COMMAND...] + +Options: + --distro NAME Base distro: ubuntu, centos, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, selinux) + --install PATH Install a .deb or .rpm package; repeatable + --copy SRC:DEST Copy an executable to an absolute guest path; repeatable + --ssh-port PORT Use a specific loopback SSH forwarding port + --keep Keep the disposable disk and logs after shutdown + --list List distros and configurations + -h, --help Show this help + +With no COMMAND, the runner opens an interactive SSH session. +EOF +} + +if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || [ ! -r "${OPENSHELL_TEST_VM_CATALOG:-}" ]; then + echo "run this script through 'nix run .#test-vm -- ...'" >&2 + exit 2 +fi + +# The generated catalog contains only trusted Nix store paths and profile data. +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_CATALOG}" + +distro= +requested_ssh_port= +keep=0 +list=0 +configurations=() +packages=() +copies=() +guest_command=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --distro) + distro=${2:-} + shift 2 + ;; + --with) + configurations+=("${2:-}") + shift 2 + ;; + --install) + packages+=("${2:-}") + shift 2 + ;; + --copy) + copies+=("${2:-}") + shift 2 + ;; + --ssh-port) + requested_ssh_port=${2:-} + shift 2 + ;; + --keep) + keep=1 + shift + ;; + --list) + list=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + --) + shift + guest_command=("$@") + break + ;; + *) + echo "unknown test VM argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ "${list}" -eq 1 ]; then + echo "Distros:" + test_vm_list_distros | sed 's/^/ /' + echo "Configurations:" + test_vm_list_configurations | sed 's/^/ /' + exit 0 +fi + +if [ -z "${distro}" ]; then + echo "--distro is required" >&2 + usage >&2 + exit 2 +fi +if ! test_vm_load_distro "${distro}"; then + echo "unknown distro: ${distro}" >&2 + exit 2 +fi + +for item in "${configurations[@]}"; do + if [ -z "${item}" ] || ! test_vm_load_configuration "${item}"; then + echo "unknown configuration: ${item:-}" >&2 + exit 2 + fi +done + +if [ -n "${requested_ssh_port}" ] && { + [[ ! ${requested_ssh_port} =~ ^[0-9]+$ ]] || + [ "${requested_ssh_port}" -lt 1024 ] || + [ "${requested_ssh_port}" -gt 65535 ] + }; then + echo "--ssh-port must be an integer between 1024 and 65535" >&2 + exit 2 +fi + +for package in "${packages[@]}"; do + if [ ! -f "${package}" ]; then + echo "package does not exist: ${package}" >&2 + exit 2 + fi + case "${TEST_VM_PACKAGE_FAMILY}:${package}" in + deb:*.deb | rpm:*.rpm) ;; + *) + echo "${package} does not match the ${TEST_VM_PACKAGE_FAMILY} package family" >&2 + exit 2 + ;; + esac +done + +for copy_spec in "${copies[@]}"; do + source_path=${copy_spec%%:*} + destination=${copy_spec#*:} + if [ "${source_path}" = "${copy_spec}" ] || [ ! -f "${source_path}" ]; then + echo "invalid --copy source: ${copy_spec}" >&2 + exit 2 + fi + case "${destination}" in + /*) + if [[ ${destination} == *"/../"* ]] || [[ ${destination} == */.. ]]; then + echo "--copy destination must not contain '..': ${destination}" >&2 + exit 2 + fi + ;; + *) + echo "--copy destination must be absolute: ${destination}" >&2 + exit 2 + ;; + esac +done + +if [ "${TEST_VM_ACCELERATOR}" = kvm ]; then + if [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + echo "KVM requires a readable and writable /dev/kvm" >&2 + exit 1 + fi +fi + +echo "==> Realizing the pinned ${distro} cloud image" +TEST_VM_IMAGE=$(nix build --no-link --print-out-paths "${TEST_VM_IMAGE_DRV}^out") + +umask 077 +run_parent=${TMPDIR:-/tmp}/openshell-test-vm +mkdir -p "${run_parent}" +run_dir=$(mktemp -d "${run_parent%/}/run.XXXXXX") +overlay=${run_dir}/disk.qcow2 +seed=${run_dir}/seed.iso +vars=${run_dir}/firmware-vars.fd +private_key=${run_dir}/id_ed25519 +serial_log=${run_dir}/serial.log +qemu_log=${run_dir}/qemu.log +qemu_pid= +ssh_port= +ssh_args=() +scp_args=() +ansible_config=${run_dir}/ansible.cfg +ansible_inventory=${run_dir}/inventory.ini + +show_logs() { + if [ -s "${qemu_log}" ]; then + echo "=== QEMU log ===" >&2 + tail -n 100 "${qemu_log}" >&2 || true + fi + if [ -s "${serial_log}" ]; then + echo "=== serial log ===" >&2 + tail -n 200 "${serial_log}" >&2 || true + fi +} + +cleanup() { + status=$? + trap - EXIT INT TERM + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + if [ "${status}" -ne 0 ]; then + show_logs + fi + if [ "${keep}" -eq 1 ]; then + echo "Kept test VM state at ${run_dir}" >&2 + else + rm -rf "${run_dir}" + fi + exit "${status}" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +pick_free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()' +} + +ssh-keygen -q -t ed25519 -N "" -f "${private_key}" +public_key=$(<"${private_key}.pub") + +cat >"${run_dir}/meta-data" <"${run_dir}/user-data" <"${qemu_log}" + echo "==> Booting ${distro} (${TEST_VM_ARCHITECTURE}) with QEMU/${TEST_VM_ACCELERATOR}" + "${TEST_VM_QEMU}" \ + -name "openshell-test-${distro}" \ + -machine "${TEST_VM_MACHINE},accel=${TEST_VM_ACCELERATOR}" \ + -cpu host \ + -smp 4 \ + -m 4096 \ + -drive "if=pflash,format=raw,readonly=on,file=${TEST_VM_FIRMWARE_CODE}" \ + -drive "if=pflash,format=raw,file=${vars}" \ + -drive "if=none,format=qcow2,file=${overlay},id=osdisk" \ + -device virtio-blk-pci,drive=osdisk \ + -drive "if=none,format=raw,readonly=on,file=${seed},id=seed" \ + -device virtio-blk-pci,drive=seed \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -monitor none \ + -serial "file:${serial_log}" \ + -no-reboot \ + >/dev/null 2>"${qemu_log}" & + qemu_pid=$! + + sleep 1 + if kill -0 "${qemu_pid}" 2>/dev/null; then + break + fi + wait "${qemu_pid}" || true + qemu_pid= + if ! grep -q "Could not set up host forwarding rule" "${qemu_log}" || + [ -n "${requested_ssh_port}" ]; then + echo "QEMU exited during startup" >&2 + exit 1 + fi + echo "SSH port was claimed concurrently; retrying (${attempt}/5)" >&2 +done + +if [ -z "${qemu_pid}" ]; then + echo "QEMU could not allocate an SSH forwarding port" >&2 + exit 1 +fi + +ssh_args=( + -F /dev/null + -i "${private_key}" + -p "${ssh_port}" + -o BatchMode=yes + -o ConnectTimeout=5 + -o LogLevel=ERROR + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) +scp_args=( + -F /dev/null + -i "${private_key}" + -P "${ssh_port}" + -o BatchMode=yes + -o ConnectTimeout=5 + -o LogLevel=ERROR + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) + +echo "==> Waiting up to 180 seconds for SSH on 127.0.0.1:${ssh_port}" +ssh_ready=0 +for _ in $(seq 1 90); do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + wait "${qemu_pid}" || true + qemu_pid= + echo "QEMU exited before SSH became ready" >&2 + exit 1 + fi + if ssh "${ssh_args[@]}" openshell@127.0.0.1 true 2>/dev/null; then + ssh_ready=1 + break + fi + sleep 2 +done +if [ "${ssh_ready}" -ne 1 ]; then + echo "SSH did not become ready within 180 seconds" >&2 + exit 1 +fi + +echo "==> Validating ${distro}" +# Profile values come from the trusted Nix-generated catalog. +# shellcheck disable=SC2029 +ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "set -eu; . /etc/os-release; test \"\${ID}\" = '${TEST_VM_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_VM_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_VM_ARCHITECTURE}'" +ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo cloud-init status --wait >/dev/null' + +cat >"${ansible_config}" <"${ansible_inventory}" < Applying configuration: ${item}" + ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ + ansible-playbook "${TEST_VM_CONFIGURATION_PLAYBOOK}" +done + +for package in "${packages[@]}"; do + package_name=$(basename "${package}") + if [[ ! ${package_name} =~ ^[A-Za-z0-9._+~-]+$ ]]; then + echo "package filename contains unsupported characters: ${package_name}" >&2 + exit 2 + fi + remote_package="/home/openshell/${package_name}" + echo "==> Installing package: ${package_name}" + scp "${scp_args[@]}" "${package}" "openshell@127.0.0.1:${remote_package}" + case "${TEST_VM_PACKAGE_FAMILY}" in + deb) + # The remote path uses a restricted package filename. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo apt-get update -qq && sudo env DEBIAN_FRONTEND=noninteractive apt-get install -qq -y '${remote_package}' && rm -f '${remote_package}'" + ;; + rpm) + # The remote path uses a restricted package filename. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo dnf install -q -y '${remote_package}' && rm -f '${remote_package}'" + ;; + esac +done + +for copy_spec in "${copies[@]}"; do + source_path=${copy_spec%%:*} + destination=${copy_spec#*:} + copy_name=$(basename "${source_path}") + if [[ ! ${copy_name} =~ ^[A-Za-z0-9._+~-]+$ ]]; then + echo "copy filename contains unsupported characters: ${copy_name}" >&2 + exit 2 + fi + remote_copy="/home/openshell/${copy_name}" + echo "==> Copying ${copy_name} to ${destination}" + scp "${scp_args[@]}" "${source_path}" "openshell@127.0.0.1:${remote_copy}" + # The destination was restricted to an absolute, traversal-free path. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo install -D -m 0755 '${remote_copy}' '${destination}'; rm -f '${remote_copy}'" +done + +echo "==> Test VM ready: ${distro} (SSH port ${ssh_port})" +if [ "${#guest_command[@]}" -eq 0 ]; then + ssh -t "${ssh_args[@]}" openshell@127.0.0.1 +else + printf -v quoted_command '%q ' "${guest_command[@]}" + # quoted_command is shell-escaped locally before it reaches the guest. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 "bash -lc $(printf '%q' "${quoted_command}")" +fi + +echo "==> Shutting down ${distro}" +ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo systemctl poweroff' >/dev/null 2>&1 || true +for _ in $(seq 1 30); do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + wait "${qemu_pid}" || true + qemu_pid= + break + fi + sleep 1 +done diff --git a/tasks/scripts/package-deb.sh b/tasks/scripts/package-deb.sh index 9d7e3d3281..3e20f6256e 100755 --- a/tasks/scripts/package-deb.sh +++ b/tasks/scripts/package-deb.sh @@ -167,22 +167,4 @@ dpkg-deb --build --root-owner-group "$pkgroot" "$package_file" dpkg-deb --info "$package_file" dpkg-deb --contents "$package_file" -# --------------------------------------------------------------------------- -# Smoke tests -# --------------------------------------------------------------------------- - -extract_dir="${tmpdir}/extract" -mkdir -p "$extract_dir" -dpkg-deb -x "$package_file" "$extract_dir" -"$extract_dir/usr/bin/openshell" --version -"$extract_dir/usr/bin/openshell-gateway" --version -"$extract_dir/usr/libexec/openshell/openshell-driver-vm" --version - -if command -v systemd-analyze >/dev/null 2>&1; then - # verify --user catches user-scope-specific issues like StateDirectory= - # resolution and the %h/%S specifiers used in this unit. - systemd-analyze --user verify "$extract_dir/usr/lib/systemd/user/openshell-gateway.service" \ - || echo "warning: systemd-analyze verify failed in the build environment" >&2 -fi - echo "Wrote ${package_file}" From b505b5ff583a7d5e333cd7642668c42671aabc21 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 11:05:19 -0700 Subject: [PATCH 2/5] docs(vm): describe test VM directory structure Signed-off-by: Drew Newberry --- nix/vm/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/nix/vm/README.md b/nix/vm/README.md index 862b2ba0e2..4288bc229a 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -16,6 +16,31 @@ This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linu The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. +## Directory structure + +```text +nix/vm/ +├── README.md +├── default.nix +├── run.sh +├── distros/ +│ ├── ubuntu.nix +│ ├── centos.nix +│ └── rocky.nix +└── configuration/ + ├── docker.yml + ├── podman.yml + └── selinux.yml +``` + +- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, imports the distro catalog, and maps `--with` names to playbooks. +- `run.sh` owns the disposable VM lifecycle: argument validation, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. +- `distros/*.nix` define the immutable base-image catalog. Each record pins the image URL and hash and declares the expected OS ID, version, and package family. Distro definitions do not provision extra software. +- `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. +- `README.md` documents the supported combinations and developer interface. + +The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` app. Debian artifact creation remains outside the VM harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. + ## Supported configurations | Distro | Docker | Podman | SELinux | Package format | From f21b33976d2a60747f2cf9f0e32467e424b6d830 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 11:17:34 -0700 Subject: [PATCH 3/5] refactor(vm): replace shell catalog functions Signed-off-by: Drew Newberry --- nix/vm/README.md | 2 +- nix/vm/default.nix | 98 +++++++++++++++++----------------------------- nix/vm/run.sh | 28 +++++++------ 3 files changed, 53 insertions(+), 75 deletions(-) diff --git a/nix/vm/README.md b/nix/vm/README.md index 4288bc229a..918296f5f5 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -33,7 +33,7 @@ nix/vm/ └── selinux.yml ``` -- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, imports the distro catalog, and maps `--with` names to playbooks. +- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, and exposes distro profiles and configuration playbooks as Nix-store catalogs. - `run.sh` owns the disposable VM lifecycle: argument validation, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. - `distros/*.nix` define the immutable base-image catalog. Each record pins the image URL and hash and declares the expected OS ID, version, and package family. Distro definitions do not provision extra software. - `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. diff --git a/nix/vm/default.nix b/nix/vm/default.nix index 18fb2f0c2b..44acd01e66 100644 --- a/nix/vm/default.nix +++ b/nix/vm/default.nix @@ -13,73 +13,38 @@ let qemuBinary = if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; - ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; - centos = import ./distros/centos.nix { inherit pkgs architecture; }; - rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; - - catalog = pkgs.writeText "openshell-test-vm-catalog.sh" '' - test_vm_list_distros() { - printf '%s\n' ubuntu centos rocky - } + distros = { + ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + centos = import ./distros/centos.nix { inherit pkgs architecture; }; + rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; + }; - test_vm_load_distro() { - case "$1" in - ubuntu) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext ubuntu.image.drvPath} - TEST_VM_OS_ID=${ubuntu.osId} - TEST_VM_OS_VERSION=${ubuntu.osVersion} - TEST_VM_PACKAGE_FAMILY=${ubuntu.packageFamily} - ;; - centos) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext centos.image.drvPath} - TEST_VM_OS_ID=${centos.osId} - TEST_VM_OS_VERSION=${centos.osVersion} - TEST_VM_PACKAGE_FAMILY=${centos.packageFamily} - ;; - rocky) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext rocky.image.drvPath} - TEST_VM_OS_ID=${rocky.osId} - TEST_VM_OS_VERSION=${rocky.osVersion} - TEST_VM_PACKAGE_FAMILY=${rocky.packageFamily} - ;; - *) - return 1 - ;; - esac + configurations = { + docker = ./configuration/docker.yml; + podman = ./configuration/podman.yml; + selinux = ./configuration/selinux.yml; + }; - TEST_VM_QEMU=${qemuBinary} - TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} - TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} - TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} - TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} - TEST_VM_ARCHITECTURE=${architecture} + mkDistroProfile = + name: distro: + pkgs.writeText "openshell-test-vm-${name}" '' + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} + TEST_VM_OS_ID=${pkgs.lib.escapeShellArg distro.osId} + TEST_VM_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} + TEST_VM_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} export TEST_VM_IMAGE_DRV TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY - export TEST_VM_QEMU TEST_VM_FIRMWARE_CODE TEST_VM_FIRMWARE_VARS - export TEST_VM_MACHINE TEST_VM_ACCELERATOR TEST_VM_ARCHITECTURE - } + ''; - test_vm_list_configurations() { - printf '%s\n' docker podman selinux - } + distroCatalog = pkgs.linkFarm "openshell-test-vm-distros" ( + pkgs.lib.mapAttrsToList (name: distro: { + inherit name; + path = mkDistroProfile name distro; + }) distros + ); - test_vm_load_configuration() { - case "$1" in - docker) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/docker.yml} - ;; - podman) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/podman.yml} - ;; - selinux) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/selinux.yml} - ;; - *) - return 1 - ;; - esac - export TEST_VM_CONFIGURATION_PLAYBOOK - } - ''; + configurationCatalog = pkgs.linkFarm "openshell-test-vm-configurations" ( + pkgs.lib.mapAttrsToList (name: path: { inherit name path; }) configurations + ); runner = pkgs.writeShellApplication { name = "openshell-test-vm"; @@ -95,7 +60,14 @@ let ]; text = '' export OPENSHELL_TEST_VM_RUNTIME=1 - export OPENSHELL_TEST_VM_CATALOG=${catalog} + export OPENSHELL_TEST_VM_DISTROS=${distroCatalog} + export OPENSHELL_TEST_VM_CONFIGURATIONS=${configurationCatalog} + export TEST_VM_QEMU=${qemuBinary} + export TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} + export TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} + export TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} + export TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + export TEST_VM_ARCHITECTURE=${architecture} exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" ''; }; diff --git a/nix/vm/run.sh b/nix/vm/run.sh index 4a8136a55e..ccf052ba27 100644 --- a/nix/vm/run.sh +++ b/nix/vm/run.sh @@ -25,15 +25,13 @@ With no COMMAND, the runner opens an interactive SSH session. EOF } -if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || [ ! -r "${OPENSHELL_TEST_VM_CATALOG:-}" ]; then +if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || + [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || + [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ]; then echo "run this script through 'nix run .#test-vm -- ...'" >&2 exit 2 fi -# The generated catalog contains only trusted Nix store paths and profile data. -# shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_CATALOG}" - distro= requested_ssh_port= keep=0 @@ -92,9 +90,13 @@ done if [ "${list}" -eq 1 ]; then echo "Distros:" - test_vm_list_distros | sed 's/^/ /' + for entry in "${OPENSHELL_TEST_VM_DISTROS}"/*; do + printf ' %s\n' "${entry##*/}" + done echo "Configurations:" - test_vm_list_configurations | sed 's/^/ /' + for entry in "${OPENSHELL_TEST_VM_CONFIGURATIONS}"/*; do + printf ' %s\n' "${entry##*/}" + done exit 0 fi @@ -103,13 +105,18 @@ if [ -z "${distro}" ]; then usage >&2 exit 2 fi -if ! test_vm_load_distro "${distro}"; then +if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2 fi +# Distro profiles contain only trusted values generated into the Nix store. +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_DISTROS}/${distro}" for item in "${configurations[@]}"; do - if [ -z "${item}" ] || ! test_vm_load_configuration "${item}"; then + if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" ]; then echo "unknown configuration: ${item:-}" >&2 exit 2 fi @@ -360,10 +367,9 @@ guest ansible_host=127.0.0.1 ansible_port=${ssh_port} ansible_user=openshell ans EOF for item in "${configurations[@]}"; do - test_vm_load_configuration "${item}" echo "==> Applying configuration: ${item}" ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ - ansible-playbook "${TEST_VM_CONFIGURATION_PLAYBOOK}" + ansible-playbook "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" done for package in "${packages[@]}"; do From 9a2f9fced0732db5cc12c129f85abb70e056ce18 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 12:50:52 -0700 Subject: [PATCH 4/5] test(vm): add Fedora release guest support Signed-off-by: Drew Newberry --- nix/vm/README.md | 19 +++++++++++-------- nix/vm/configuration/docker.yml | 13 +++++++++++++ nix/vm/configuration/podman.yml | 2 +- nix/vm/configuration/selinux.yml | 2 +- nix/vm/default.nix | 1 + nix/vm/distros/fedora.nix | 19 +++++++++++++++++++ nix/vm/run.sh | 29 ++++++++++++++++++----------- 7 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 nix/vm/distros/fedora.nix diff --git a/nix/vm/README.md b/nix/vm/README.md index 918296f5f5..444c9c5b4f 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -5,14 +5,14 @@ SPDX-License-Identifier: Apache-2.0 # Test VMs -This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS and KVM on native-architecture Linux hosts. +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. ## Requirements - Nix with flakes enabled. -- Apple Silicon macOS with HVF, or Linux with readable and writable `/dev/kvm`. +- Apple Silicon macOS with HVF, or a native-architecture Linux host. Linux uses KVM when `/dev/kvm` is available and falls back to QEMU TCG otherwise. - Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay. -- Native-architecture artifacts. The runner does not use software emulation. +- Native-architecture artifacts. TCG emulates the guest CPU on Linux but does not enable cross-architecture guests. The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. @@ -26,6 +26,7 @@ nix/vm/ ├── distros/ │ ├── ubuntu.nix │ ├── centos.nix +│ ├── fedora.nix │ └── rocky.nix └── configuration/ ├── docker.yml @@ -47,6 +48,7 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` | --- | --- | --- | --- | --- | | Ubuntu 24.04 | Yes | Yes | No | `.deb` | | CentOS Stream 10 | No | Yes | Yes | `.rpm` | +| Fedora 44 | No | Yes | Yes | `.rpm` | | Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | List the available distros and configurations: @@ -74,6 +76,7 @@ Other combinations use the same interface: ```shell nix run .#test-vm -- --distro rocky --with docker nix run .#test-vm -- --distro centos --with podman +nix run .#test-vm -- --distro fedora --with podman ``` Configurations are repeatable: @@ -85,7 +88,7 @@ nix run .#test-vm -- \ --with podman ``` -Ensure SELinux is enforcing on CentOS or Rocky: +Ensure SELinux is enforcing on CentOS, Fedora, or Rocky: ```shell nix run .#test-vm -- \ @@ -128,7 +131,7 @@ nix run .#test-vm -- \ For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture. -`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS and Rocky Linux. This prototype can install an existing RPM but does not build one. +`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS, Fedora, and Rocky Linux. This prototype can install an existing RPM but does not build one. ## Copy binaries directly @@ -146,7 +149,7 @@ The destination must be an absolute guest path. Copied files are installed with ## Runner options ```text ---distro NAME Base distro: ubuntu, centos, or rocky +--distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable into the guest; repeatable @@ -162,7 +165,7 @@ Arguments after `--` are executed inside the guest. Without a command, the runne Nix downloads and caches the selected, hash-pinned cloud image. Each invocation then: 1. Creates a temporary QCOW2 overlay. -2. Boots QEMU with HVF or KVM acceleration. +2. Boots QEMU with HVF, KVM, or the Linux TCG fallback. 3. Creates an ephemeral `openshell` user and SSH key through cloud-init. 4. Applies the selected Ansible configurations. 5. Installs or copies the supplied artifacts. @@ -174,7 +177,7 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f ## Current limitations - Host and guest architectures must match. -- Software emulation without HVF or KVM is not supported. +- TCG is slower than hardware virtualization and uses a longer SSH readiness timeout. - Configurations are applied fresh on every run; prepared VM caching is not implemented. - Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. - The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/vm/configuration/docker.yml b/nix/vm/configuration/docker.yml index 1c158d3653..1b25e365f4 100644 --- a/nix/vm/configuration/docker.yml +++ b/nix/vm/configuration/docker.yml @@ -60,6 +60,19 @@ groups: - docker append: true + register: docker_group_membership + + - name: Read the test user ID + ansible.builtin.command: + cmd: id -u openshell + register: test_user_id + changed_when: false + + - name: Refresh the test user's systemd group membership + ansible.builtin.systemd_service: + name: "user@{{ test_user_id.stdout }}.service" + state: restarted + when: docker_group_membership.changed - name: Verify Docker ansible.builtin.command: diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml index 37ef04ea58..c7851ce141 100644 --- a/nix/vm/configuration/podman.yml +++ b/nix/vm/configuration/podman.yml @@ -13,7 +13,7 @@ - name: Validate Podman support ansible.builtin.assert: that: - - ansible_facts.distribution in ["Ubuntu", "CentOS", "Rocky"] + - ansible_facts.distribution in ["Ubuntu", "CentOS", "Fedora", "Rocky"] fail_msg: >- Podman is unsupported on {{ ansible_facts.distribution }}. diff --git a/nix/vm/configuration/selinux.yml b/nix/vm/configuration/selinux.yml index 3b4e32dd7b..6f34e16c67 100644 --- a/nix/vm/configuration/selinux.yml +++ b/nix/vm/configuration/selinux.yml @@ -15,7 +15,7 @@ that: - ansible_facts.os_family == "RedHat" fail_msg: >- - SELinux configuration is supported only on CentOS and Rocky, + SELinux configuration is supported only on CentOS, Fedora, and Rocky, not {{ ansible_facts.distribution }}. - name: Install SELinux tools diff --git a/nix/vm/default.nix b/nix/vm/default.nix index 44acd01e66..ec82b4b8be 100644 --- a/nix/vm/default.nix +++ b/nix/vm/default.nix @@ -16,6 +16,7 @@ let distros = { ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; centos = import ./distros/centos.nix { inherit pkgs architecture; }; + fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; }; diff --git a/nix/vm/distros/fedora.nix b/nix/vm/distros/fedora.nix new file mode 100644 index 0000000000..d8701ed166 --- /dev/null +++ b/nix/vm/distros/fedora.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "fedora"; + osVersion = "44"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + url = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/${architecture}/images/Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs=" + else + "sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38="; + }; +} diff --git a/nix/vm/run.sh b/nix/vm/run.sh index ccf052ba27..1a54ea4a33 100644 --- a/nix/vm/run.sh +++ b/nix/vm/run.sh @@ -12,7 +12,7 @@ Usage: nix run .#test-vm -- --distro DISTRO [OPTIONS] [-- COMMAND...] Options: - --distro NAME Base distro: ubuntu, centos, or rocky + --distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply a configuration; repeatable (docker, podman, selinux) --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable to an absolute guest path; repeatable @@ -166,11 +166,14 @@ for copy_spec in "${copies[@]}"; do esac done -if [ "${TEST_VM_ACCELERATOR}" = kvm ]; then - if [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - echo "KVM requires a readable and writable /dev/kvm" >&2 - exit 1 - fi +test_vm_cpu=host +ssh_wait_seconds=180 +if [ "${TEST_VM_ACCELERATOR}" = kvm ] && + { [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; }; then + echo "==> /dev/kvm is unavailable; falling back to QEMU/TCG" + TEST_VM_ACCELERATOR=tcg + test_vm_cpu=max + ssh_wait_seconds=600 fi echo "==> Realizing the pinned ${distro} cloud image" @@ -266,7 +269,7 @@ for attempt in $(seq 1 5); do "${TEST_VM_QEMU}" \ -name "openshell-test-${distro}" \ -machine "${TEST_VM_MACHINE},accel=${TEST_VM_ACCELERATOR}" \ - -cpu host \ + -cpu "${test_vm_cpu}" \ -smp 4 \ -m 4096 \ -drive "if=pflash,format=raw,readonly=on,file=${TEST_VM_FIRMWARE_CODE}" \ @@ -324,9 +327,9 @@ scp_args=( -o UserKnownHostsFile=/dev/null ) -echo "==> Waiting up to 180 seconds for SSH on 127.0.0.1:${ssh_port}" +echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}" ssh_ready=0 -for _ in $(seq 1 90); do +for _ in $(seq 1 "$((ssh_wait_seconds / 2))"); do if ! kill -0 "${qemu_pid}" 2>/dev/null; then wait "${qemu_pid}" || true qemu_pid= @@ -340,7 +343,7 @@ for _ in $(seq 1 90); do sleep 2 done if [ "${ssh_ready}" -ne 1 ]; then - echo "SSH did not become ready within 180 seconds" >&2 + echo "SSH did not become ready within ${ssh_wait_seconds} seconds" >&2 exit 1 fi @@ -349,7 +352,11 @@ echo "==> Validating ${distro}" # shellcheck disable=SC2029 ssh "${ssh_args[@]}" openshell@127.0.0.1 \ "set -eu; . /etc/os-release; test \"\${ID}\" = '${TEST_VM_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_VM_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_VM_ARCHITECTURE}'" -ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo cloud-init status --wait >/dev/null' +# cloud-init returns 2 when it completes with recoverable errors. Fedora can +# report that status for an initial transient-hostname warning even though the +# requested user and SSH configuration were applied successfully. +ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + 'sudo cloud-init status --wait >/dev/null; status=$?; [ "$status" -eq 0 ] || [ "$status" -eq 2 ]' cat >"${ansible_config}" < Date: Fri, 24 Jul 2026 16:00:46 -0700 Subject: [PATCH 5/5] test(vm): enable rootless Podman socket Signed-off-by: Drew Newberry --- nix/vm/configuration/podman.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml index c7851ce141..3012c59f3e 100644 --- a/nix/vm/configuration/podman.yml +++ b/nix/vm/configuration/podman.yml @@ -27,6 +27,14 @@ name: podman state: present + - name: Enable the rootless Podman API socket + ansible.builtin.systemd_service: + name: podman.socket + scope: user + enabled: true + state: started + become: false + - name: Verify rootless Podman ansible.builtin.command: cmd: podman info