From a1d8d05bea0efd462b0114e1958271c8ccdbd5ea Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Thu, 16 Oct 2025 18:38:24 +0200 Subject: [PATCH 1/7] move example unikernel building from shell.nix to example.nix --- example.nix | 29 +++++++++++++++++++++++++---- shell.nix | 28 ---------------------------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/example.nix b/example.nix index 50dbc5c82..995697ec5 100644 --- a/example.nix +++ b/example.nix @@ -1,12 +1,24 @@ -{ withCcache ? false, +{ - doCheck ? true, # boot unikernel after building it - includeos ? import ./default.nix { inherit withCcache; }, + # The unikernel to build + unikernel ? ./example, + + # Boot unikernel after building it + doCheck ? true, + + # Enable multicore suport. + smp ? false, + + # Enable ccache support. See overlay.nix for details. + withCcache ? false, + + # The includeos library to build and link against + includeos ? import ./default.nix { inherit withCcache; inherit smp; }, }: includeos.stdenv.mkDerivation rec { pname = "includeos_example"; - src = includeos.pkgs.lib.cleanSource ./example; + src = includeos.pkgs.lib.cleanSource "${unikernel}"; dontStrip = true; inherit doCheck; @@ -20,13 +32,22 @@ includeos.stdenv.mkDerivation rec { includeos.chainloader ]; + cmakeFlags = [ + "-DARCH=x86_64" + "-DINCLUDEOS_PACKAGE=${includeos}" + "-DCMAKE_MODULE_PATH=${includeos}/cmake" + "-DFOR_PRODUCTION=OFF" + ]; + nativeCheckInputs = [ includeos.vmrunner includeos.pkgs.qemu ]; checkPhase = '' + runHook preCheck boot *.elf.bin + runHook postCheck ''; version = "dev"; diff --git a/shell.nix b/shell.nix index b26dbef07..ec9ac0ae4 100644 --- a/shell.nix +++ b/shell.nix @@ -3,9 +3,6 @@ # nix-shell --argstr buildpath . buildpath ? "", - # The unikernel to build - unikernel ? "./example", - # vmrunner path, for vmrunner development vmrunner ? "", @@ -16,7 +13,6 @@ smp ? false, includeos ? import ./default.nix { inherit withCcache; inherit smp; } - }: includeos.pkgs.mkShell.override { inherit (includeos) stdenv; } rec { @@ -48,31 +44,7 @@ includeos.pkgs.mkShell.override { inherit (includeos) stdenv; } rec { ]; shellHook = '' - - unikernel=$(realpath ${unikernel}) - echo -e "Attempting to build unikernel: \n$unikernel" - if [ ! -d "$unikernel" ]; then - echo "$unikernel is not a valid directory" - exit 1 - fi - export BUILDPATH=${buildpath} - if [ -z "${buildpath}" ]; then - export BUILDPATH="$(mktemp -d)" - pushd "$BUILDPATH" - else - mkdir -p "$BUILDPATH" - pushd "$BUILDPATH" - fi - cmake "$unikernel" -DARCH=x86_64 -DINCLUDEOS_PACKAGE=${includeos} -DCMAKE_MODULE_PATH=${includeos}/cmake \ - -DFOR_PRODUCTION=OFF - make -j $NIX_BUILD_CORES echo -e "\n====================== IncludeOS nix-shell =====================" - if [ -z "${buildpath}" ]; then - echo -e "\nWorking directory, generated by this script:" - echo $BUILDPATH - echo -e "\nTo use another directory pass in 'buildpath' to nix:" - echo "nix-shell --argstr buildpath you/build/path" - fi echo -e "\nThe C++ compiler set to:" echo $(which $CXX) echo -e "\nIncludeOS package:" From a14e84ed0fbae812940c19904f8af591a0c0a7fa Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Thu, 16 Oct 2025 18:50:28 +0200 Subject: [PATCH 2/7] permit more options --- example.nix | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/example.nix b/example.nix index 995697ec5..b135c6cbd 100644 --- a/example.nix +++ b/example.nix @@ -6,18 +6,25 @@ # Boot unikernel after building it doCheck ? true, + # Which architecture to build against + arch ? "x86_64", + # Enable multicore suport. smp ? false, # Enable ccache support. See overlay.nix for details. withCcache ? false, + # Enable stricter requirements + forProduction ? false, + # The includeos library to build and link against includeos ? import ./default.nix { inherit withCcache; inherit smp; }, }: includeos.stdenv.mkDerivation rec { pname = "includeos_example"; + version = "dev"; src = includeos.pkgs.lib.cleanSource "${unikernel}"; dontStrip = true; inherit doCheck; @@ -33,10 +40,10 @@ includeos.stdenv.mkDerivation rec { ]; cmakeFlags = [ - "-DARCH=x86_64" + "-DARCH=${arch}" "-DINCLUDEOS_PACKAGE=${includeos}" "-DCMAKE_MODULE_PATH=${includeos}/cmake" - "-DFOR_PRODUCTION=OFF" + "-DFOR_PRODUCTION=${if forProduction then "ON" else "OFF"}" ]; nativeCheckInputs = [ @@ -49,6 +56,4 @@ includeos.stdenv.mkDerivation rec { boot *.elf.bin runHook postCheck ''; - - version = "dev"; } From 714ac614ec7085851682156889d1b093f27e5d49 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Thu, 16 Oct 2025 19:10:29 +0200 Subject: [PATCH 3/7] name is misleading since it can build more unikernels --- example.nix => unikernel.nix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename example.nix => unikernel.nix (100%) diff --git a/example.nix b/unikernel.nix similarity index 100% rename from example.nix rename to unikernel.nix From 51bd9d488b0d6b1447b3193e4edcc16ae8c8c28a Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Fri, 17 Oct 2025 16:05:33 +0200 Subject: [PATCH 4/7] add shellHook to permit tests with capability requirements --- test.sh | 25 ++++++++++------- unikernel.nix | 74 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/test.sh b/test.sh index 52e8275c0..16a0f9ad3 100755 --- a/test.sh +++ b/test.sh @@ -67,23 +67,23 @@ build_chainloader(){ } build_example(){ - nix-build $CCACHE_FLAG example.nix + nix-build $CCACHE_FLAG unikernel.nix } multicore_subset(){ - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --run ./test.py + nix-build ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --arg doCheck true # The following tests are not using multiple CPU's, but have been equippedd with some anyway # to make sure core functionality is not broken by missing locks etc. when waking up more cores. - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --run ./test.py - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --run ./test.py + nix-shell ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp + nix-build ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --arg doCheck true } smoke_tests(){ - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/net/integration/tcp --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --run ./test.py + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/tcp + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --arg doCheck true + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --arg doCheck true } run unittests "Build and run unit tests" @@ -112,6 +112,7 @@ fi # Continuing from here will run all integration tests. +sandboxed=true run_testsuite() { local base_folder="$1" shift @@ -151,7 +152,11 @@ run_testsuite() { # The command to run, as string to be able to print the fully expanded command - cmd="nix-shell --pure $CCACHE_FLAG --argstr unikernel $subfolder --run ./test.py" + if $sandboxed; then + cmd="nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ${subfolder%/} --arg doCheck true" + else + cmd="nix-shell ./unikernel.nix $CCACHE_FLAG --argstr unikernel ${subfolder%/}" + fi echo "" echo "🚧 Step $steps.$substeps" @@ -223,7 +228,9 @@ exclusions=( "websocket" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute ) +sandboxed=false run_testsuite "./test/net/integration" "${exclusions[@]}" +sandboxed=true echo -e "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" diff --git a/unikernel.nix b/unikernel.nix index b135c6cbd..1aab8d755 100644 --- a/unikernel.nix +++ b/unikernel.nix @@ -1,10 +1,12 @@ { - # The unikernel to build unikernel ? ./example, + # The test file to run + test ? "test.py", + # Boot unikernel after building it - doCheck ? true, + doCheck ? false, # Which architecture to build against arch ? "x86_64", @@ -20,12 +22,27 @@ # The includeos library to build and link against includeos ? import ./default.nix { inherit withCcache; inherit smp; }, + + # vmrunner path, for vmrunner development + vmrunner ? "", }: +let + absolutePathOf = base: p: + if p == null then null else + if builtins.isPath p then p + else builtins.toPath (base + "/${p}"); + unikernelPath = absolutePathOf ./. unikernel; + vmrunnerPkg = + if vmrunner == "" then + includeos.vmrunner + else + includeos.pkgs.callPackage (builtins.toPath /. + vmrunner) {}; +in includeos.stdenv.mkDerivation rec { pname = "includeos_example"; version = "dev"; - src = includeos.pkgs.lib.cleanSource "${unikernel}"; + src = includeos.pkgs.lib.cleanSource unikernelPath; dontStrip = true; inherit doCheck; @@ -46,14 +63,63 @@ includeos.stdenv.mkDerivation rec { "-DFOR_PRODUCTION=${if forProduction then "ON" else "OFF"}" ]; + installPhase = '' + runHook preInstall + # we want to place any files required by the test into the output + find -mindepth 1 -maxdepth 1 -type f -exec install -v -D -t "$out/" {} \; + + # especially the unikernel image, in case it wasn't at the rootdir already + find -mindepth 2 -name '*.elf.bin' -exec install -v -t "$out/" {} \; + runHook postInstall + ''; + + nativeCheckInputs = [ includeos.vmrunner + includeos.pkgs.grub2 + includeos.pkgs.python3 includeos.pkgs.qemu + includeos.pkgs.iputils + includeos.pkgs.xorriso ]; + checkInputs = [ + includeos.lest + ]; + + # use `nix-build --arg doCheck true` to run tests normally checkPhase = '' runHook preCheck - boot *.elf.bin + set -e + if [ -e "${src}/${test}" ]; then + echo "Running IncludeOS test: ${src}/${test}" + python3 "${src}/${test}" + else + echo "Default test script '${test}', but no test was found 😟" + echo "For a custom path, consider specifying the path to the test script:" + echo " --argstr test 'path/to/test.py'" + exit 1 + fi runHook postCheck ''; + + # this is a hack + # some tests need to be run through a shell because of net_cap_raw+ep and net_cap_admin+ep + # replace nix-build with nix-shell to test without dropping capabilities + packages = [ + (includeos.pkgs.python3.withPackages (p: [ + vmrunnerPkg + ])) + ]; + shellHook = '' + set -eu + pkg="$(nix-build ./unikernel.nix --arg doCheck false --arg unikernel ${unikernel})" + + testPath="$(realpath "${unikernel}/${test}")" + cd "$pkg" + "$testPath" || exit 1 + + exit 0 + ''; + } From f4029ecba1367c60e929e91b7bf667953e30d9eb Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Fri, 17 Oct 2025 16:50:20 +0200 Subject: [PATCH 5/7] permit unsandoxing specific tests --- test.sh | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/test.sh b/test.sh index 16a0f9ad3..052f3d9ad 100755 --- a/test.sh +++ b/test.sh @@ -75,7 +75,7 @@ multicore_subset(){ # The following tests are not using multiple CPU's, but have been equippedd with some anyway # to make sure core functionality is not broken by missing locks etc. when waking up more cores. - nix-shell ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp + nix-shell ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --arg doCheck true nix-build ./unikernel.nix --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --arg doCheck true } @@ -112,7 +112,6 @@ fi # Continuing from here will run all integration tests. -sandboxed=true run_testsuite() { local base_folder="$1" shift @@ -138,6 +137,7 @@ run_testsuite() { for subfolder in "$base_folder"/*/; do local skip=false + local sandboxed=true for exclude in "${exclusion_list[@]}"; do if [[ "$subfolder" == *"$exclude"* ]]; then @@ -145,17 +145,22 @@ run_testsuite() { break fi done - if [ "$skip" = true ]; then continue fi + for unsandbox in "${unsandbox_list[@]}"; do + if [[ "$subfolder" == *"$unsandbox"* ]]; then + sandboxed=false + break + fi + done # The command to run, as string to be able to print the fully expanded command if $sandboxed; then cmd="nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ${subfolder%/} --arg doCheck true" else - cmd="nix-shell ./unikernel.nix $CCACHE_FLAG --argstr unikernel ${subfolder%/}" + cmd="nix-shell ./unikernel.nix $CCACHE_FLAG --argstr unikernel ${subfolder%/} --arg doCheck true" fi echo "" @@ -199,7 +204,11 @@ exclusions=( "modules" # Requires 32-bit build, which our shell.nix is not set up for ) +unsandbox_list=( + "term" # fails to create the tun device, like the net integration tests +) run_testsuite "./test/kernel/integration" "${exclusions[@]}" +unsandbox_list=() # # C++ STL runtime tests @@ -228,9 +237,20 @@ exclusions=( "websocket" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute ) -sandboxed=false +# all the following fail with the following error: +# failed to create tun device: Operation not permitted +# qemu-system-x86_64: -netdev bridge,id=net0,br=bridge43: bridge helper failed +unsandbox_list=( + "./test/net/integration/configure" + "./test/net/integration/icmp" + "./test/net/integration/icmp6" + "./test/net/integration/slaac" + "./test/net/integration/tcp" + "./test/net/integration/udp" + "./test/net/integration/dns" # except this one which times out instead +) run_testsuite "./test/net/integration" "${exclusions[@]}" -sandboxed=true +unsandbox_list=() echo -e "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 633b8f8f559cdcb311c4a7a9ccc6e10c543bf5ab Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Fri, 17 Oct 2025 17:09:31 +0200 Subject: [PATCH 6/7] clarify network privileges, expose paths to underlying tooling --- shell.nix | 59 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/shell.nix b/shell.nix index ec9ac0ae4..f7861a820 100644 --- a/shell.nix +++ b/shell.nix @@ -44,25 +44,44 @@ includeos.pkgs.mkShell.override { inherit (includeos) stdenv; } rec { ]; shellHook = '' - echo -e "\n====================== IncludeOS nix-shell =====================" - echo -e "\nThe C++ compiler set to:" - echo $(which $CXX) - echo -e "\nIncludeOS package:" - echo ${includeos} - echo -e "\n---------------------- Network privileges ---------------------" - echo "The vmrunner for IncludeOS tests requires bridged networking for full functionality." - echo "The following commands requiring sudo privileges can be used to set this up:" - echo "1. the qemu-bridge-helper needs sudo to create a bridge. Can be enabled with:" - echo " sudo chmod u+s ${includeos.pkgs.qemu}/libexec/qemu-bridge-helper" - echo "2. bridge43 must exist. Can be set up with vmrunner's create_bridge.sh script:" - echo " ${vmrunnerPkg.create_bridge}" - echo "3. /etc/qemu/bridge.conf must contain this line:" - echo " allow bridge43" - echo "" - echo "Some tests require ping, which requires premissions to send raw packets. On some hosts" - echo "this is not enabled by default for iputils provided by nix. It can be enabled with:" - echo "4. sudo setcap cap_net_raw+ep ${includeos.pkgs.iputils}/bin/ping" - echo " " - echo + cat <<-EOF +================================== IncludeOS nix-shell ================================== +Packages: + IncludeOS: ${includeos} + vmrunner: ${vmrunnerPkg} + chainloader: ${includeos.chainloader} + +Tooling: + CXX $(command -v $CXX) + cmake: $(command -v cmake) + nasm: $(command -v nasm) + qemu-system-x86: $(command -v qemu-system-x86_64) + grub-mkrescue: $(command -v grub-mkrescue) + xorriso: $(command -v xorriso) + ping: $(command -v ping) + +---------------------------------- Network privileges ---------------------------------- +The vmrunner for IncludeOS tests requires bridged networking for full functionality. +The following checklist can be used to set this up from the host: + +1. The qemu-bridge-helper needs root escalation to manipulate bridges. You can provide this + either through capabilities or through root execution. Pick one: + sudo chmod u+s ${includeos.pkgs.qemu}/libexec/qemu-bridge-helper + sudo setcap cap_net_admin+ep ${includeos.pkgs.qemu}/libexec/qemu-bridge-helper + +2. bridge43 must exist. Can be set up with vmrunner's create_bridge.sh script (not as root): + ${vmrunnerPkg.create_bridge} + +3. /etc/qemu/bridge.conf must contain this line: + allow bridge43 + Also note that /etc/qemu needs specific permissions, so it might be easiest to install + qemu on the host to generate these directories for you, despite not using its executable here. + +4. Some tests also perform ICMP pings, which requires permissions to send raw packets. On some + hosts this is not enabled by default for iputils provided by nix. + It can be enabled with: + sudo setcap cap_net_raw+ep ${includeos.pkgs.iputils}/bin/ping + +EOF ''; } From 7fde7e410bdbcb0b53c13bd792ac0bc114ad8420 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Sat, 18 Oct 2025 12:40:58 +0200 Subject: [PATCH 7/7] actually run the phase check for udp,tcp tests --- test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.sh b/test.sh index 052f3d9ad..671d1efd9 100755 --- a/test.sh +++ b/test.sh @@ -80,8 +80,8 @@ multicore_subset(){ } smoke_tests(){ - nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp - nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/tcp + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --arg doCheck true + nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/net/integration/tcp --arg doCheck true nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --arg doCheck true nix-build ./unikernel.nix $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --arg doCheck true }