From aa4a9811b5c07df3e9a0e4b72d7bdf45b09dad04 Mon Sep 17 00:00:00 2001
From: Tino Reichardt <milky-zfs@mcmilk.de>
Date: Sun, 3 Mar 2024 21:36:30 +0100
Subject: [PATCH] ZTS: Use QEMU for tests on Linux and FreeBSD

-----------------------------------------------------------------
Do not merge - this is my testing version
-Tony Hutter

Requires-builders: none
----------------------------------------------------------------

This commit adds functional tests for these systems:
- AlmaLinux 8, AlmaLinux 9
- ArchLinux
- CentOS Stream 8, CentOS Stream 9
- Fedora 38, Fedora 39
- Debian 11, Debian 12
- FreeBSD 13, FreeBSD 14, FreeBSD 15
- Ubuntu 22.04, Ubuntu 24.04

Workflow for each operating system:
- install QEMU on the github runner
- download cloud image for this system
- start and init that image via cloud-init
- install deps, build openzfs, load the module
- do the functional testings, hopefully < 5h

Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
---
 .github/workflows/scripts/generate-summary.sh | 178 ++++++++++-----
 .github/workflows/scripts/qemu-1-setup.sh     |  30 +++
 .github/workflows/scripts/qemu-2-start.sh     | 178 +++++++++++++++
 .github/workflows/scripts/qemu-3-deps.sh      | 208 +++++++++++++++++
 .github/workflows/scripts/qemu-4-build.sh     | 122 ++++++++++
 .github/workflows/scripts/qemu-5-load.sh      |  64 ++++++
 .github/workflows/scripts/qemu-6-tests.sh     |  35 +++
 .../workflows/scripts/setup-dependencies.sh   |   4 +-
 .github/workflows/scripts/setup-functional.sh |   6 +-
 .github/workflows/zfs-linux.yml               | 211 +++++++++++++++---
 .../cli_root/zfs_share/zfs_share_005_pos.ksh  |   1 +
 .../redacted_send/redacted_deleted.ksh        |   2 +
 12 files changed, 942 insertions(+), 97 deletions(-)
 create mode 100755 .github/workflows/scripts/qemu-1-setup.sh
 create mode 100755 .github/workflows/scripts/qemu-2-start.sh
 create mode 100755 .github/workflows/scripts/qemu-3-deps.sh
 create mode 100755 .github/workflows/scripts/qemu-4-build.sh
 create mode 100755 .github/workflows/scripts/qemu-5-load.sh
 create mode 100755 .github/workflows/scripts/qemu-6-tests.sh

diff --git a/.github/workflows/scripts/generate-summary.sh b/.github/workflows/scripts/generate-summary.sh
index b5d89208a5d8..39ad7ec0d4f9 100755
--- a/.github/workflows/scripts/generate-summary.sh
+++ b/.github/workflows/scripts/generate-summary.sh
@@ -1,60 +1,107 @@
 #!/usr/bin/env bash
 
-# for runtime reasons we split functional testings into N parts
-# - use a define to check for missing tarfiles
-FUNCTIONAL_PARTS="4"
-
-ZTS_REPORT="tests/test-runner/bin/zts-report.py"
-chmod +x $ZTS_REPORT
+######################################################################
+# generate github summary page of all the testings
+######################################################################
 
 function output() {
-  echo -e $* >> Summary.md
+  echo -e $* >> "out-$logfile.md"
+}
+
+function outfile() {
+  cat "$1" >> "out-$logfile.md"
+}
+
+function send2github() {
+  test -f "$1" && dd if="$1" bs=999k count=1 >> $GITHUB_STEP_SUMMARY
 }
 
 function error() {
   output ":bangbang: $* :bangbang:\n"
 }
 
-# this function generates the real summary
-# - expects a logfile "log" in current directory
+# generate summary of one test
 function generate() {
   # we issued some error already
   test ! -s log && return
 
-  # for overview and zts-report
-  cat log | grep '^Test' > list
+  ######################################################
+  # input:
+  # - log     -> full debug output
+  # - results -> full list with summary in the end
+  ######################################################
+  # output:
+  # - info.txt  -> short summary list (zts-report)
+  # - list.txt  -> full list, but without debugging
+  # - debug.txt -> full list with debugging info
+  ######################################################
+
+  if [ -s results ]; then
+    cat results | grep '^Test[: ]' > list.txt
+    cat results | grep -v '^Test[: ]' > info.txt
+  else
+    cat log | grep '^Test[: ]' > list.txt
+    ./zts-report.py --no-maybes ./list.txt > info.txt
+  fi
 
   # error details
   awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
-    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err
+    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > debug.txt
 
-  # summary of errors
-  if [ -s err ]; then
+  # headline of this summary
+  output "\n## $headline\n"
+
+  if [ -s uname.txt ]; then
     output "<pre>"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
+    outfile uname.txt
     output "</pre>"
+  fi
 
-    # generate seperate error logfile
-    ERRLOGS=$((ERRLOGS+1))
-    errfile="err-$ERRLOGS.md"
-    echo -e "\n## $headline (debugging)\n" >> $errfile
-    echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "</pre></details>" >> $errfile
+  if [ -s info.txt ]; then
+    output "<pre>"
+    outfile info.txt
+    output "</pre>"
   else
     output "All tests passed :thumbsup:"
   fi
 
-  output "<details><summary>Full Listing</summary><pre>"
-  cat list >> Summary.md
-  output "</pre></details>"
+  if [ -s dmesg-prerun.txt ]; then
+    output "<details><summary>Dmesg - systemstart</summary><pre>"
+    outfile dmesg-prerun.txt
+    output "</pre></details>"
+  fi
+
+  if [ -s dmesg-module-load.txt ]; then
+    output "<details><summary>Dmesg - module loading</summary><pre>"
+    outfile dmesg-module-load.txt
+    output "</pre></details>"
+  fi
+
+  if [ -s make-stderr.txt ]; then
+    output "<details><summary>Stderr of make</summary><pre>"
+    outfile make-stderr.txt
+    output "</pre></details>"
+  fi
+
+  if [ -s list.txt ]; then
+    output "<details><summary>List of all tests</summary><pre>"
+    outfile list.txt
+    output "</pre></details>"
+  fi
+
+  if [ -s debug.txt ]; then
+    output "<details><summary>Debug list with dmesg and dbgmsg</summary><pre>"
+    outfile debug.txt
+    output "</pre></details>"
+  fi
 
   # remove tmp files
-  rm -f err list log
+  rm -f log results *.txt
+  logfile=$((logfile+1))
 }
 
 # check tarfiles and untar
-function check_tarfile() {
+function my_untar() {
   if [ -f "$1" ]; then
     tar xf "$1" || error "Tarfile $1 returns some error"
   else
@@ -62,58 +109,71 @@ function check_tarfile() {
   fi
 }
 
-# check logfile and concatenate test results
-function check_logfile() {
+# check file and copy
+function my_copy() {
   if [ -f "$1" ]; then
-    cat "$1" >> log
+    cat "$1" >> "$2"
   else
-    error "Logfile $1 not found"
+    error "File $1 not found"
   fi
 }
 
-# sanity
-function summarize_s() {
-  headline="$1"
-  output "\n## $headline\n"
+# sanity checks on ubuntu runner
+function summarize_sanity() {
+  headline="Sanity Tests Ubuntu $1"
   rm -rf testfiles
-  check_tarfile "$2/sanity.tar"
-  check_logfile "testfiles/log"
+  my_untar "Logs-$1-sanity/sanity.tar"
+  my_copy "testfiles/log" log
   generate
 }
 
-# functional
-function summarize_f() {
-  headline="$1"
-  output "\n## $headline\n"
+# functional on ubuntu runner matrix
+function summarize_functional() {
+  headline="Functional Tests Ubuntu $1"
   rm -rf testfiles
-  for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2-part$i/part$i.tar"
-    check_tarfile "$tarfile"
-    check_logfile "testfiles/log"
+  for i in $(seq 1 4); do
+    tarfile="Logs-$1-functional-part$i/part$i.tar"
+    my_untar "$tarfile"
+    my_copy "testfiles/log" log
   done
   generate
 }
 
+# functional tests via qemu
+function summarize_qemu() {
+  for tarfile in Logs-functional*/qemu-*.tar; do
+    rm -rf current
+    my_untar "$tarfile"
+    osname=`cat osname.txt`
+    headline="Functional Tests: $osname"
+    my_copy "current/log" log
+    my_copy "current/results" results
+    generate
+  done
+}
+
 # https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
 # Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB.
 # [ ] can not show all error findings here
 # [x] split files into smaller ones and create additional steps
 
-ERRLOGS=0
-if [ ! -f Summary/Summary.md ]; then
-  # first call, we do the default summary (~500k)
-  echo -n > Summary.md
-  summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity
-  summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity
-  summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional
-  summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional
-
-  cat Summary.md >> $GITHUB_STEP_SUMMARY
-  mkdir -p Summary
-  mv *.md Summary
+# first call, generate all summaries
+if [ ! -f out-0.md ]; then
+  # create ./zts-report.py for generate()
+  TEMPLATE="tests/test-runner/bin/zts-report.py.in"
+  cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py
+  chmod +x ./zts-report.py
+
+  logfile="0"
+  summarize_sanity "20.04"
+  summarize_sanity "22.04"
+  summarize_functional "20.04"
+  summarize_functional "22.04"
+  summarize_qemu
+
+  send2github out-0.md
 else
-  # here we get, when errors where returned in first call
-  test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY
+  send2github out-$1.md
 fi
 
 exit 0
diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh
new file mode 100755
index 000000000000..7b904429ce74
--- /dev/null
+++ b/.github/workflows/scripts/qemu-1-setup.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup the action runner to start some qemu instance
+######################################################################
+
+set -eu
+
+# docker isn't needed, free some memory
+sudo systemctl stop docker.socket
+sudo apt-get remove docker-ce-cli docker-ce podman
+
+# remove snapd, not needed
+for x in lxd core20 snapd; do sudo snap remove $x; done
+sudo apt-get remove google-chrome-stable firefox snapd
+
+# only install qemu
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils guestfs-tools virt-manager
+
+# no swap needed
+sudo swapoff -a
+
+# disk usage afterwards
+sudo df -h /
+sudo df -h /mnt
+sudo fstrim -a
+
+# generate ssh keys
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh
new file mode 100755
index 000000000000..8c5de514dc1d
--- /dev/null
+++ b/.github/workflows/scripts/qemu-2-start.sh
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+OS="$1"
+
+# valid ostypes: virt-install --os-variant list
+OSv=$OS
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    ;;
+  centos-stream8)
+    OSNAME="CentOS Stream 8"
+    URL="https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2"
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora38)
+    OSNAME="Fedora 38"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/Fedora-Cloud-Base-38-1.6.x86_64.qcow2"
+    ;;
+  fedora39)
+    OSNAME="Fedora 39"
+    OSv="fedora38"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+    ;;
+  freebsd13)
+    OSNAME="FreeBSD 13"
+    OSv="freebsd13.0"
+    # URL="https://download.freebsd.org/ftp/snapshots/amd64"
+    # freebsd images don't have clout-init within it! :(
+    # -> workaround: provide own images
+    URL_ZS="https://openzfs.de/freebsd/amd64-freebsd-13.3.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14)
+    OSNAME="FreeBSD 14"
+    OSv="freebsd13.0"
+    URL_ZS="https://openzfs.de/freebsd/amd64-freebsd-14.0.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15)
+    OSNAME="FreeBSD 15"
+    OSv="freebsd13.0"
+    URL_ZS="https://openzfs.de/freebsd/amd64-freebsd-15.0.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for variable OS!"
+    exit 111
+    ;;
+esac
+
+IMG="/mnt/original.qcow2"
+DISK="/mnt/openzfs.qcow2"
+sudo chown -R $(whoami) /mnt
+
+if [ ! -z "$URL_ZS" ]; then
+  echo "Loading image $URL_ZS ..."
+  axel -q "$URL_ZS" -o "$IMG.zst" || exit 111
+  zstd -d "$IMG.zst" && rm -f "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  axel -q "$URL" -o "$IMG" || exit 111
+fi
+
+# we use zstd for faster IO on the testing runner
+echo "Converting image ..."
+qemu-img convert -q -f qcow2 -O qcow2 -c -o compression_type=zstd,preallocation=off $IMG $DISK || exit 111
+rm -f $IMG || exit 111
+
+echo "Resizing image to 60GiB ..."
+qemu-img resize -q $DISK 60G || exit 111
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+
+write_files:
+  - path: /tmp/runner-init.sh
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env bash
+
+      cd \$HOME
+      exec 2>stderr-init.log
+      echo "$OSNAME" > /var/tmp/osname.txt
+
+runcmd:
+  - sudo -u zfs /tmp/runner-init.sh
+EOF
+
+
+#   --console pty,target_type=virtio \
+
+# Our instance has 16GB RAM which is way more than we need.  Make three 1GB
+# ramdisks for ZTS to use.
+# if [ "$OS" == "freebsd*" ] ; then
+#    sudo mdconfig -s 1g
+#
+#    # Ramdisks are /dev/md{0,1,2}
+#    DEV="md"
+#else
+#    # Note: rd_size is in units of KB.
+#    sudo modprobe brd rd_nr=3 rd_size=$((1024 * 1024))
+#
+#    # Ramdisks are /dev/ram{0,1,2}
+#    DEV="ram"
+#fi
+
+# we could extend this with more virtual disks for example /TR
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm \
+  --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*8)) \
+  --graphics none \
+  --network bridge=virbr0,model=virtio \
+  --cloud-init user-data=/tmp/user-data \
+  --disk $DISK,format=qcow2,bus=virtio \
+  --import --noautoconsole -v
+#  --disk /dev/${DEV}0,size=1,bus=virtio \
+#  --disk /dev/${DEV}1,size=1,bus=virtio \
+#  --disk /dev/${DEV}2,size=1,bus=virtio \
+
+sudo rm -f /tmp/user-data
diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh
new file mode 100755
index 000000000000..7da2b42fbde3
--- /dev/null
+++ b/.github/workflows/scripts/qemu-3-deps.sh
@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 3) install dependencies for compiling and loading
+######################################################################
+
+set -eu
+
+function archlinux() {
+  echo "##[group]Running pacman -Syu"
+  sudo pacman -Syu --noconfirm
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo pacman -Sy --noconfirm base-devel bc cpio dkms fakeroot fio \
+    inetutils  linux linux-headers lsscsi nfs-utils parted pax perf \
+    python-packaging python-setuptools ksh samba sysstat rng-tools \
+    rsync wget
+  echo "##[endgroup]"
+}
+
+function debian() {
+  export DEBIAN_FRONTEND=noninteractive
+
+  echo "##[group]Running apt-get update+upgrade"
+  sudo apt-get update -y
+  sudo apt-get upgrade -y
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  yes | sudo apt-get install -y build-essential autoconf libtool \
+    libtool-bin gdb lcov git alien fakeroot wget curl bc fio acl \
+    sysstat lsscsi parted gdebi attr dbench watchdog ksh nfs-kernel-server \
+    samba rng-tools dkms rsync linux-headers-$(uname -r) \
+    zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
+    xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
+    libssl-dev libaio-dev libffi-dev libelf-dev libmount-dev \
+    libpam0g-dev pamtester python3-dev python3-setuptools python3 \
+    python3-dev python3-setuptools python3-cffi libcurl4-openssl-dev \
+    python3-packaging python3-distlib dh-python python3-all-dev python3-sphinx
+
+  # dkms related
+  yes | sudo apt-get install -y build-essential:native dh-sequence-dkms \
+    libpam0g-dev rpm2cpio cpio || true
+  echo "##[endgroup]"
+}
+
+function freebsd() {
+  export ASSUME_ALWAYS_YES="YES"
+
+  echo "##[group]Install Development Tools"
+  sudo pkg install -y autoconf automake autotools base64 fio gdb git gmake \
+    gsed python python3 gettext gettext-runtime checkbashisms lcov libtool \
+    lscpu ksh93 pamtester pamtester rsync
+
+  sudo pkg install -xy \
+    '^samba4[[:digit:]]+$' \
+    '^py3[[:digit:]]+-cffi$' \
+    '^py3[[:digit:]]+-sysctl$' \
+    '^py3[[:digit:]]+-packaging$'
+
+  echo "##[endgroup]"
+
+  # the image has /usr/src already
+  #echo "##[group]Install Kernel Headers"
+  #VERSION=$(freebsd-version -r)
+  #sudo mkdir -p /usr/src
+  #sudo chown -R $(whoami) /usr/src
+  #git clone --depth=1 -b releng/${VERSION%%-*} https://github.com/freebsd/freebsd-src /usr/src ||
+  #git clone --depth=1 -b stable/${VERSION%%.*} https://github.com/freebsd/freebsd-src /usr/src ||
+  #git clone --depth=1 -b main https://github.com/freebsd/freebsd-src /usr/src
+  #echo "##[endgroup]"
+}
+
+function rhel() {
+  echo "##[group]Running dnf update"
+  sudo dnf update -y
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo dnf group install -y "Development Tools"
+  sudo dnf install -y libtool rpm-build libtirpc-devel libblkid-devel \
+    libuuid-devel libudev-devel openssl-devel zlib-devel libaio-devel \
+    libattr-devel elfutils-libelf-devel python3 python3-devel \
+    python3-setuptools python3-cffi libffi-devel git ncompress \
+    libcurl-devel python3-packaging systemd
+  sudo dnf install -y kernel-devel-$(uname -r)
+
+
+  # Required development tools.
+  sudo -E yum -y --skip-broken install gcc make autoconf libtool gdb \
+    kernel-rpm-macros kernel-abi-whitelists
+
+  echo "##[endgroup]"
+
+  echo "##[group]Install utilities for ZTS"
+  sudo dnf install -y acl ksh bc bzip2 fio sysstat mdadm lsscsi parted attr \
+    nfs-utils samba rng-tools perf rsync dbench pamtester
+  echo "##[endgroup]"
+}
+
+# Enable and startup nfs and smb
+function enable_services() {
+  echo "##[group]starting services"
+
+    case "$1" in
+    ubuntu*|debian11*)
+        sudo -E systemctl enable nfs-kernel-server
+        sudo -E systemctl enable smbd
+        ;;
+    debian*)
+        sudo -E systemctl enable nfs-kernel-server
+        sudo -E systemctl enable samba
+        ;;
+    freebsd*)
+        sudo -E touch /etc/zfs/exports
+        sudo -E sysrc mountd_flags="/etc/zfs/exports"
+        sudo -E service nfsd enable
+        echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null
+        sudo -E service samba_server enable
+        ;;
+
+    # Fedora, Alma, most other distros
+    *)
+        sudo -E systemctl enable nfs-server
+        sudo -E systemctl enable smb
+        ;;
+  esac
+
+  echo "##[endgroup]"
+}
+
+case "$1" in
+  almalinux8|centos-stream8)
+    echo "##[group]Enable epel and powertools repositories"
+    sudo dnf config-manager -y --set-enabled powertools
+    sudo dnf install -y epel-release
+    echo "##[endgroup]"
+    rhel
+    ;;
+  almalinux9|centos-stream9)
+    echo "##[group]Enable epel and crb repositories"
+    sudo dnf config-manager -y --set-enabled crb
+    sudo dnf install -y epel-release
+
+   sudo dnf -y install kernel-abi-stablelists
+
+    # To minimize EPEL leakage, disable by default...
+    sudo -E sed -e "s/enabled=1/enabled=0/g" -i /etc/yum.repos.d/epel.repo
+
+     # Required utilities.
+    sudo -E yum -y --skip-broken install --enablerepo=epel git rpm-build \
+        wget curl bc fio acl sysstat mdadm lsscsi parted attr dbench watchdog \
+        ksh nfs-utils samba rng-tools dkms pamtester ncompress rsync
+
+    # Required development libraries
+    sudo -E yum -y --skip-broken install kernel-devel \
+        zlib-devel libuuid-devel libblkid-devel libselinux-devel \
+        xfsprogs-devel libattr-devel libacl-devel libudev-devel \
+        openssl-devel libargon2-devel libffi-devel pam-devel libaio-devel libcurl-devel
+
+    sudo -E yum -y --skip-broken install --enablerepo=powertools \
+        python3-packaging rpcgen
+
+    echo "##[endgroup]"
+    rhel
+    ;;
+  archlinux)
+    archlinux
+    ;;
+  debian*)
+    debian
+    echo "##[group]Install linux-perf"
+    sudo apt-get install -yq linux-perf
+    echo "##[endgroup]"
+    ;;
+  fedora*)
+    rhel
+
+    ;;
+  freebsd*)
+    freebsd
+    ;;
+  ubuntu*)
+    debian
+    echo "##[group]Install linux-tools-common"
+    sudo apt-get install -yq linux-tools-common libtirpc-dev
+    echo "##[endgroup]"
+    ;;
+esac
+
+enable_services "$1"
+
+# Enable serial console and remove 'quiet' from cmdline so we see all kernel
+# messages.
+#
+# This is most certainly overkill, but designed to work on all distos
+if [ "$1" != "freebsd*" ] ; then
+    sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0,115200n8 /g; s/quiet //g' /etc/default/grub || true
+fi
+
+for i in /boot/grub/grub.cfg /etc/grub2.cfg /etc/grub2-efi.cfg /boot/grub2/grub.cfg ; do
+    if [ -e $i ] ; then
+        sudo grub-mkconfig -o $i
+    fi
+done
+
+sudo poweroff
diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh
new file mode 100755
index 000000000000..f91acbcd638a
--- /dev/null
+++ b/.github/workflows/scripts/qemu-4-build.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 4) configure and build openzfs modules
+######################################################################
+
+set -eu
+cd $HOME/zfs
+
+function freebsd() {
+  echo "##[group]Autogen.sh"
+  MAKE="gmake" ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  MAKE="gmake" ./configure \
+    --prefix=/usr/local \
+    --with-libintl-prefix=/usr/local \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  gmake -j`sysctl -n hw.ncpu` 2>/var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  sudo gmake install 2>>/var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+}
+
+function linux() {
+  echo "##[group]Autogen.sh"
+  ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  make -j$(nproc) 2>/var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  sudo make install 2>>/var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+}
+
+function rpm_build_and_install() {
+  EXTRA_CONFIG="${1:-}"
+  echo "##[group]Autogen.sh"
+  ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+   ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  make pkg-kmod pkg-utils 2>&1 | tee /var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  sudo yum -y localinstall $(ls *.rpm | grep -v src.rpm)
+  echo "##[endgroup]"
+
+}
+
+function deb_build_and_install() {
+echo "##[group]Autogen.sh"
+  ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  make native-deb-kmod native-deb-utils  2>&1 | tee /var/tmp/make-stderr.txt
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+
+  # Do kmod install.  Note that when you build the native debs, the
+  # packages themselves are placed in parent directory '../' rather than
+  # in the source directory like the rpms are.
+  sudo apt-get -y install `find ../ | grep -E '\.deb$' | grep -Ev 'dkms|dracut'`
+  echo "##[endgroup]"
+}
+
+if [ -e /proc/cmdline ] ; then
+    cat /proc/cmdline || true
+fi
+
+export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
+case "$1" in
+  freebsd*)
+    freebsd
+    ;;
+  alma*|centos*)
+    rpm_build_and_install "--with-spec=redhat"
+    ;;
+  fedora*)
+    rpm_build_and_install
+    ;;
+  debian*|ubuntu*)
+    deb_build_and_install
+    ;;
+  *)
+    linux
+    ;;
+esac
diff --git a/.github/workflows/scripts/qemu-5-load.sh b/.github/workflows/scripts/qemu-5-load.sh
new file mode 100755
index 000000000000..08dfcaf2846e
--- /dev/null
+++ b/.github/workflows/scripts/qemu-5-load.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 5) load openzfs modules
+######################################################################
+
+set -eu
+cd $HOME/zfs
+
+OS="$1"
+
+function freebsd() {
+  echo "##[group]Load modules"
+  # when freebsd zfs is loaded, unload this one
+  kldstat -n zfs 2>/dev/null && sudo kldunload zfs
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+  sudo -E ./scripts/zfs.sh
+  sudo dmesg
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
+  echo "Loaded module: "
+  sudo kldstat -n openzfs
+  uname -a > /var/tmp/uname.txt
+  echo "##[endgroup]"
+}
+
+function linux_forceload() {
+  echo "Need to force the module loading!"
+  # -f tells modprobe to ignore wrong version numbers
+  sudo modprobe -v -f spl || echo "!! Loading module spl is failing !!"
+  sudo modprobe -v -f zfs || echo "!! Loading module zfs is failing !!"
+}
+
+function linux() {
+  echo "##[group]Load modules"
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+  sudo -E ./scripts/zfs.sh
+  test -d /proc/spl/kstat/zfs || linux_forceload
+  sudo dmesg
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
+  uname -a > /var/tmp/uname.txt
+  echo "##[endgroup]"
+}
+
+function modprobe_load() {
+  echo "##[group]Load modules"
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+  sudo -E modprobe zfs
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
+  uname -a > /var/tmp/uname.txt
+  echo "##[endgroup]"
+
+}
+
+export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
+case "$1" in
+  freebsd*)
+    freebsd
+    ;;
+  *)
+    modprobe_load
+    ;;
+esac
+
+ps aux | grep cat || true
diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh
new file mode 100755
index 000000000000..edbf513d821f
--- /dev/null
+++ b/.github/workflows/scripts/qemu-6-tests.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 6) configure and build openzfs modules
+######################################################################
+
+set -eu
+set -o pipefail
+cd $HOME/zfs
+
+# OPTS="-T casenorm"
+OPTS=""
+OPTS="-T pool_checkpoint"
+
+# Our VM has already created some ramdisks for us to use
+# if [ -e /dev/vdb ] && [ -e /dev/vdc ] && [ -e /dev/vdd ] ; then
+#     DISKS="/dev/vdb /dev/vdc /dev/vdd"
+#     export DISKS
+# elif [ -e /dev/vtbd1 ] && [ -e /dev/vtbd2 ] && [ -e /dev/vtbd3 ] ; then
+#     DISKS="/dev/vtbd1 /dev/vtbd2 /dev/vtbd3"
+#     export DISKS
+# else
+#     echo "no pre-created disks"
+# fi
+
+case "$1" in
+  freebsd*)
+    /usr/local/share/zfs/zfs-tests.sh -vKR -s 3G $OPTS \
+      | scripts/zfs-tests-color.sh
+    ;;
+  *)
+    /usr/share/zfs/zfs-tests.sh -vKR -s 3G $OPTS \
+      | scripts/zfs-tests-color.sh
+    ;;
+esac
diff --git a/.github/workflows/scripts/setup-dependencies.sh b/.github/workflows/scripts/setup-dependencies.sh
index b40f9290f914..57e51dcaed70 100755
--- a/.github/workflows/scripts/setup-dependencies.sh
+++ b/.github/workflows/scripts/setup-dependencies.sh
@@ -13,7 +13,7 @@ function prerun() {
   sudo apt upgrade
   sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq
   sudo apt-get clean
-  sudo dmesg -c > /var/tmp/dmesg-prerun
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
   echo "::endgroup::"
 }
 
@@ -47,7 +47,7 @@ function mod_install() {
   sudo depmod -a
   sudo modprobe zfs
   sudo dmesg
-  sudo dmesg -c > /var/tmp/dmesg-module-load
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
   echo "::endgroup::"
 
   echo "::group::Report CPU information"
diff --git a/.github/workflows/scripts/setup-functional.sh b/.github/workflows/scripts/setup-functional.sh
index 08c4d872abdf..efeb1e2852ac 100755
--- a/.github/workflows/scripts/setup-functional.sh
+++ b/.github/workflows/scripts/setup-functional.sh
@@ -6,15 +6,15 @@ TDIR="/usr/share/zfs/zfs-tests/tests/functional"
 echo -n "TODO="
 case "$1" in
   part1)
-    # ~1h 20m
+    # ~1h 10m
     echo "cli_root"
     ;;
   part2)
-    # ~1h
+    # ~1h 30m
     ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ','
     ;;
   part3)
-    # ~1h
+    # ~40m
     ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ','
     ;;
   part4)
diff --git a/.github/workflows/zfs-linux.yml b/.github/workflows/zfs-linux.yml
index e6b705c86055..1bb09dbd0887 100644
--- a/.github/workflows/zfs-linux.yml
+++ b/.github/workflows/zfs-linux.yml
@@ -6,59 +6,204 @@ on:
 
 jobs:
 
-  build:
-    name: Build
+#  build:
+#    name: Build
+#    strategy:
+#      fail-fast: false
+#      matrix:
+#        os: [20.04]
+#    runs-on: ubuntu-${{ matrix.os }}
+#    steps:
+#    - uses: actions/checkout@v4
+#      with:
+#        ref: ${{ github.event.pull_request.head.sha }}
+#    - name: Build modules
+#      run: .github/workflows/scripts/setup-dependencies.sh build
+#    - name: Prepare modules upload
+#      run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
+#    - uses: actions/upload-artifact@v4
+#      with:
+#        name: modules-${{ matrix.os }}
+#        path: modules-${{ matrix.os }}.tgz
+#        retention-days: 14
+#
+#  tests:
+#    name: Tests
+#    strategy:
+#      fail-fast: false
+#      matrix:
+#        os: [20.04]
+#    needs: build
+#    uses: ./.github/workflows/zfs-linux-tests.yml
+#    with:
+#      os: ${{ matrix.os }}
+
+  qemu-vm:
+    name: QEMU
     strategy:
       fail-fast: false
       matrix:
-        os: [20.04, 22.04]
-    runs-on: ubuntu-${{ matrix.os }}
+        os: [almalinux8, almalinux9, archlinux, centos-stream8, centos-stream9, fedora38, fedora39, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24]
+
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - name: Build modules
-      run: .github/workflows/scripts/setup-dependencies.sh build
-    - name: Prepare modules upload
-      run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
+    - name: Setup QEMU
+      run: .github/workflows/scripts/qemu-1-setup.sh
+    - name: Start QEMU machine
+      run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }}
+    - name: Install dependencies in QEMU machine
+      timeout-minutes: 15
+      run: |
+        echo "Install dependencies in QEMU machine"
+        echo "StrictHostKeyChecking no" >> $HOME/.ssh/config
+        echo "ConnectTimeout 1" >> $HOME/.ssh/config
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          sleep 1
+          IP=`arp | grep "^192.168.122."| cut -d' ' -f1`
+          test -z "$IP" && continue
+          ssh -v zfs@$IP "uname -a" && break
+        done
+        echo IP=$IP >> $GITHUB_ENV
+        scp .github/workflows/scripts/qemu-3-deps.sh zfs@$IP:qemu-3-deps.sh
+        ssh zfs@$IP '$HOME/qemu-3-deps.sh' ${{ matrix.os }} && true
+        # restart vm with new kernel and so on
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do sleep 5; done
+        sleep 1
+        sudo virsh start openzfs
+        sleep 5
+    - name: Build modules in QEMU machine
+      timeout-minutes: 30
+      run: |
+        echo "Build modules in QEMU machine"
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          sleep 1
+          ssh 2>/dev/null zfs@${{ env.IP }} "uname -a" && break
+        done
+        rsync -ar $HOME/work/zfs/zfs zfs@${{ env.IP }}:./
+        ssh zfs@${{ env.IP }} '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }}
+    - name: Load modules in QEMU machine
+      timeout-minutes: 2
+      run: |
+        ssh zfs@${{ env.IP }} '$HOME/zfs/.github/workflows/scripts/qemu-5-load.sh' ${{ matrix.os }}
+    - name: Run tests in QEMU machine
+      timeout-minutes: 400
+      run: |
+        echo "Start logging serial console before running tests"
+        # Save the VM's serial output (ttyS0) to /var/tmp/console-$OS.txt
+        # ttyS0 on the VM corresponds to a local /dev/pty/N entry.
+        # Use 'virsh ttyconsole' to lookup the /dev/pty/N entry.
+        OS=${{ matrix.os }}
+        read pty <<< $(sudo virsh ttyconsole "openzfs")
+        sudo mkdir -p /var/tmp
+        echo "Begin console for $OS" | sudo tee /var/tmp/console-$OS.txt
+        sudo nohup bash -c "cat $pty > /var/tmp/console-$OS.txt" &
+        echo "pty $pty, console started"
+        ssh zfs@${{ env.IP }} '$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh' ${{ matrix.os }}
+    - name: Prepare artifacts
+      if: success() || failure()
+      run: |
+        RESPATH="/var/tmp/test_results"
+        echo "rsyncing"
+        rsync -arL zfs@${{ env.IP }}:$RESPATH/current $RESPATH || true
+        echo "scp-ing"
+        scp zfs@$IP:"/var/tmp/*.txt" /var/tmp || true 
+        grep -A 100 'Results Summary' $RESPATH/current/log || true
+        cp -f /var/tmp/*.txt $RESPATH || true
+        ls -l $RESPATH || true
+        tar cf qemu-${{ matrix.os }}.tar -C $RESPATH -h current uname.txt \
+          osname.txt dmesg-prerun.txt dmesg-module-load.txt make-stderr.txt \
+          console-${{ matrix.os }}.txt || true
+        echo "done tarring, files in pwd $(pwd)"
+        ls -l
     - uses: actions/upload-artifact@v4
+      if: success() || failure()
       with:
-        name: modules-${{ matrix.os }}
-        path: modules-${{ matrix.os }}.tgz
-        retention-days: 14
-
-  testings:
-    name: Testing
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [20.04, 22.04]
-    needs: build
-    uses: ./.github/workflows/zfs-linux-tests.yml
-    with:
-      os: ${{ matrix.os }}
-
+        name: Logs-functional-${{ matrix.os }} 
+        path: qemu-${{ matrix.os }}.tar
+        if-no-files-found: ignore
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: console-${{ matrix.os }}.txt
+        path: /var/tmp/console-${{ matrix.os }}.txt
+        if-no-files-found: ignore
+        compression-level: 0
   cleanup:
     if: always()
     name: Cleanup
     runs-on: ubuntu-22.04
-    needs: testings
+#    needs: [ qemu-vm, tests ]
+    needs: [ qemu-vm ]
+
     steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
     - uses: actions/download-artifact@v4
     - name: Generating summary
-      run: |
-        tar xzf modules-22.04/modules-22.04.tgz .github tests
-        .github/workflows/scripts/generate-summary.sh
-    # up to 4 steps, each can have 1 MiB output (for debugging log files)
-    - name: Summary for errors #1
+      run: .github/workflows/scripts/generate-summary.sh
+    - name: Generating summary...
       run: .github/workflows/scripts/generate-summary.sh 1
-    - name: Summary for errors #2
+    - name: Generating summary...
       run: .github/workflows/scripts/generate-summary.sh 2
-    - name: Summary for errors #3
+    - name: Generating summary...
       run: .github/workflows/scripts/generate-summary.sh 3
-    - name: Summary for errors #4
+    - name: Generating summary...
       run: .github/workflows/scripts/generate-summary.sh 4
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 5
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 6
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 7
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 8
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 9
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 10
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 11
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 12
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 13
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 14
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 15
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 16
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 17
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 18
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 19
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 20
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 21
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 22
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 23
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 24
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 25
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 26
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 27
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 28
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 29
     - uses: actions/upload-artifact@v4
       with:
         name: Summary Files
-        path: Summary/
+        path: out-*
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
index 85854085f560..b5d8a4fc32ae 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
@@ -67,6 +67,7 @@ log_onexit cleanup
 
 cleanup
 
+
 typeset -i i=0
 while (( i < ${#shareopts[*]} ))
 do
diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
index 3e2aeb733546..18cd440e8431 100755
--- a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
+++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
@@ -68,6 +68,7 @@ log_must mount_redacted -f $recvfs
 # when we re-enable redaction blkptrs.
 #
 #log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1
+log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt"
 log_must diff $send_mnt/f2 $recv_mnt/f2
 log_must zfs rollback -R $clone@snap
 log_must zfs destroy -R $recvfs
@@ -96,6 +97,7 @@ log_must zfs destroy -R $clone2
 log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream"
 log_must eval "zfs recv $recvfs <$stream"
 log_must mount_redacted -f $recvfs
+log_note "send_mnt2=$send_mnt, recv_mnt2=$recv_mnt"
 log_must diff <(ls $send_mnt) <(ls $recv_mnt)
 log_must zfs destroy -R $recvfs
 log_must zfs rollback -R $sendfs@snap