diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7a6b567..cea6d51 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -384,3 +384,58 @@ steps: provider: gcp machineType: n2-standard-2 enableNestedVirtualization: true + + - label: "quark-test on ubuntu 18.04 (no bpf)" + key: test_ubuntu_18_04 + command: "./.buildkite/runtest_distro.sh ubuntu 18.04 -k" + depends_on: + - make_docker + agents: + image: family/core-ubuntu-2404 + provider: gcp + machineType: n2-standard-2 + enableNestedVirtualization: true + + - label: "quark-test on ubuntu 20.04" + key: test_ubuntu_20_04 + command: "./.buildkite/runtest_distro.sh ubuntu 20.04" + depends_on: + - make_docker + agents: + image: family/core-ubuntu-2404 + provider: gcp + machineType: n2-standard-2 + enableNestedVirtualization: true + + - label: "quark-test on ubuntu 22.04" + key: test_ubuntu_22_04 + command: "./.buildkite/runtest_distro.sh ubuntu 22.04" + depends_on: + - make_docker + agents: + image: family/core-ubuntu-2404 + provider: gcp + machineType: n2-standard-2 + enableNestedVirtualization: true + + - label: "quark-test on ubuntu 24.04" + key: test_ubuntu_24_04 + command: "./.buildkite/runtest_distro.sh ubuntu 24.04" + depends_on: + - make_docker + agents: + image: family/core-ubuntu-2404 + provider: gcp + machineType: n2-standard-2 + enableNestedVirtualization: true + + - label: "quark-test on ubuntu 25.04" + key: test_ubuntu_25_04 + command: "./.buildkite/runtest_distro.sh ubuntu 25.04" + depends_on: + - make_docker + agents: + image: family/core-ubuntu-2404 + provider: gcp + machineType: n2-standard-2 + enableNestedVirtualization: true diff --git a/.buildkite/runtest_distro.sh b/.buildkite/runtest_distro.sh index 3abfb3c..d4b8b19 100755 --- a/.buildkite/runtest_distro.sh +++ b/.buildkite/runtest_distro.sh @@ -36,5 +36,6 @@ sudo kvm-ok case "$DISTRO" in fedora) sudo ./krun-fedora.sh initramfs.gz "$DISTROVER" quark-test $@;; rhel) sudo ./krun-rhel.sh initramfs.gz "$DISTROVER" quark-test $@;; +ubuntu) sudo ./krun-ubuntu.sh initramfs.gz "$DISTROVER" quark-test $@;; *) echo bad distribution "$DISTROVER" 1>&2;; esac diff --git a/krun-fedora.sh b/krun-fedora.sh index e70dbad..88f6159 100755 --- a/krun-fedora.sh +++ b/krun-fedora.sh @@ -3,13 +3,36 @@ set -euo pipefail SCRIPT=${0##*/} +VERBOSE=0 + +log() { (( VERBOSE )) && printf '%s\n' "INFO: $*" >&2 || true; } +log_error() { printf '%s\n' "ERROR: $*" >&2; } +die() { log_error "$*"; exit 1; } function usage { - echo "usage: $SCRIPT initramfs.gz FEDORAVERSION command" 1>&2 + echo "usage: $SCRIPT [-v] initramfs.gz FEDORAVERSION command..." 1>&2 + echo + echo " -v Verbose output" + echo " initramfs.gz Path to initramfs image" + echo " FEDORAVERSION Fedora version (e.g. 39, 40, rawhide)" + echo " command... Command to run in guest" + echo + echo "Examples:" + echo " $SCRIPT initramfs.gz 40 /bin/bash" + echo " $SCRIPT -v initramfs.gz rawhide quark-test -vvv" exit 1 } +while getopts "vh" opt; do + case $opt in + v) VERBOSE=1 ;; + h) usage ;; + *) usage ;; + esac +done +shift $((OPTIND - 1)) + if [ $# -lt 3 ]; then usage fi @@ -18,16 +41,24 @@ INITRAMFS="$1" FEDORAVER="$2" shift 2 +[[ -f $INITRAMFS ]] || die "Initramfs not found: $INITRAMFS" +[[ -f ./krun.sh ]] || die "Required launcher ./krun.sh is missing" + case $FEDORAVER in 2?|3?) URL="https://archives.fedoraproject.org/pub/archive/fedora/linux/updates/$FEDORAVER/Everything/x86_64/Packages/k";; 43|rawhide) URL="https://ftp.fau.de/fedora/linux/development/$FEDORAVER/Everything/x86_64/os/Packages/k";; 4?) URL="https://ftp.fau.de/fedora/linux/updates/$FEDORAVER/Everything/x86_64/Packages/k";; -*) echo bad version "$FEDORAVER" 1>&2;; +*) die "Unsupported Fedora version: $FEDORAVER";; esac +log "Searching for Fedora $FEDORAVER kernel..." + TMPDIR=$(mktemp -d "/tmp/$SCRIPT.XXXXXXXXXX") -trap 'rm -rf "$TMPDIR"' EXIT +readonly TMPDIR +cleanup() { [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR"; } +trap cleanup EXIT +log "Fetching package list from $URL" RPMURL=$(lynx -dump -listonly "$URL"|grep kernel-core) RPMURL=${RPMURL##* } RPM=$(basename "$RPMURL") @@ -35,13 +66,18 @@ VMLINUZ=${RPM##kernel-core-} VMLINUZ=${VMLINUZ%%.rpm} VMLINUZ=$TMPDIR/lib/modules/$VMLINUZ/vmlinuz -# echo URL $URL -# echo RPMURL $RPMURL -# echo RPM $RPM -# echo VMLINUZ $VMLINUZ +log "URL: $URL" +log "RPMURL: $RPMURL" +log "Downloading kernel RPM: $RPM" +log "Target vmlinuz: $VMLINUZ" cd "$TMPDIR" curl -s "$RPMURL" | rpm2cpio - | cpio -idm cd - +[[ -f "$VMLINUZ" ]] || die "vmlinuz not found: $VMLINUZ" + +log "Kernel ready: $VMLINUZ" +log "Handing off to ./krun.sh" + ./krun.sh "$INITRAMFS" "$VMLINUZ" "$@" diff --git a/krun-rhel.sh b/krun-rhel.sh index fd7caea..db53132 100755 --- a/krun-rhel.sh +++ b/krun-rhel.sh @@ -3,13 +3,36 @@ set -euo pipefail SCRIPT=${0##*/} +VERBOSE=0 + +log() { (( VERBOSE )) && printf '%s\n' "INFO: $*" >&2 || true; } +log_error() { printf '%s\n' "ERROR: $*" >&2; } +die() { log_error "$*"; exit 1; } function usage { - echo "usage: $SCRIPT initramfs.gz RHELVER command" 1>&2 + echo "usage: $SCRIPT [-v] initramfs.gz RHELVER command..." 1>&2 + echo + echo " -v Verbose output" + echo " initramfs.gz Path to initramfs image" + echo " RHELVER RHEL version (e.g. 8, 9, 8.4, 9.1)" + echo " command... Command to run in guest" + echo + echo "Examples:" + echo " $SCRIPT initramfs.gz 9 /bin/bash" + echo " $SCRIPT -v initramfs.gz 8.4 quark-test -vvv" exit 1 } +while getopts "vh" opt; do + case $opt in + v) VERBOSE=1 ;; + h) usage ;; + *) usage ;; + esac +done +shift $((OPTIND - 1)) + if [ $# -lt 3 ]; then usage fi @@ -18,16 +41,24 @@ INITRAMFS="$1" RHELVER="$2" shift 2 +[[ -f $INITRAMFS ]] || die "Initramfs not found: $INITRAMFS" +[[ -f ./krun.sh ]] || die "Required launcher ./krun.sh is missing" + case $RHELVER in 8|9) URL="https://ftp.fau.de/rockylinux/$RHELVER/BaseOS/x86_64/os/Packages/k";; 8.[34]) URL="https://dl.rockylinux.org/vault/rocky/$RHELVER/BaseOS/x86_64/os/Packages";; 8.?|9.?) URL="https://dl.rockylinux.org/vault/rocky/$RHELVER/BaseOS/x86_64/os/Packages/k";; -*) echo bad version "$RHELVER" 1>&2;; +*) die "Unsupported RHEL version: $RHELVER";; esac +log "Searching for RHEL $RHELVER kernel..." + TMPDIR=$(mktemp -d "/tmp/$SCRIPT.XXXXXXXXXX") -trap 'rm -rf "$TMPDIR"' EXIT +readonly TMPDIR +cleanup() { [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR"; } +trap cleanup EXIT +log "Fetching package list from $URL" RPMURL=$(lynx -dump -listonly "$URL"|grep kernel-core) RPMURL=${RPMURL##* } RPM=$(basename "$RPMURL") @@ -35,13 +66,16 @@ VMLINUZ=${RPM##kernel-core-} VMLINUZ=${VMLINUZ%%.rpm} VMLINUZ=$TMPDIR/lib/modules/$VMLINUZ/vmlinuz -# echo URL $URL -# echo RPMURL $RPMURL -# echo RPM $RPM -# echo VMLINUZ $VMLINUZ +log "Downloading kernel RPM: $RPM" +log "Target vmlinuz: $VMLINUZ" cd "$TMPDIR" curl -s "$RPMURL" | rpm2cpio - | cpio -idm cd - +[[ -f "$VMLINUZ" ]] || die "vmlinuz not found: $VMLINUZ" + +log "Kernel ready: $VMLINUZ" +log "Handing off to ./krun.sh" + ./krun.sh "$INITRAMFS" "$VMLINUZ" "$@" diff --git a/krun-ubuntu.sh b/krun-ubuntu.sh new file mode 100755 index 0000000..74bb612 --- /dev/null +++ b/krun-ubuntu.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# +# krun-ubuntu.sh +# ----------------------------------------- +# Download the newest *generic* Linux kernel for an Ubuntu release +# (amd64 | arm64), unpack vmlinuz, and hand it off to ./krun.sh. +# +# Usage: +# ./krun-ubuntu.sh [-a arch] [-v] [command...] +# +# --------------------------------------------------------------------- + +set -Eeuo pipefail + +SCRIPT=${0##*/} +VERBOSE=0 + +# ---------- helpers -------------------------------------------------- + +log() { (( $VERBOSE )) && printf '%s\n' "INFO: $*" >&2 || true; } +log_error() { printf '%s\n' "ERROR: $*" >&2; } +die() { log_error "$*"; exit 1; } + +usage() { + echo "usage: $SCRIPT [-v] initramfs.gz UBUNTUVERSION command..." 1>&2 + echo + echo " -v Verbose output" + echo " initramfs.gz Path to initramfs image" + echo " UBUNTUVERSION Ubuntu version (e.g. 18.04, noble)" + echo " command... Command to run in guest" + echo + echo "Examples:" + echo " $SCRIPT initramfs.gz 40 /bin/bash" + echo " $SCRIPT -v initramfs.gz rawhide quark-test -vvv" + exit 1 +} + +need_bins() { for b; do command -v "$b" >/dev/null || die "Missing $b"; done; } + +# ---------- parse args ----------------------------------------------- + +ARCH=amd64 # Only amd64 for now +VERBOSE=0 + +while getopts "vh" opt; do + case $opt in + v) VERBOSE=1 ;; + h) usage ;; + *) usage ;; + esac +done +shift $((OPTIND - 1)) + +[[ $# -lt 2 ]] && usage +[[ $ARCH =~ ^(amd64|arm64)$ ]] || die "Invalid architecture: $ARCH" + +INITRAMFS="$1"; shift +UBUNTU_VERSION="$1"; shift + +[[ -f $INITRAMFS ]] || die "Initramfs not found: $INITRAMFS" +[[ -f ./krun.sh ]] || die "Required launcher ./krun.sh is missing" + +readonly UBUNTU_VERSION ARCH INITRAMFS + +# ---------- prerequisites -------------------------------------------- + +need_bins curl awk ar tar gunzip sha256sum +command -v zstd >/dev/null && HAVE_ZSTD=1 || HAVE_ZSTD=0 + +CURL_OPTS=(--fail --silent --show-error --location --proto '=https' --tlsv1.2) + +# ---------- version → codename --------------------------------------- + +case "$UBUNTU_VERSION" in + 18.04) CODENAME=bionic ;; + 20.04) CODENAME=focal ;; + 22.04) CODENAME=jammy ;; + 24.04) CODENAME=noble ;; + 25.04) CODENAME=oracular ;; + *) die "Unsupported Ubuntu version: $UBUNTU_VERSION" ;; +esac + +if [[ $ARCH == amd64 ]]; then + BASE_URL="https://archive.ubuntu.com/ubuntu" +else + BASE_URL="https://ports.ubuntu.com/ubuntu-ports" +fi + +# ---------- main ----------------------------------------------------- + +REPOS=("${CODENAME}-updates" "${CODENAME}-security" "${CODENAME}") + +TMPDIR=$(mktemp -d "/tmp/$SCRIPT.XXXXXXXXXX") +readonly TMPDIR +cleanup() { [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR"; } +trap cleanup EXIT + +log "Searching latest *generic* kernel for Ubuntu $UBUNTU_VERSION ($ARCH)…" + +LATEST_PATH="" + +for repo in "${REPOS[@]}"; do + # Full path for the download … + PKG_PATH="dists/$repo/main/binary-$ARCH/Packages.gz" + # … and relative path as it appears inside the Release file + PKG_PATH_REL="main/binary-$ARCH/Packages.gz" + REL_URL="$BASE_URL/dists/$repo/Release" + log "Checking $REL_URL" + + EXPECTED_HASH="" + if curl "${CURL_OPTS[@]}" "$REL_URL" -o "$TMPDIR/Release"; then + EXPECTED_HASH=$(awk -v pk="$PKG_PATH_REL" ' + $1=="SHA256:" {tbl=1;next} + tbl && $NF==pk {print $1;exit}' "$TMPDIR/Release") || true + fi + + curl "${CURL_OPTS[@]}" "$BASE_URL/$PKG_PATH" -o "$TMPDIR/Packages.gz" || + { log "No Packages.gz for $repo (arch not built?)"; continue; } + + if [[ -n $EXPECTED_HASH ]]; then + [[ $(sha256sum "$TMPDIR/Packages.gz" | awk '{print $1}') == "$EXPECTED_HASH" ]] || + die "Checksum mismatch for Packages.gz (possible MITM)" + else + log "Release digest unavailable; skipping checksum validation (less secure)" + fi + + if ! gunzip -c "$TMPDIR/Packages.gz" >"$TMPDIR/Packages" 2>/dev/null; then + die "gunzip failed – archive not gzip? (unexpected for Ubuntu mirrors)" + fi + + awk -v RS='' ' + /(^|\n)Package: linux-image-[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-generic($|\n)/ { + ver=""; file="" + for (i=1;i<=NF;i++){ + if ($i=="Version:") ver=$(i+1) + if ($i=="Filename:") file=$(i+1) + } + if (ver && file) print ver, file + }' "$TMPDIR/Packages" > "$TMPDIR/candidates" + + if [[ -s $TMPDIR/candidates ]]; then + LATEST_PATH=$(sort -V -k1,1 "$TMPDIR/candidates" | tail -1 | awk '{print $2}') + log "Found candidate in $repo: ${LATEST_PATH##*/}" + break + fi +done + +[[ -n $LATEST_PATH ]] || die "No linux-image-generic package found for $UBUNTU_VERSION/$ARCH" + +DEB_URL="$BASE_URL/$LATEST_PATH" +DEB_FILE="$TMPDIR/${DEB_URL##*/}" + +log "Downloading: $DEB_URL" +curl "${CURL_OPTS[@]}" "$DEB_URL" -o "$DEB_FILE" + +log "Extracting .deb" +( + cd "$TMPDIR" + ar x "${DEB_FILE##*/}" + if [[ -f data.tar.xz ]]; then tar -xf data.tar.xz + elif [[ -f data.tar.zst ]]; then + [[ $HAVE_ZSTD -eq 1 ]] || die "zstd archive but zstd binary missing" + tar --use-compress-program=zstd -xf data.tar.zst + elif [[ -f data.tar.gz ]]; then tar -xf data.tar.gz + else die "Unknown data archive inside .deb" + fi +) + +KERNEL_VER=$(sed -nE 's/linux-image-([0-9]+\.[0-9]+\.[0-9]+-[0-9]+-generic)_.*/\1/p' \ + <<< "${DEB_FILE##*/}") +VMLINUZ="$TMPDIR/boot/vmlinuz-$KERNEL_VER" +[[ -f $VMLINUZ ]] || die "vmlinuz not found: $VMLINUZ" + +if file -b "$VMLINUZ" | grep -qi '^gzip compressed'; then + log "Kernel is gzip-compressed; decompressing…" + DECOMPRESSED="$TMPDIR/Image-$KERNEL_VER" + gunzip -c "$VMLINUZ" > "$DECOMPRESSED" \ + || die "Failed to gunzip kernel" + VMLINUZ="$DECOMPRESSED" +fi + +log "Kernel ready: $VMLINUZ" +log "Handing off to ./krun.sh" + +./krun.sh "$INITRAMFS" "$VMLINUZ" "$@"