cidata: split user-data

docs/internal.md

@@ -17,7 +17,7 @@ Metadata:
 - `lima.yaml`: the YAML
 
 cloud-init:
-- `cidata.iso`: cloud-init ISO9660 image. (`user-data`, `meta-data`, `lima-guestagent.Linux-<ARCH>`)
+- `cidata.iso`: cloud-init ISO9660 image. See [`cidata.iso`](#cidata-iso).
 
 disk:
 - `basedisk`: the base image
@@ -58,3 +58,30 @@ The directory contains the following files:
 
 - `$LIMA_INSTANCE`: `lima ...` is expanded to `limactl shell ${LIMA_INSTANCE} ...`.
   - Default : `default`
+
+## `cidata.iso`
+`cidata.iso` contains the following files:
+
+- `user-data`: [Cloud-init user-data](https://cloudinit.readthedocs.io/en/latest/topics/format.html)
+- `meta-data`: [Cloud-init meta-data](https://cloudinit.readthedocs.io/en/latest/topics/instancedata.html)
+- `lima.env`: the environment variables
+- `lima-guestagent`: Lima guest agent binary
+- `nerdctl-full.tgz`: [`nerdctl-full-<VERSION>-linux-<ARCH>.tar.gz`](https://github.com/containerd/nerdctl/releases)
+- `boot.sh`: Boot script
+- `boot/*`: Boot script modules
+- `provision.system/*`: Custom provision scripts (system)
+- `provision.user/*`: Custom provision scripts (user)
+
+Max file name length = 30
+
+### Volume label
+The volume label is "cidata", as defined by [cloud-init NoCloud](https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html).
+
+### Environment variables
+- `LIMA_CIDATA_MNT`: the mount point of the disk. `/mnt/lima-cidata`.
+- `LIMA_CIDATA_USER`: the user name string
+- `LIMA_CIDATA_UID`: the numeric UID
+- `LIMA_CIDATA_MOUNTS`: the number of the Lima mounts
+- `LIMA_CIDATA_MOUNTS_%d_MOUNTPOINT`: the N-th mount point of Lima mounts (N=0, 1, ...)
+- `LIMA_CIDATA_CONTAINERD_USER`: set to "1" if rootless containerd to be set up
+- `LIMA_CIDATA_CONTAINERD_SYSTEM`: set to "1" if system-wide containerd to be set up

pkg/cidata/cidata.TEMPLATE.d/boot.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+INFO(){
+  echo "LIMA| $*"
+}
+
+WARNING(){
+  echo "LIMA| WARNING: $*"
+}
+
+# shellcheck disable=SC2163
+while read -r line; do export "$line"; done <"${LIMA_CIDATA_MNT}"/lima.env
+
+CODE=0
+
+for f in "${LIMA_CIDATA_MNT}"/boot/*; do
+	INFO "Executing $f"
+	if ! "$f"; then
+		WARNING "Failed to execute $f"
+		CODE=1
+	fi
+done
+
+if [ -d "${LIMA_CIDATA_MNT}"/provision.system ]; then
+	for f in "${LIMA_CIDATA_MNT}"/provision.system/*; do
+		INFO "Executing $f"
+		if ! "$f"; then
+			WARNING "Failed to execute $f"
+			CODE=1
+		fi
+	done
+fi
+
+if [ -d "${LIMA_CIDATA_MNT}"/provision.user ]; then
+	until [ -e "/run/user/${LIMA_CIDATA_UID}}/systemd/private" ]; do sleep 3; done
+	for f in "${LIMA_CIDATA_MNT}"/provision.user/*; do
+		INFO "Executing $f (as user ${LIMA_CIDATA_USER})"
+		if ! sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" "$f"; then
+			WARNING "Failed to execute $f (as user ${LIMA_CIDATA_USER})"
+			CODE=1
+		fi
+	done
+fi
+
+INFO "Exiting with code $CODE"
+exit "$CODE"

pkg/cidata/cidata.TEMPLATE.d/boot/00-alpine-ash-as-bash.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# This script pretends that /bin/ash can be used as /bin/bash, so all following
+# cloud-init scripts can use `#!/bin/bash` and `set -o pipefail`.
+test -f /etc/alpine-release || exit 0
+
+# Redirect bash to ash (built with CONFIG_ASH_BASH_COMPAT) and hope for the best :)
+# It does support `set -o pipefail`, but not `[[`.
+# /bin/bash can't be a symlink because /bin/ash is a symlink to /bin/busybox
+cat >/bin/bash <<'EOF'
+#!/bin/sh
+exec /bin/ash "$@"
+EOF
+chmod +x /bin/bash

pkg/cidata/cidata.TEMPLATE.d/boot/05-persistent-data-volume.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# Restrict the rest of this script to Alpine until it has been tested with other distros
+test -f /etc/alpine-release || exit 0
+
+# Data directories that should be persisted across reboots
+DATADIRS="/home /usr/local /etc/containerd /var/lib/containerd"
+
+# When running from RAM try to move persistent data to data-volume
+# FIXME: the test for tmpfs mounts is probably Alpine-specific
+if [ "$(awk '$2 == "/" {print $3}' /proc/mounts)" == "tmpfs" ]; then
+  mkdir -p /mnt/data
+  if [ -e /dev/disk/by-label/data-volume ]; then
+    mount -t ext4 /dev/disk/by-label/data-volume /mnt/data
+  else
+    # Find an unpartitioned disk and create data-volume
+    DISKS=$(lsblk --list --noheadings --output name,type | awk '$2 == "disk" {print $1}')
+    for DISK in ${DISKS}; do
+      IN_USE=false
+      # Looking for a disk that is not mounted or partitioned
+      for PART in $(awk '/^\/dev\// {gsub("/dev/", ""); print $1}' /proc/mounts); do
+        if [ "${DISK}" == "${PART}" -o -e /sys/block/${DISK}/${PART} ]; then
+          IN_USE=true
+          break
+        fi
+      done
+      if [ "${IN_USE}" == "false" ]; then
+        echo 'type=83' | sfdisk --label dos /dev/${DISK}
+        PART=$(lsblk --list /dev/${DISK} --noheadings --output name,type | awk '$2 == "part" {print $1}')
+        mkfs.ext4 -L data-volume /dev/${PART}
+        mount -t ext4 /dev/disk/by-label/data-volume /mnt/data
+        for DIR in ${DATADIRS}; do
+          DEST="/mnt/data$(dirname ${DIR})"
+          mkdir -p ${DIR} ${DEST}
+          mv ${DIR} ${DEST}
+        done
+        break
+      fi
+    done
+  fi
+  for DIR in ${DATADIRS}; do
+    if [ -d /mnt/data${DIR} ]; then
+      [ -e ${DIR} ] && rm -rf ${DIR}
+      ln -s /mnt/data${DIR} ${DIR}
+    fi
+  done
+fi

pkg/cidata/cidata.TEMPLATE.d/boot/10-alpine-prep.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# This script prepares Alpine for lima; there is nothing in here for other distros
+test -f /etc/alpine-release || exit 0
+
+# Configure apk repos
+BRANCH=edge
+VERSION_ID=$(awk -F= '$1=="VERSION_ID" {print $2}' /etc/os-release)
+case ${VERSION_ID} in
+*_alpha*|*_beta*) BRANCH=edge;;
+*.*.*) BRANCH=v${VERSION_ID%.*};;
+esac
+
+for REPO in main community; do
+  URL="https://dl-cdn.alpinelinux.org/alpine/${BRANCH}/${REPO}"
+  if ! grep -q "^${URL}$" /etc/apk/repositories; then
+    echo "${URL}" >> /etc/apk/repositories
+  fi
+done
+
+# Alpine doesn't use PAM so we need to explicitly allow public key auth
+usermod -p '*' "${LIMA_CIDATA_USER}"
+
+# Alpine disables TCP forwarding, which is needed by the lima-guestagent
+sed -i 's/AllowTcpForwarding no/AllowTcpForwarding yes/g' /etc/ssh/sshd_config
+rc-service sshd reload
+
+# Create directory for the lima-guestagent socket (normally done by systemd)
+mkdir -p /run/user/${LIMA_CIDATA_UID}
+chown "${LIMA_CIDATA_USER}" /run/user/${LIMA_CIDATA_UID}
+chmod 700 /run/user/${LIMA_CIDATA_UID}
+
+# Install the openrc lima-guestagent service script
+cat >/etc/init.d/lima-guestagent <<'EOF'
+#!/sbin/openrc-run
+supervisor=supervise-daemon
+
+name="lima-guestagent"
+description="Forward ports to the lima-hostagent"
+
+export XDG_RUNTIME_DIR="/run/user/${LIMA_CIDATA_UID}"
+command=/usr/local/bin/lima-guestagent
+command_args="daemon"
+command_background=true
+command_user="${LIMA_CIDATA_USER}:${LIMA_CIDATA_USER}"
+pidfile="${XDG_RUNTIME_DIR}/lima-guestagent.pid"
+EOF
+chmod 755 /etc/init.d/lima-guestagent
+
+# mount /sys/fs/cgroup
+rc-service cgroups start
+
+# `limactl stop` tells acpid to powerdown
+rc-update add acpid
+rc-service acpid start

pkg/cidata/cidata.TEMPLATE.d/boot/20-rootless-base.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# This script does not work unless systemd is available
+command -v systemctl 2>&1 >/dev/null || exit 0
+
+# Set up env
+for f in .profile .bashrc; do
+  if ! grep -q "# Lima BEGIN" "/home/${LIMA_CIDATA_USER}.linux/$f"; then
+    cat >>"/home/${LIMA_CIDATA_USER}.linux/$f" <<EOF
+# Lima BEGIN
+# Make sure iptables and mount.fuse3 are available
+PATH="$PATH:/usr/sbin:/sbin"
+# fuse-overlayfs is the most stable snapshotter for rootless
+CONTAINERD_SNAPSHOTTER="fuse-overlayfs"
+export PATH CONTAINERD_SNAPSHOTTER
+# Lima END
+EOF
+    chown "${LIMA_CIDATA_USER}" "/home/${LIMA_CIDATA_USER}.linux/$f"
+  fi
+done
+# Enable cgroup delegation (only meaningful on cgroup v2)
+if [ ! -e "/etc/systemd/system/user@.service.d/lima.conf" ]; then
+  mkdir -p "/etc/systemd/system/user@.service.d"
+  cat >"/etc/systemd/system/user@.service.d/lima.conf"  <<EOF
+[Service]
+Delegate=yes
+EOF
+fi
+systemctl daemon-reload
+
+# Set up sysctl
+sysctl_conf="/etc/sysctl.d/99-lima.conf"
+if [ ! -e "${sysctl_conf}" ]; then
+  if [ -e "/proc/sys/kernel/unprivileged_userns_clone" ]; then
+    echo "kernel.unprivileged_userns_clone=1" >> "${sysctl_conf}"
+  fi
+  echo "net.ipv4.ping_group_range = 0 2147483647" >> "${sysctl_conf}"
+  echo "net.ipv4.ip_unprivileged_port_start=0" >> "${sysctl_conf}"
+  sysctl --system
+fi
+
+# Set up subuid
+for f in /etc/subuid /etc/subgid; do
+  grep -qw "${LIMA_CIDATA_USER}" $f || echo "${LIMA_CIDATA_USER}:100000:65536" >> $f
+done
+
+# Start systemd session
+loginctl enable-linger "${LIMA_CIDATA_USER}"

pkg/cidata/cidata.TEMPLATE.d/boot/25-guestagent-base.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -eux
+
+# Create mount points
+# NOTE: Busybox sh does not support `for ((i=0;i<$N;i++))` form
+for f in $(seq 0 $((LIMA_CIDATA_MOUNTS - 1))); do
+  mountpointvar="LIMA_CIDATA_MOUNTS_${f}_MOUNTPOINT"
+  mountpoint="$(eval echo \$$mountpointvar)"
+  mkdir -p "${mountpoint}"
+  chown "${LIMA_CIDATA_USER}" "${mountpoint}"
+done
+
+# Install or update the guestagent binary
+install -m 755 "${LIMA_CIDATA_MNT}"/lima-guestagent /usr/local/bin/lima-guestagent
+
+# Launch the guestagent service
+if [ -f /etc/alpine-release ]; then
+  rc-update add lima-guestagent default
+  rc-service lima-guestagent start
+else
+  until [ -e "/run/user/${LIMA_CIDATA_UID}/systemd/private" ]; do sleep 3; done
+  sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" lima-guestagent install-systemd
+fi

pkg/cidata/cidata.TEMPLATE.d/boot/30-install-packages.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# Install minimum dependencies
+if command -v apt-get 2>&1 >/dev/null; then
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update
+  if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ]; then
+    apt-get install -y sshfs
+  fi
+  if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+    apt-get install -y iptables
+  fi
+  if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+    apt-get install -y uidmap fuse3 dbus-user-session
+  fi
+elif command -v dnf 2>&1 >/dev/null; then
+  if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ]; then
+    dnf install -y fuse-sshfs
+  fi
+  if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+    dnf install -y iptables
+  fi
+  if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+    dnf install -y shadow-utils fuse3
+    if [ ! -f /usr/bin/fusermount ]; then
+      # Workaround for https://github.com/containerd/stargz-snapshotter/issues/340
+      ln -s fusermount3 /usr/bin/fusermount
+    fi
+  fi
+elif command -v apk 2>&1 >/dev/null; then
+  if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ]; then
+    if ! command -v sshfs 2>&1 >/dev/null; then
+      apk update
+      apk add sshfs
+    fi
+    modprobe fuse
+  fi
+fi
+# Modify /etc/fuse.conf to allow "-o allow_root"
+
+if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ]; then
+  if ! grep -q "^user_allow_other" /etc/fuse.conf ; then
+    echo "user_allow_other" >> /etc/fuse.conf
+  fi
+fi

pkg/cidata/cidata.TEMPLATE.d/boot/40-install-containerd.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -eux -o pipefail
+
+
+if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" != 1 ] && [ "${LIMA_CIDATA_CONTAINERD_USER}" != 1 ]; then
+  exit 0
+fi
+
+# This script does not work unless systemd is available
+command -v systemctl 2>&1 >/dev/null || exit 0
+
+if [ ! -x /usr/local/bin/nerdctl ]; then
+  tar Cxzf /usr/local "${LIMA_CIDATA_MNT}"/nerdctl-full.tgz
+fi
+
+if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ]; then
+  mkdir -p /etc/containerd
+  cat >"/etc/containerd/config.toml" <<EOF
+  version = 2
+  [proxy_plugins]
+    [proxy_plugins."stargz"]
+      type = "snapshot"
+      address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+EOF
+  systemctl enable --now containerd buildkit stargz-snapshotter
+fi
+
+if [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+  modprobe tap || true
+  if [ ! -e "/home/${LIMA_CIDATA_USER}.linux/.config/containerd/config.toml" ]; then
+    mkdir -p "/home/${LIMA_CIDATA_USER}.linux/.config/containerd"
+    cat >"/home/${LIMA_CIDATA_USER}.linux/.config/containerd/config.toml" <<EOF
+  version = 2
+  [proxy_plugins]
+    [proxy_plugins."fuse-overlayfs"]
+      type = "snapshot"
+      address = "/run/user/${LIMA_CIDATA_UID}/containerd-fuse-overlayfs.sock"
+    [proxy_plugins."stargz"]
+      type = "snapshot"
+      address = "/run/user/${LIMA_CIDATA_UID}/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+EOF
+  chown -R "${LIMA_CIDATA_USER}" "/home/${LIMA_CIDATA_USER}.linux/.config"
+  fi
+  selinux=
+  if command -v selinuxenabled 2>&1 >/dev/null && selinuxenabled; then
+    selinux=1
+  fi
+  if [ ! -e "/home/${LIMA_CIDATA_USER}}}.linux/.config/systemd/user/containerd.service" ]; then
+    until [ -e "/run/user/${LIMA_CIDATA_UID}/systemd/private" ]; do sleep 3; done
+    if [ -n "$selinux" ]; then
+      echo "Temporarily disabling SELinux, during installing containerd units"
+      setenforce 0
+    fi
+    sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" systemctl --user enable --now dbus
+    sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" containerd-rootless-setuptool.sh install
+    sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" containerd-rootless-setuptool.sh install-buildkit
+    sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" containerd-rootless-setuptool.sh install-fuse-overlayfs
+    if ! sudo -iu "${LIMA_CIDATA_USER}" "XDG_RUNTIME_DIR=/run/user/${LIMA_CIDATA_UID}" containerd-rootless-setuptool.sh install-stargz; then
+      echo >&2 "WARNING: rootless stargz does not seem supported on this host (kernel older than 5.11?)"
+    fi
+    if [ -n "$selinux" ]; then
+      echo "Restoring SELinux"
+      setenforce 1
+    fi
+  fi
+fi