Reduce amount of containers in local-up and improve dependency checks (…

…#119)

* Detect os in local-up

* Remove some containers from local-up

* Restore setup.Dockerfile

* Improve README

* Typos

.gitignore

@@ -7,7 +7,6 @@ node_modules
 core*
 app/styles
 .yarn-cache
-ops/resources/gitpod-ip.sh
 buildkite.ext4
 buildkite.qcow2
 vmlinux-*.bin

README.md

@@ -32,9 +32,10 @@ Hocus integrates with any Git provider that uses the SSH protocol, like GitHub,
 
 ### Requirements
 
-- x86_64 Linux
+- x86_64 Linux, preferably with at least the 5.10 kernel
 - KVM support on the host
-- Git and Git LFS
+- A Linux distribution which supports the `target_core_user` and `tcm_loop` kernel modules (generally available with the notable exception of WSL https://github.com/microsoft/WSL/issues/9511)
+- Git
 - Docker, Docker Compose, and Buildx
 
 That's it! Hocus is fully containerized and won't install anything directly on your host system. The following script will check if your system meets the requirements, prompt you to install any missing dependencies, and set up Hocus with `docker-compose`.

app/agent/runtime/qemu/qemu.service.ts

@@ -313,6 +313,7 @@ export class QemuService implements HocusRuntime {
   ): Promise<T> {
     const t1 = performance.now();
     const kernelPath = config.kernelPath ?? this.agentConfig.defaultKernel;
+    const initrdPath = this.agentConfig.defaultInitrd;
     const shouldPoweroff = config.shouldPoweroff ?? true;
     let vmStarted = false;
     if (config.fs["/"] === void 0) {
@@ -393,7 +394,7 @@ export class QemuService implements HocusRuntime {
            * Here is the code:
            * https://github.com/hocus-dev/tiny-initramfs/
            */
-          "/srv/jailer/resources/initrd.img",
+          initrdPath,
           "-qmp",
           `unix:${this.qmpSocketPath},server,nowait`,
           "-machine",

app/config/index.ts

@@ -40,7 +40,8 @@ export const config = makeConfig()({
   agent: () => ({
     temporalAddress: get("AGENT_TEMPORAL_ADDRESS", "localhost:7233"),
     databaseUrl: get("AGENT_DATABASE_URL", "postgres://postgres:pass@localhost:5432/rooms"),
-    defaultKernel: get("AGENT_KERNEL_PATH", "/srv/jailer/resources/vmlinux-6.2-x86_64.bin"),
+    defaultKernel: get("AGENT_KERNEL_PATH", "/kernel/vmlinux-6.2-x86_64.bin"),
+    defaultInitrd: get("AGENT_INITRD_PATH", "/kernel/initrd.img"),
     hostBuildfsResourcesDir: get("AGENT_HOST_BUILDFS_RESOURCES_DIR", "/app/resources"),
     // `get` is not used here because users usually will not want to set these manually
     // in production. `get` would throw an error if the env var was not set.

ops/bin/dev/setup-agent-dependencies.sh

@@ -15,29 +15,5 @@ export REPO_DIR="$(realpath "${SCRIPT_DIR}/../../..")"
 if ! mountpoint -q /sys/kernel/config/; then
     mount -t configfs none /sys/kernel/config
 fi
-# Check if kernel has target_core_user compiled in
-if ! [ -d /sys/kernel/config/target ] ; then
-    echo "Please run a workspace with the target_core_user module compiled in";
-    exit 1
-fi
-# Check if kernel has tcm_loop compiled in
-if ! [ -d /sys/kernel/config/target/loopback/ ] ; then
-    # Oh perhaps the module is there but not started?
-    mkdir /sys/kernel/config/target/loopback/ || true
-    # If the directory is not there then the kernel doesn't have tcm_loop
-    if ! [ -d /sys/kernel/config/target/loopback/ ] ; then
-        echo "Please run a workspace with the tcm_loop module compiled in";
-        exit 1
-    fi
-fi
-# Check if kernel has scsi disk support
-if ! [ -d /sys/bus/scsi/drivers/sd ] ; then
-    echo "No scsi disk support detected";
-    exit 1
-fi
-
-# Just a sanity check
-cat /sys/kernel/config/target/version
-cat /sys/kernel/config/target/loopback/version
 
-/bin/bash "$REPO_DIR/ops/bin/dev/download-kernel.sh" "/home/hocus/dev/hocus-resources/resources"
+"$REPO_DIR"/ops/bin/setup-tcmu.sh

ops/bin/local-up.sh

@@ -6,17 +6,49 @@ SCRIPT_DIR="$(dirname "$0")"
 export REPO_DIR="$(realpath "${SCRIPT_DIR}/../..")"
 export HOCUS_RESOURCES_DIR="$(realpath ${REPO_DIR}/../hocus-resources)"
 
+# Send an optional telemetry event
 PHOG_EVENT_NAME=local-up ENABLE_SENTRY=1 eval "$(cat "$REPO_DIR"/ops/bin/phog-telemetry.hook)"
 
-# Try to make the script as idiotproof as it gets
+# First check the OS
+if [ "$(uname)" = 'Darwin' ]; then
+  echo "Unsupported environment detected. MacOS is not supported due to the lack of KVM support."
+  echo "Hocus currently only works on Linux 🐧."
+  echo "Please deploy Hocus on a Linux 🐧 server and then use it from the web interface using any web browser"
+  echo "For a demo deployment please reach out to the founders in the Hocus Slack"
+  exit 1
+fi
+
+if [ -f "/proc/sys/fs/binfmt_misc/WSLInterop" ]; then
+  echo "Unsupported environment detected. WSL 2.0 is not supported due to https://github.com/microsoft/WSL/issues/9511."
+  echo "Hocus currently only works on Linux 🐧."
+  echo "Please deploy Hocus on a Linux 🐧 server and then use it from the web interface using any web browser"
+  echo "For a demo deployment please reach out to the founders in the Hocus Slack"
+  exit 1
+fi
+
+# Check if the kernel is new enough
+KERNEL_SEMVER=$(uname -r)
+KERNEL_MAJOR=$(echo $KERNEL_SEMVER | cut -d. -f1)
+KERNEL_MINOR=$(echo $KERNEL_SEMVER | cut -d. -f2)
+if [[ KERNEL_MAJOR -lt 5 ]] || { [[ KERNEL_MAJOR -eq 5 ]] && [[ KERNEL_MINOR -lt 10 ]]; }; then
+  echo "[WARNING] Host kernel *might* be too old. If you encounter issues with nested virtualization please first try running Hocus on at least the 5.10 kernel"
+fi
+
+# Now check if KVM is available
+KVM_DIAG=$(${REPO_DIR}/ops/bin/kvm-ok 2>/dev/null)
+if ! [[ $? -eq 0 ]]; then
+  echo "😭 $KVM_DIAG"
+  exit 1
+fi
+
+# Ok Hocus should work on this machine - time to check for software dependencies
 # First check if docker is installed on the system
 docker -v &>/dev/null
 if ! [[ $? -eq 0 ]]; then
   echo "Looks like docker is not installed 😭"
   echo "Try running one of the following commands to install it:"
   echo "Ubuntu/Debian: curl https://get.docker.com/ | sudo bash -"
   echo "Arch/Manjaro: sudo pacman -S docker"
-  echo "Windows/macOS: buy/rent some 🐧🐧🐧"
   exit 1
 fi
 
@@ -37,7 +69,6 @@ if ! [[ $? -eq 0 ]]; then
   echo "Try running one of the following commands to install it:"
   echo "Ubuntu/Debian: sudo apt-get install docker-buildx-plugin"
   echo "Arch/Manjaro: sudo pacman -S docker-buildx"
-  echo "Windows/macOS: buy/rent some 🐧🐧🐧"
   exit 1
 fi
 
@@ -48,43 +79,6 @@ if ! [[ $? -eq 0 ]]; then
   echo "Try running one of the following commands to install it:"
   echo "Ubuntu/Debian: sudo apt-get install docker-compose-plugin"
   echo "Arch/Manjaro: sudo pacman -S docker-compose"
-  echo "Windows/macOS: buy/rent some 🐧🐧🐧"
-  exit 1
-fi
-
-# Check if git lfs is installed on the platform :)
-git lfs &>/dev/null
-if ! [[ $? -eq 0 ]]; then
-  echo "Looks like git lfs is not installed 😭"
-  echo "Try running one of the following commands to install it:"
-  echo "Ubuntu/Debian: sudo apt-get install git-lfs"
-  echo "Arch/Manjaro: sudo pacman -S git-lfs"
-  echo "Windows/macOS: buy/rent some 🐧🐧🐧"
-  exit 1
-fi
-
-# Check if loop devices are available - otherwise building the base vm will fail!
-if ! { [[ -c "/dev/loop-control" ]] || [[ -c "/dev/loop" ]] || [[ -b "/dev/loop0" ]];  } then
-  echo "Looks like loop devices are not available on this machine 😭"
-  echo "Try running one of the following commands to enable them:"
-  echo "Ubuntu/Debian: sudo modprobe loop"
-  echo "Arch/Manjaro: sudo modprobe loop"
-  echo "Windows/macOS: buy/rent some 🐧🐧🐧"
-  exit 1
-fi
-
-# Check if the kernel is new enough
-KERNEL_SEMVER=$(uname -r)
-KERNEL_MAJOR=$(echo $KERNEL_SEMVER | cut -d. -f1)
-KERNEL_MINOR=$(echo $KERNEL_SEMVER | cut -d. -f2)
-if [[ KERNEL_MAJOR -lt 5 ]] || { [[ KERNEL_MAJOR -eq 5 ]] && [[ KERNEL_MINOR -lt 10 ]]; }; then
-  echo "[WARNING] Host kernel *might* be too old. If you encounter issues with nested virtualization please first try running Hocus on at least the 5.10 kernel"
-fi
-
-# Now check if KVM is available
-KVM_DIAG=$(${REPO_DIR}/ops/bin/kvm-ok 2>/dev/null)
-if ! [[ $? -eq 0 ]]; then
-  echo "😭 $KVM_DIAG"
   exit 1
 fi
 
@@ -101,13 +95,6 @@ if [[ ! -v HOCUS_HOSTNAME ]]; then
   exit 1
 fi
 
-if [[ $(file --mime-type -b ${REPO_DIR}/public/user-icon.jpg) == text/plain ]]; then
-  echo "Checking out Git LFS assets"
-  git lfs install
-  git lfs fetch --all
-  git lfs pull
-fi
-
 export HOCUS_DEV_GIT_NAME=$(git config --get user.name)
 export HOCUS_DEV_GIT_EMAIL=$(git config --get user.email)
 
@@ -162,7 +149,6 @@ build_service () {
 
 # Building images
 echo "Building docker images 👷📦"
-build_service download-kernel vm-dependencies-setup
 build_service setup-keycloak db-autosetup
 build_service keycloak keycloak
 build_service temporal-hocus-codec temporal-codec
@@ -184,23 +170,6 @@ else
   echo -e "\r\033[KPulling docker images 📥 - ✅ in $DT s"
 fi
 
-echo -n "Downloading VM dependencies 🚄 "
-T0=$(date +%s%N | cut -b1-13)
-VM_BUILD_LOG=$($REPO_DIR/ops/bin/local-cmd.sh run --rm download-kernel 2>&1)
-if ! [[ $? -eq 0 ]]; then
-  T1=$(date +%s%N | cut -b1-13)
-  DT=$(printf %.2f\\n "$(( $T1 - $T0 ))e-3")
-  echo -e "\r\033[KDownloading VM dependencies 🚄 - ❌ in $DT"
-
-  echo -e "$VM_BUILD_LOG" | grep --color -E '^|ERROR:.*'
-  echo -e "\nAbove you will find the vm build logs with the errors highlighted"
-  fatal_error
-else
-  T1=$(date +%s%N | cut -b1-13)
-  DT=$(printf %.2f\\n "$(( $T1 - $T0 ))e-3")
-  echo -e "\r\033[KDownloading VM dependencies 🚄 - ✅ in $DT s"
-fi
-
 echo -n "Seeding the DB 🌱"
 T0=$(date +%s%N | cut -b1-13)
 SEED_LOG=$($REPO_DIR/ops/bin/local-cmd.sh run --rm setup-keycloak 2>&1)

ops/bin/worker-in-hocus-entrypoint.sh

@@ -12,5 +12,6 @@ export AGENT_DATABASE_URL="postgres://postgres:pass@db:5432/rooms"
 export AGENT_DEV_CREATE_DEVELOPMENT_PROJECTS="true"
 export NODE_ENV="development"
 
+mkdir /kernel/ && ops/bin/download-kernel.sh /kernel/
 yarn run regen
 yarn ts-node -r tsconfig-paths/register entrypoints/agent.ts

ops/buildkite/test.yml

@@ -163,6 +163,8 @@ steps:
 
   - label: ":fire: E2E agent tests"
     command: |
+      mkdir /kernel/ && ops/bin/download-kernel.sh /kernel/
+      ops/bin/setup-tcmu.sh
       export DB_HOST="db";
       export AGENT_DATABASE_URL="postgres://postgres:pass@db:5432/rooms";
       ops/docker/resources/setup-network.sh;

ops/docker/agent-tests.Dockerfile

@@ -4,9 +4,10 @@ FROM quay.io/hocus/hocus-prebuilds-agent-dev:88597e72173bcb830fb69805bcfb9e0ebae
 
 WORKDIR /app
 COPY ops/bin/override-prisma-types.sh ops/bin/override-prisma-types.sh
+COPY ops/bin/download-kernel.sh ops/bin/download-kernel.sh
 COPY package.json yarn.lock ./
 RUN --mount=type=cache,id=hocus-tests-yarn,sharing=locked,target=/usr/local/share/.cache/yarn \
    yarn --frozen-lockfile
 COPY prisma prisma
-RUN yarn run regen
+RUN yarn run regen && mkdir /kernel/ && ops/bin/download-kernel.sh /kernel
 COPY . .

ops/docker/agent-tests.docker-compose.yml

@@ -25,44 +25,13 @@ services:
       timeout: 10s
       retries: 30
 
-  setup-vm-images:
-    privileged: true
-    build:
-      dockerfile: ops/docker/setup.Dockerfile
-      context: ${REPO_DIR}
-    restart: "no"
-    command: |
-      bash -o errexit -o nounset -o pipefail -o xtrace -c "
-        mkdir -pv /agent_data/qemu
-        mkdir -pv /agent_data/resources
-
-        ops/bin/dev/download-kernel.sh /agent_data/resources
-      "
-    volumes:
-      - agent-test-data:/agent_data:rw
-
-  setup-tcmu:
-    build:
-      dockerfile: ops/docker/setup-tcmu.Dockerfile
-      context: ${REPO_DIR}
-    privileged: true
-    restart: "no"
-    command: ops/bin/setup-tcmu.sh
-    volumes:
-      - /lib/modules:/lib/modules:ro
-      - /sys/kernel/config:/sys/kernel/config
-
   agent:
     build:
       dockerfile: ops/docker/agent-tests.Dockerfile
       context: ${REPO_DIR}
     depends_on:
       db:
         condition: service_healthy
-      setup-vm-images:
-        condition: service_completed_successfully
-      setup-tcmu:
-        condition: service_completed_successfully
     restart: "no"
     privileged: true
     ports:
@@ -73,5 +42,6 @@ services:
       - "agent-test-data:/srv/jailer"
       - /dev/kvm:/dev/kvm
       - /dev:/dev/hocus
+      - /lib/modules:/lib/modules:ro
       - /sys/kernel/config:/sys/kernel/config
     command: sleep infinity

ops/docker/deploy-smoke-test.yml

@@ -9,8 +9,6 @@ services:
         condition: service_healthy
       temporal:
         condition: service_healthy
-      download-kernel:
-        condition: service_completed_successfully
       hocus-ui:
         condition: service_healthy
       hocus-agent: