Add compose tests

Signed-off-by: Kyle Harding <kyle@balena.io>
balena-io-experimental · Oct 12, 2023 · 9ac9614 · 9ac9614
1 parent 5e3aa62
commit 9ac9614
Show file tree

Hide file tree

Showing 10 changed files with 344 additions and 250 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -7,11 +7,20 @@ ARG DEBIAN_FRONTEND=noninteractive
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+    awscli \
     ca-certificates \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$(uname -m)/kernels/vmlinux.bin" -O
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Print the available kernels in S3 in case the filenames change
+RUN aws s3 ls --no-sign-request "s3://spec.ccfc.min/firecracker-ci/v1.6/$(uname -m)/"
+
+# RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$(uname -m)/kernels/vmlinux.bin" -o vmlinux.bin
+# RUN curl -fsSL "http://mirror.archlinuxarm.org/aarch64/core/linux-aarch64-6.2.10-1-aarch64.pkg.tar.xz" -o vmlinux.bin
+# RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/hello/kernel/hello-vmlinux.bin" -o vmlinux.bin
+RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.6/$(uname -m)/vmlinux-5.10.197" -o vmlinux.bin
 
 ###############################################
 
@@ -51,6 +60,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+    bridge-utils \
     ca-certificates \
     curl \
     e2fsprogs \
@@ -62,6 +72,7 @@ RUN apt-get update \
     jq \
     procps \
     rsync \
+    tcpdump \
     uuid-runtime \
     && rm -rf /var/lib/apt/lists/*
 
@@ -82,100 +93,60 @@ RUN chmod +x start.sh overlay/sbin/* overlay/usr/local/bin/*
 
 ENTRYPOINT [ "/usr/src/app/start.sh" ]
 
-# Default command to exec after init.
-# This should be a long-running process or service, and get overriden by the user.
-# hadolint ignore=DL3025
-CMD 'curl http://artscene.textfiles.com/asciiart/unicorn && sleep infinity'
+CMD [ "/usr/local/bin/usage.sh" ]
 
 ###############################################
 
+# Example alpine rootfs for testing, with some debug utilities
 FROM alpine:3.18 AS alpine-rootfs
 
-# WORKDIR /src
-
-# # hadolint ignore=DL3018
-# RUN apk add --no-cache openrc util-linux
+# hadolint ignore=DL3018
+RUN apk add --no-cache bash ca-certificates ca-certificates curl iproute2 iputils-ping lsblk
 
-# # Set up a login terminal on the serial console (ttyS0)
-# RUN ln -s agetty /etc/init.d/agetty.ttyS0 \
-#     && echo ttyS0 > /etc/securetty \
-#     && rc-update add agetty.ttyS0 default
+FROM jailer AS alpine-test
 
-# # Make sure special file systems are mounted on boot
-# RUN rc-update add devfs boot \
-#     && rc-update add procfs boot \
-#     && rc-update add sysfs boot
+COPY --from=alpine-rootfs / /usr/src/app/rootfs/
 
-# # Create a tarball of the root file system
-# RUN tar cf /rootfs.tar /bin /etc /lib /root /sbin /usr
+CMD "/usr/local/bin/healthcheck.sh && sleep infinity"
 
-# hadolint ignore=DL3018
-RUN apk add --no-cache curl iproute2
+# Use livepush directives to conditionally run this test stage
+# for livepush, but not for default builds used in publishing.
+#dev-cmd-live="/usr/local/bin/healthcheck.sh && sleep infinity"
 
 ###############################################
 
-# Use the official Ubuntu image as a base
-FROM ubuntu:jammy AS ubuntu-rootfs
-
-# # Set environment variables to avoid prompts
-# ENV DEBIAN_FRONTEND=noninteractive
-
-# # Install the necessary packages
-# # hadolint ignore=DL3008
-# RUN apt-get update \
-#     && apt-get install -y --no-install-recommends curl systemd systemd-sysv \
-#     && rm -rf /var/lib/apt/lists/*
-
-# # Remove unnecessary services
-# RUN find /etc/systemd/system \
-#         /lib/systemd/system \
-#         \( \
-#         -name "*udev*" \
-#         -o -name "*resolved*" \
-#         -o -name "*logind*" \
-#         -o -name "*getty*" \
-#         -o -name "*networkd*" \
-#         \) \
-#         -exec rm -f {} \;
-
-# # Set systemd as the entrypoint
-# STOPSIGNAL SIGRTMIN+3
-# CMD [ "/sbin/init" ]
-
-# # Set up necessary mount points
-# VOLUME [ "/sys/fs/cgroup" ]
-
-# # Copy the updated systemd service file
-# COPY entrypoint.service /etc/systemd/system/entrypoint.service
-# RUN systemctl enable entrypoint.service
-
-# COPY init /init
-# RUN chmod +x /init
+# Example debian rootfs for testing, with some debug utilities
+FROM debian:bookworm AS debian-rootfs
 
 # hadolint ignore=DL3008
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl iproute2 \
+    && apt-get install -y --no-install-recommends curl iproute2 iputils-ping ca-certificates util-linux \
     && rm -rf /var/lib/apt/lists/*
 
+FROM jailer AS debian-test
+
+COPY --from=debian-rootfs / /usr/src/app/rootfs/
+
+CMD "/usr/local/bin/healthcheck.sh && sleep infinity"
+
 ###############################################
 
-FROM ghcr.io/product-os/self-hosted-runners:v3.3.3 AS self-hosted-runners
+# Example ubuntu rootfs for testing, with some debug utilities
+FROM ubuntu:jammy AS ubuntu-rootfs
 
 # hadolint ignore=DL3008
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl iproute2 \
+    && apt-get install -y --no-install-recommends ca-certificates curl iproute2 iputils-ping util-linux \
     && rm -rf /var/lib/apt/lists/*
 
-CMD [ "/init" ]
+FROM jailer AS ubuntu-test
 
-###############################################
+COPY --from=ubuntu-rootfs / /usr/src/app/rootfs/
 
-# Include firecracker wrapper and scripts
-FROM jailer AS runtime
+CMD "/usr/local/bin/healthcheck.sh && sleep infinity"
 
-# Copy the root file system from your container final stage
-COPY --from=alpine-rootfs / /usr/src/app/rootfs/
-# COPY --from=ubuntu-rootfs / /usr/src/app/rootfs/
-# COPY --from=self-hosted-runners / /usr/src/app/rootfs/
+###############################################
 
-CMD 'curl http://artscene.textfiles.com/asciiart/unicorn ; echo $SECRET_KEY ; sleep infinity'
+# This is the stage we want to publish, but it has no rootfs
+# so we can't use it for livepush testing.
+FROM jailer AS default
diff --git a/README.md b/README.md
@@ -96,35 +96,41 @@ Reference: <https://github.com/firecracker-microvm/firecracker/blob/main/docs/ge
 Since traditional container environment variables are not available in the VM, this wrapper will
 inject them into the VM rootfs and export them at runtime.
 
-Provide environment variables or secrets with the `CTR_` prefix, like `CTR_SECRETKEY=secretvalue`.
+Provide environment variables or secrets with the `CTR_` prefix, like `CTR_SECRET_KEY=secretvalue`.
 
 If the values have spaces, or special characters, it is recommended to encode your secret values
 with `base64` and have your init service decode them.
 
 After being exported to the running process, the files are removed so they can safely
-be used for secrets if the init stage of your service runs `unset` after using them.
+be used for secrets as long as the init stage of your service calls `unset <SECRET_KEY>` after using them.
 
 ### Networking
 
 A TAP/TUN device will be automatically created for the guest to have network access.
 
+The IP address/netmask can be configured via `TAP_IP`, otherwise a random address in the 10.x.x.1/30 range will be assigned.
+
+The host interface for routing can be configured via `INTERFACE` otherwise the default route interface will be used.
+
+In order to create the TAP device, and update iptables rules, the container jailer must be run in host networking mode.
+
 Reference: <https://github.com/firecracker-microvm/firecracker/blob/main/docs/network-setup.md>
 
-Because of this, the container jailer wrapper must be run in host networking mode.
+Exposing ports is TBD.
 
 ### Resources
 
-Resources like virtual CPUs and Memory can be overprovisioned and adjusted
-via the env vars `VCPU_COUNT` and `MEM_SIZE_MIB`.
+Resources like virtual CPUs and Memory can be overprovisioned and adjusted via the env vars `VCPU_COUNT` and `MEM_SIZE_MIB`.
+
 The default is the maximum available on the host.
 
 ### Persistent Storage
 
-The rootfs is recreated on every run, so anything written to the rootfs will not persist and
-is considered ephemeral like container layers.
+The root filesystem is recreated on every run, so anything written to the root partition will not persist restarts and
+is considered ephemeral similar to container layers.
 
-However an additional data drive will be created for optional use
-and it can be made persistent by mounting a container volume to `/jail/data`.
+However an optional data drive `/dev/vdb` will be created and can be made persistent by mounting a volume
+or host path to `/jail/data`.
 
 ## Contributing
 

diff --git a/config.json b/config.json
@@ -39,5 +39,11 @@
   "logger": null,
   "metrics": null,
   "mmds-config": null,
-  "entropy": null
+  "rate_limiter": {
+    "bandwidth": {
+      "size": 1000,
+      "one_time_burst": 0,
+      "refill_time": 100
+    }
+  }
 }
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -2,7 +2,69 @@ version: "2.4"
 
 services:
   sut:
+    image: docker:stable
+    environment:
+      DOCKER_HOST: unix:///var/run/docker.sock
+      COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-ctr-jailer}
+      COMPOSE_FILE: docker-compose.yml:docker-compose.test.yml
+    depends_on:
+      - alpine-test
+      - debian-test
+      - ubuntu-test
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./:/usr/src/app:ro
+    working_dir: /usr/src/app
+    command:
+      - /bin/sh
+      - -c
+      - |
+        set -e
+        apk add --no-cache docker-compose
+
+        count=0
+        while [ "$$(docker-compose logs | grep "Healthchecks passed!" | wc -l)" -lt 3 ]; do
+          if [ $$count -gt 10 ]; then
+            echo "Timed out waiting for 3 passed healthchecks"
+            exit 1
+          fi
+          sleep 5
+        done
+
+  alpine-test:
+    extends:
+      file: docker-compose.yml
+      service: firecracker
+    build:
+      context: .
+      target: alpine-test
+    volumes:
+      - alpine-data:/jail/data
+    command: /usr/local/bin/healthcheck.sh
+
+  debian-test:
     extends:
       file: docker-compose.yml
       service: firecracker
-    image: localhost:5000/sut
+    build:
+      context: .
+      target: debian-test
+    volumes:
+      - debian-data:/jail/data
+    command: /usr/local/bin/healthcheck.sh
+
+  ubuntu-test:
+    extends:
+      file: docker-compose.yml
+      service: firecracker
+    build:
+      context: .
+      target: ubuntu-test
+    volumes:
+      - ubuntu-data:/jail/data
+    command: /usr/local/bin/healthcheck.sh
+
+volumes:
+  alpine-data: {}
+  debian-data: {}
+  ubuntu-data: {}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,14 +7,10 @@ services:
     # but permissions are dropped to a chroot in order to start your VM
     privileged: true
     network_mode: host
-    # Optionally run the VM rootfs and kernel in-memory to save storage wear
+    # Optionally run the VM jail in-memory to save storage wear
     tmpfs:
       - /tmp
       - /run
       - /srv
-    volumes:
-      - persistent-data:/jail/data
-
-volumes:
-  persistent-data: {}
-
+    environment:
+      - CTR_SECRET_KEY=secretvalue
diff --git a/overlay/sbin/init b/overlay/sbin/init
@@ -4,20 +4,52 @@ exec 1>/dev/console
 exec 2>/dev/console
 
 # Mount essential file systems
-mount -t proc none /proc
-mount -t sysfs none /sys
-mount -t devtmpfs none /dev
+if ! mountpoint -q /proc; then
+  mount -t proc none /proc
+fi
 
-# Mount tmpfs
-mount -o remount,rw,exec tmpfs /tmp
-mount -o remount,rw,exec tmpfs /var/run
+if ! mountpoint -q /sys; then
+  mount -t sysfs none /sys
+fi
 
-# Bring up networking
+if ! mountpoint -q /dev; then
+  mount -t devtmpfs none /dev
+fi
+
+if ! mountpoint -q /tmp; then
+  mount -t tmpfs none /tmp
+fi
+
+if ! mountpoint -q /run; then
+  mount -t tmpfs none /run
+fi
+
+# rngd -b
+
+# The IP is assigned by converting the last 4 hexa groups of the MAC into decimals.
 # https://github.com/firecracker-microvm/firecracker/blob/main/resources/overlay/usr/local/bin/fcnet-setup.sh
-/usr/local/bin/fcnet-setup.sh
+for dev in $(ip link list | awk /'^[0-9]+:/ {print $2}' | sed 's/://'); do
+    dev="$(basename "$dev")"
+    case $dev in
+    *lo) continue ;;
+    esac
+    for octet in $(
+        ip link show dev "$dev" |
+            awk '/link\/ether/ {print $2}' |
+            awk -F: '{print $3" "$4" "$5" "$6}'
+    ); do
+        ip=$ip$(printf "%d" 0x"$octet").
+    done
+    ip=${ip%?}
+    ip addr add "$ip/30" dev "$dev"
+    ip link set "$dev" up
+    ip route add default via "${ip%?}1" dev "$dev"
+done
 
 # Export secrets to the environment and remove the files
-for f in /var/secrets/*; do
-    eval "export $(basename "${f}")=$(cat "${f}")"
-    rm -f "${f}"
-done
+if [ "$(ls /var/secrets)" ]; then
+    for f in /var/secrets/*; do
+        eval "export $(basename "${f}")=$(cat "${f}")"
+        rm -f "${f}"
+    done
+fi