Skip to content

Commit

Permalink
Unshare the environment for dockerd
Browse files Browse the repository at this point in the history
Makes the cleanup much less messy, allows us to use localhost:80 as
local web server allowing multiple builds to run in parallel.
  • Loading branch information
mlschroe committed Jul 17, 2017
1 parent b635c6d commit 8a06286
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 86 deletions.
1 change: 1 addition & 0 deletions Makefile
Expand Up @@ -71,6 +71,7 @@ install:
telnet_login_wrapper \
build-validate-params \
openstack-console \
startdockerd \
$(DESTDIR)$(pkglibdir)
install -m755 emulator/emulator.sh $(DESTDIR)$(pkglibdir)/emulator/
install -m644 Build/*.pm $(DESTDIR)$(pkglibdir)/Build
Expand Down
94 changes: 8 additions & 86 deletions build-recipe-docker
Expand Up @@ -43,38 +43,6 @@ recipe_prepare_docker() {
:
}

docker_umount_setup() {
if test -z "$BUILD_ROOT" -o "$BUILD_ROOT" = / ; then
return
fi
mount --make-rslave "$BUILD_ROOT/sys/fs/cgroup" 2>/dev/null
umount -n -R $BUILD_ROOT/sys/fs/cgroup 2>/dev/null
# get rid of the bind mount
umount -n $BUILD_ROOT/dev/shm 2>/dev/null
umount -n -l $BUILD_ROOT
}

docker_mount_setup() {
if test -z "$BUILD_ROOT" -o "$BUILD_ROOT" = / ; then
return
fi
test -d /sys/fs/cgroup || cleanup_and_exit 1 "/sys/fs/cgroup does not exist"

# make build root a mount point
mount --rbind --make-private "$BUILD_ROOT" "$BUILD_ROOT"
mount --make-rprivate "$BUILD_ROOT"

if ! test -e $BUILD_ROOT/sys/block; then
mkdir -p $BUILD_ROOT/sys
mount -n -tsysfs sys $BUILD_ROOT/sys
fi
mount --rbind /sys/fs/cgroup "$BUILD_ROOT/sys/fs/cgroup"
mount --make-rslave "$BUILD_ROOT/sys/fs/cgroup"

# work around docker pivot root failure (to be investigated)
export DOCKER_RAMDISK=true
}

# Variables:
# $BUILD_ROOT is the chroot
# $TOPDIR/SOURCES includes the docker sources
Expand All @@ -88,26 +56,8 @@ recipe_build_docker() {
base_image_path=$(find containers -regextype egrep -regex ".*\.(tgz|tar|tar\.xz|tar\.gz)$" -print -quit)
test -f "$base_image_path" || cleanup_and_exit 1 "base image not found"

# load needed kernel modules
modprobe bridge br_netfilter
modprobe nf_nat
modprobe xt_conntrack
modprobe ip_tables

docker_mount_setup

echo "Starting container daemon"
chroot $BUILD_ROOT /usr/sbin/containerd --listen unix:///run/containerd/containerd.sock &
echo "Starting docker daemon"
chroot $BUILD_ROOT /usr/bin/dockerd --containerd /run/containerd/containerd.sock --add-runtime oci=/usr/bin/docker-runc &
echo "Waiting for docker daemon to start"
for i in 1 2 3 4 5 6 7 8 9 10 ; do
chroot $BUILD_ROOT docker version >/dev/null 2>&1 && break
sleep 1
done
if ! chroot $BUILD_ROOT docker version >/dev/null 2>&1 ; then
docker_umount_setup
cleanup_and_exit 1 "Docker is dead"
if ! $BUILD_DIR/startdockerd --root "$BUILD_ROOT" --webserver "$TOPDIR/SOURCES/packages" ; then
cleanup_and_exit 1
fi

echo "Loading base image"
Expand Down Expand Up @@ -136,28 +86,9 @@ recipe_build_docker() {
# TODO [TBD] What about SLES? Are dependencies fetched automatically from osc/OBS?
# What about registration?

# This is where we copied the downloaded dependencies in the setup method above.
REPOPATH=$TOPDIR/SOURCES/packages
# Prepare the zypper repository
# TODO: Handle other distribution repositories as well (deb, arch etc)
chroot $BUILD_ROOT createrepo $REPOPATH >/dev/null
# TODO Add Python as a default building dependency.
chroot $BUILD_ROOT bash -c "cd $REPOPATH && python -m SimpleHTTPServer 8080 &"

# Build the repository url

if test -x /sbin/ip ; then
REPO_URL=$(ip addr show docker0 | sed -n -e 's@.*inet \([^ /]*\).*@\1@p')
else
REPO_URL=$(ifconfig docker0 | sed -n -e 's@.*inet addr:\([^ /]*\).*@\1@p')
fi
if test -z "$REPO_URL" ; then
echo "network setup error, no docker0 interface?"
cleanup_build_processes
BUILD_SUCCEEDED=false
return
fi
REPO_URL="http://$REPO_URL:8080"
chroot $BUILD_ROOT createrepo "$TOPDIR/SOURCES/packages" >/dev/null

# Create the needed file to publish the generated image
IMAGE_NAME=
Expand All @@ -169,7 +100,7 @@ recipe_build_docker() {

if [ -z "$IMAGE_NAME" ] || [ -z "$TAG" ]; then
echo "TAG file missing of contents invalid (use image_name:tag format)"
cleanup_build_processes
$BUILD_DIR/startdockerd --root "$BUILD_ROOT" --kill
BUILD_SUCCEEDED=false
return
fi
Expand All @@ -179,9 +110,9 @@ recipe_build_docker() {
FILENAME="${FILENAME//[\/:]/-}"

echo "Building image"
if ! chroot $BUILD_ROOT docker build -t "$IMAGE_NAME:$TAG" --build-arg OBS_REPOSITORY_URL=$REPO_URL $TOPDIR/SOURCES/ ; then
if ! chroot $BUILD_ROOT docker build --network=host -t "$IMAGE_NAME:$TAG" --build-arg OBS_REPOSITORY_URL=http://localhost:80 $TOPDIR/SOURCES/ ; then
echo "Docker build command failed"
cleanup_build_processes
$BUILD_DIR/startdockerd --root "$BUILD_ROOT" --kill
BUILD_SUCCEEDED=false
return
fi
Expand All @@ -190,27 +121,18 @@ recipe_build_docker() {
mkdir -p $BUILD_ROOT$TOPDIR/DOCKER
if ! chroot $BUILD_ROOT docker save --output "$TOPDIR/DOCKER/$FILENAME.tar" "$IMAGE_NAME:$TAG" ; then
echo "Docker save command failed"
cleanup_build_processes
$BUILD_DIR/startdockerd --root "$BUILD_ROOT" --kill
BUILD_SUCCEEDED=false
return
fi

# Create containerinfo
perl -I$BUILD_DIR -MBuild::Docker -e Build::Docker::showcontainerinfo "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE" "$FILENAME.tar" "$IMAGE_NAME:$TAG" containers/annotation> "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.containerinfo"

cleanup_build_processes
$BUILD_DIR/startdockerd --root "$BUILD_ROOT" --kill
BUILD_SUCCEEDED=true
}

cleanup_build_processes() {
echo "Stopping local repository server"
kill $(pgrep -f "python -m SimpleHTTPServer 8080")
echo "Stopping the docker daemon"
kill $(pidof dockerd)
kill $(pidof containerd)
docker_umount_setup
}

recipe_resultdirs_docker() {
echo DOCKER
}
Expand Down
149 changes: 149 additions & 0 deletions startdockerd
@@ -0,0 +1,149 @@
#!/bin/bash

export BUILD_DIR=${BUILD_DIR:-/usr/lib/build}

cleanup_and_exit() {
test -z "$1" && set 0
if test -n "$2" ; then
if test "$1" -ne 0 ; then
echo "$2" >&2
else
echo "$2"
fi
fi
exit $1
}


BUILD_ROOT=
IS_UNSHARED=
KILL=
WEBSERVER=

CONTAINERD_PID=0
DOCKERD_PID=0
WEBSERVER_PID=0

while test -n "$1" ; do
case "$1" in
--root)
BUILD_ROOT="$2"
shift 2
;;
--webserver)
WEBSERVER="$2"
shift 2
;;
--isunshared)
IS_UNSHARED=true
shift
;;
--kill)
KILL=true
shift
;;
*)
break
;;
esac
done

if test -n "$1" -o -z "$BUILD_ROOT" ; then
cleanup_and_exit 1 "Usage: startdockerd --root <buildroot>"
echo "Usage: startdockerd --root <buildroot>"
fi

if test -n "$KILL" ; then
if test -e "$BUILD_ROOT/.startdockerd.pids" ; then
read CONTAINERD_PID DOCKERD_PID WEBSERVER_PID < $BUILD_ROOT/.startdockerd.pids
if test -n "$WEBSERVER_PID" -a "$WEBSERVER_PID" != 0 ; then
echo "Stopping local repository server"
kill "$WEBSERVER_PID"
fi
if test -n "$DOCKERD_PID" -a "$DOCKERD_PID" != 0 ; then
echo "Stopping docker daemon"
kill "$DOCKERD_PID"
fi
if test -n "$CONTAINERD_PID" -a "$CONTAINERD_PID" != 0 ; then
echo "Stopping container daemon"
kill "$CONTAINERD_PID"
fi
rm -f "$BUILD_ROOT/.startdockerd.pids"
fi
exit 0
fi

rm -f $BUILD_ROOT/.startdockerd.pids

if test -z "$IS_UNSHARED" ; then
echo "Unsharing environment"
# unshare mounts and network
exec unshare -m -n $BUILD_DIR/startdockerd --isunshared --root "$BUILD_ROOT" --webserver "$WEBSERVER" "$@"
cleanup_and_exit 1 "exec unshare returned"
fi

# load needed kernel modules
modprobe bridge br_netfilter
modprobe nf_nat
modprobe xt_conntrack
modprobe ip_tables

# setup cgroups
if test "$BUILD_ROOT" != '/' ; then
test -d /sys/fs/cgroup || cleanup_and_exit 1 "/sys/fs/cgroup does not exist"

test -n "$IS_UNSHARED" && mount --make-rprivate /

# make build root a mount point
mount --rbind --make-private "$BUILD_ROOT" "$BUILD_ROOT"
mount --make-rprivate "$BUILD_ROOT"
if ! test -e $BUILD_ROOT/sys/block; then
mkdir -p $BUILD_ROOT/sys
mount -n -tsysfs sys $BUILD_ROOT/sys
fi
mount --rbind /sys/fs/cgroup "$BUILD_ROOT/sys/fs/cgroup"
mount --make-rslave "$BUILD_ROOT/sys/fs/cgroup"
export DOCKER_RAMDISK=true
fi

# create loopback interface
if test -x /sbin/ip ; then
ip addr add 127.0.0.1/8 dev lo
ip addr add ::1/128 dev lo
ip link set lo up
else
ifconfig lo 127.0.0.1 up
ifconfig lo add ::1/128
fi

# setup mounts
test -e "$BUILD_ROOT/proc/self" || mount -n -tproc none $BUILD_ROOT/proc

echo "Starting container daemon"
chroot $BUILD_ROOT /usr/sbin/containerd --listen unix:///run/containerd/containerd.sock &
CONTAINERD_PID=$!
echo "$CONTAINERD_PID $DOCKERD_PID $WEBSERVER_PID" > $BUILD_ROOT/.startdockerd.pids

echo "Starting docker daemon"
chroot $BUILD_ROOT /usr/bin/dockerd --containerd /run/containerd/containerd.sock --bridge=none --add-runtime oci=/usr/bin/docker-runc &
DOCKERD_PID=$!
echo "$CONTAINERD_PID $DOCKERD_PID $WEBSERVER_PID" > $BUILD_ROOT/.startdockerd.pids

echo "Waiting for docker daemon to complete startup"
for i in 1 2 3 4 5 6 7 8 9 10 ; do
chroot $BUILD_ROOT docker version >/dev/null 2>&1 && break
sleep 1
done
if ! chroot $BUILD_ROOT docker version >/dev/null 2>&1 ; then
cleanup_and_exit 1 "Docker is dead"
fi

if test -n "$WEBSERVER" ; then
echo "Starting local repository server"
chroot $BUILD_ROOT bash -c "cd $WEBSERVER && python -m SimpleHTTPServer 80 &"
WEBSERVER_PID=$!
echo "$CONTAINERD_PID $DOCKERD_PID $WEBSERVER_PID" > $BUILD_ROOT/.startdockerd.pids
fi

echo "Docker is running"
exit 0

0 comments on commit 8a06286

Please sign in to comment.