build-probe-binaries

#!/bin/bash
#
# This script builds a precompiled version of sysdig-probe for a bunch of kernels
# The precompiled binary is then obtained at runtime by sysdig-probe-loader
# Ideally, the community should expand this stuff with better support
#
set -euo pipefail

#
# For continuous integration log purposes, wget prints its own output to stderr
# so it can be convenient to redirect it to a file. This can be done directly
# at runtime.
#

PROBE_NAME=$1
PROBE_VERSION=$2
REPOSITORY_NAME=$3
KERNEL_TYPE=
BASEDIR=$(pwd)
ARCH=$(uname -m)
URL_TIMEOUT=300
RETRY=10

if [ $# -eq 4 ]; then
	KERNEL_TYPE=$4
fi

if [ ! -d $BASEDIR/output ]; then
	mkdir $BASEDIR/output
fi

if [ $PROBE_NAME = "sysdigcloud-probe" ]; then
        PROBE_REPO_NAME="agent"
else
        PROBE_REPO_NAME=$(echo $PROBE_NAME | cut -f1 -d-)
fi

function update_code_for {
        repo=$1
	if [ ! -d $repo ]; then
                git clone git@github.com:draios/$repo.git
	fi

	cd $repo
	git checkout master
	# The UEK builder container doesn't have git credentials
	# It relies on the non-UEK builds doing the pull earlier
	if [[ ! "$KERNEL_TYPE" =~ "UEK" ]]; then
	        git pull
	fi

	if [ $PROBE_REPO_NAME = $repo ]; then
	        git checkout $PROBE_VERSION
	else
	        git checkout $PROBE_REPO_NAME/$PROBE_VERSION
	fi

	# Remove everything other than the files actually belonging to
	# the repo.
	git clean -d -f -x

	# Reset the state of the files belonging to the repo to the
	# state associated with the tag.
	git reset --hard

	cd ..
}

function build_probe {

	if [ ! -f $BASEDIR/output/$PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH.ko ] || [ ! -f $BASEDIR/output/$PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH_ORIG.ko ]; then

		echo Building $PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH.ko [${FUNCNAME[1]}]

		update_code_for sysdig

		if [ $PROBE_NAME != "sysdig-probe" ]; then
		        update_code_for falco
		fi

		if [ $PROBE_NAME = "sysdigcloud-probe" ]; then
		        update_code_for agent
		fi

		cd $PROBE_REPO_NAME
		mkdir build
		cd build
		version_name=-D$(echo $PROBE_REPO_NAME | tr [a-z] [A-Z])_VERSION

		cmake -DCMAKE_BUILD_TYPE=Release $version_name=$PROBE_VERSION ..
		make driver
		strip -g driver/$PROBE_NAME.ko

		KO_VERSION=$(/sbin/modinfo driver/$PROBE_NAME.ko | grep vermagic | tr -s " " | cut -d " " -f 2)
		if [ "$KO_VERSION" != "$KERNEL_RELEASE" ]; then
			echo "Corrupted probe, KO_VERSION " $KO_VERSION ", KERNEL_RELEASE " $KERNEL_RELEASE
			exit 1
		fi

		cp driver/$PROBE_NAME.ko $BASEDIR/output/$PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH.ko
		cp driver/$PROBE_NAME.ko $BASEDIR/output/$PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH_ORIG.ko
	else
		echo Skipping $PROBE_NAME-$PROBE_VERSION-$ARCH-$KERNEL_RELEASE-$HASH.ko \(already built\)
	fi

	cd $BASEDIR
}

function coreos_build_old {
	VERSION_URL=$1
	VERSION_NUMBER=$2
	COREOS_DIR="coreos-"$VERSION_NUMBER

	if [ ! -d $COREOS_DIR ]; then
		mkdir $COREOS_DIR
	fi

	cd $COREOS_DIR

	if [ ! -f config_orig ]; then
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${VERSION_URL}coreos_developer_container.bin.bz2
		bunzip2 coreos_developer_container.bin.bz2
		sudo kpartx -asv coreos_developer_container.bin
		LOOPDEV=$(sudo kpartx -asv coreos_developer_container.bin | cut -d\  -f 3)
		sudo mkdir /tmp/loop || true
		sudo mount /dev/mapper/$LOOPDEV /tmp/loop
		cp /tmp/loop/usr/boot/config-* .
		sudo umount /tmp/loop
		sudo kpartx -dv coreos_developer_container.bin
		rm -rf coreos_developer_container.bin
		cp config-* config_orig
	fi

	KERNEL_RELEASE=$(ls config-* | sed s/config-//)
	VANILLA=$(echo $KERNEL_RELEASE | sed s/[-+].*// | sed s/\.0$//)
	MAJOR=$(echo $KERNEL_RELEASE | head -c1)
	EXTRAVERSION=$(echo $KERNEL_RELEASE | sed s/[^-+]*//)
	TGZ_NAME=linux-${VANILLA}.tar.xz
	DIR_NAME=linux-${VANILLA}
	KERNEL_URL=https://www.kernel.org/pub/linux/kernel/v${MAJOR}.x/$TGZ_NAME

	if [ ! -f $TGZ_NAME ]; then
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $KERNEL_URL
	fi

	if [ ! -d $DIR_NAME ]; then
		tar xf $TGZ_NAME
		cd $DIR_NAME
		make distclean
		sed -i "s/^EXTRAVERSION.*/EXTRAVERSION = $EXTRAVERSION/" Makefile
		cp ../config_orig .config
		make modules_prepare
		mv .config ../config
		cd ..
	fi

	HASH=$(md5sum config | cut -d' ' -f1)
	HASH_ORIG=$(md5sum config_orig | cut -d' ' -f1)

	cd $BASEDIR

	export KERNELDIR=$BASEDIR/$COREOS_DIR/$DIR_NAME
	build_probe
}

function coreos_build_new {
	VERSION_URL=$1
	VERSION_NUMBER=$2
	COREOS_DIR="coreos-"$VERSION_NUMBER

	if [ ! -d $COREOS_DIR ]; then
		mkdir $COREOS_DIR
	fi

	cd $COREOS_DIR

	if [ ! -f coreos_developer_container.bin ]; then
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${VERSION_URL}coreos_developer_container.bin.bz2
		bunzip2 coreos_developer_container.bin.bz2
	fi
	sudo kpartx -asv coreos_developer_container.bin
	LOOPDEV=$(sudo kpartx -asv coreos_developer_container.bin | cut -d\  -f 3)
	sudo mkdir /tmp/loop || true
	sudo mount /dev/mapper/$LOOPDEV /tmp/loop

	cp /tmp/loop/usr/boot/config-* .
	cp config-* config_orig
	cp config_orig config
	# https://groups.google.com/forum/#!topic/coreos-dev/Z8Q7sIy6YwE
	sed -i 's/CONFIG_INITRAMFS_SOURCE=""/CONFIG_INITRAMFS_SOURCE="bootengine.cpio"\nCONFIG_INITRAMFS_ROOT_UID=0\nCONFIG_INITRAMFS_ROOT_GID=0/' config
	KERNEL_RELEASE=$(ls config-* | sed s/config-//)
	HASH_ORIG=$(md5sum config_orig | cut -d' ' -f1)
	HASH=$(md5sum config | cut -d' ' -f1)

	cd $BASEDIR
	export KERNELDIR=$(readlink -f /tmp/loop/lib/modules/$KERNEL_RELEASE/build)

	# If build fails the script should exit immediately
	# but in this case it's important to umount the loop device
	# and release resources. So keep running and fail afterwards.
	set +e
	build_probe
	BUILD_RET=$?
	set -e

	cd $COREOS_DIR
	# cleanup
	sudo umount /tmp/loop
	sudo kpartx -dv coreos_developer_container.bin

	cd $BASEDIR

	return $BUILD_RET
}

function boot2docker_build {
	CONFIGURATION_NAME=$1
	KERNEL_RELEASE=$2
	KERNEL_URL=$3
	KERNEL_CONFIG=$4
	AUFS_REPO=$5
	AUFS_BRANCH=$6
	AUFS_COMMIT=$7
	TGZ_NAME=$(echo $KERNEL_URL | awk -F"/" '{print $NF }')
	DIR_NAME=$(echo $TGZ_NAME | sed 's/.tar.xz//')

	if [ ! -d $CONFIGURATION_NAME ]; then
		mkdir $CONFIGURATION_NAME
	fi

	cd $CONFIGURATION_NAME

	if [ ! -f $TGZ_NAME ]; then
		echo Downloading $TGZ_NAME [Boot2Docker]
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $KERNEL_URL
	fi

	if [ ! -d $DIR_NAME ]; then
		tar xf $TGZ_NAME
		cd $DIR_NAME
		make distclean
		git clone -b "$AUFS_BRANCH" "$AUFS_REPO" aufs-standalone
		cd aufs-standalone 
		git checkout -q "$AUFS_COMMIT"
		cd ..
		cp -r aufs-standalone/Documentation . 
		cp -r aufs-standalone/fs .
		cp -r aufs-standalone/include/uapi/linux/aufs_type.h include/uapi/linux/
		set -e && for patch in \
			aufs-standalone/aufs*-kbuild.patch \
			aufs-standalone/aufs*-base.patch \
			aufs-standalone/aufs*-mmap.patch \
			aufs-standalone/aufs*-standalone.patch \
			aufs-standalone/aufs*-loopback.patch \
		; do \
	        patch -p1 < "$patch"; \
		done
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} -O .config $KERNEL_CONFIG
		cp .config ../config-orig
		make olddefconfig
		make modules_prepare
		mv .config ../config
		cd ..
	fi

	HASH=$(md5sum config | cut -d' ' -f1)
	HASH_ORIG=$(md5sum config-orig | cut -d' ' -f1)

	cd $BASEDIR

	export KERNELDIR=$BASEDIR/$CONFIGURATION_NAME/$DIR_NAME
	build_probe
}

function ubuntu_build {

	URL=$1
	DEB=$(echo $URL | grep -o '[^/]*$')
	KERNEL_RELEASE_FULL=$(echo $DEB | grep -E -o "[0-9]{1}\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+")		# ex. 3.13.0-24.47
	KERNEL_RELEASE=$(echo $KERNEL_RELEASE_FULL | grep -E -o "[0-9]{1}\.[0-9]+\.[0-9]+-[0-9]+")	# ex. 3.13.0-24
	KERNEL_UPDATE=$(echo $KERNEL_RELEASE_FULL | grep -E -o "[0-9]+$")							# ex. 47

	if [ ! -d $KERNEL_RELEASE ]; then
		mkdir $KERNEL_RELEASE
	fi

	cd $KERNEL_RELEASE

	if [ ! -d $KERNEL_UPDATE ]; then
		mkdir $KERNEL_UPDATE
	fi

	cd $KERNEL_UPDATE

	if [ ! -f $DEB ]; then
		echo Downloading $DEB [Ubuntu]
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $URL
		dpkg -x $DEB ./
	fi

	NUM_DEB=$(ls linux-*.deb -1 | wc -l)

	if [ $NUM_DEB -eq 3 ]; then

		local KERNEL_FOLDER=$KERNEL_RELEASE
		KERNEL_RELEASE=$(ls -1 linux-image-* | grep -E -o "[0-9]{1}\.[0-9]+\.[0-9]+-[0-9]+-[a-z]+")

		HASH=$(md5sum boot/config-$KERNEL_RELEASE | cut -d' ' -f1)
		HASH_ORIG=$HASH

		cd $BASEDIR

		export KERNELDIR=$BASEDIR/$KERNEL_FOLDER/$KERNEL_UPDATE/usr/src/linux-headers-$KERNEL_RELEASE
		build_probe
	fi

	cd $BASEDIR
}

function rhel_build {

	#
	# The function just requires the rpm url
	#

	# Get all the parameters needed
	URL=$1
	RPM=$(echo $URL | grep -o '[^/]*$')
	KERNEL_RELEASE=$(echo $RPM | awk 'match($0, /[^kernel\-(uek\-)?(core\-|devel\-)?].*[^(\.rpm)]/){ print substr($0, RSTART, RLENGTH) }')

	if [ ! -d $KERNEL_RELEASE ]; then
		mkdir $KERNEL_RELEASE
	fi

	cd $KERNEL_RELEASE

	if [ ! -f $RPM ]; then
		echo Downloading $RPM [RHEL and CentOS]
		wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $URL
		rpm2cpio $RPM | cpio -idm
	fi

	NUM_RPM=$(ls kernel-*.rpm -1 | wc -l)

	if [ $NUM_RPM -eq 2 ]; then

		#echo Building $KERNEL_RELEASE

		if [ -f boot/config-$KERNEL_RELEASE ]; then
			HASH=$(md5sum boot/config-$KERNEL_RELEASE | cut -d' ' -f1)
		else
			HASH=$(md5sum lib/modules/$KERNEL_RELEASE/config | cut -d' ' -f1)
		fi

		HASH_ORIG=$HASH

		cd $BASEDIR

		export KERNELDIR=$BASEDIR/$KERNEL_RELEASE/usr/src/kernels/$KERNEL_RELEASE
		build_probe
	fi

	cd $BASEDIR
}

function debian_build {

	URL=${1}
	DEB=$(echo ${URL} | grep -o '[^/]*$')

	if [[ ${DEB} == *"kbuild"* ]]; then
		if [[ ! -d ${BASEDIR}/common-dependencies/debian/kbuild/ ]]; then
			mkdir -p ${BASEDIR}/common-dependencies/debian/kbuild
		fi
		if [ ! -f ${BASEDIR}/common-dependencies/debian/kbuild/${DEB} ]; then
			echo Downloading ${DEB} [Ubuntu]
			wget --timeout=${URL_TIMEOUT} --tries=${RETRY} -P ${BASEDIR}/common-dependencies/debian/kbuild ${URL}
		fi
		return
	else
		KERNEL_RELEASE=$(echo ${DEB} | grep -E -o "[0-9]{1}\.[0-9]+\.[0-9]+(-[0-9]+)?"| head -1)
		KERNEL_MAJOR=$(echo ${KERNEL_RELEASE} | grep -E -o "[0-9]{1}\.[0-9]+")
		PACKAGE=$(echo ${DEB} | grep -E -o "(common_[0-9]{1}\.[0-9]+.*amd64|amd64_[0-9]{1}\.[0-9]+.*amd64)" | sed -E 's/(common_|amd64_|_amd64)//g')

		if [[ ! -d ${KERNEL_RELEASE} ]]; then
			mkdir ${KERNEL_RELEASE}
		fi

		cd ${KERNEL_RELEASE}

		if [ ! -d ${PACKAGE} ]; then
			mkdir ${PACKAGE}
		fi

		cd ${PACKAGE}

		if [ ! -f ${DEB} ]; then
			echo Downloading ${DEB} [Ubuntu]
			wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${URL}
			dpkg -x ${DEB} ./
		fi
	fi

	NUM_DEB=$(ls linux-*.deb -1| grep -v kbuild | wc -l)

	if [[ ${NUM_DEB} -eq 3 ]]; then
		set +e
			KBUILD_PACKAGE=$(ls -t ${BASEDIR}/common-dependencies/debian/kbuild | grep  "kbuild\-${KERNEL_MAJOR}" | head -1)
		set -e
		if [[ ! -z ${KBUILD_PACKAGE} ]]; then
			cp ${BASEDIR}/common-dependencies/debian/kbuild/${KBUILD_PACKAGE} .
			dpkg -x ${KBUILD_PACKAGE} ./

			local KERNEL_FOLDER=${KERNEL_RELEASE}
			KERNEL_RELEASE=$(ls boot/config-* | sed 's|boot/config-||')

			HASH=$(md5sum boot/config-${KERNEL_RELEASE} | cut -d' ' -f1)
			HASH_ORIG=${HASH}

			export KERNELDIR=${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE}

			#fix symbolic links
			unlink ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/lib/modules/${KERNEL_RELEASE}/build
			ln -s ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE} ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/lib/modules/${KERNEL_RELEASE}/build

			common_folder=$(ls ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/ | egrep '*common')
			unlink ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/lib/modules/${KERNEL_RELEASE}/source
			ln -s ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/${common_folder}  ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/lib/modules/${KERNEL_RELEASE}/source

			#hack Makefile
			sed -i '0,/MAKEARGS.*$/s||MAKEARGS := -C '"${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/${common_folder}"' O='"${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE}"'|' ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE}/Makefile
			sed -i 's/@://' ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE}/Makefile
			sed -i 's|$(cmd) %.*$|$(cmd) : all|' ${BASEDIR}/${KERNEL_FOLDER}/${PACKAGE}/usr/src/linux-headers-${KERNEL_RELEASE}/Makefile

			cd ${BASEDIR}

			build_probe
		fi
	fi

	cd ${BASEDIR}
}

if [ -z "$KERNEL_TYPE" ]; then
	#
	# Ubuntu build
	#

	echo Building Ubuntu
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/kernel-crawler.py Ubuntu)"

	for URL in $URLS
	do
		ubuntu_build $URL
	done

	#
	# RHEL build
	#

	echo Building RHEL
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/kernel-crawler.py CentOS)"

	for URL in $URLS
	do
		rhel_build $URL
	done

	#
	# Fedora build
	#

	echo Building Fedora
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/kernel-crawler.py Fedora)"

	for URL in $URLS
	do
		rhel_build $URL
	done

	#
	# CoreOS build
	#
	echo Building CoreOS
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/kernel-crawler.py CoreOS)"

	for URL in $URLS
	do
		set +e
		eval $(curl -s ${URL}version.txt)
		if [ ${?} -ne 0 ]; then
			echo "### Error fetching ${URL}version.txt ###"
			continue
		fi
		set -e
		if [ $COREOS_BUILD -gt 890 ]; then
			coreos_build_new $URL $COREOS_VERSION
		else
			coreos_build_old $URL $COREOS_VERSION
		fi
	done

	#
	# boot2docker build
	#
	echo Building boot2docker
	DIR=$(dirname $(readlink -f $0))
	$DIR/boot2docker-kernel-crawler.py | \
		while read KERNEL_ARGS
		do
			boot2docker_build $KERNEL_ARGS
		done

	#
	# Debian build
	#
	echo Building Debian
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/kernel-crawler.py Debian)"

	for URL in $URLS
	do
		debian_build $URL
	done
	# XXX agent/434 - We need to force a build for certain kernel versions
	# because they are still in use by some GCE customers but the headers
	# are no longer available from the mirror. We pass the URL but nothing
	# needs to be downloaded because we already have it cached on the builder.
	debian_build https://mirrors.kernel.org/debian/pool/main/l/linux/linux-headers-3.16.0-4-amd64_3.16.36-1+deb8u2_amd64.deb

	#
	# Oracle RHCK build
	#
	echo Building Oracle RHCK
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/oracle-kernel-crawler.py Oracle-RHCK)"

	for URL in $URLS
	do
		rhel_build $URL
	done

	#
	# The UEK builds needs to happen in a UEK environment, so we launch
	# a container running OL and do the builds there. This container
	# doesn't have git credentials, so it relies on previous builds
	# to do the sysdig/falco/agent git clone/pulls.
	#

	#
	# Oracle Linux 6 UEK build
	#
	echo "Creating Oracle Linux 6 UEK builder"
	docker ps -q -f 'name=ol6-build' | xargs --no-run-if-empty docker rm
	docker build -t ol6-builder:latest -f ../sysdig/scripts/Dockerfile.ol6 ../sysdig/scripts
	docker images -q -f 'dangling=true' | xargs --no-run-if-empty docker rmi
	docker run -i --rm --name ol6-build -v $BASEDIR:/build/probe ol6-builder $PROBE_NAME $PROBE_VERSION $REPOSITORY_NAME OL6-UEK

	#
	# Oracle Linux 7 UEK build
	#
	echo "Creating Oracle Linux 7 UEK builder"
	docker ps -q -f 'name=ol7-build' | xargs --no-run-if-empty docker rm
	docker build -t ol7-builder:latest -f ../sysdig/scripts/Dockerfile.ol7 ../sysdig/scripts
	docker images -q -f 'dangling=true' | xargs --no-run-if-empty docker rmi
	docker run -i --rm --name ol7-build -v $BASEDIR:/build/probe ol7-builder $PROBE_NAME $PROBE_VERSION $REPOSITORY_NAME OL7-UEK

	#
	# Upload modules
	#
	aws s3 sync ./output/ s3://download.draios.com/$REPOSITORY_NAME/sysdig-probe-binaries/ --acl public-read

	echo "Success."

elif [ "OL6-UEK" = "$KERNEL_TYPE" ]; then
	# This should only run in the ol6-builder container context
	echo Building Oracle Linux 6 UEK
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/oracle-kernel-crawler.py OL6-UEK)"

	for URL in $URLS
	do
		rhel_build $URL
	done
elif [ "OL7-UEK" = "$KERNEL_TYPE" ]; then
	# This should only run in the ol7-builder container context
	echo Building Oracle Linux 7 UEK
	DIR=$(dirname $(readlink -f $0))
	URLS="$($DIR/oracle-kernel-crawler.py OL7-UEK)"

	for URL in $URLS
	do
		rhel_build $URL
	done
else
	exit 1
fi