Merge pull request #16 from StaticRocket/dockerfile-rework

Dockerfile rework and id remapping

Dockerbuild.md

@@ -5,13 +5,6 @@ Quite often we need to build quite a bunch of applications to get the very
 latest tools and environment. Instead of hand holding every developer to get the
 latest environment, let's use Docker to bootstrap a common build environment.
 
-Proxy setup
-===========
-
-If you are working in an environment where an http proxy is necessary, update
-the files in `proxy-configuration` as required. With out this, you cannot
-install the various packages needed to build the Docker image.
-
 Versions of packages
 ====================
 

Dockerfile

@@ -3,145 +3,98 @@ FROM $BASE_DISTRO
 
 ARG INSTALL_GCC=0
 
-# In case of Proxy based environment, leave the following enabled.
-# in Direct internet cases, comment out the following two lines.
-#--- PROXY SETUP START
-# COPY proxy-configuration/ /
-# RUN  export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils
-#--- END START
-
 ARG DEBIAN_FRONTEND noninteractive
-RUN apt-get update && \
-    apt-get install -y \
-        build-essential \
-        wget \
-        gcc \
-        ccache \
-        ncurses-dev \
-        xz-utils \
-        libssl-dev \
-        bc \
-        flex \
-        libelf-dev \
-        bison \
-        libyaml-dev \
-        python3-pip \
-        libcurl4-gnutls-dev \
-        libexpat1-dev \
-        gettext \
-        libz-dev \
-        libssl-dev \
-        flex \
-        bison \
-        pkg-config \
-        sqlite3 \
-        libsqlite3-dev \
-        llvm \
-        autoconf \
-        pkg-config \
-        ocaml-nox \
-        ocaml-findlib \
-        menhir \
-        libmenhir-ocaml-dev \
-        ocaml-native-compilers \
-        libpcre-ocaml-dev \
-        libparmap-ocaml-dev \
-        libpython3.11 \
-        libpython3.11-dev \
-        libgmp-dev \
-        libmpc-dev \
-        diffstat \
-        yamllint \
-        swig \
-        python3 \
-        python3-ruamel.yaml \
-        aria2
-
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-
-COPY other-configs/ /
-
-COPY build-env.sh /tmp
-RUN  INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh
-
-FROM $BASE_DISTRO
-
-ARG INSTALL_GCC=0
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		aria2 \
+		autoconf \
+		automake \
+		bc \
+		bison \
+		build-essential \
+		ca-certificates \
+		ccache \
+		diffstat \
+		dumb-init \
+		flex \
+		gcc \
+		gettext \
+		gnupg2 \
+		gosu \
+		libcurl4-gnutls-dev \
+		libelf-dev \
+		libexpat1-dev \
+		libgmp-dev \
+		libmenhir-ocaml-dev \
+		libmpc-dev \
+		libparmap-ocaml-dev \
+		libpcre-ocaml-dev \
+		libpython3.11 \
+		libpython3.11-dev \
+		libsqlite3-dev \
+		libssl-dev \
+		libyaml-dev \
+		libz-dev \
+		menhir \
+		ncurses-dev \
+		ocaml-findlib \
+		ocaml-native-compilers \
+		ocaml-nox \
+		pkg-config \
+		python-is-python3 \
+		python3 \
+		python3-dev \
+		python3-pip \
+		python3-ruamel.yaml \
+		sqlite3 \
+		swig \
+		wget \
+		xz-utils \
+		yamllint \
+	&& echo "**** cleanup ****" \
+	&& apt-get autoremove \
+	&& apt-get clean \
+	&& rm -rf \
+		/tmp/* \
+		/var/lib/apt/lists/* \
+		/var/tmp/* \
+		/var/log/*
+
+RUN --mount=type=bind,source=build-env.sh,target=/tmp/build-env.sh \
+	INSTALL_GCC=$INSTALL_GCC /tmp/build-env.sh
 
 # Publish the source repository
 LABEL org.opencontainers.image.source https://github.com/nmenon/kernel_patch_verify
 
-#--- PROXY SETUP START
-# COPY proxy-configuration/ /
-# RUN  export DEBIAN_FRONTEND=noninteractive;apt-get update;apt-get install -y apt-transport-https socket corkscrew apt-utils
-#--- END START
-
-
-ARG DEBIAN_FRONTEND noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ca-certificates \
-        gnupg2 \
-        build-essential \
-        wget \
-        gcc \
-        ccache \
-        ncurses-dev \
-        xz-utils \
-        libssl-dev \
-        bc \
-        flex \
-        libelf-dev \
-        bison \
-        libyaml-dev \
-        python3-pip \
-        libcurl4-gnutls-dev \
-        libexpat1-dev \
-        gettext \
-        libz-dev \
-        libssl-dev \
-        flex \
-        bison \
-        pkg-config \
-        sqlite3 \
-        libsqlite3-dev \
-        autoconf \
-        pkg-config \
-        ocaml-nox \
-        ocaml-findlib \
-        menhir \
-        libmenhir-ocaml-dev \
-        ocaml-native-compilers \
-        libpcre-ocaml-dev \
-        libparmap-ocaml-dev \
-        libpython3.11 \
-        libpython3.11-dev \
-        libgmp-dev \
-        libmpc-dev \
-        diffstat \
-        yamllint \
-        swig \
-        python3 \
-        python3-dev \
-        python3-ruamel.yaml
-
 # Add our llvm repo configs
 COPY llvm-config /
-RUN apt-get update && apt-get install -y --no-install-recommends\
-        llvm \
-        clang \
-        lld
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		llvm \
+		clang \
+		lld \
+	&& echo "**** cleanup ****" \
+	&& apt-get autoremove \
+	&& apt-get clean \
+	&& rm -rf \
+		/tmp/* \
+		/var/lib/apt/lists/* \
+		/var/tmp/* \
+		/var/log/*
 
-RUN apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+COPY other-configs/ /
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+RUN echo "**** create developer user and make our folders ****" \
+	&& useradd -u 1000 -U -d /config -s /bin/false developer \
+	&& usermod -G users developer \
+	&& mkdir /workdir && chown developer:developer /workdir \
+	&& mkdir /config && chown developer:developer /config
 
-COPY other-configs/ /
+ENTRYPOINT ["/init"]
 
-COPY --from=0 /usr/local/ /usr/local/
+CMD ["/usr/bin/bash"]
 
-RUN ldconfig /usr/local/lib
+VOLUME /workdir
 
 COPY kernel_patch_verify /usr/bin/kernel_patch_verify
 

build-env.sh

@@ -18,7 +18,7 @@ export COCCI_TAG=1.1.1
 # https://github.com/devicetree-org/dt-schema/tags
 export DTSCHEMA_REV=v2024.02
 
-ARIA_OPTS=(--timeout=180 --retry-wait=10 -m 0 -x 10 -j 10)
+ARIA_OPTS=( --summary-interval=5 --timeout=180 --retry-wait=10 -m 0 -x 10 -j 10 )
 
 download_build_install_git()
 {
@@ -47,12 +47,13 @@ download_build_install_python_deps()
 	# scripts/spdxcheck.py dependencies
 	python -m pip install --upgrade  --break-system-packages ply gitpython yamllint rfc3987 pylibfdt
 	python -m pip install  --break-system-packages git+https://github.com/devicetree-org/dt-schema.git@$DTSCHEMA_REV
+	rm -rf "/tmp/.cache/"
 }
 
 clone_and_cd()
 {
 	cd /tmp &&
-	git clone --depth=1 --branch "$1" "$2" "$3" &&
+	git clone --progress --depth=1 --branch "$1" "$2" "$3" &&
 	cd /tmp/"$3"
 	return $?
 }
@@ -118,7 +119,7 @@ download_and_install_armgcc_64()
 	cd /tmp
 	mkdir -p /opt/cross-gcc-linux-9/
 	aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL"
-	tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE"
+	tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE"
 	rm -f /tmp/"$FILE"
 }
 
@@ -131,7 +132,7 @@ download_and_install_armgcc_32()
 	cd /tmp
 	mkdir -p /opt/cross-gcc-linux-9/
 	aria2c "${ARIA_OPTS[@]}" -o "$FILE" "$URL"
-	tar -C /usr/local/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE"
+	tar -C /opt/cross-gcc-linux-9/ --strip-components=1 -xf "$FILE"
 	rm -f /tmp/"$FILE"
 }
 

kernel_patch_verify

@@ -26,6 +26,11 @@
 source /etc/profile
 shopt -s expand_aliases
 
+# extend the path with the supplied extra directories
+if [ -n "$KP_PATH" ]; then
+       export PATH=${KP_PATH}:${PATH}
+fi
+
 ccache=$(which ccache)
 
 # We would rather that we hit cache more often, than rebuild..

kp_common

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export USER_ID GROUP_ID PATH DOCKER_MOUNT_DIRS IMAGE_ID
+export DOCKER_ARGS IMG_NAME KP_PATH
 
 # Check if Docker image exists
 
@@ -38,30 +38,46 @@ else
 	fi
 fi
 
-USER_ID=$(id -u)
-GROUP_ID=$(id -g)
-
-DOCKER_MOUNT_DIRS=()
-DOCKER_MOUNT_DIRS+=(-v /tmp:/tmp)
-DOCKER_MOUNT_DIRS+=(-v /opt:/opt)
-DOCKER_MOUNT_DIRS+=(-v "$CCACHEDIR":/ccache)
-DOCKER_MOUNT_DIRS+=(-v "$(pwd)":/workdir)
+DOCKER_ARGS=()
+DOCKER_ARGS+=(-v /tmp:/tmp)
+DOCKER_ARGS+=(-v /opt:/opt)
+DOCKER_ARGS+=(-v "$CCACHEDIR":/ccache)
 
 # Check if current directory is a git directory
-if ! GIT_WORKTREE_COMMONDIR=$(git rev-parse --git-common-dir); then
+if ! git rev-parse --is-inside-work-tree > /dev/null; then
 	exit 1
 fi
 
-# Mount parent directory if its a worktree
-if [ "$GIT_WORKTREE_COMMONDIR" != ".git" ]; then
-	DOCKER_MOUNT_DIRS+=(-v "$GIT_WORKTREE_COMMONDIR":"$GIT_WORKTREE_COMMONDIR")
+ROOT_GIT=$(git rev-parse --show-toplevel)
+# if a submodule or worktree then we need to fetch and preserve the gitdir path
+if [ -f "${ROOT_GIT}/.git" ]; then
+	gitdir_str=$(grep -P -o '(?<=gitdir:\s).*' "${ROOT_GIT}/.git")
+	relative_str=$(echo "$gitdir_str" | grep -P -o '^(../)*')
+	if [ -n "$relative_str" ]; then
+		# relative paths need to be preserved
+		common_path=$(realpath "$ROOT_GIT/$relative_str")
+		relative_pwd=$(realpath --relative-to="$common_path" "$PWD")
+		DOCKER_ARGS+=(-v "$common_path":"/workdir")
+		DOCKER_ARGS+=(-w "/workdir/${relative_pwd}")
+	else
+		# absolute paths can be passed through
+		gitdir_path=$(git rev-parse --git-common-dir)
+		DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir")
+		DOCKER_ARGS+=(-v "$gitdir_path":"$gitdir_path")
+	fi
+else
+	# normal git directory, just toss everything in
+	DOCKER_ARGS+=(-v "$ROOT_GIT":"/workdir")
 fi
 
-# Run our image to add our swuser
-docker run "$IMG_NAME" /bin/bash -c "groupadd -r swuser -g $GROUP_ID && useradd -u $USER_ID -r -g swuser -d /workdir -s /sbin/nologin -c \"Docker kernel patch user\" swuser"
-# Get the container ID of the last run container (above)
-CONTAINER_ID=$(docker ps -lq)
-# Commit the container state (returns an image_id with sha256: prefix cut off)
-IMAGE_ID=$(docker commit "$CONTAINER_ID" | cut -c8-)
+# list of paths to append to the PATH variable in the container
+extra_paths=(
+	/workdir/scripts/dtc
+	/opt/cross-gcc-linux-13/bin
+	/opt/cross-gcc-linux-12/bin
+	/opt/cross-gcc-linux-11/bin
+	/opt/cross-gcc-linux-10/bin
+	/opt/cross-gcc-linux-9/bin
+)
 
-PATH=/workdir/scripts/dtc:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/opt/cross-gcc-linux-13/bin:/opt/cross-gcc-linux-12/bin:/opt/cross-gcc-linux-11/bin:/opt/cross-gcc-linux-10/bin:/opt/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-9/bin:/usr/local/cross-gcc-linux-10/bin
+KP_PATH=$(IFS=:; printf '%s' "${extra_paths[*]}")

kps

@@ -4,11 +4,7 @@ source "$(dirname "$(readlink -f "$0")")/kp_common"
 
 # If we wanted to get to bash shell:
 docker run --rm -ti \
-	--user "$USER_ID":"$GROUP_ID" \
-	-e PATH \
-	"${DOCKER_MOUNT_DIRS[@]}" \
-	"$IMAGE_ID" \
+	-e KP_PATH -e CROSS_COMPILE -e ARCH \
+	"${DOCKER_ARGS[@]}" \
+	"$IMG_NAME" \
 	bash --init-file /etc/profile
-
-# TODO: we can reuse this image for future runs, for now just clean up after ourselves
-docker rmi "$IMAGE_ID"

kpv

@@ -3,11 +3,7 @@
 source "$(dirname "$(readlink -f "$0")")/kp_common"
 
 docker run --rm -ti \
-	--user "$USER_ID":"$GROUP_ID" \
-	-e PATH \
-	"${DOCKER_MOUNT_DIRS[@]}" \
-	"$IMAGE_ID" \
+	-e KP_PATH -e CROSS_COMPILE -e ARCH \
+	"${DOCKER_ARGS[@]}" \
+	"$IMG_NAME" \
 	kernel_patch_verify -S /usr/local/smatch/bin/k_sm_check_script "$@"
-
-# TODO: we can reuse this image for future runs, for now just clean up after ourselves
-docker rmi "$IMAGE_ID"