Merge pull request #1135 from lkollar/musl

Add image for musllinux_1_1

.github/workflows/build.yml

@@ -34,6 +34,10 @@ jobs:
             platform: "i686"
           - policy: "manylinux_2_24"
             platform: "x86_64"
+          - policy: "musllinux_1_1"
+            platform: "x86_64"
+          - policy: "musllinux_1_1"
+            platform: "i686"
 
     env:
       POLICY: ${{ matrix.policy }}

.travis.yml

@@ -38,6 +38,14 @@ jobs:
       env: POLICY="manylinux_2_24" PLATFORM="s390x"
     - arch: ppc64le
       env: POLICY="manylinux_2_24" PLATFORM="ppc64le" MANYLINUX_BUILD_FRONTEND="buildkit"
+    - arch: arm64-graviton2
+      virt: vm
+      group: edge
+      env: POLICY="musllinux_1_1" PLATFORM="aarch64"
+    - arch: s390x
+      env: POLICY="musllinux_1_1" PLATFORM="s390x"
+    - arch: ppc64le
+      env: POLICY="musllinux_1_1" PLATFORM="ppc64le" MANYLINUX_BUILD_FRONTEND="buildkit"
 
 before_install:
   - if [ -d "${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM}" ]; then cp -rlf ${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM} ./; fi

build.sh

@@ -62,6 +62,11 @@ elif [ "${POLICY}" == "manylinux_2_24" ]; then
 	DEVTOOLSET_ROOTPATH=
 	PREPEND_PATH=
 	LD_LIBRARY_PATH_ARG=
+elif [ "${POLICY}" == "musllinux_1_1" ]; then
+	BASEIMAGE="${MULTIARCH_PREFIX}alpine:3.12"
+	DEVTOOLSET_ROOTPATH=
+	PREPEND_PATH=
+	LD_LIBRARY_PATH_ARG=
 else
 	echo "Unsupported policy: '${POLICY}'"
 	exit 1

docker/Dockerfile

@@ -25,12 +25,19 @@ ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
 COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
 
 # setup entrypoint, this will wrap commands with `linux32` with i686 images
-COPY build_scripts/install-entrypoint.sh build_scripts/update-system-packages.sh /build_scripts/
-RUN bash /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
+COPY build_scripts/install-entrypoint.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/build_utils.sh \
+     /build_scripts/
+
+RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
 COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
 ENTRYPOINT ["manylinux-entrypoint"]
 
-COPY build_scripts/install-runtime-packages.sh build_scripts/update-system-packages.sh /build_scripts/
+COPY build_scripts/install-runtime-packages.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/build_utils.sh \
+     /build_scripts/
 RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
 
 COPY build_scripts/build_utils.sh /build_scripts/
@@ -68,7 +75,6 @@ RUN export LIBXCRYPT_VERSION=4.4.26 && \
     export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
     manylinux-entrypoint /build_scripts/install-libxcrypt.sh
 
-
 FROM runtime_base AS build_base
 COPY build_scripts/install-build-packages.sh /build_scripts/
 RUN manylinux-entrypoint /build_scripts/install-build-packages.sh

docker/build_scripts/build-git.sh

@@ -7,6 +7,10 @@ set -exuo pipefail
 # Get script directory
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
+if [ "${POLICY}" == "musllinux_1_1" ]; then
+	export NO_REGEX=NeedsStartEnd
+fi
+
 # Get build utilities
 source $MY_DIR/build_utils.sh
 

docker/build_scripts/build-openssl.sh

@@ -27,6 +27,8 @@ fi
 
 if which yum; then
 	yum erase -y openssl-devel
+elif which apk; then
+	apk del openssl-dev
 else
 	apt-get remove -y libssl-dev
 fi

docker/build_scripts/build-sqlite3.sh

@@ -30,7 +30,7 @@ strip_ /manylinux-rootfs
 
 # Install
 cp -rlf /manylinux-rootfs/* /
-ldconfig
+ldconfig /
 
 # Clean-up for runtime
 rm -rf /manylinux-rootfs/usr/local/share

docker/build_scripts/build_utils.sh

@@ -10,6 +10,10 @@ MANYLINUX_CFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -
 MANYLINUX_CXXFLAGS="-g -O2 -Wall -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
 MANYLINUX_LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now"
 
+export BASE_POLICY=manylinux
+if [ "${POLICY:0:9}" == "musllinux" ]; then
+	export BASE_POLICY=musllinux
+fi
 
 function check_var {
     if [ -z "$1" ]; then

docker/build_scripts/finalize.sh

@@ -75,7 +75,13 @@ deactivate
 pushd $MY_DIR/requirements-tools
 for TOOL_PATH in $(find . -type f); do
 	TOOL=$(basename ${TOOL_PATH})
-	pipx install --pip-args="--require-hashes -r" ${TOOL}
+	if [ "${TOOL}" == "cmake" ] && [ "${POLICY}" == "musllinux_1_1" ]; then
+		# TODO remove this exception once https://github.com/scikit-build/cmake-python-distributions
+		# provides a musllinux wheel
+		apk add --no-cache cmake
+	else
+		pipx install --pip-args="--require-hashes -r" ${TOOL}
+	fi
 done
 popd
 

docker/build_scripts/install-build-packages.sh

@@ -19,6 +19,9 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" ==
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	PACKAGE_MANAGER=apt
 	COMPILE_DEPS="libz-dev libbz2-dev libexpat1-dev libncurses5-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-kernel-headers"
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	PACKAGE_MANAGER=apk
+	COMPILE_DEPS="zlib-dev bzip2-dev expat-dev ncurses-dev readline-dev tk-dev gdbm-dev libpcap-dev xz-dev openssl openssl-dev keyutils-dev krb5-dev libcom_err libidn-dev curl-dev util-linux-dev libffi-dev linux-headers"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -35,6 +38,8 @@ elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
 	apt-get install -qq -y --no-install-recommends ${COMPILE_DEPS}
 	apt-get clean -qq
 	rm -rf /var/lib/apt/lists/*
+elif [ "${PACKAGE_MANAGER}" == "apk" ]; then
+	apk add --no-cache ${COMPILE_DEPS}
 else
 	echo "Not implemented"
 	exit 1

docker/build_scripts/install-entrypoint.sh

@@ -1,15 +1,16 @@
-#!/bin/bash
+#!/bin/sh
 # Install entrypoint:
-# make sure yum is configured correctly and linux32 is available on i686
+#   - make sure yum is configured correctly and linux32 is available on i686
+#   - install bash on Alpine as most scripts require it
 
 # Stop at any error, show all commands
-set -exuo pipefail
+set -exu
 
 # Set build environment variables
-MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+MY_DIR=$(dirname "$0")
 
 
-if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then
+if [ "${AUDITWHEEL_PLAT}" = "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" = "manylinux2014_i686" ]; then
 	echo "i386" > /etc/yum/vars/basearch
 	fixup-mirrors
 	yum -y update
@@ -19,5 +20,9 @@ if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" ==
 	# the following script takes care of cleaning-up some things
 	# and since it's also needed in the finalize step, everything's
 	# centralized in this script to avoid code duplication
-	LC_ALL=C ${MY_DIR}/update-system-packages.sh
+	LC_ALL=C "${MY_DIR}/update-system-packages.sh"
+fi
+
+if [ "${AUDITWHEEL_POLICY}" = "musllinux_1_1" ]; then
+	apk add --no-cache bash
 fi

docker/build_scripts/install-libxcrypt.sh

@@ -10,6 +10,11 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
+if [ "$BASE_POLICY" == "musllinux" ]; then
+	echo "Skip libxcrypt installation on musllinux"
+	exit 0
+fi
+
 # We need perl 5.14+
 if ! perl -e 'use 5.14.0' &> /dev/null; then
 	check_var ${PERL_ROOT}

docker/build_scripts/install-pypy.sh

@@ -9,6 +9,10 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
+if [ "${BASE_POLICY}" == "musllinux" ]; then
+	echo "Skip PyPy build on musllinux"
+	exit 0
+fi
 
 PYTHON_VERSION=$1
 PYPY_VERSION=$2

docker/build_scripts/install-runtime-packages.sh

@@ -7,6 +7,9 @@ set -exuo pipefail
 # Set build environment variables
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
+# Get build utilities
+source $MY_DIR/build_utils.sh
+
 # Libraries that are allowed as part of the manylinux2014 profile
 # Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy
 # On RPM-based systems, they are provided by these packages:
@@ -28,6 +31,8 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" ==
 	MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev"
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	MANYLINUX_DEPS="musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -43,15 +48,17 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" ==
 	fi
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	RUNTIME_DEPS="zlib1g libbz2-1.0 libexpat1 libncurses5 libreadline7 tk libgdbm3 libdb5.3 libpcap0.8 liblzma5 libssl1.1 libkeyutils1 libkrb5-3 libcomerr2 libidn2-0 libcurl3 uuid libffi6"
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	RUNTIME_DEPS="zlib bzip2 expat ncurses5-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
 fi
 
-BASETOOLS="autoconf automake bison bzip2 diffutils file hardlink make patch unzip"
+BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip"
 if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
 	PACKAGE_MANAGER=yum
-	BASETOOLS="${BASETOOLS} which"
+	BASETOOLS="${BASETOOLS} hardlink which"
 	# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
 	echo "multilib_policy=best" >> /etc/yum.conf
 	fixup-mirrors
@@ -72,7 +79,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
 	fi
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	PACKAGE_MANAGER=yum
-	BASETOOLS="${BASETOOLS} hostname which"
+	BASETOOLS="${BASETOOLS} hardlink hostname which"
 	# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
 	echo "multilib_policy=best" >> /etc/yum.conf
 	# Error out if requested packages do not exist
@@ -101,13 +108,18 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	fi
 elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	PACKAGE_MANAGER=apt
-	BASETOOLS="${BASETOOLS} hostname"
+	BASETOOLS="${BASETOOLS} hardlink hostname"
 	export DEBIAN_FRONTEND=noninteractive
 	sed -i 's/none/en_US/g' /etc/apt/apt.conf.d/docker-no-languages
 	apt-get update -qq
 	apt-get upgrade -qq -y
 	apt-get install -qq -y --no-install-recommends ca-certificates gpg curl locales
 	TOOLCHAIN_DEPS="binutils gcc g++ gfortran"
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	TOOLCHAIN_DEPS="binutils gcc g++ gfortran"
+	BASETOOLS="${BASETOOLS} curl util-linux"
+	PACKAGE_MANAGER=apk
+	apk add --no-cache ca-certificates gnupg
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -117,6 +129,8 @@ if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 	yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
 elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
 	apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
+elif [ "${PACKAGE_MANAGER}" == "apk" ]; then
+	apk add --no-cache ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
 else
 	echo "Not implemented"
 	exit 1
@@ -128,9 +142,11 @@ fi
 # centralized in this script to avoid code duplication
 LC_ALL=C ${MY_DIR}/update-system-packages.sh
 
-# we'll be removing libcrypt.so.1 later on
-# this is needed to ensure the new one will be found
-# as LD_LIBRARY_PATH does not seem enough.
-# c.f. https://github.com/pypa/manylinux/issues/1022
-echo "/usr/local/lib" > /etc/ld.so.conf.d/00-manylinux.conf
-ldconfig
+if [ "${BASE_POLICY}" == "manylinux" ]; then
+	# we'll be removing libcrypt.so.1 later on
+	# this is needed to ensure the new one will be found
+	# as LD_LIBRARY_PATH does not seem enough.
+	# c.f. https://github.com/pypa/manylinux/issues/1022
+	echo "/usr/local/lib" > /etc/ld.so.conf.d/00-manylinux.conf
+	ldconfig
+fi

docker/build_scripts/update-system-packages.sh

@@ -4,6 +4,11 @@
 # Stop at any error, show all commands
 set -exuo pipefail
 
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Get build utilities
+source $MY_DIR/build_utils.sh
 
 fixup-mirrors
 if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
@@ -21,32 +26,36 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	apt-get upgrade -qq -y
 	apt-get clean -qq
 	rm -rf /var/lib/apt/lists/*
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	apk upgrade --no-cache
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
 fi
 fixup-mirrors
 
 # do we want to update locales ?
-LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
-TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp
-if [ ! -f ${TIMESTAMP_FILE} ] || [ ${LOCALE_ARCHIVE} -nt ${TIMESTAMP_FILE} ]; then
-	# upgrading glibc-common can end with removal on en_US.UTF-8 locale
-	localedef -i en_US -f UTF-8 en_US.UTF-8
-
-	# if we updated glibc, we need to strip locales again...
-	if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
-		localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
-	fi
-	if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
-		mv -f ${LOCALE_ARCHIVE} ${LOCALE_ARCHIVE}.tmpl
-		build-locale-archive --install-langs="en_US.utf8"
-	elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
-		rm ${LOCALE_ARCHIVE}
+if [ "${BASE_POLICY}" == "manylinux" ]; then
+	LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
+	TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp
+	if [ ! -f ${TIMESTAMP_FILE} ] || [ ${LOCALE_ARCHIVE} -nt ${TIMESTAMP_FILE} ]; then
+		# upgrading glibc-common can end with removal on en_US.UTF-8 locale
 		localedef -i en_US -f UTF-8 en_US.UTF-8
-		update-locale LANG=en_US.UTF-8
+
+		# if we updated glibc, we need to strip locales again...
+		if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
+			localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
+		fi
+		if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then
+			mv -f ${LOCALE_ARCHIVE} ${LOCALE_ARCHIVE}.tmpl
+			build-locale-archive --install-langs="en_US.utf8"
+		elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+			rm ${LOCALE_ARCHIVE}
+			localedef -i en_US -f UTF-8 en_US.UTF-8
+			update-locale LANG=en_US.UTF-8
+		fi
+		touch ${TIMESTAMP_FILE}
 	fi
-	touch ${TIMESTAMP_FILE}
 fi
 
 if [ -d /usr/share/locale ]; then

tests/run_tests.sh

@@ -12,6 +12,8 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
 	export DEBIAN_FRONTEND=noninteractive
 	PACKAGE_MANAGER=apt
 	apt-get update -qq
+elif [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ]; then
+	PACKAGE_MANAGER=apk
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -61,6 +63,8 @@ if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 	yum -y install openssh-clients
 elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
 	apt-get install -qq -y --no-install-recommends openssh-client
+elif [ "${PACKAGE_MANAGER}" == "apk" ]; then
+	apk add --no-cache openssh-client
 else
 	echo "Unsupported package manager: '${PACKAGE_MANAGER}'"
 	exit 1