Skip to content

Commit

Permalink
chore: rework PyPy installation
Browse files Browse the repository at this point in the history
PyPy installation is just a tar extract.
As such, it might be interesting to install PyPy at runtime rather than at build time.
This is especially true for EOL versions of PyPy which would allow to reduce the image size,
and thus, overall build time for users not using EOL PyPy versions.

This commit does not modify default installed PyPy versions yet.

This commit will also be useful for GraalPy installation.
  • Loading branch information
mayeut committed Aug 21, 2023
1 parent 323dd42 commit 87815c6
Show file tree
Hide file tree
Showing 12 changed files with 508 additions and 183 deletions.
52 changes: 20 additions & 32 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc1

FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
FROM build_cpython AS all_cpython
COPY build_scripts/finalize-python.sh /build_scripts/
RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
--mount=type=bind,target=/build_cpython37,from=build_cpython37 \
--mount=type=bind,target=/build_cpython38,from=build_cpython38 \
--mount=type=bind,target=/build_cpython39,from=build_cpython39 \
--mount=type=bind,target=/build_cpython310,from=build_cpython310 \
--mount=type=bind,target=/build_cpython311,from=build_cpython311 \
--mount=type=bind,target=/build_cpython312,from=build_cpython312 \
mkdir -p /opt/_internal && \
cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
manylinux-entrypoint /build_scripts/finalize-python.sh


FROM runtime_base
COPY --from=build_git /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_python /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh \
build_scripts/update-system-packages.sh \
build_scripts/python-tag-abi-tag.py \
build_scripts/requirements3.6.txt \
build_scripts/requirements3.7.txt \
build_scripts/requirements3.8.txt \
build_scripts/requirements3.9.txt \
build_scripts/requirements3.10.txt \
build_scripts/requirements3.11.txt \
build_scripts/requirements3.12.txt \
build_scripts/requirements-base-tools.txt \
/build_scripts/
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
COPY build_scripts /opt/_internal/build_scripts/
RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
pp37-pypy37_pp73 \
pp38-pypy38_pp73 \
pp39-pypy39_pp73 \
pp310-pypy310_pp73

ENV SSL_CERT_FILE=/opt/_internal/certs.pem

Expand Down
30 changes: 30 additions & 0 deletions docker/build_scripts/download-and-install-interpreter.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash

# Stop at any error, show all commands
set -exuo pipefail

# Get script directory
MY_DIR=$(dirname "${BASH_SOURCE[0]}")

ABI_TAG=$1
DOWNLOAD_URL=$2
SHA256=$3

PREFIX="/opt/_internal/${ABI_TAG}"

case ${DOWNLOAD_URL} in
*.tar) COMP=;;
*.tar.gz) COMP=z;;
*.tar.bz2) COMP=j;;
*.tar.xz) COMP=J;;
*) echo "unsupported archive"; exit 1;;
esac

mkdir ${PREFIX}

curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")

# remove debug symbols if any
find ${PREFIX}/bin -name '*.debug' -delete

${MY_DIR}/finalize-one.sh ${PREFIX}
37 changes: 37 additions & 0 deletions docker/build_scripts/finalize-one.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

# Stop at any error, show all commands
set -exuo pipefail

PREFIX=$1

# Get script directory
MY_DIR=$(dirname "${BASH_SOURCE[0]}")

# Some python's install as bin/python3. Make them available as
# bin/python.
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
ln -s python3 ${PREFIX}/bin/python
fi
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")

# Install pinned packages for this python version.
# Use the already intsalled cpython pip to bootstrap pip if available
if [ -f /usr/local/bin/python${PY_VER} ]; then
/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
else
${PREFIX}/bin/python -m ensurepip
${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
fi
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
ln -s pip3 ${PREFIX}/bin/pip
fi
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
ln -s ${PREFIX} /opt/python/${ABI_TAG}
# Make versioned python commands available directly in environment.
if [[ "${PY_IMPL}" == "cpython" ]]; then
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
fi
ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}
36 changes: 14 additions & 22 deletions docker/build_scripts/finalize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh

mkdir /opt/python
for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
# Some python's install as bin/python3. Make them available as
# bin/python.
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
ln -s python3 ${PREFIX}/bin/python
fi
${PREFIX}/bin/python -m ensurepip
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
ln -s pip3 ${PREFIX}/bin/pip
fi
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
# Since we fall back on a canned copy of pip, we might not have
# the latest pip and friends. Upgrade them to make sure.
${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
ln -s ${PREFIX} /opt/python/${ABI_TAG}
# Make versioned python commands available directly in environment.
if [[ "${PREFIX}" == *"/pypy"* ]]; then
ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
else
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
fi
${MY_DIR}/finalize-one.sh ${PREFIX}
done

# create manylinux-interpreters script
cat <<EOF > /usr/local/bin/manylinux-interpreters
#!/bin/bash
set -euo pipefail
/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
EOF
chmod 755 /usr/local/bin/manylinux-interpreters

MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"

# Create venv for auditwheel & certifi
TOOLS_PATH=/opt/_internal/tools
/opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
Expand Down Expand Up @@ -84,6 +75,7 @@ clean_pyc /opt/_internal

# remove cache
rm -rf /root/.cache
rm -rf /tmp/* || true

hardlink -cv /opt/_internal

Expand Down
72 changes: 0 additions & 72 deletions docker/build_scripts/install-pypy.sh

This file was deleted.

Loading

0 comments on commit 87815c6

Please sign in to comment.