From 9ba64a477be0c3654d888c607ef0861e416cad25 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 10 Nov 2025 10:23:37 +0100 Subject: [PATCH] feat: add zstd compression module for Python 3.14+ --- docker/Dockerfile | 18 ++++++++++-- docker/build_scripts/build-zstd.sh | 43 +++++++++++++++++++++++++++++ docker/tests/modules-check.py | 13 +++++++++ tools/update_native_dependencies.py | 11 ++++++-- 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100755 docker/build_scripts/build-zstd.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 4683d7cf..9587cb2e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -135,18 +135,28 @@ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ FROM build_base AS build_mpdecimal COPY build_scripts/build-mpdecimal.sh /build_scripts/ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ - export MPDECIMAL_ROOT=mpdecimal-4.0.0 && \ - export MPDECIMAL_HASH=942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c && \ + export MPDECIMAL_ROOT=mpdecimal-4.0.1 && \ + export MPDECIMAL_HASH=96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8 && \ export MPDECIMAL_DOWNLOAD_URL=https://www.bytereef.org/software/mpdecimal/releases && \ /tmp/cross-compiler/entrypoint /build_scripts/build-mpdecimal.sh +FROM build_base AS build_zstd +COPY build_scripts/build-zstd.sh /build_scripts/ +RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ + export ZSTD_VERSION=1.5.7 && \ + export ZSTD_HASH=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 && \ + export ZSTD_DOWNLOAD_URL=https://github.com/facebook/zstd/releases/download && \ + /tmp/cross-compiler/entrypoint /build_scripts/build-zstd.sh + + FROM --platform=${BUILDPLATFORM} ghcr.io/sigstore/cosign/cosign:v${MANYLINUX_COSIGN_VERSION} AS cosign-bin FROM build_base AS build_cpython COPY --from=build_tcl_tk /manylinux-buildfs / COPY --from=build_mpdecimal /manylinux-buildfs / +COPY --from=build_zstd /manylinux-buildfs / COPY --from=build_sqlite3 /manylinux-buildfs / RUN if command -v apk >/dev/null 2>&1; then ldconfig /; else ldconfig; fi COPY build_scripts/build-openssl.sh /build_scripts/ @@ -208,6 +218,7 @@ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ FROM runtime_base COPY --from=build_tcl_tk /manylinux-rootfs / COPY --from=build_mpdecimal /manylinux-rootfs / +COPY --from=build_zstd /manylinux-rootfs / COPY --from=build_sqlite3 /manylinux-rootfs / COPY --from=build_git /manylinux-rootfs / COPY build_scripts /opt/_internal/build_scripts/ @@ -221,7 +232,8 @@ RUN --mount=type=bind,target=/build_cpython38,from=build_cpython38 \ --mount=type=bind,target=/build_cpython314,from=build_cpython314 \ --mount=type=bind,target=/build_cpython314_nogil,from=build_cpython314_nogil \ mkdir -p /opt/_internal && \ - cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \ + cp -rf /build_cpython*/opt/_internal/cpython* /opt/_internal/ && \ + if test -n "$(find /build_cpython314/opt/_internal -maxdepth 1 -name 'openssl*' -print -quit)"; then cp -rf /build_cpython314/opt/_internal/openssl* /opt/_internal/; fi && \ manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \ pp311-pypy311_pp73 diff --git a/docker/build_scripts/build-zstd.sh b/docker/build_scripts/build-zstd.sh new file mode 100755 index 00000000..fb4e5586 --- /dev/null +++ b/docker/build_scripts/build-zstd.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +# shellcheck source-path=SCRIPTDIR +source "${MY_DIR}/build_utils.sh" + +# Install a more recent mpdecimal +check_var "${ZSTD_VERSION}" +check_var "${ZSTD_HASH}" +check_var "${ZSTD_DOWNLOAD_URL}" +ZSTD_ROOT="zstd-${ZSTD_VERSION}" + +PREFIX=/opt/_internal/${ZSTD_ROOT%%.*} + +fetch_source "${ZSTD_ROOT}.tar.gz" "${ZSTD_DOWNLOAD_URL}/v${ZSTD_VERSION}" +check_sha256sum "${ZSTD_ROOT}.tar.gz" "${ZSTD_HASH}" +tar xfz "${ZSTD_ROOT}.tar.gz" +pushd "${ZSTD_ROOT}/lib" +# add rpath +sed -i "s|^Libs:|Libs: -Wl,--enable-new-dtags,-rpath=\${libdir} |g" ./libzstd.pc.in +DESTDIR=/manylinux-rootfs MT=1 prefix=${PREFIX} make install-includes install-pc install-shared V=1 CPPFLAGS="${MANYLINUX_CPPFLAGS} -DZSTD_MULTITHREAD" CFLAGS="${MANYLINUX_CFLAGS} -DZSTD_MULTITHREAD -pthread -fPIC -fvisibility=hidden" LDFLAGS="${MANYLINUX_LDFLAGS} -shared -pthread" +popd +rm -rf "${ZSTD_ROOT}" "${ZSTD_ROOT}.tar.gz" + +# Strip what we can +strip_ /manylinux-rootfs + +# Install for build +mkdir /manylinux-buildfs +cp -rlf /manylinux-rootfs/* /manylinux-buildfs/ +# copy pkgconfig +mkdir -p /manylinux-buildfs/usr/local/lib/pkgconfig/ +ln -s "${PREFIX}/lib/pkgconfig/libzstd.pc" /manylinux-buildfs/usr/local/lib/pkgconfig/libzstd.pc + +# Clean-up for runtime +rm -rf "/manylinux-rootfs/${PREFIX}/lib/pkgconfig" "/manylinux-rootfs/${PREFIX}/include" diff --git a/docker/tests/modules-check.py b/docker/tests/modules-check.py index 7a807707..b23233ba 100644 --- a/docker/tests/modules-check.py +++ b/docker/tests/modules-check.py @@ -62,6 +62,19 @@ def test_sysconfig(self): assert config_vars["LDSHARED"] == f"{cc} -shared", config_vars["LDSHARED"] assert config_vars["LDCXXSHARED"] == f"{cxx} -shared", config_vars["LDCXXSHARED"] + @unittest.skipIf(sys.version_info[:2] < (3, 14), reason="not supported in this version") + def test_zstd(self): + from compression import zstd + + print(f"{zstd.zstd_version_info=}", end=" ", file=sys.stderr) + assert zstd.zstd_version_info[:3] >= (1, 5, 7) + + def test_ssl(self): + import ssl + + print(f"{ssl.OPENSSL_VERSION_INFO=}", end=" ", file=sys.stderr) + assert ssl.OPENSSL_VERSION_INFO[:3] >= (1, 1, 1) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tools/update_native_dependencies.py b/tools/update_native_dependencies.py index f2da70db..cf29e7c6 100644 --- a/tools/update_native_dependencies.py +++ b/tools/update_native_dependencies.py @@ -145,6 +145,11 @@ def _update_sqlite(dry_run): def _update_with_gh(tool, dry_run): repo = { "libxcrypt": "besser82/libxcrypt", + "zstd": "facebook/zstd", + } + extension = { + "libxcrypt": "tar.xz", + "zstd": "tar.gz", } lines = DOCKERFILE.read_text().splitlines() re_ = re.compile(f"^ export {tool.upper()}_VERSION=(?P\\S+) && \\\\$") @@ -153,13 +158,13 @@ def _update_with_gh(tool, dry_run): if match is None: continue current_version = Version(match["version"]) - latest_tag = latest(repo[tool], output_format="tag") + latest_tag = latest(repo[tool], output_format="tag", having_asset=True) latest_version = Version(latest_tag) if latest_version > current_version: url = re.match( f"^ export {tool.upper()}_DOWNLOAD_URL=(?P\\S+) && \\\\$", lines[i + 2] )["url"] - sha256 = _sha256(f"{url}/{latest_tag}/libxcrypt-{latest_version}.tar.xz") + sha256 = _sha256(f"{url}/{latest_tag}/{tool}-{latest_version}.{extension[tool]}") lines[i + 0] = f" export {tool.upper()}_VERSION={latest_version} && \\" lines[i + 1] = f" export {tool.upper()}_HASH={sha256} && \\" message = f"Bump {tool} {current_version} → {latest_version}" @@ -257,7 +262,7 @@ def main(): _update_with_root(tool, args.dry_run) except Exception as e: print(f"::warning::update: {e}\n", file=sys.stderr) - for tool in ["libxcrypt"]: + for tool in ["libxcrypt", "zstd"]: try: _update_with_gh(tool, args.dry_run) except Exception as e: