Merge pull request #1964 from minrk/cross-docs

add cross-compile examples to docs

docs/source/howto/build.md

@@ -104,6 +104,10 @@ You can tell pyzmq to skip searching for libzmq and always build the bundled ver
 When building a bundled libzmq, pyzmq downloads and builds libzmq and libsodium as static libraries.
 These static libraries are then linked to by the pyzmq extension and discarded.
 
+Bundled libzmq is supported on a best-effort basis, and isn't expected to work everywhere with zero configuration.
+If you have trouble building bundled libzmq, please do [report it](https://github.com/zeromq/pyzmq/issues).
+But the best solution is usually to install libzmq yourself via the appropriate mechanism _before_ building pyzmq.
+
 ### Building bundled libsodium
 
 libsodium is built first, with `configure` most places:
@@ -401,5 +405,48 @@ ZMQ_OUTPUT_BASENAME:STRING=zmq
 
 </details>
 
+## Cross-compiling pyzmq
+
+Cross-compiling Python extensions is tricky!
+
+To cross-compile pyzmq, in general you need:
+
+- Python built for the 'build' machine
+- Python built for the 'host' machine (identical version)
+- cross-compiling toolchain (e.g. `aarch64-linux-gnu-gcc`)
+- Python setup to cross-compile ([crossenv] is the popular tool these days, and includes lots of info for cross-compiling for Python, but pyzmq makes no assumptions)
+
+It is probably a good idea to build libzmq/libsodium separately and link them with ZMQ_PREFIX,
+as cross-compiling bundled libzmq is not guaranteed to work.
+
+I don't have a lot of experience cross-compiling,
+but we have two example Dockerfiles that appear to work to cross-compile pyzmq.
+These aren't official or supported, but they appear to work and may be useful as reference to get you started.
+
+<details>
+
+<summary>Dockerfile for building x86_64 on aarch64</summary>
+
+```{literalinclude} cross.Dockerfile
+---
+language: Dockerfile
+---
+```
+
+</details>
+
+<details>
+
+<summary>Dockerfile for building for android-aarch64 on x86_64</summary>
+
+```{literalinclude} cross-android.Dockerfile
+---
+language: Dockerfile
+---
+```
+
+</details>
+
+[crossenv]: https://crossenv.readthedocs.io/
 [fetchcontent]: https://cmake.org/cmake/help/latest/module/FetchContent.html
 [scikit-build-core]: https://scikit-build-core.readthedocs.io

docs/source/howto/cross-android.Dockerfile

@@ -0,0 +1,104 @@
+FROM ubuntu:22.04
+RUN apt-get -y update \
+ && apt-get -y install curl unzip cmake ninja-build openssl xz-utils build-essential libz-dev libssl-dev
+
+ENV BUILD_PREFIX=/opt/build
+ENV PATH=${BUILD_PREFIX}/bin:$PATH
+
+ARG PYTHON_VERSION=3.11.8
+WORKDIR /src
+RUN curl -L -o python.tar.xz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz \
+ && tar -xf python.tar.xz \
+ && rm python.tar.xz \
+ && mv Python-* cpython
+
+# build our 'build' python
+WORKDIR /src/cpython
+RUN ./configure --prefix=${BUILD_PREFIX}
+RUN make -j4
+RUN make install
+
+# sanity check
+RUN python3 -c 'import ssl' \
+ && python3 -m ensurepip \
+ && python3 -m pip install --upgrade pip
+
+# get our cross-compile toolchain from NDK
+WORKDIR /opt
+RUN curl -L -o ndk.zip https://dl.google.com/android/repository/android-ndk-r26c-linux.zip \
+ && unzip ndk.zip \
+ && rm ndk.zip \
+ && mv android-* ndk
+ENV BUILD="x86_64-linux-gnu"
+ENV HOST="aarch64-linux-android34"
+ENV PATH=/opt/ndk/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
+ENV CC=$HOST-clang \
+    CXX=$HOST-clang++ \
+    READELF=llvm-readelf
+
+# build our 'host' python
+WORKDIR /src/cpython
+RUN make clean
+ENV HOST_PREFIX=/opt/host
+RUN ./configure \
+    --prefix=${HOST_PREFIX} \
+    --host=$HOST \
+    --build=$BUILD \
+    --with-build-python=$BUILD_PREFIX/bin/python3 \
+    --without-ensurepip \
+    ac_cv_buggy_getaddrinfo=no \
+    ac_cv_file__dev_ptmx=yes \
+    ac_cv_file__dev_ptc=no
+RUN make -j4
+RUN make install
+
+# (optional) cross-compile libsodium, libzmq
+WORKDIR /src
+ENV LIBSODIUM_VERSION=1.0.19
+RUN curl -L -O "https://download.libsodium.org/libsodium/releases/libsodium-${LIBSODIUM_VERSION}.tar.gz" \
+ && tar -xzf libsodium-${LIBSODIUM_VERSION}.tar.gz \
+ && mv libsodium-stable libsodium \
+ && rm libsodium*.tar.gz
+
+WORKDIR /src/libsodium
+# need CFLAGS or libsodium >= 1.0.20 https://github.com/android/ndk/issues/1945
+ENV CFLAGS="-march=armv8-a+crypto"
+RUN ./configure --prefix="${HOST_PREFIX}" --host=$HOST
+RUN make -j4
+RUN make install
+
+# build libzmq
+WORKDIR /src
+ENV LIBZMQ_VERSION=4.3.5
+RUN curl -L -O "https://github.com/zeromq/libzmq/releases/download/v${LIBZMQ_VERSION}/zeromq-${LIBZMQ_VERSION}.tar.gz" \
+ && tar -xzf zeromq-${LIBZMQ_VERSION}.tar.gz \
+ && mv zeromq-${LIBZMQ_VERSION} zeromq
+WORKDIR /src/zeromq
+RUN ./configure --prefix="$HOST_PREFIX" --host=$HOST --disable-perf --disable-Werror --without-docs --enable-curve --with-libsodium=$HOST_PREFIX --disable-drafts --disable-libsodium_randombytes_close
+RUN make -j4
+RUN make install
+
+# setup crossenv
+ENV CROSS_PREFIX=/opt/cross
+RUN python3 -m pip install crossenv \
+ && python3 -m crossenv ${HOST_PREFIX}/bin/python3 ${CROSS_PREFIX}
+ENV PATH=${CROSS_PREFIX}/bin:$PATH
+
+# install build dependencies in crossenv
+RUN . ${CROSS_PREFIX}/bin/activate \
+ && build-pip install build pyproject_metadata scikit-build-core pathspec cython
+
+ENV ZMQ_PREFIX=${HOST_PREFIX}
+# if pyzmq is bundling libsodium, tell it to cross-compile
+# not required if libzmq is already installed
+ENV PYZMQ_LIBSODIUM_CONFIGURE_ARGS="--host $HOST"
+ARG PYZMQ_VERSION=26.0.0b2
+# build wheel of pyzmq
+WORKDIR /src
+RUN python3 -m pip download --no-binary pyzmq --pre pyzmq==$PYZMQ_VERSION \
+ && tar -xzf pyzmq-*.tar.gz \
+ && rm pyzmq-*.tar.gz \
+ && . ${CROSS_PREFIX}/bin/activate \
+ && cross-python -m build --no-isolation --skip-dependency-check --wheel ./pyzmq*
+
+# there is now a pyzmq wheel in /src/pyzmq-$VERSION/dist/pyzmq-$VERSION-cp311-cp311-linux_aarch64.whl

docs/source/howto/cross.Dockerfile

@@ -0,0 +1,100 @@
+FROM ubuntu:22.04
+RUN apt-get -y update \
+ && apt-get -y install curl unzip cmake ninja-build openssl xz-utils build-essential libz-dev libssl-dev
+
+ENV BUILD_PREFIX=/opt/build
+ENV PATH=${BUILD_PREFIX}/bin:$PATH
+
+ARG PYTHON_VERSION=3.11.8
+WORKDIR /src
+RUN curl -L -o python.tar.xz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz \
+ && tar -xf python.tar.xz \
+ && rm python.tar.xz \
+ && mv Python-* cpython
+
+# build our 'build' python
+WORKDIR /src/cpython
+RUN ./configure --prefix=${BUILD_PREFIX}
+RUN make -j4
+RUN make install
+
+# sanity check
+RUN python3 -c 'import ssl' \
+ && python3 -m ensurepip \
+ && python3 -m pip install --upgrade pip
+
+# get our cross-compile toolchain
+# I'm on aarch64, so use x86_64 as host
+ENV BUILD="aarch64-linux-gnu"
+ENV HOST="x86_64-linux-gnu"
+RUN HOST_PKG=$(echo $HOST | sed s@_@-@g) \
+ && apt-get -y install binutils-$HOST_PKG gcc-$HOST_PKG g++-$HOST_PKG
+ENV CC=$HOST-gcc \
+    CXX=$HOST-g++
+
+# build our 'host' python
+WORKDIR /src/cpython
+RUN make clean
+ENV HOST_PREFIX=/opt/host
+RUN ./configure \
+    --prefix=${HOST_PREFIX} \
+    --host=$HOST \
+    --build=$BUILD \
+    --with-build-python=$BUILD_PREFIX/bin/python3 \
+    --without-ensurepip \
+    ac_cv_buggy_getaddrinfo=no \
+    ac_cv_file__dev_ptmx=yes \
+    ac_cv_file__dev_ptc=no
+RUN make -j4
+RUN make install
+
+WORKDIR /src
+
+# # (optional) cross-compile libsodium, libzmq
+# WORKDIR /src
+# ENV LIBSODIUM_VERSION=1.0.19
+# RUN curl -L -O "https://download.libsodium.org/libsodium/releases/libsodium-${LIBSODIUM_VERSION}.tar.gz" \
+#  && tar -xzf libsodium-${LIBSODIUM_VERSION}.tar.gz \
+#  && mv libsodium-stable libsodium \
+#  && rm libsodium*.tar.gz
+#
+# WORKDIR /src/libsodium
+# RUN ./configure --prefix="${HOST_PREFIX}" --host=$HOST
+# RUN make -j4
+# RUN make install
+#
+# # build libzmq
+# WORKDIR /src
+# ENV LIBZMQ_VERSION=4.3.5
+# RUN curl -L -O "https://github.com/zeromq/libzmq/releases/download/v${LIBZMQ_VERSION}/zeromq-${LIBZMQ_VERSION}.tar.gz" \
+#  && tar -xzf zeromq-${LIBZMQ_VERSION}.tar.gz \
+#  && mv zeromq-${LIBZMQ_VERSION} zeromq
+# WORKDIR /src/zeromq
+# RUN ./configure --prefix="$HOST_PREFIX" --host=$HOST --disable-perf --disable-Werror --without-docs --enable-curve --with-libsodium=$HOST_PREFIX --disable-drafts --disable-libsodium_randombytes_close
+# RUN make -j4
+# RUN make install
+
+# setup crossenv
+WORKDIR /src
+ENV CROSS_PREFIX=/opt/cross
+RUN python3 -m pip install crossenv \
+ && python3 -m crossenv ${HOST_PREFIX}/bin/python3 ${CROSS_PREFIX}
+ENV PATH=${CROSS_PREFIX}/bin:$PATH
+
+# install build dependencies in crossenv
+RUN . ${CROSS_PREFIX}/bin/activate \
+ && build-pip install build pyproject_metadata scikit-build-core pathspec cython
+
+# if pyzmq is bundling libsodium, tell it to cross-compile
+# not required if libzmq is already installed
+ENV PYZMQ_LIBSODIUM_CONFIGURE_ARGS="--host $HOST"
+ARG PYZMQ_VERSION=26.0.0b2
+# build wheel of pyzmq
+WORKDIR /src
+RUN python3 -m pip download --no-binary pyzmq --pre pyzmq==$PYZMQ_VERSION \
+ && tar -xzf pyzmq-*.tar.gz \
+ && rm pyzmq-*.tar.gz \
+ && . ${CROSS_PREFIX}/bin/activate \
+ && cross-python -m build --no-isolation --skip-dependency-check --wheel ./pyzmq*
+
+# there is now a pyzmq wheel in /src/pyzmq-$version/dist/pyzmq-$VERSION-cp311-cp311-linux_x86_64.whl