Build wheels on CI with cibuildwheel (#365)

pygeos · Aug 19, 2021 · c6d04df · c6d04df
1 parent a12e511
commit c6d04df
Show file tree

Hide file tree

Showing 26 changed files with 309 additions and 62 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,13 +1,18 @@
-name: Release and publish pygeos to GitHub
+name: Build and publish
 
 on:
   push:
+    branches:
+      - master  # just build the sdist & wheel, skip release
     tags:
       - "*"
+  pull_request:  # also build on PRs touching this file
+    paths:
+      - ".github/workflows/release.yml"
 
 jobs:
-  build-n-publish:
-    name: Build and publish pygeos to GitHub
+  build_sdist:
+    name: Build sdist
     runs-on: ubuntu-latest
 
     steps:
@@ -17,19 +22,134 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.x"
+          python-version: "3.8"
 
       - name: Build a source tarball
         run: |
           python -m pip install --upgrade pip
           pip install setuptools
           python setup.py sdist
 
-      # - name: Publish distribution to PyPI
-      #   uses: pypa/gh-action-pypi-publish@master
-      #   with:
-      #     user: __token__
-      #     password: ${{ secrets.PYPI_API_TOKEN }}
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./dist/*.tar.gz
+          retention-days: 30
+
+  build_wheels:
+    name: Build ${{ matrix.archs }} wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GEOS_VERSION: "3.9.1"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        archs: [auto32, auto64]
+        exclude:
+        # osx doesn't have 32-bit builds
+        - os: macos-10.15
+          archs: auto32
+        include:
+        # Only numpy >= 1.19 has aarch64 wheels
+        - os: ubuntu-20.04
+          archs: aarch64
+        # Numpy doesn't have ppc64le wheels:
+        # - os: ubuntu-20.04
+        #   archs: ppc64le
+        # We know that tests don't pass on s390x:
+        # - os: ubuntu-20.04
+        #   archs: s390x  
+        # On ARM64, the GEOS libraries do not end up in the wheel:
+        # - os: macos-10.15
+        #   archs: arm64
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Cache GEOS build
+        uses: actions/cache@v2
+        with:
+          path: |
+            geos-${{ env.GEOS_VERSION }}
+          key: ${{ matrix.os }}-${{ matrix.archs }}-2
+
+      - name: Add GEOS LICENSE
+        run: |
+          cp ci/wheelbuilder/LICENSE_GEOS .
+        shell: bash
+
+      - name: Add MSVC LICENSE
+        run: |
+          cp ci/wheelbuilder/LICENSE_win32 .
+        shell: bash
+        if: ${{ matrix.os == 'windows-2019' }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+        if: ${{ matrix.os == 'ubuntu-20.04' && matrix.archs == 'aarch64' }}
+
+      - name: Activate MSVC 32-bit
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x86
+        if: ${{ matrix.os == 'windows-2019' && matrix.archs == 'auto32' }}
+
+      - name: Activate MSVC 64-bit
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+        if: ${{ matrix.os == 'windows-2019' && matrix.archs == 'auto64' }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.0.0
+        env:
+          CIBW_ARCHS: ${{ matrix.archs }}
+          CIBW_SKIP: pp* cp36-manylinux_aarch64 cp37-manylinux_aarch64 cp38-manylinux_aarch64
+          CIBW_ENVIRONMENT_LINUX:
+            GEOS_VERSION=${{ env.GEOS_VERSION }}
+            GEOS_INSTALL=/host${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}
+            GEOS_CONFIG=/host${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}/bin/geos-config
+            LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/host${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}/lib
+          CIBW_ENVIRONMENT_MACOS:
+            GEOS_INSTALL=${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}
+            GEOS_CONFIG=${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}/bin/geos-config
+            DYLD_FALLBACK_LIBRARY_PATH=${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}/lib
+            LDFLAGS=-Wl,-rpath,${{ github.workspace }}/geos-${{ env.GEOS_VERSION }}/lib
+          CIBW_ENVIRONMENT_WINDOWS:
+            GEOS_INSTALL='${{ github.workspace }}\geos-${{ env.GEOS_VERSION }}'
+            GEOS_LIBRARY_PATH='${{ github.workspace }}\geos-${{ env.GEOS_VERSION }}\lib'
+            GEOS_INCLUDE_PATH='${{ github.workspace }}\geos-${{ env.GEOS_VERSION }}\include'
+          CIBW_BEFORE_ALL: ./ci/install_geos.sh
+          CIBW_BEFORE_ALL_WINDOWS: ci\install_geos.cmd
+          CIBW_BEFORE_BUILD_WINDOWS: pip install delvewheel
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: delvewheel repair --add-path ${{ github.workspace }}\geos-${{ env.GEOS_VERSION }}\bin -w {dest_dir} {wheel}
+          CIBW_TEST_REQUIRES: pytest
+          # The pytest command has the custom setting import-mode=importlib
+          # so that pytest does not mess up the PATH.
+          # If you do not do that, the (uncompiled) contents of
+          # {package}/pygeos will be tested against instead of the
+          # installed contents of the wheel that was built.
+          CIBW_TEST_COMMAND: pytest --pyargs pygeos.tests
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+          retention-days: 5
+
+  publish:
+    name: Publish on GitHub and PyPI
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    # release on every tag
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
 
       - name: Create GitHub Release
         id: create_release
@@ -57,4 +177,11 @@ jobs:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
           asset_path: dist/${{ env.name }}
           asset_name: ${{ env.name }}
-          asset_content_type: application/zip
+          asset_content_type: application/zip      
+
+      - name: Upload Release Assets to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_password }}
+          # To test: repository_url: https://test.pypi.org/legacy/
diff --git a/.gitignore b/.gitignore
@@ -24,6 +24,9 @@ var/
 *.egg
 *.log
 *.whl
+# to make binary distributions (that pack additional licenses) not 'dirty':
+/LICENSE_GEOS
+/LICENSE_win32
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -4,11 +4,21 @@ Changelog
 Version 0.11 (unreleased)
 -------------------------
 
-**Major enhancements**
+**Distribution**
 
-**API Changes**
+Unittests are now included in the pygeos distribution. Run them by 1) installing
+``pytest`` (or ``pygeos[test]``) and invoking ``pytest --pyargs pygeos.tests``.
+
+We started using a new tool for building binary wheels: ``cibuildwheel``. This
+resulted into the following improvements in the distributed binary wheels:
 
-**Added GEOS Functions**
+* Windows binary wheels now contain mangled DLLs, which avoids conflicts
+  with other GEOS versions present on the system (a.k.a. 'DLL hell') (#365).
+* Windows binary wheels now contain the Microsoft Visual C++ Runtime Files
+  (msvcp140.dll) for usage on systems without the C++ redistributable (#365).
+* Linux x86_64 and i686 wheels are now built using the manylinux2010 image
+  instead of manylinux1 (#365).
+* Linux aarch64 wheels are now available for Python 3.9 (manylinux2014, #365).
 
 **Bug fixes**
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,7 +2,6 @@ include src/*
 include pygeos/*.pyx pygeos/*.pxd
 include versioneer.py
 include pygeos/_version.py
-include pygeos/*.dll
 include LICENSE
 include pyproject.toml
 exclude pygeos/*.c

diff --git a/ci/install_geos.cmd b/ci/install_geos.cmd
@@ -14,7 +14,7 @@ if exist %GEOS_INSTALL% (
   7z x geos-%GEOS_VERSION%.tar
   cd geos-%GEOS_VERSION% || exit /B 1
 
-  pip install ninja
+  pip install ninja cmake
   cmake --version
 
   mkdir build

diff --git a/ci/install_geos.sh b/ci/install_geos.sh
@@ -16,11 +16,13 @@ elif [ -z "$GEOS_VERSION" ]; then
     exit 1
 fi
 
-# Create directories, if they don't exit
+# Create directories, if they don't exist
 mkdir -p $GEOS_INSTALL
 
 # Download and build GEOS outside other source tree
-GEOS_BUILD=$HOME/geosbuild
+if [ -z "$GEOS_BUILD" ]; then
+    GEOS_BUILD=$HOME/geosbuild
+fi
 
 prepare_geos_build_dir(){
   rm -rf $GEOS_BUILD
@@ -29,12 +31,15 @@ prepare_geos_build_dir(){
 }
 
 build_geos(){
+    echo "Installing cmake"
+    pip install cmake
+
     echo "Building geos-$GEOS_VERSION"
     mkdir build
     cd build
     cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$GEOS_INSTALL ..
     make -j 2
-    ctest .
+    # ctest .
     make install
 }
 
@@ -63,7 +68,7 @@ else
         echo "Using cached install $GEOS_INSTALL"
     else
         prepare_geos_build_dir
-        wget -q http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2
+        curl -OL http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2
         tar xfj geos-$GEOS_VERSION.tar.bz2
         cd geos-$GEOS_VERSION
         build_geos

diff --git a/GEOS_LICENSE → ci/wheelbuilder/LICENSE_GEOS b/GEOS_LICENSE → ci/wheelbuilder/LICENSE_GEOS
@@ -1,3 +1,10 @@
+This binary distribution of pygeos also bundles the following software:
+
+Name: Geometry Engine Open Source (GEOS)
+Files: libgeos-*.so.*, libgeos_c-*.so.*, libgeos.dylib, libgeos_c.dylib, geos-*.dll, geos_c-*.dll
+Availability: https://trac.osgeo.org/geos/
+License: LGPLv2.1
+
                   GNU LESSER GENERAL PUBLIC LICENSE
                        Version 2.1, February 1999
 

diff --git a/ci/wheelbuilder/LICENSE_win32 b/ci/wheelbuilder/LICENSE_win32
@@ -0,0 +1,26 @@
+This binary distribution of pygeos also bundles the following software:
+
+Name: Microsoft Visual C++ Runtime Files
+Files: msvcp140.dll
+License: MSVC
+  https://www.visualstudio.com/license-terms/distributable-code-microsoft-visual-studio-2015-rc-microsoft-visual-studio-2015-sdk-rc-includes-utilities-buildserver-files/#visual-c-runtime
+
+  Subject to the License Terms for the software, you may copy and
+  distribute with your program any of the files within the followng
+  folder and its subfolders except as noted below. You may not modify
+  these files.
+
+    C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist
+
+  You may not distribute the contents of the following folders:
+
+    C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist\debug_nonredist
+    C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\redist\onecore\debug_nonredist
+
+  Subject to the License Terms for the software, you may copy and
+  distribute the following files with your program in your program’s
+  application local folder or by deploying them into the Global
+  Assembly Cache (GAC):
+
+  VC\atlmfc\lib\mfcmifc80.dll
+  VC\atlmfc\lib\amd64\mfcmifc80.dll
diff --git a/docs/conf.py b/docs/conf.py
@@ -43,7 +43,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-# html_theme = "sphinx_rtd_theme"
+html_theme = "sphinx_rtd_theme"
 # html_theme_path = ["_themes", ]
 
 # Add any paths that contain custom static files (such as style sheets) here,

diff --git a/docs/environment.yml b/docs/environment.yml
@@ -7,3 +7,4 @@ dependencies:
   - numpy==1.19.*
   - numpydoc==1.1.*
   - Cython==0.29.*
+  - docutils==0.16.*