Github Actions parallel builds, python 3.10 CI, pip wheel caching and other minor fixes

.github/workflows/build.yml

@@ -0,0 +1,117 @@
+name: Build
+
+# Run CI only when a release is created, on changes to main branch, or any PR 
+# to main. Do not run CI on any other branch. Also, skip any non-source changes 
+# from running on CI
+on:
+  release:
+    types: [created]
+  push:
+    branches: main
+    paths-ignore:
+    - 'docs/**'
+    - 'examples/**'
+    - '.gitignore'
+    - 'README.rst'
+    - '.github/workflows/manylinux.yml'
+    - '.github/workflows/sdl1-sdist.yml'
+
+  pull_request:
+    branches: main
+    paths-ignore:
+    - 'docs/**'
+    - 'examples/**'
+    - '.gitignore'
+    - 'README.rst'
+    - '.github/workflows/manylinux.yml'
+    - '.github/workflows/sdl1-sdist.yml'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false  # if a particular matrix build fails, don't skip the rest
+      matrix:
+        os: [windows-latest, macos-10.15]
+
+        # seperate into matrix for concurrency. 
+        pyver: ['cp27', 'cp35', 'cp36', 'cp37', 'cp38', 'cp39', 'cp310', 'pp27', 'pp36', 'pp37']
+
+        # exclude py2 builds on windows, because they don't work due to the
+        # fact that py2 builds use old compiler. There is a workaround for this
+        # but is not implemented here because Appveyor can already handle py2
+        # on windows
+        exclude:
+          - os: windows-latest
+            pyver: 'cp27'
+          - os: windows-latest
+            pyver: 'pp27'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # use cibuildwheel v1 to build on older python versions
+      - name: Build wheels with CIBuildWheel v1
+        if: matrix.pyver == 'cp27' || matrix.pyver == 'cp35' || matrix.pyver == 'pp27' || matrix.pyver == 'pp36'
+        uses: pypa/cibuildwheel@v1.12.0
+        env:
+          SDL_VIDEODRIVER: "dummy"
+          SDL_AUDIODRIVER: "disk"
+
+          CIBW_BUILD: ${{ matrix.pyver }}-*
+
+          # Install some dependencies
+          CIBW_BEFORE_BUILD_MACOS: pip install requests numpy && brew install sdl2 sdl2_image sdl2_mixer sdl2_ttf zlib freetype portmidi
+          CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy
+
+          CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300
+
+          # Increase pip debugging output
+          CIBW_BUILD_VERBOSITY: 2
+
+      # use latest cibuildwheel for newer python versions
+      - name: Build wheels with CIBuildWheel v2
+        if: matrix.pyver != 'cp27' && matrix.pyver != 'cp35' && matrix.pyver != 'pp27' && matrix.pyver != 'pp36'
+        uses: pypa/cibuildwheel@v2.1.2
+        env:
+          SDL_VIDEODRIVER: "dummy"
+          SDL_AUDIODRIVER: "disk"
+
+          CIBW_BUILD: ${{ matrix.pyver }}-*
+
+          # Build universal2 wheels too on an Intel runner.
+          # Note that the `arm64` part of the `universal2` wheel cannot be 
+          # tested in this configuration.
+          # uncomment when arm builds are working
+          # CIBW_ARCHS_MACOS: "x86_64 universal2"
+
+          # Install some dependencies
+          CIBW_BEFORE_BUILD_MACOS: pip install requests numpy && brew install sdl2 sdl2_image sdl2_mixer sdl2_ttf zlib freetype portmidi
+          CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy
+
+          CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300
+
+          # Increase pip debugging output
+          CIBW_BUILD_VERBOSITY: 2
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: pygame-wheels
+          path: ./wheelhouse/*.whl
+
+#   - name: Upload binaries to Github Releases
+#     if: github.event_name == 'release'
+#     uses: svenstaro/upload-release-action@v2
+#     with:
+#       repo_token: ${{ secrets.GITHUB_TOKEN }}
+#       file: ./wheelhouse/*.whl
+#       tag: ${{ github.ref }}
+#
+#   - name: Upload binaries to PyPI
+#     if: github.event_name == 'release'
+#     env:
+#      TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+#      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+#     run: |
+#       python3 -m pip install twine
+#       twine upload ./wheelhouse/*.whl

.github/workflows/manylinux.yml

@@ -13,36 +13,46 @@ on:
     - 'examples/**'
     - '.gitignore'
     - 'README.rst'
-    - '.github/workflows/macos.yml'
-    - '.github/workflows/windows.yml'
+    - '.github/workflows/build.yml'
     - '.github/workflows/sdl1-sdist.yml'
+
   pull_request:
     branches: main
     paths-ignore:
     - 'docs/**'
     - 'examples/**'
     - '.gitignore'
     - 'README.rst'
-    - '.github/workflows/macos.yml'
-    - '.github/workflows/windows.yml'
+    - '.github/workflows/build.yml'
     - '.github/workflows/sdl1-sdist.yml'
 
-# TODO: Parallelize this build
 jobs:
   build-manylinux:
     runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false  # if a particular matrix build fails, don't skip the rest
+      matrix:
+        arch: ['x64', 'x86', 'aarch64']
+        tag: ['-', '-manylinux-'] # one tag for CPython builds, and another for PyPy
+
+        exclude:
+          # no pypy builds on aarch64 for now
+          - arch: 'aarch64'
+            tag: '-manylinux-'
 
     steps:
     - uses: actions/checkout@v2.3.4
 
+    # setup QEMU only on aarch64 matrix
     - name: Set up QEMU
+      if: ${{ matrix.arch == 'aarch64' }}
       id: qemu
       uses: docker/setup-qemu-action@v1
 
     - name: Build manylinux wheels
       run: |
         cd buildconfig/manylinux-build
-        make pull pull-manylinux wheels wheels-manylinux
+        make pull${{ matrix.tag }}${{ matrix.arch }} wheels${{ matrix.tag }}${{ matrix.arch }}
         cd ../..
         mkdir -p dist/
         cp buildconfig/manylinux-build/wheelhouse/*.whl dist/
@@ -51,6 +61,7 @@ jobs:
     - name: Upload dist
       uses: actions/upload-artifact@v2
       with:
+        name: pygame-manylinux-wheels
         path: dist/*.whl
 
 #   - name: Upload binaries to Github Releases

.github/workflows/sdl1-sdist.yml

@@ -13,18 +13,17 @@ on:
     - 'examples/**'
     - '.gitignore'
     - 'README.rst'
-    - '.github/workflows/macos.yml'
-    - '.github/workflows/windows.yml'
+    - '.github/workflows/build.yml'
     - '.github/workflows/manylinux.yml'
+
   pull_request:
     branches: main
     paths-ignore:
     - 'docs/**'
     - 'examples/**'
     - '.gitignore'
     - 'README.rst'
-    - '.github/workflows/macos.yml'
-    - '.github/workflows/windows.yml'
+    - '.github/workflows/build.yml'
     - '.github/workflows/manylinux.yml'
 
 jobs:
@@ -36,6 +35,8 @@ jobs:
 
     - name: Install deps
       run: |
+        sudo apt-get update --fix-missing
+        sudo apt-get upgrade
         sudo apt-get install python3-dev python3-setuptools python3-numpy python3-opengl libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev libsdl1.2-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev libtiff5-dev libx11-6 libx11-dev fluid-soundfont-gm timgm6mb-soundfont xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic fontconfig fonts-freefont-ttf libfreetype6-dev
         python3 -m pip install -U pip
         python3 -m pip install -U wheel requests
@@ -78,6 +79,7 @@ jobs:
     - name: Upload sdist
       uses: actions/upload-artifact@v2
       with:
+        name: pygame-sdist
         path: dist/*.tar.gz
 
 #   - name: Upload binaries to Github Releases
@@ -95,4 +97,4 @@ jobs:
 #      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
 #     run: |
 #       python3 -m pip install twine
-#       twine upload dist/*.tar.gz
+#       twine upload dist/*.tar.gz