Port & upstream mypyc wheel build workflow (#3197)

.github/mypyc-requirements.txt

@@ -1,4 +1,4 @@
-mypy == 0.920
+mypy == 0.971
 
 # A bunch of packages for type information
 mypy-extensions >= 0.4.3

.github/workflows/pypi_upload.yml

@@ -1,4 +1,4 @@
-name: pypi_upload
+name: Publish to PyPI
 
 on:
   release:
@@ -8,14 +8,14 @@ permissions:
   contents: read
 
 jobs:
-  build:
-    name: PyPI Upload
+  main:
+    name: sdist + pure wheel
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python
+      - name: Set up latest Python
         uses: actions/setup-python@v4
         with:
           python-version: "*"
@@ -26,11 +26,51 @@ jobs:
           python -m pip install --upgrade build twine
 
       - name: Build wheel and source distributions
-        run: |
-          python -m build
+        run: python -m build
 
       - name: Upload to PyPI via Twine
         env:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          twine upload --verbose -u '__token__' dist/*
+        run: twine upload --verbose -u '__token__' dist/*
+
+  mypyc:
+    name: mypyc wheels (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            name: linux-x86_64
+          - os: windows-2019
+            name: windows-amd64
+          - os: macos-11
+            name: macos-x86_64
+            macos_arch: "x86_64"
+          - os: macos-11
+            name: macos-arm64
+            macos_arch: "arm64"
+          - os: macos-11
+            name: macos-universal2
+            macos_arch: "universal2"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build wheels via cibuildwheel
+        uses: pypa/cibuildwheel@v2.8.1
+        env:
+          CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}"
+          # This isn't supported in pyproject.toml which makes sense (but is annoying).
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6.2"
+
+      - name: Upload wheels as workflow artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.name }}-mypyc-wheels
+          path: ./wheelhouse/*.whl
+
+      - name: Upload wheels to PyPI via Twine
+        env:
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl

pyproject.toml

@@ -29,6 +29,58 @@ preview = true
 requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"]
 build-backend = "setuptools.build_meta"
 
+[tool.cibuildwheel]
+build-verbosity = 1
+# So these are the environments we target:
+# - Python: CPython 3.6+ only
+# - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64
+# - OS: Linux (no musl), Windows, and macOS
+build = "cp3*-*"
+skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"]
+before-build = ["pip install -r .github/mypyc-requirements.txt"]
+# This is the bare minimum needed to run the test suite. Pulling in the full
+# test_requirements.txt would download a bunch of other packages not necessary
+# here and would slow down the testing step a fair bit.
+test-requires = ["pytest>=6.1.1"]
+test-command = 'pytest {project} -k "not incompatible_with_mypyc"'
+test-extras = ["d"," jupyter"]
+# Skip trying to test arm64 builds on Intel Macs. (so cross-compilation doesn't
+# straight up crash)
+test-skip = ["*-macosx_arm64", "*-macosx_universal2:arm64"]
+
+[tool.cibuildwheel.environment]
+BLACK_USE_MYPYC = "1"
+MYPYC_OPT_LEVEL = "3"
+MYPYC_DEBUG_LEVEL = "0"
+# The dependencies required to build wheels with mypyc aren't specified in
+# [build-system].requires so we'll have to manage the build environment ourselves.
+PIP_NO_BUILD_ISOLATION = "no"
+
+[tool.cibuildwheel.linux]
+before-build = [
+    "pip install -r .github/mypyc-requirements.txt",
+    "yum install -y clang",
+]
+# Newer images break the builds, not sure why. We'll need to investigate more later.
+manylinux-x86_64-image = "quay.io/pypa/manylinux2014_x86_64:2021-11-20-f410d11"
+
+[tool.cibuildwheel.linux.environment]
+BLACK_USE_MYPYC = "1"
+MYPYC_OPT_LEVEL = "3"
+MYPYC_DEBUG_LEVEL = "0"
+PIP_NO_BUILD_ISOLATION = "no"
+# Black needs Clang to compile successfully on Linux.
+CC = "clang"
+
+[tool.cibuildwheel.windows]
+# For some reason, (compiled) mypyc is failing to start up with "ImportError: DLL load
+# failed: A dynamic link library (DLL) initialization routine failed." on Windows for
+# at least 3.6. Let's just use interpreted mypy[c].
+# See also: https://github.com/mypyc/mypyc/issues/819.
+before-build = [
+    "pip install -r .github/mypyc-requirements.txt --no-binary mypy"
+]
+
 [tool.isort]
 atomic = true
 profile = "black"

setup.py

@@ -67,7 +67,10 @@ def find_python_files(base: Path) -> List[Path]:
     ]
 
     opt_level = os.getenv("MYPYC_OPT_LEVEL", "3")
-    ext_modules = mypycify(mypyc_targets, opt_level=opt_level, verbose=True)
+    debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "3")
+    ext_modules = mypycify(
+        mypyc_targets, opt_level=opt_level, debug_level=debug_level, verbose=True
+    )
 else:
     ext_modules = []