Merge pull request #1846 from gevent/issue1837

Adding musllinux binary wheels and testing.

.github/workflows/ci.yml

@@ -486,37 +486,41 @@ jobs:
           python -c 'import gevent.libuv.loop as CF; assert not CF.libuv.LIBUV_EMBED'
           python -mgevent.tests --second-chance
 
-  manylinux_x86_64:
+  linux_wheels:
     runs-on: ubuntu-latest
-    # We use a regular Python matrix entry to share as much code as possible.
     strategy:
       matrix:
-        python-version: [3.9]
-
+        config:
+          # Python version, docker image, short name
+          - ["3.9", "quay.io/pypa/manylinux2010_x86_64", "manylinux"]
+          - ["3.9", "quay.io/pypa/musllinux_1_1_x86_64", "musllinux"]
+    name: ${{ matrix.config[2] }}
     steps:
       - name: checkout
         uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.config[0] }}
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.config[0] }}
       - name: Cache ~/.ccache
         uses: actions/cache@v2
         with:
           path: ~/.ccache/
-          key: ${{ runner.os }}-ccache_manylinux2-${{ matrix.python-version }}
+          key: ${{ runner.os }}-ccache_${{ matrix.config[2] }}-${{ matrix.config[0] }}
       - name: Get pip cache dir
         id: pip-cache
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: pip cache
         uses: actions/cache@v2
+        # Be careful not to accidentally share caches. The wheel names
+        # may be the same (e.g.,
+        # psutil-5.8.0-cp310-cp310-linux_x86_64.whl) but they aren't
+        # compatible across glibc/musl.
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip_manylinux_x8664-${{ matrix.python-version }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          key: ${{ runner.os }}-pip_${{ matrix.config[2] }}_x8664-${{ matrix.config[0] }}
 
       - name: Update pip
         run: pip install -U pip
@@ -529,13 +533,15 @@ jobs:
         # Unfortunately, this does not include Python 3.10, and the
         # images that include Python 3.10 don't have 2.7. Sigh.
         env:
-          DOCKER_IMAGE: quay.io/pypa/manylinux2010_x86_64
+          DOCKER_IMAGE: ${{ matrix.config[1] }}
+          GEVENT_MANYLINUX_NAME: ${{ matrix.config[2] }}
+
         run: scripts/releases/make-manylinux
       - name: Upload gevent wheels
         uses: actions/upload-artifact@v2
         with:
           path: wheelhouse/*whl
-          name: manylinux_x86_64_wheels.zip
+          name: ${{ matrix.config[2] }}_x86_64_wheels.zip
       - name: Restore pip cache permissions
         run: sudo chown -R $(whoami) ${{ steps.pip-cache.outputs.dir }}
       - name: Publish package to PyPI
@@ -547,6 +553,5 @@ jobs:
           skip_existing: true
           packages_dir: wheelhouse/
 
-
 # TODO:
 # * Use YAML syntax to share snippets, like the old .travis.yml did

docs/changes/1822.misc

@@ -1,4 +1,5 @@
 Build and upload CPython 3.10 binary manylinux wheels.
 
-Stop building and uploading CPython 2.7 binary manylinux wheels.
-Binary wheels for 2.7 continue to be available for Windows and macOS.
+Unfortunately, this required us to stop building and uploading CPython
+2.7 binary manylinux wheels. Binary wheels for 2.7 continue to be
+available for Windows and macOS.

docs/changes/1837.misc

@@ -0,0 +1 @@
+Test and distribute musllinux_1_1 wheels.

scripts/releases/make-manylinux

@@ -78,18 +78,24 @@ if [ -d /gevent -a -d /opt/python ]; then
     fi
     ls -ld /cache
     ls -ld /cache/pip
-    yum -y install libffi-devel
-    # Some images/archs (specificaly 2014_aarch64) don't have ccache;
-    # This also seems to have vanished for manylinux_2010 x64/64 after November 30
-    # 2020 when the OS went EOL and the package repos switched to the "vault"
-    if [ -n "$SLOW_ARM" ]; then
-        # This provides access to ccache for the 2014 image
-        echo Installing epel
-        rpm -Uvh https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm
+    if [ -e /usr/bin/yum ]; then
+        yum -y install libffi-devel
+        # Some images/archs (specificaly 2014_aarch64) don't have ccache;
+        # This also seems to have vanished for manylinux_2010 x64/64 after November 30
+        # 2020 when the OS went EOL and the package repos switched to the "vault"
+        if [ -n "$SLOW_ARM" ]; then
+            # This provides access to ccache for the 2014 image
+            echo Installing epel
+            rpm -Uvh https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm
+        fi
+        yum -y install ccache || export CC=gcc LDSHARED="gcc -shared" LDCXXSHARED="gcc -shared"
+        # On Fedora Rawhide (F33)
+        # yum install python39 python3-devel gcc kernel-devel kernel-headers make diffutils file
+    fi
+    if [ -e /sbin/apk ]; then
+        # the muslinux image
+        apk add --no-cache build-base libffi-dev ccache
     fi
-    yum -y install ccache || export CC=gcc LDSHARED="gcc -shared" LDCXXSHARED="gcc -shared"
-    # On Fedora Rawhide (F33)
-    # yum install python39 python3-devel gcc kernel-devel kernel-headers make diffutils file
 
     echo Current environment
     env | sort
@@ -166,5 +172,5 @@ fi
 # Travis CI and locally we want `-ti`, but github actions doesn't have a TTY, so one
 # or the other of the arguments causes this to fail with 'input device is not a TTY'
 # Pas through whether we're running on github or not to help with caching.
-docker run --rm -e GEVENTSETUP_DISABLE_ARES -e GITHUB_ACTIONS -e GEVENTTEST_SKIP_ALL -e DOCKER_IMAGE -v "$(pwd):/gevent" -v "$LCACHE:/cache" -v "$HOME/.ccache:/ccache" ${DOCKER_IMAGE:-quay.io/pypa/manylinux2010_x86_64} /gevent/scripts/releases/$(basename $0)
+docker run --rm -e GEVENT_MANYLINUX_NAME -e GEVENTSETUP_DISABLE_ARES -e GITHUB_ACTIONS -e GEVENTTEST_SKIP_ALL -e DOCKER_IMAGE -v "$(pwd):/gevent" -v "$LCACHE:/cache" -v "$HOME/.ccache:/ccache" ${DOCKER_IMAGE:-quay.io/pypa/manylinux2010_x86_64} /gevent/scripts/releases/$(basename $0)
 ls -l wheelhouse

src/gevent/subprocess.py

@@ -626,6 +626,9 @@ class Popen(object):
        it may make many system calls and/or trips around the event loop to accomplish this).
        See :issue:`1711`.
 
+    .. versionchanged:: NEXT
+       Added the ``pipesize`` argument for compatibility with Python 3.10.
+       This is ignored on all platforms.
     """
 
     if GenericAlias is not None:
@@ -652,6 +655,8 @@ def __init__(self, args,
                  # Added in 3.9
                  group=None, extra_groups=None, user=None,
                  umask=-1,
+                 # Added in 3.10, but ignored.
+                 pipesize=-1,
                  # gevent additions
                  threadpool=None):
 

src/gevent/testing/patched_tests_setup.py

@@ -19,6 +19,7 @@
 from .sysinfo import RESOLVER_ARES
 from .sysinfo import RESOLVER_DNSPYTHON
 from .sysinfo import RUNNING_ON_CI
+from .sysinfo import RUNNING_ON_MUSLLINUX
 from .sysinfo import RUN_COVERAGE
 
 
@@ -1408,6 +1409,9 @@ def test(*args, **kwargs):
         # This wants two true threads to work, but a CPU bound loop
         # in a greenlet can't be interrupted.
         'test_threading.InterruptMainTests.test_can_interrupt_tight_loops',
+        # We don't currently implement pipesize.
+        'test_subprocess.ProcessTestCase.test_pipesize_default',
+        'test_subprocess.ProcessTestCase.test_pipesizes',
     ]
 
     if TRAVIS:
@@ -1432,6 +1436,13 @@ def test(*args, **kwargs):
 
     ]
 
+if RUNNING_ON_MUSLLINUX:
+    disabled_tests += [
+        # This is supposed to *not* crash, but on the muslilnux image, it
+        # does crash.
+        'test_threading.ThreadingExceptionTests.test_recursion_limit',
+    ]
+
 
 # Now build up the data structure we'll use to actually find disabled tests
 # to avoid a linear scan for every file (it seems the list could get quite large)

src/gevent/testing/sysinfo.py

@@ -121,6 +121,9 @@
 RUNNING_ON_APPVEYOR = os.environ.get('APPVEYOR')
 RUNNING_ON_CI = RUNNING_ON_TRAVIS or RUNNING_ON_APPVEYOR
 RUNNING_ON_MANYLINUX = os.environ.get('GEVENT_MANYLINUX')
+# I'm not sure how to reliably auto-detect this, without
+# importing platform, something we don't want to do.
+RUNNING_ON_MUSLLINUX = os.environ.get('GEVENT_MANYLINUX_NAME') == 'musllinux'
 
 if RUNNING_ON_APPVEYOR:
     # We can't exec corecext on appveyor if we haven't run setup.py in

src/gevent/tests/test__socket.py

@@ -478,6 +478,7 @@ class TestCreateConnection(greentest.TestCase):
 
     def test_refuses(self, **conn_args):
         connect_port = support.find_unused_port()
+
         with self.assertRaisesRegex(
                 socket.error,
                 # We really expect "connection refused". It's unclear
@@ -488,7 +489,9 @@ def test_refuses(self, **conn_args):
                 # use', which makes even less sense. The manylinux
                 # 2010 environment produces 'errno 99 Cannot assign
                 # requested address', which, I guess?
-                'refused|not known|already in use|assign'
+                # Meanwhile, the musllinux_1 environment produces
+                # '[Errno 99] Address not available'
+                'refused|not known|already in use|assign|not available'
         ):
             socket.create_connection(
                 (greentest.DEFAULT_BIND_ADDR, connect_port),