Backport fixes for PyPy (GH-5429) (#5465)

* Fixes for PyPy (GH-5429) Currently require a nightly PyPy build for Py3.9 to support async iteration and finalisation. Avoid PyIter_Next() and call tp_iternext() instead because PyIter_Next() swallows StopIteration exceptions and thus looses the return value. See https://foss.heptapod.net/pypy/pypy/-/issues/3280 See https://foss.heptapod.net/pypy/pypy/-/issues/3935 * change MACOSX_DEPLOYMENT_TARGET to 11.0
cython · Jun 11, 2023 · 51078b3 · 51078b3
1 parent 863e931
commit 51078b3
Show file tree

Hide file tree

Showing 8 changed files with 32 additions and 13 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -96,7 +96,7 @@ jobs:
             allowed_failure: true
             extra_hash: "-allowed_failures"
           - os: ubuntu-20.04
-            python-version: pypy-3.9
+            python-version: pypy-3.9-nightly
             backend: c
             env: { NO_CYTHON_COMPILE: 1 }
             allowed_failure: true
@@ -123,7 +123,7 @@ jobs:
       CCACHE_SLOPPINESS: "pch_defines,time_macros"
       CCACHE_COMPRESS: 1
       CCACHE_MAXSIZE: "200M"
-      MACOSX_DEPLOYMENT_TARGET: "11"
+      MACOSX_DEPLOYMENT_TARGET: "11.0"
 
     steps:
       - name: Checkout repo

diff --git a/Cython/Utility/AsyncGen.c b/Cython/Utility/AsyncGen.c
@@ -202,7 +202,9 @@ __Pyx_async_gen_repr(__pyx_CoroutineObject *o)
 static int
 __Pyx_async_gen_init_hooks(__pyx_PyAsyncGenObject *o)
 {
+#if !CYTHON_COMPILING_IN_PYPY
     PyThreadState *tstate;
+#endif
     PyObject *finalizer;
     PyObject *firstiter;
 
@@ -212,15 +214,22 @@ __Pyx_async_gen_init_hooks(__pyx_PyAsyncGenObject *o)
 
     o->ag_hooks_inited = 1;
 
+#if CYTHON_COMPILING_IN_PYPY
+    finalizer = _PyEval_GetAsyncGenFinalizer();
+#else
     tstate = __Pyx_PyThreadState_Current;
-
     finalizer = tstate->async_gen_finalizer;
+#endif
     if (finalizer) {
         Py_INCREF(finalizer);
         o->ag_finalizer = finalizer;
     }
 
+#if CYTHON_COMPILING_IN_PYPY
+    firstiter = _PyEval_GetAsyncGenFirstiter();
+#else
     firstiter = tstate->async_gen_firstiter;
+#endif
     if (firstiter) {
         PyObject *res;
 #if CYTHON_UNPACK_METHODS

diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
@@ -83,7 +83,7 @@
     #define CYTHON_PEP489_MULTI_PHASE_INIT 1
   #endif
   #undef CYTHON_USE_TP_FINALIZE
-  #define CYTHON_USE_TP_FINALIZE 0
+  #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1 && PYPY_VERSION_NUM >= 0x07030C00)
   #undef CYTHON_USE_DICT_VERSIONS
   #define CYTHON_USE_DICT_VERSIONS 0
   #undef CYTHON_USE_EXC_INFO_STACK

diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c
@@ -194,11 +194,11 @@ static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject*
     // We always do a quick slot check because calling PyIter_Check() is so wasteful.
     iternextfunc iternext = Py_TYPE(iterator)->tp_iternext;
     if (likely(iternext)) {
-#if CYTHON_USE_TYPE_SLOTS
+#if CYTHON_USE_TYPE_SLOTS || CYTHON_COMPILING_IN_PYPY
         next = iternext(iterator);
         if (likely(next))
             return next;
-        #if PY_VERSION_HEX >= 0x02070000
+        #if PY_VERSION_HEX >= 0x02070000 && CYTHON_COMPILING_IN_CPYTHON
         if (unlikely(iternext == &_PyObject_NextNotImplemented))
             return NULL;
         #endif

diff --git a/tests/pypy_bugs.txt b/tests/pypy_bugs.txt
@@ -18,7 +18,7 @@ memoryview_in_subclasses
 external_ref_reassignment
 run.exttype_dealloc
 
-# bugs in cpyext
+# bugs in cpyext: PyNumber_InPlacePower with non-None modulus is not supported
 run.special_methods_T561
 run.special_methods_T561_py2
 
@@ -35,4 +35,3 @@ double_dealloc_T796
 run.exceptionrefcount
 run.capiimpl
 run.refcount_in_meth
-
diff --git a/tests/run/async_iter_pep492.pyx b/tests/run/async_iter_pep492.pyx
@@ -211,6 +211,9 @@ cdef class Iterable:
         self.i += 1
         return self.i
 
+def has_getrefcount():
+    import sys
+    return hasattr(sys, "getrefcount")
 
 def test_with_for():
     """
@@ -223,8 +226,9 @@ def test_with_for():
 
     manager = Manager(I)
     iterable = Iterable()
-    mrefs_before = sys.getrefcount(manager)
-    irefs_before = sys.getrefcount(iterable)
+    if has_getrefcount():
+        mrefs_before = sys.getrefcount(manager)
+        irefs_before = sys.getrefcount(iterable)
 
     async def main():
         async with manager:
@@ -235,8 +239,9 @@ def test_with_for():
     run_async(main())
     print(I[0])
 
-    assert sys.getrefcount(manager) == mrefs_before
-    assert sys.getrefcount(iterable) == irefs_before
+    if has_getrefcount():
+        assert sys.getrefcount(manager) == mrefs_before
+        assert sys.getrefcount(iterable) == irefs_before
 
     ##############
 

diff --git a/tests/run/error_pos.srctree b/tests/run/error_pos.srctree
@@ -21,4 +21,4 @@ proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
 _, err = proc.communicate()
 # The error should contain the line number and the line text where the
 # undefined identifier is used.
-assert b'line 3, in init error_pos' and b'abcdefg(line)' in err, err
+assert b'line 3, in init error_pos' in err and b'abcdefg(line)' in err, err
diff --git a/tests/run/test_asyncgen.py b/tests/run/test_asyncgen.py
@@ -46,6 +46,11 @@ def needs_py36_asyncio(f):
     from unittest import skip
     return skip("needs Python 3.6 or later")(f)
 
+def not_pypy(f):
+    if getattr(sys, "pypy_version_info", False):
+        from unittest import skip
+        return skip("cannot run on PyPy due to to finalizer")(f)
+    return f
 
 try:
     from types import coroutine as types_coroutine
@@ -765,6 +770,7 @@ async def run():
         t.cancel()
         self.loop.run_until_complete(asyncio.sleep(0.01))
 
+    @not_pypy
     @needs_py36_asyncio
     def test_async_gen_asyncio_gc_aclose_09(self):
         DONE = 0