Pure-Python approach to closure cell rewriting (#522)

* Pure-Python approach to closure cell rewriting

* Re-add mistakenly removed @skipif(PYPY)

* no-cover the 'should be impossible' lines and the functions that are only defined for their side effects on closure cells

changelog.d/522.change.rst

@@ -0,0 +1 @@
+Slotted classes now use a pure Python mechanism to rewrite the ``__class__`` cell when rebuilding the class, so ``super()`` works even on environments where ``ctypes`` is not installed.

src/attr/_compat.py

@@ -106,7 +106,8 @@ def just_warn(*args, **kw):
         consequences of not setting the cell on Python 2.
         """
         warnings.warn(
-            "Missing ctypes.  Some features like bare super() or accessing "
+            "Running interpreter doesn't sufficiently support code object "
+            "introspection.  Some features like bare super() or accessing "
             "__class__ will not work with slotted classes.",
             RuntimeWarning,
             stacklevel=2,
@@ -124,36 +125,98 @@ def metadata_proxy(d):
         return types.MappingProxyType(dict(d))
 
 
-def import_ctypes():
-    """
-    Moved into a function for testability.
-    """
-    import ctypes
-
-    return ctypes
-
-
 def make_set_closure_cell():
+    """Return a function of two arguments (cell, value) which sets
+    the value stored in the closure cell `cell` to `value`.
     """
-    Moved into a function for testability.
-    """
+    # pypy makes this easy. (It also supports the logic below, but
+    # why not do the easy/fast thing?)
     if PYPY:  # pragma: no cover
 
         def set_closure_cell(cell, value):
             cell.__setstate__((value,))
 
+        return set_closure_cell
+
+    # Otherwise gotta do it the hard way.
+
+    # Create a function that will set its first cellvar to `value`.
+    def set_first_cellvar_to(value):
+        x = value
+        return
+
+        # This function will be eliminated as dead code, but
+        # not before its reference to `x` forces `x` to be
+        # represented as a closure cell rather than a local.
+        def force_x_to_be_a_cell():  # pragma: no cover
+            return x
+
+    try:
+        # Extract the code object and make sure our assumptions about
+        # the closure behavior are correct.
+        if PY2:
+            co = set_first_cellvar_to.func_code
+        else:
+            co = set_first_cellvar_to.__code__
+        if co.co_cellvars != ("x",) or co.co_freevars != ():
+            raise AssertionError  # pragma: no cover
+
+        # Convert this code object to a code object that sets the
+        # function's first _freevar_ (not cellvar) to the argument.
+        args = [co.co_argcount]
+        if not PY2:
+            args.append(co.co_kwonlyargcount)
+        args.extend(
+            [
+                co.co_nlocals,
+                co.co_stacksize,
+                co.co_flags,
+                co.co_code,
+                co.co_consts,
+                co.co_names,
+                co.co_varnames,
+                co.co_filename,
+                co.co_name,
+                co.co_firstlineno,
+                co.co_lnotab,
+                # These two arguments are reversed:
+                co.co_cellvars,
+                co.co_freevars,
+            ]
+        )
+        set_first_freevar_code = types.CodeType(*args)
+
+        def set_closure_cell(cell, value):
+            # Create a function using the set_first_freevar_code,
+            # whose first closure cell is `cell`. Calling it will
+            # change the value of that cell.
+            setter = types.FunctionType(
+                set_first_freevar_code, {}, "setter", (), (cell,)
+            )
+            # And call it to set the cell.
+            setter(value)
+
+        # Make sure it works on this interpreter:
+        def make_func_with_cell():
+            x = None
+
+            def func():
+                return x  # pragma: no cover
+
+            return func
+
+        if PY2:
+            cell = make_func_with_cell().func_closure[0]
+        else:
+            cell = make_func_with_cell().__closure__[0]
+        set_closure_cell(cell, 100)
+        if cell.cell_contents != 100:
+            raise AssertionError  # pragma: no cover
+
+    except Exception:
+        return just_warn
     else:
-        try:
-            ctypes = import_ctypes()
-
-            set_closure_cell = ctypes.pythonapi.PyCell_Set
-            set_closure_cell.argtypes = (ctypes.py_object, ctypes.py_object)
-            set_closure_cell.restype = ctypes.c_int
-        except Exception:
-            # We try best effort to set the cell, but sometimes it's not
-            # possible.  For example on Jython or on GAE.
-            set_closure_cell = just_warn
-    return set_closure_cell
+        return set_closure_cell
 
 
 set_closure_cell = make_set_closure_cell()

tests/test_slots.py

@@ -2,6 +2,7 @@
 Unit tests for slots-related functionality.
 """
 
+import types
 import weakref
 
 import pytest
@@ -411,14 +412,17 @@ def statmethod():
 
         assert D.statmethod() is D
 
-    @pytest.mark.skipif(PYPY, reason="ctypes are used only on CPython")
-    def test_missing_ctypes(self, monkeypatch):
+    @pytest.mark.skipif(PYPY, reason="set_closure_cell always works on PyPy")
+    def test_code_hack_failure(self, monkeypatch):
         """
-        Keeps working if ctypes is missing.
+        Keeps working if function/code object introspection doesn't work
+        on this (nonstandard) interpeter.
 
         A warning is emitted that points to the actual code.
         """
-        monkeypatch.setattr(attr._compat, "import_ctypes", lambda: None)
+        # This is a pretty good approximation of the behavior of
+        # the actual types.CodeType on Brython.
+        monkeypatch.setattr(types, "CodeType", lambda: None)
         func = make_set_closure_cell()
 
         with pytest.warns(RuntimeWarning) as wr:
@@ -427,7 +431,8 @@ def test_missing_ctypes(self, monkeypatch):
         w = wr.pop()
         assert __file__ == w.filename
         assert (
-            "Missing ctypes.  Some features like bare super() or accessing "
+            "Running interpreter doesn't sufficiently support code object "
+            "introspection.  Some features like bare super() or accessing "
             "__class__ will not work with slotted classes.",
         ) == w.message.args