-
Notifications
You must be signed in to change notification settings - Fork 244
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #318 from python-greenlet/gevent-1909
Save/restore the trash_delete_nesting; add tests for this.
- Loading branch information
Showing
8 changed files
with
255 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| # -*- coding: utf-8 -*- | ||
| """ | ||
| Tests for greenlets interacting with the CPython trash can API. | ||
| The CPython trash can API is not designed to be re-entered from a | ||
| single thread. But this can happen using greenlets, if something | ||
| during the object deallocation process switches greenlets, and this second | ||
| greenlet then causes the trash can to get entered again. Here, we do this | ||
| very explicitly, but in other cases (like gevent) it could be arbitrarily more | ||
| complicated: for example, a weakref callback might try to acquire a lock that's | ||
| already held by another greenlet; that would allow a greenlet switch to occur. | ||
| See https://github.com/gevent/gevent/issues/1909 | ||
| This test is fragile and relies on details of the CPython | ||
| implementation (like most of the rest of this package): | ||
| - We enter the trashcan and deferred deallocation after | ||
| ``_PyTrash_UNWIND_LEVEL`` calls. This constant, defined in | ||
| CPython's object.c, is generally 50. That's basically how many objects are required to | ||
| get us into the deferred deallocation situation. | ||
| - The test fails by hitting an ``assert()`` in object.c; if the | ||
| build didn't enable assert, then we don't catch this. | ||
| - If the test fails in that way, the interpreter crashes. | ||
| """ | ||
| from __future__ import print_function, absolute_import, division | ||
|
|
||
| import sys | ||
| import unittest | ||
|
|
||
|
|
||
|
|
||
| class TestTrashCanReEnter(unittest.TestCase): | ||
|
|
||
| @unittest.skipUnless( | ||
| sys.version_info[0] > 2, | ||
| "Python 2 tracks this slightly differently, so our test doesn't catch a problem there. " | ||
| ) | ||
| def test_it(self): | ||
| # Try several times to trigger it, because it isn't 100% | ||
| # reliable. | ||
| for _ in range(10): | ||
| self.check_it() | ||
|
|
||
| def check_it(self): # pylint:disable=too-many-statements | ||
| import greenlet | ||
| from greenlet._greenlet import get_tstate_trash_delete_nesting # pylint:disable=no-name-in-module | ||
|
|
||
| main = greenlet.getcurrent() | ||
|
|
||
| assert get_tstate_trash_delete_nesting() == 0 | ||
|
|
||
| # We expect to be in deferred deallocation after this many | ||
| # deallocations have occurred. TODO: I wish we had a better way to do | ||
| # this --- that was before get_tstate_trash_delete_nesting; perhaps | ||
| # we can use that API to do better? | ||
| TRASH_UNWIND_LEVEL = 50 | ||
| # How many objects to put in a container; it's the container that | ||
| # queues objects for deferred deallocation. | ||
| OBJECTS_PER_CONTAINER = 500 | ||
|
|
||
| class Dealloc: # define the class here because we alter class variables each time we run. | ||
| """ | ||
| An object with a ``__del__`` method. When it starts getting deallocated | ||
| from a deferred trash can run, it switches greenlets, allocates more objects | ||
| which then also go in the trash can. If we don't save state appropriately, | ||
| nesting gets out of order and we can crash the interpreter. | ||
| """ | ||
|
|
||
| #: Has our deallocation actually run and switched greenlets? | ||
| #: When it does, this will be set to the current greenlet. This should | ||
| #: be happening in the main greenlet, so we check that down below. | ||
| SPAWNED = False | ||
|
|
||
| #: Has the background greenlet run? | ||
| BG_RAN = False | ||
|
|
||
| BG_GLET = None | ||
|
|
||
| #: How many of these things have ever been allocated. | ||
| CREATED = 0 | ||
|
|
||
| #: How many of these things have ever been deallocated. | ||
| DESTROYED = 0 | ||
|
|
||
| #: How many were destroyed not in the main greenlet. There should always | ||
| #: be some. | ||
| #: If the test is broken or things change in the trashcan implementation, | ||
| #: this may not be correct. | ||
| DESTROYED_BG = 0 | ||
|
|
||
| def __init__(self, sequence_number): | ||
| """ | ||
| :param sequence_number: The ordinal of this object during | ||
| one particular creation run. This is used to detect (guess, really) | ||
| when we have entered the trash can's deferred deallocation. | ||
| """ | ||
| self.i = sequence_number | ||
| Dealloc.CREATED += 1 | ||
|
|
||
| def __del__(self): | ||
| if self.i == TRASH_UNWIND_LEVEL and not self.SPAWNED: | ||
| Dealloc.SPAWNED = greenlet.getcurrent() | ||
| other = Dealloc.BG_GLET = greenlet.greenlet(background_greenlet) | ||
| x = other.switch() | ||
| assert x == 42 | ||
| # It's important that we don't switch back to the greenlet, | ||
| # we leave it hanging there in an incomplete state. But we don't let it | ||
| # get collected, either. If we complete it now, while we're still | ||
| # in the scope of the initial trash can, things work out and we | ||
| # don't see the problem. We need this greenlet to complete | ||
| # at some point in the future, after we've exited this trash can invocation. | ||
| del other | ||
| elif self.i == 40 and greenlet.getcurrent() is not main: | ||
| Dealloc.BG_RAN = True | ||
| try: | ||
| main.switch(42) | ||
| except greenlet.GreenletExit as ex: | ||
| # We expect this; all references to us go away | ||
| # while we're still running, and we need to finish deleting | ||
| # ourself. | ||
| Dealloc.BG_RAN = type(ex) | ||
| del ex | ||
|
|
||
| # Record the fact that we're dead last of all. This ensures that | ||
| # we actually get returned too. | ||
| Dealloc.DESTROYED += 1 | ||
| if greenlet.getcurrent() is not main: | ||
| Dealloc.DESTROYED_BG += 1 | ||
|
|
||
|
|
||
| def background_greenlet(): | ||
| # We direct through a second function, instead of | ||
| # directly calling ``make_some()``, so that we have complete | ||
| # control over when these objects are destroyed: we need them | ||
| # to be destroyed in the context of the background greenlet | ||
| t = make_some() | ||
| del t # Triggere deletion. | ||
|
|
||
| def make_some(): | ||
| t = () | ||
| i = OBJECTS_PER_CONTAINER | ||
| while i: | ||
| # Nest the tuples; it's the recursion that gets us | ||
| # into trash. | ||
| t = (Dealloc(i), t) | ||
| i -= 1 | ||
| return t | ||
|
|
||
|
|
||
| some = make_some() | ||
| self.assertEqual(Dealloc.CREATED, OBJECTS_PER_CONTAINER) | ||
| self.assertEqual(Dealloc.DESTROYED, 0) | ||
|
|
||
| # If we're going to crash, it should be on the following line. | ||
| # We only crash if ``assert()`` is enabled, of course. | ||
| del some | ||
|
|
||
| # For non-debug builds of CPython, we won't crash. The best we can do is check | ||
| # the nesting level explicitly. | ||
| self.assertEqual(0, get_tstate_trash_delete_nesting()) | ||
|
|
||
| # Discard this, raising GreenletExit into where it is waiting. | ||
| Dealloc.BG_GLET = None | ||
| # The same nesting level maintains. | ||
| self.assertEqual(0, get_tstate_trash_delete_nesting()) | ||
|
|
||
| # We definitely cleaned some up in the background | ||
| self.assertGreater(Dealloc.DESTROYED_BG, 0) | ||
|
|
||
| # Make sure all the cleanups happened. | ||
| self.assertIs(Dealloc.SPAWNED, main) | ||
| self.assertTrue(Dealloc.BG_RAN) | ||
| self.assertEqual(Dealloc.BG_RAN, greenlet.GreenletExit) | ||
| self.assertEqual(Dealloc.CREATED, Dealloc.DESTROYED ) | ||
| self.assertEqual(Dealloc.CREATED, OBJECTS_PER_CONTAINER * 2) | ||
|
|
||
| import gc | ||
| gc.collect() | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main() |