Merge 222ffb2 into 305b60f

.coveragerc

@@ -37,9 +37,13 @@ exclude_lines =
     if sys.platform == 'win32':
     if mswindows:
     if is_windows:
+    if WIN:
+    self.fail
 omit =
     # local.so sometimes gets included, and it can't be parsed
     # as source, so it fails the whole process.
     # coverage 4.5 needs this specified here, 4.4.2 needed it in [run]
     *.so
     /tmp/test_*
+    # Third-party vendored code
+    src/gevent/_tblib.py

CHANGES.rst

@@ -58,6 +58,11 @@
 - Make `gevent.util.assert_switches` produce more informative messages
   when the assertion fails.
 
+- Python 2: If a `gevent.socket` was closed asynchronously (in a
+  different greenlet or a hub callback), `AttributeError` could result
+  if the socket was already in use. Now the correct socket.error
+  should be raised.
+
 1.3.7 (2018-10-12)
 ==================
 

src/gevent/__init__.py

@@ -159,8 +159,10 @@ def __new__(cls, *args, **kwargs):
 
 # the following makes hidden imports visible to freezing tools like
 # py2exe. see https://github.com/gevent/gevent/issues/181
+# This is not well maintained or tested, though, so it likely becomes
+# outdated on each major release.
 
-def __dependencies_for_freezing():
+def __dependencies_for_freezing(): # pragma: no cover
     # pylint:disable=unused-variable
     from gevent import core
     from gevent import resolver_thread

src/gevent/_abstract_linkable.py

@@ -181,7 +181,9 @@ def _wait_core(self, timeout, catch=Timeout):
 
     def _wait_return_value(self, waited, wait_success):
         # pylint:disable=unused-argument
-        return None
+        # Subclasses should override this to return a value from _wait.
+        # By default we return None.
+        return None # pragma: no cover all extent subclasses override
 
     def _wait(self, timeout=None):
         if self.ready():

src/gevent/_hub_primitives.py

@@ -282,6 +282,10 @@ def _primitive_wait(watcher, timeout, timeout_exc, hub):
 
 # Suitable to be bound as an instance method
 def wait_on_socket(socket, watcher, timeout_exc=None):
+    if socket is None or watcher is None:
+        # test__hub TestCloseSocketWhilePolling, on Python 2; Python 3
+        # catches the EBADF differently.
+        raise ConcurrentObjectUseError("The socket has already been closed by another greenlet")
     _primitive_wait(watcher, socket.timeout,
                     timeout_exc if timeout_exc is not None else _NONE,
                     socket.hub)

src/gevent/_socket2.py

@@ -185,10 +185,9 @@ def _set_ref(self, value):
     _wait = _wait_on_socket
 
     def accept(self):
-        sock = self._sock
-        while True:
+        while 1:
             try:
-                client_socket, address = sock.accept()
+                client_socket, address = self._sock.accept()
                 break
             except error as ex:
                 if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
@@ -212,10 +211,17 @@ def _drop_events(self, cancel_wait_ex=cancel_wait_ex):
     def close(self, _closedsocket=_closedsocket):
         # This function should not reference any globals. See Python issue #808164.
 
-        # Also break any reference to the loop.io objects. Our fileno, which they were
-        # tied to, is now free to be reused, so these objects are no longer functional.
+        # Also break any reference to the loop.io objects. Our fileno,
+        # which they were tied to, is now free to be reused, so these
+        # objects are no longer functional.
         self._drop_events()
         s = self._sock
+
+        # Note that we change self._sock at this point. Methods *must not*
+        # cache `self._sock` separately from self._write_event/self._read_event,
+        # or they will be out of sync and we may get inappropriate errors.
+        # (See test__hub:TestCloseSocketWhilePolling for an example).
+
         self._sock = _closedsocket()
         if PYPY:
             s._drop()
@@ -227,16 +233,16 @@ def closed(self):
     def connect(self, address):
         if self.timeout == 0.0:
             return self._sock.connect(address)
-        sock = self._sock
-        address = _socketcommon._resolve_addr(sock, address)
+
+        address = _socketcommon._resolve_addr(self._sock, address)
 
         timer = Timeout._start_new_or_dummy(self.timeout, timeout('timed out'))
         try:
-            while True:
-                err = sock.getsockopt(SOL_SOCKET, SO_ERROR)
+            while 1:
+                err = self._sock.getsockopt(SOL_SOCKET, SO_ERROR)
                 if err:
                     raise error(err, strerror(err))
-                result = sock.connect_ex(address)
+                result = self._sock.connect_ex(address)
                 if not result or result == EISCONN:
                     break
                 elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
@@ -283,10 +289,9 @@ def makefile(self, mode='r', bufsize=-1):
         return fobj
 
     def recv(self, *args):
-        sock = self._sock  # keeping the reference so that fd is not closed during waiting
-        while True:
+        while 1:
             try:
-                return sock.recv(*args)
+                return self._sock.recv(*args)
             except error as ex:
                 if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
                     raise
@@ -295,51 +300,47 @@ def recv(self, *args):
             self._wait(self._read_event)
 
     def recvfrom(self, *args):
-        sock = self._sock
-        while True:
+        while 1:
             try:
-                return sock.recvfrom(*args)
+                return self._sock.recvfrom(*args)
             except error as ex:
                 if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
                     raise
                 sys.exc_clear()
             self._wait(self._read_event)
 
     def recvfrom_into(self, *args):
-        sock = self._sock
-        while True:
+        while 1:
             try:
-                return sock.recvfrom_into(*args)
+                return self._sock.recvfrom_into(*args)
             except error as ex:
                 if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
                     raise
                 sys.exc_clear()
             self._wait(self._read_event)
 
     def recv_into(self, *args):
-        sock = self._sock
-        while True:
+        while 1:
             try:
-                return sock.recv_into(*args)
+                return self._sock.recv_into(*args)
             except error as ex:
                 if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
                     raise
                 sys.exc_clear()
             self._wait(self._read_event)
 
     def send(self, data, flags=0, timeout=timeout_default):
-        sock = self._sock
         if timeout is timeout_default:
             timeout = self.timeout
         try:
-            return sock.send(data, flags)
+            return self._sock.send(data, flags)
         except error as ex:
             if ex.args[0] not in _socketcommon.GSENDAGAIN or timeout == 0.0:
                 raise
             sys.exc_clear()
             self._wait(self._write_event)
             try:
-                return sock.send(data, flags)
+                return self._sock.send(data, flags)
             except error as ex2:
                 if ex2.args[0] == EWOULDBLOCK:
                     return 0
@@ -354,16 +355,15 @@ def sendall(self, data, flags=0):
         return _socketcommon._sendall(self, data_memory, flags)
 
     def sendto(self, *args):
-        sock = self._sock
         try:
-            return sock.sendto(*args)
+            return self._sock.sendto(*args)
         except error as ex:
             if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0:
                 raise
             sys.exc_clear()
             self._wait(self._write_event)
             try:
-                return sock.sendto(*args)
+                return self._sock.sendto(*args)
             except error as ex2:
                 if ex2.args[0] == EWOULDBLOCK:
                     return 0

src/gevent/testing/__init__.py

@@ -68,6 +68,7 @@
 from .skipping import skipOnPyPyOnCI
 from .skipping import skipOnPyPy3
 from .skipping import skipIf
+from .skipping import skipUnless
 from .skipping import skipOnLibev
 from .skipping import skipOnLibuv
 from .skipping import skipOnLibuvOnWin
@@ -122,6 +123,12 @@
 from .flaky import reraises_flaky_timeout
 from .flaky import reraises_flaky_race_condition
 
+def gc_collect_if_needed():
+    "Collect garbage if necessary for destructors to run"
+    import gc
+    if PYPY: # pragma: no cover
+        gc.collect()
+
 try:
     from unittest import mock
 except ImportError: # Python 2

src/gevent/testing/patched_tests_setup.py

@@ -466,6 +466,8 @@ def get_switch_expected(fullname):
             'test_socket.BufferIOTest.testRecvFromIntoBytearray',
             'test_socket.BufferIOTest.testRecvFromIntoArray',
             'test_socket.BufferIOTest.testRecvFromIntoEmptyBuffer',
+            'test_socket.BufferIOTest.testRecvFromIntoMemoryview',
+            'test_socket.BufferIOTest.testRecvFromIntoSmallBuffer',
         ]
 
         if PY3:

src/gevent/testing/skipping.py

@@ -92,6 +92,7 @@ def _do_not_skip(reason):
 skipUnderCoverage = unittest.skip if sysinfo.RUN_COVERAGE else _do_not_skip
 
 skipIf = unittest.skipIf
+skipUnless = unittest.skipUnless
 
 
 

src/gevent/tests/test___monitor.py

@@ -106,7 +106,7 @@ def test_add_monitoring_function(self):
         self.assertRaises(ValueError, self.pmt.add_monitoring_function, lambda: None, -1)
 
         def f():
-            pass
+            "Does nothing"
 
         # Add
         self.pmt.add_monitoring_function(f, 1)

src/gevent/tests/test__environ.py

@@ -8,7 +8,7 @@
     os.environ['GEVENT_BACKEND'] = 'select'
     popen = subprocess.Popen([sys.executable, __file__, '1'])
     assert popen.wait() == 0, popen.poll()
-else:
+else: # pragma: no cover
     hub = gevent.get_hub()
     if 'select' in gevent.core.supported_backends():
         assert hub.loop.backend == 'select', hub.loop.backend

src/gevent/tests/test__exc_info.py

@@ -35,8 +35,8 @@ def test1(self):
             g = gevent.spawn(hello, expected_error)
             g.join()
             self.assert_error(ExpectedError, expected_error)
-            if not isinstance(g.exception, ExpectedError):
-                raise g.exception
+            self.assertIsInstance(g.exception, ExpectedError)
+
             try:
                 raise
             except: # pylint:disable=bare-except

src/gevent/tests/test__greenlet.py

@@ -165,9 +165,6 @@ def return25():
     return 25
 
 
-def sleep0():
-    return sleep(0)
-
 
 class TestReturn_link(LinksTestCase):
     link_method = 'link'
@@ -748,7 +745,7 @@ def get():
             raise ValueError("call stack is not deep enough")
         try:
             ogf = greenlet.sys_getframe
-        except AttributeError:
+        except AttributeError: # pragma: no cover
             # Must be running cython compiled
             raise unittest.SkipTest("Cannot mock when Cython compiled")
         greenlet.sys_getframe = get
@@ -806,7 +803,7 @@ def test_kill_started(self):
 
 
 @greentest.skipOnPurePython("Needs C extension")
-class TestCExt(greentest.TestCase):
+class TestCExt(greentest.TestCase): # pragma: no cover (we only do coverage on pure-Python)
 
     def test_c_extension(self):
         self.assertEqual(greenlet.Greenlet.__module__,

src/gevent/tests/test__greenletset.py

@@ -149,13 +149,11 @@ def f():
         s = set()
         s.add(p1)
         s.add(p2)
-        try:
+        with self.assertRaises(Timeout):
             gevent.killall(s, timeout=0.5)
-        except Timeout:
-            for g in s:
-                assert not g.dead
-        else:
-            self.fail("Should raise timeout")
+
+        for g in s:
+            self.assertFalse(g.dead, g)
 
 
 class GreenletSubclass(gevent.Greenlet):

src/gevent/tests/test__hub.py

@@ -36,10 +36,11 @@
 class TestCloseSocketWhilePolling(greentest.TestCase):
 
     def test(self):
-        with self.assertRaises(Exception):
-            sock = socket.socket()
-            self._close_on_teardown(sock)
-            t = get_hub().loop.timer(0, sock.close)
+        sock = socket.socket()
+        self._close_on_teardown(sock)
+        t = get_hub().loop.timer(0)
+        t.start(sock.close)
+        with self.assertRaises(socket.error):
             try:
                 sock.connect(('python.org', 81))
             finally:

src/gevent/tests/test__issue330.py

@@ -14,7 +14,7 @@ def setUp(self):
         self.switched_to = [False, False]
         self.caught = None
 
-    def runner(self, i):
+    def should_never_run(self, i): # pragma: no cover
         self.switched_to[i] = True
 
     def check(self, g, g2):
@@ -33,25 +33,25 @@ def check(self, g, g2):
 
 
     def test_gevent_kill(self):
-        g = gevent.spawn(self.runner, 0) # create but do not switch to
-        g2 = gevent.spawn(self.runner, 1) # create but do not switch to
+        g = gevent.spawn(self.should_never_run, 0) # create but do not switch to
+        g2 = gevent.spawn(self.should_never_run, 1) # create but do not switch to
         # Using gevent.kill
         gevent.kill(g)
         gevent.kill(g2)
         self.check(g, g2)
 
     def test_greenlet_kill(self):
         # killing directly
-        g = gevent.spawn(self.runner, 0)
-        g2 = gevent.spawn(self.runner, 1)
+        g = gevent.spawn(self.should_never_run, 0)
+        g2 = gevent.spawn(self.should_never_run, 1)
         g.kill()
         g2.kill()
         self.check(g, g2)
 
     def test_throw(self):
         # throwing
-        g = gevent.spawn(self.runner, 0)
-        g2 = gevent.spawn(self.runner, 1)
+        g = gevent.spawn(self.should_never_run, 0)
+        g2 = gevent.spawn(self.should_never_run, 1)
         g.throw(gevent.GreenletExit)
         g2.throw(gevent.GreenletExit)
         self.check(g, g2)

src/gevent/tests/test__issue6.py

@@ -16,7 +16,7 @@
     #    or __package__, falling back on __name__ and __path__\n  return f(*args, **kwds)\n'
     assert err == b'' or b'sys.excepthook' in err or b'Warning' in err, (out, err, code)
 
-elif sys.argv[1:] == ['subprocess']:
+elif sys.argv[1:] == ['subprocess']: # pragma: no cover
     import gevent
     import gevent.monkey
     gevent.monkey.patch_all(sys=True)
@@ -30,5 +30,5 @@ def printline():
 
     gevent.spawn(printline).join()
 
-else:
+else: # pragma: no cover
     sys.exit('Invalid arguments: %r' % (sys.argv, ))