More resources cleanup.

README.rst

@@ -83,9 +83,9 @@ Running Tests
 There are a few different ways to run the tests. To simply run the
 tests on one version of Python during development, try this::
 
-  python setup.py develop
-  cd src/greentest
-  PYTHONPATH=.. python testrunner.py --config known_failures.py
+  (env) $ pip install -e .
+  (env) $ cd src/greentest
+  (env) $ python ./testrunner.py
 
 Before submitting a pull request, it's a good idea to run the tests
 across all supported versions of Python, and to check the code quality
@@ -100,7 +100,7 @@ coverage metrics through the `coverage.py`_ package. That would go
 something like this::
 
   cd src/greentest
-  PYTHONPATH=.. python testrunner.py --config known_failures.py --coverage
+  python testrunner.py --coverage
   coverage combine
   coverage html -i
   <open htmlcov/index.html>

ci-requirements.txt

@@ -26,6 +26,7 @@ psutil
 perf
 # Used in a test
 zope.interface
+requests
 # For viewing README.rst (restview --long-description),
 # CONTRIBUTING.rst, etc.
 # https://github.com/mgedmin/restview

examples/concurrent_download.py

@@ -9,26 +9,21 @@
 # patches stdlib (including socket and ssl modules) to cooperate with other greenlets
 monkey.patch_all()
 
-import sys
+import requests
 
-# Note that all of these redirect to HTTPS, so
+# Note that we're using HTTPS, so
 # this demonstrates that SSL works.
 urls = [
-    'http://www.google.com',
-    'http://www.apple.com',
-    'http://www.python.org'
+    'https://www.google.com/',
+    'https://www.apple.com/',
+    'https://www.python.org/'
 ]
 
 
-if sys.version_info[0] == 3:
-    from urllib.request import urlopen # pylint:disable=import-error,no-name-in-module
-else:
-    from urllib2 import urlopen # pylint: disable=import-error
-
 
 def print_head(url):
     print('Starting %s' % url)
-    data = urlopen(url).read()
+    data = requests.get(url).text
     print('%s: %s bytes: %r' % (url, len(data), data[:50]))
 
 jobs = [gevent.spawn(print_head, _url) for _url in urls]

src/gevent/_compat.py

@@ -38,10 +38,13 @@ def reraise(t, value, tb=None): # pylint:disable=unused-argument
         if value.__traceback__ is not tb and tb is not None:
             raise value.with_traceback(tb)
         raise value
+    def exc_clear():
+        pass
 
 else:
     from gevent._util_py2 import reraise # pylint:disable=import-error,no-name-in-module
     reraise = reraise # export
+    exc_clear = sys.exc_clear
 
 ## Functions
 if PY3:

src/gevent/socket.py

@@ -13,8 +13,8 @@
 # Our import magic sadly makes this warning useless
 # pylint: disable=undefined-variable
 
-import sys
 from gevent._compat import PY3
+from gevent._compat import exc_clear
 from gevent._util import copy_globals
 
 
@@ -60,16 +60,21 @@ def getfqdn(*args):
 
 
 def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
-    """Connect to *address* and return the socket object.
+    """
+    create_connection(address, timeout=None, source_address=None) -> socket
+
+    Connect to *address* and return the :class:`gevent.socket.socket`
+    object.
 
-    Convenience function.  Connect to *address* (a 2-tuple ``(host,
-    port)``) and return the socket object.  Passing the optional
+    Convenience function. Connect to *address* (a 2-tuple ``(host,
+    port)``) and return the socket object. Passing the optional
     *timeout* parameter will set the timeout on the socket instance
-    before attempting to connect.  If no *timeout* is supplied, the
-    global default timeout setting returned by :func:`getdefaulttimeout`
-    is used. If *source_address* is set it must be a tuple of (host, port)
-    for the socket to bind as a source address before making the connection.
-    A host of '' or port 0 tells the OS to use the default.
+    before attempting to connect. If no *timeout* is supplied, the
+    global default timeout setting returned by
+    :func:`getdefaulttimeout` is used. If *source_address* is set it
+    must be a tuple of (host, port) for the socket to bind as a source
+    address before making the connection. A host of '' or port 0 tells
+    the OS to use the default.
     """
 
     host, port = address
@@ -102,12 +107,7 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=N
             # because _socket.socket.connect() is a built-in. this is
             # similar to "getnameinfo loses a reference" failure in
             # test_socket.py
-            try:
-                c = sys.exc_clear
-            except AttributeError:
-                pass # Python 3 doesn't have this
-            else:
-                c()
+            exc_clear()
         except BaseException:
             # Things like GreenletExit,  Timeout and KeyboardInterrupt.
             # These get raised immediately, being sure to
@@ -117,7 +117,10 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=N
             sock = None
             raise
         else:
-            return sock
+            try:
+                return sock
+            finally:
+                sock = None
 
 
 # This is promised to be in the __all__ of the _source, but, for circularity reasons,

src/greentest/greentest/util.py

@@ -121,7 +121,7 @@ def getname(command, env=None, setenv=None):
     return ' '.join(result)
 
 
-def start(command, **kwargs):
+def start(command, quiet=False, **kwargs):
     timeout = kwargs.pop('timeout', None)
     preexec_fn = None
     if not os.environ.get('DO_NOT_SETPGRP'):
@@ -138,7 +138,8 @@ def start(command, **kwargs):
             env = os.environ.copy()
         env.update(setenv)
 
-    log('+ %s', name)
+    if not quiet:
+        log('+ %s', name)
     popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs)
     popen.name = name
     popen.setpgrp_enabled = preexec_fn is not None
@@ -176,11 +177,12 @@ def run(command, **kwargs):
     buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT)
     quiet = kwargs.pop('quiet', QUIET)
     verbose = not quiet
+    nested = kwargs.pop('nested', False)
     if buffer_output:
         assert 'stdout' not in kwargs and 'stderr' not in kwargs, kwargs
         kwargs['stderr'] = subprocess.STDOUT
         kwargs['stdout'] = subprocess.PIPE
-    popen = start(command, **kwargs)
+    popen = start(command, quiet=nested, **kwargs)
     name = popen.name
     try:
         time_start = time.time()
@@ -204,7 +206,7 @@ def run(command, **kwargs):
                 log('| %s\n%s', name, out)
         if result:
             log('! %s [code %s] [took %.1fs]', name, result, took)
-        else:
+        elif not nested:
             log('- %s [took %.1fs]', name, took)
     if took >= MIN_RUNTIME:
         runtimelog.append((-took, name))

src/greentest/test___monkey_patching.py

@@ -6,7 +6,6 @@
 # subprocess: include in subprocess tests
 
 from greentest import util
-from greentest.sysinfo import PY2
 
 TIMEOUT = 120
 directory = '%s.%s' % sys.version_info[:2]
@@ -53,8 +52,10 @@ def TESTRUNNER(tests=None):
             # debug produces resource tracking warnings for the
             # CFFI backends. On Python 2, many of the stdlib tests
             # rely on refcounting to close sockets so they produce
-            # lots of noise
-            'GEVENT_DEBUG': 'error' if PY2 else 'debug',
+            # lots of noise. Python 3 is not completely immune;
+            # test_ftplib.py tends to produce warnings---and the Python 3
+            # test framework turns those into test failures!
+            'GEVENT_DEBUG': 'error',
         }
     }
 

src/greentest/test__examples.py

@@ -3,54 +3,59 @@
 import glob
 import time
 
+import greentest
 from greentest import util
 
 
 cwd = '../../examples/'
-ignore = ['wsgiserver.py',
-          'wsgiserver_ssl.py',
-          'webproxy.py',
-          'webpy.py',
-          'unixsocket_server.py',
-          'unixsocket_client.py',
-          'psycopg2_pool.py',
-          'geventsendfile.py']
+ignore = [
+    'wsgiserver.py',
+    'wsgiserver_ssl.py',
+    'webproxy.py',
+    'webpy.py',
+    'unixsocket_server.py',
+    'unixsocket_client.py',
+    'psycopg2_pool.py',
+    'geventsendfile.py',
+]
 ignore += [x[14:] for x in glob.glob('test__example_*.py')]
 
 default_time_range = (2, 4)
 time_ranges = {
     'concurrent_download.py': (0, 30),
-    'processes.py': (0, 4)}
+    'processes.py': (0, 4)
+}
 
+class _AbstractTestMixin(object):
+    time_range = (2, 4)
+    filename = None
 
-def main(tests=None):
-    if not tests:
-        tests = set(os.path.basename(x) for x in glob.glob(cwd + '/*.py'))
-        tests = sorted(tests)
-
-    failed = []
-
-    for filename in tests:
-        if filename in ignore:
-            continue
-        min_time, max_time = time_ranges.get(filename, default_time_range)
-
+    def test_runs(self):
         start = time.time()
-        if util.run([sys.executable, '-u', filename], timeout=max_time, cwd=cwd):
-            failed.append(filename)
+        min_time, max_time = self.time_range
+        if util.run([sys.executable, '-u', self.filename],
+                    timeout=max_time,
+                    cwd=cwd,
+                    quiet=True,
+                    buffer_output=True,
+                    nested=True,
+                    setenv={'GEVENT_DEBUG': 'error'}):
+            self.fail("Failed example: " + self.filename)
         else:
             took = time.time() - start
-            if took < min_time:
-                util.log('! Failed example %s: exited too quickly, after %.1fs (expected %.1fs)', filename, took, min_time)
-                failed.append(filename)
-
-    if failed:
-        util.log('! Failed examples:\n! - %s', '\n! - '.join(failed))
-        sys.exit(1)
-
-    if not tests:
-        sys.exit('No tests.')
-
+            self.assertGreaterEqual(took, min_time)
+
+for filename in glob.glob(cwd + '/*.py'):
+    bn = os.path.basename(filename)
+    if bn in ignore:
+        continue
+    tc = type('Test_' + bn,
+              (_AbstractTestMixin, greentest.TestCase),
+              {
+                  'filename': bn,
+                  'time_range': time_ranges.get(bn, _AbstractTestMixin.time_range)
+              })
+    locals()[tc.__name__] = tc
 
 if __name__ == '__main__':
-    main()
+    greentest.main()

src/greentest/test__pywsgi.py

@@ -199,7 +199,7 @@ def read(cls, fd, code=200, reason='default', version='1.1',
 
 
 class TestCase(greentest.TestCase):
-
+    server = None
     validator = staticmethod(validator)
     application = None
 
@@ -248,12 +248,10 @@ def _tearDownCloseOnTearDown(self):
 
     def tearDown(self):
         greentest.TestCase.tearDown(self)
-        timeout = gevent.Timeout.start_new(0.5)
-        try:
+        with gevent.Timeout.start_new(0.5):
             self.server.stop()
-        finally:
-            timeout.close()
-        # XXX currently listening socket is kept open in gevent.wsgi
+        self.server = None
+
 
     def connect(self):
         conn = socket.create_connection((self.connect_addr, self.port))
@@ -787,9 +785,13 @@ class TestNonLatin1HeaderFromApplication(TestCase):
     header = b'\xe1\xbd\x8a3' # bomb in utf-8 bytes
     should_error = PY3 # non-native string under Py3
 
-    def init_server(self, application):
-        TestCase.init_server(self, application)
-        self.errors = list()
+    def setUp(self):
+        super(TestNonLatin1HeaderFromApplication, self).setUp()
+        self.errors = []
+
+    def tearDown(self):
+        self.errors = []
+        super(TestNonLatin1HeaderFromApplication, self).tearDown()
 
     def application(self, environ, start_response):
         # We return a header that cannot be encoded in latin-1
@@ -1287,6 +1289,11 @@ class TestLeakInput(TestCase):
     _leak_wsgi_input = None
     _leak_environ = None
 
+    def tearDown(self):
+        TestCase.tearDown(self)
+        self._leak_wsgi_input = None
+        self._leak_environ = None
+
     def application(self, environ, start_response):
         pi = environ["PATH_INFO"]
         self._leak_wsgi_input = environ["wsgi.input"]
@@ -1302,13 +1309,13 @@ def test_connection_close_leak_simple(self):
         fd = self.connect().makefile(bufsize=1)
         fd.write(b"GET / HTTP/1.0\r\nConnection: close\r\n\r\n")
         d = fd.read()
-        assert d.startswith(b"HTTP/1.1 200 OK"), "bad response: %r" % d
+        self.assertTrue(d.startswith(b"HTTP/1.1 200 OK"), d)
 
     def test_connection_close_leak_frame(self):
         fd = self.connect().makefile(bufsize=1)
         fd.write(b"GET /leak-frame HTTP/1.0\r\nConnection: close\r\n\r\n")
         d = fd.read()
-        assert d.startswith(b"HTTP/1.1 200 OK"), "bad response: %r" % d
+        self.assertTrue(d.startswith(b"HTTP/1.1 200 OK"), d)
         self._leak_environ.pop('_leak')
 
 class TestHTTPResponseSplitting(TestCase):
@@ -1326,6 +1333,10 @@ def setUp(self):
         self.status = TestHTTPResponseSplitting.status
         self.headers = TestHTTPResponseSplitting.headers
 
+    def tearDown(self):
+        TestCase.tearDown(self)
+        self.start_exc = None
+
     def application(self, environ, start_response):
         try:
             start_response(self.status, self.headers)