Merge ae6f90e into 994d3f4

lib/cheroot/_compat.py

@@ -3,10 +3,28 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
+import platform
 import re
 
 import six
 
+try:
+    import ssl
+    IS_ABOVE_OPENSSL10 = ssl.OPENSSL_VERSION_INFO >= (1, 1)
+    del ssl
+except ImportError:
+    IS_ABOVE_OPENSSL10 = None
+
+
+IS_PYPY = platform.python_implementation() == 'PyPy'
+
+
+SYS_PLATFORM = platform.system()
+IS_WINDOWS = SYS_PLATFORM == 'Windows'
+IS_LINUX = SYS_PLATFORM == 'Linux'
+IS_MACOS = SYS_PLATFORM == 'Darwin'
+
+
 if six.PY3:
     def ntob(n, encoding='ISO-8859-1'):
         """Return the native string as bytes in the given encoding."""
@@ -42,10 +60,11 @@ def ntou(n, encoding='ISO-8859-1'):
         # escapes, but without having to prefix it with u'' for Python 2,
         # but no prefix for Python 3.
         if encoding == 'escape':
-            return six.u(
-                re.sub(r'\\u([0-9a-zA-Z]{4})',
-                       lambda m: six.unichr(int(m.group(1), 16)),
-                       n.decode('ISO-8859-1')))
+            return re.sub(
+                r'\\u([0-9a-zA-Z]{4})',
+                lambda m: six.unichr(int(m.group(1), 16)),
+                n.decode('ISO-8859-1'),
+            )
         # Assume it's already in the given encoding, which for ISO-8859-1
         # is almost always what was intended.
         return n.decode(encoding)
@@ -64,3 +83,25 @@ def assert_native(n):
     """
     if not isinstance(n, str):
         raise TypeError('n must be a native str (got %s)' % type(n).__name__)
+
+
+if six.PY3:
+    """Python 3 has memoryview builtin."""
+    # Python 2.7 has it backported, but socket.write() does
+    # str(memoryview(b'0' * 100)) -> <memory at 0x7fb6913a5588>
+    # instead of accessing it correctly.
+    memoryview = memoryview
+else:
+    """Link memoryview to buffer under Python 2."""
+    memoryview = buffer  # noqa: F821
+
+
+def extract_bytes(mv):
+    """Retrieve bytes out of memoryview/buffer or bytes."""
+    if isinstance(mv, memoryview):
+        return mv.tobytes() if six.PY3 else bytes(mv)
+
+    if isinstance(mv, bytes):
+        return mv
+
+    raise ValueError

lib/cheroot/errors.py

@@ -29,10 +29,9 @@ def plat_specific_errors(*errnames):
     the specific platform (OS). This function will return the list of
     numeric values for a given list of potential names.
     """
-    errno_names = dir(errno)
-    nums = [getattr(errno, k) for k in errnames if k in errno_names]
-    # de-dupe the list
-    return list(dict.fromkeys(nums).keys())
+    missing_attr = set([None, ])
+    unique_nums = set(getattr(errno, k, None) for k in errnames)
+    return list(unique_nums - missing_attr)
 
 
 socket_error_eintr = plat_specific_errors('EINTR', 'WSAEINTR')

lib/cheroot/makefile.py

@@ -15,6 +15,11 @@
 import six
 
 from . import errors
+from ._compat import extract_bytes, memoryview
+
+
+# Write only 16K at a time to sockets
+SOCK_WRITE_BLOCKSIZE = 16384
 
 
 class BufferedWriter(io.BufferedWriter):
@@ -64,17 +69,21 @@ def _drop(self):
 
     def write(self, data):
         """Sendall for non-blocking sockets."""
-        while data:
+        bytes_sent = 0
+        data_mv = memoryview(data)
+        payload_size = len(data_mv)
+        while bytes_sent < payload_size:
             try:
-                bytes_sent = self.send(data)
-                data = data[bytes_sent:]
+                bytes_sent += self.send(
+                    data_mv[bytes_sent:bytes_sent + SOCK_WRITE_BLOCKSIZE]
+                )
             except socket.error as e:
                 if e.args[0] not in errors.socket_errors_nonblocking:
                     raise
 
     def send(self, data):
         """Send some part of message to the socket."""
-        bytes_sent = self._sock.send(data)
+        bytes_sent = self._sock.send(extract_bytes(data))
         self.bytes_written += bytes_sent
         return bytes_sent
 

lib/cheroot/server.py

@@ -96,11 +96,34 @@
 
 
 IS_WINDOWS = platform.system() == 'Windows'
+"""Flag indicating whether the app is running under Windows."""
 
 
-if not IS_WINDOWS:
-    import grp
-    import pwd
+IS_GAE = os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/')
+"""Flag indicating whether the app is running in GAE env.
+
+Ref:
+https://cloud.google.com/appengine/docs/standard/python/tools
+/using-local-server#detecting_application_runtime_environment
+"""
+
+
+IS_UID_GID_RESOLVABLE = not IS_WINDOWS and not IS_GAE
+"""Indicates whether UID/GID resolution's available under current platform."""
+
+
+if IS_UID_GID_RESOLVABLE:
+    try:
+        import grp
+        import pwd
+    except ImportError:
+        """Unavailable in the current env.
+
+        This shouldn't be happening normally.
+        All of the known cases are excluded via the if clause.
+        """
+        IS_UID_GID_RESOLVABLE = False
+        grp, pwd = None, None
     import struct
 
 
@@ -1261,7 +1284,12 @@ def _handle_no_ssl(self, req):
         if not req or req.sent_headers:
             return
         # Unwrap wfile
-        self.wfile = StreamWriter(self.socket._sock, 'wb', self.wbufsize)
+        try:
+            resp_sock = self.socket._sock
+        except AttributeError:
+            # self.socket is of OpenSSL.SSL.Connection type
+            resp_sock = self.socket._socket
+        self.wfile = StreamWriter(resp_sock, 'wb', self.wbufsize)
         msg = (
             'The client sent a plain HTTP request, but '
             'this server only speaks HTTPS on this port.'
@@ -1326,6 +1354,8 @@ def get_peer_creds(self):  # LRU cached on per-instance basis, see __init__
 
         try:
             peer_creds = self.socket.getsockopt(
+                # FIXME: Use LOCAL_CREDS for BSD-like OSs
+                # Ref: https://gist.github.com/LucaFilipozzi/e4f1e118202aff27af6aadebda1b5d91  # noqa
                 socket.SOL_SOCKET, socket.SO_PEERCRED,
                 struct.calcsize(PEERCRED_STRUCT_DEF)
             )
@@ -1371,9 +1401,11 @@ def resolve_peer_creds(self):  # LRU cached on per-instance basis
             RuntimeError: in case of UID/GID lookup unsupported or disabled
 
         """
-        if IS_WINDOWS:
+        if not IS_UID_GID_RESOLVABLE:
             raise NotImplementedError(
-                'UID/GID lookup can only be done under UNIX-like OS'
+                'UID/GID lookup is unavailable under current platform. '
+                'It can only be done under UNIX-like OS '
+                'but not under the Google App Engine'
             )
         elif not self.peercreds_resolve_enabled:
             raise RuntimeError(
@@ -1817,7 +1849,10 @@ def bind_unix_socket(self, bind_addr):
         try:
             """FreeBSD/macOS pre-populating fs mode permissions."""
             if not FS_PERMS_SET:
-                os.lchmod(bind_addr, fs_permissions)
+                try:
+                    os.lchmod(bind_addr, fs_permissions)
+                except AttributeError:
+                    os.chmod(bind_addr, fs_permissions, follow_symlinks=False)
                 FS_PERMS_SET = True
         except OSError:
             pass
@@ -1837,19 +1872,28 @@ def prepare_socket(bind_addr, family, type, proto, nodelay, ssl_adapter):
         """Create and prepare the socket object."""
         sock = socket.socket(family, type, proto)
         prevent_socket_inheritance(sock)
-        if not IS_WINDOWS:
-            # Windows has different semantics for SO_REUSEADDR,
-            # so don't set it.
-            # https://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+
+        host, port = bind_addr[:2]
+        IS_EPHEMERAL_PORT = port == 0
+
+        if not (IS_WINDOWS or IS_EPHEMERAL_PORT):
+            """Enable SO_REUSEADDR for the current socket.
+
+            Skip for Windows (has different semantics)
+            or ephemeral ports (can steal ports from others).
+
+            Refs:
+            * https://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+            * https://github.com/cherrypy/cheroot/issues/114
+            * https://gavv.github.io/blog/ephemeral-port-reuse/
+            """
             sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         if nodelay and not isinstance(bind_addr, str):
             sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
 
         if ssl_adapter is not None:
             sock = ssl_adapter.bind(sock)
 
-        host, port = bind_addr[:2]
-
         # If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
         # activate dual-stack. See
         # https://github.com/cherrypy/cherrypy/issues/871.
@@ -2056,7 +2100,7 @@ def __init__(self, req):
 
     def respond(self):
         """Process the current request. Must be overridden in a subclass."""
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
 
 # These may either be ssl.Adapter subclasses or the string names

lib/cheroot/ssl/__init__.py

@@ -38,14 +38,14 @@ def bind(self, sock):
     @abstractmethod
     def wrap(self, sock):
         """Wrap and return the given socket, plus WSGI environ entries."""
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     @abstractmethod
     def get_environ(self):
         """Return WSGI environ entries to be merged into each request."""
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     @abstractmethod
     def makefile(self, sock, mode='r', bufsize=-1):
         """Return socket file object."""
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover

lib/cheroot/ssl/builtin.py

@@ -10,6 +10,8 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
+import sys
+
 try:
     import ssl
 except ImportError:
@@ -27,6 +29,7 @@
 
 from . import Adapter
 from .. import errors
+from .._compat import IS_ABOVE_OPENSSL10
 from ..makefile import StreamReader, StreamWriter
 
 if six.PY3:
@@ -37,14 +40,17 @@
     del socket
 
 
+IS_BELOW_PY37 = sys.version_info[:2] < (3, 7)
+
+
 def _assert_ssl_exc_contains(exc, *msgs):
     """Check whether SSL exception contains either of messages provided."""
     if len(msgs) < 1:
         raise TypeError(
             '_assert_ssl_exc_contains() requires '
             'at least one message to be passed.'
         )
-    err_msg_lower = exc.args[1].lower()
+    err_msg_lower = str(exc).lower()
     return any(m.lower() in err_msg_lower for m in msgs)
 
 
@@ -131,6 +137,7 @@ def wrap(self, sock):
                     'wrong version number',
                     'no shared cipher', 'certificate unknown',
                     'ccs received early',
+                    'certificate verify failed',  # client cert w/o trusted CA
                 )
                 if _assert_ssl_exc_contains(ex, *_block_errors):
                     # Accepted error, let's pass
@@ -144,15 +151,19 @@ def wrap(self, sock):
         except generic_socket_error as exc:
             """It is unclear why exactly this happens.
 
-            It's reproducible only under Python 2 with openssl>1.0 and stdlib
-            ``ssl`` wrapper, and only with CherryPy.
-            So it looks like some healthcheck tries to connect to this socket
-            during startup (from the same process).
+            It's reproducible only under Python<=3.6 with openssl>1.0
+            and stdlib ``ssl`` wrapper.
+            In CherryPy it's triggered by Checker plugin, which connects
+            to the app listening to the socket port in TLS mode via plain
+            HTTP during startup (from the same process).
 
 
             Ref: https://github.com/cherrypy/cherrypy/issues/1618
             """
-            if six.PY2 and exc.args == (0, 'Error'):
+            is_error0 = exc.args == (0, 'Error')
+            ssl_doesnt_handle_error0 = IS_ABOVE_OPENSSL10 and IS_BELOW_PY37
+
+            if is_error0 and ssl_doesnt_handle_error0:
                 return EMPTY_RESULT
             raise
         return s, self.get_environ(s)