Merge pull request #190 from dstufft/workaround-python-23801

Include a patch from CPython's bug tracker to fix cgi.FieldStorage on 3.x < 3.4.4

webob/compat.py

@@ -129,3 +129,66 @@ def parse_qsl_text(qs, encoding='utf-8'):
     from html import escape
 else:
     from cgi import escape
+
+
+# We only need this on Python3 but the issue was fixed in Pytohn 3.4.4 and 3.5.
+if PY3 and sys.version_info[:3] < (3, 4, 4):  # pragma no cover
+    # Work around http://bugs.python.org/issue23801
+    import cgi
+    from cgi import FieldStorage as _cgi_FieldStorage
+
+    class cgi_FieldStorage(_cgi_FieldStorage):
+        # This is taken exactly from Python 3.5's cgi.py module, and patched
+        # with the patch from http://bugs.python.org/issue23801.
+        def read_multi(self, environ, keep_blank_values, strict_parsing):
+            """Internal: read a part that is itself multipart."""
+            ib = self.innerboundary
+            if not cgi.valid_boundary(ib):
+                raise ValueError(
+                    'Invalid boundary in multipart form: %r' % (ib,))
+            self.list = []
+            if self.qs_on_post:
+                query = cgi.urllib.parse.parse_qsl(
+                    self.qs_on_post, self.keep_blank_values,
+                    self.strict_parsing,
+                    encoding=self.encoding, errors=self.errors)
+                for key, value in query:
+                    self.list.append(cgi.MiniFieldStorage(key, value))
+
+            klass = self.FieldStorageClass or self.__class__
+            first_line = self.fp.readline()  # bytes
+            if not isinstance(first_line, bytes):
+                raise ValueError("%s should return bytes, got %s"
+                                 % (self.fp, type(first_line).__name__))
+            self.bytes_read += len(first_line)
+
+            # Ensure that we consume the file until we've hit our innerboundary
+            while (first_line.strip() != (b"--" + self.innerboundary) and
+                    first_line):
+                first_line = self.fp.readline()
+                self.bytes_read += len(first_line)
+
+            while True:
+                parser = cgi.FeedParser()
+                hdr_text = b""
+                while True:
+                    data = self.fp.readline()
+                    hdr_text += data
+                    if not data.strip():
+                        break
+                if not hdr_text:
+                    break
+                # parser takes strings, not bytes
+                self.bytes_read += len(hdr_text)
+                parser.feed(hdr_text.decode(self.encoding, self.errors))
+                headers = parser.close()
+                part = klass(self.fp, headers, ib, environ, keep_blank_values,
+                             strict_parsing, self.limit-self.bytes_read,
+                             self.encoding, self.errors)
+                self.bytes_read += part.bytes_read
+                self.list.append(part)
+                if part.done or self.bytes_read >= self.length > 0:
+                    break
+            self.skip_lines()
+else:
+    from cgi import FieldStorage as cgi_FieldStorage

webob/request.py

@@ -1,5 +1,4 @@
 import binascii
-import cgi
 import io
 import os
 import re
@@ -39,6 +38,7 @@
     url_unquote,
     quote_plus,
     urlparse,
+    cgi_FieldStorage
     )
 
 from webob.cookies import RequestCookies
@@ -229,13 +229,13 @@ def decode(self, charset=None, errors='strict'):
         fs_environ.setdefault('CONTENT_LENGTH', '0')
         fs_environ['QUERY_STRING'] = ''
         if PY3: # pragma: no cover
-            fs = cgi.FieldStorage(fp=self.body_file,
+            fs = cgi_FieldStorage(fp=self.body_file,
                                   environ=fs_environ,
                                   keep_blank_values=True,
                                   encoding=charset,
                                   errors=errors)
         else:
-            fs = cgi.FieldStorage(fp=self.body_file,
+            fs = cgi_FieldStorage(fp=self.body_file,
                                   environ=fs_environ,
                                   keep_blank_values=True)
 
@@ -793,14 +793,14 @@ def POST(self):
         fs_environ.setdefault('CONTENT_LENGTH', '0')
         fs_environ['QUERY_STRING'] = ''
         if PY3: # pragma: no cover
-            fs = cgi.FieldStorage(
+            fs = cgi_FieldStorage(
                 fp=self.body_file,
                 environ=fs_environ,
                 keep_blank_values=True,
                 encoding='utf8')
             vars = MultiDict.from_fieldstorage(fs)
         else:
-            fs = cgi.FieldStorage(
+            fs = cgi_FieldStorage(
                 fp=self.body_file,
                 environ=fs_environ,
                 keep_blank_values=True)
@@ -1566,7 +1566,7 @@ def _cgi_FieldStorage__repr__patch(self):
         return "FieldStorage(%r, %r)" % (self.name, self.filename)
     return "FieldStorage(%r, %r, %r)" % (self.name, self.filename, self.value)
 
-cgi.FieldStorage.__repr__ = _cgi_FieldStorage__repr__patch
+cgi_FieldStorage.__repr__ = _cgi_FieldStorage__repr__patch
 
 class FakeCGIBody(io.RawIOBase):
     def __init__(self, vars, content_type):