Account for squid proxy adding unexpected headers

Most servers (including S3) will just return the CLRF after the 100
continue response.  However, some servers (I've specifically seen this
for squid when used as a straight HTTP proxy) will also inject a
Connection: keep-alive header.  To account for this we'll read until we
read '\r\n', and ignore any headers that come immediately after the 100
continue response.

Fixes aws/aws-cli#1116

botocore/awsrequest.py

@@ -154,6 +154,18 @@ def _send_output(self, message_body=None):
             # we must run the risk of Nagle.
             self.send(message_body)
 
+    def _consume_headers(self, fp):
+        # Most servers (including S3) will just return
+        # the CLRF after the 100 continue response.  However,
+        # some servers (I've specifically seen this for squid when
+        # used as a straight HTTP proxy) will also inject a
+        # Connection: keep-alive header.  To account for this
+        # we'll read until we read '\r\n', and ignore any headers
+        # that come immediately after the 100 continue response.
+        current = None
+        while current != b'\r\n':
+            current = fp.readline()
+
     def _handle_expect_response(self, message_body):
         # This is called when we sent the request headers containing
         # an Expect: 100-continue header and received a response.
@@ -163,8 +175,7 @@ def _handle_expect_response(self, message_body):
             maybe_status_line = fp.readline()
             parts = maybe_status_line.split(None, 2)
             if self._is_100_continue_status(maybe_status_line):
-                # Read an empty line as per the RFC.
-                fp.readline()
+                self._consume_headers(fp)
                 logger.debug("100 Continue response seen, now sending request body.")
                 self._send_message_body(message_body)
             elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):

tests/unit/test_awsrequest.py

@@ -28,17 +28,25 @@
 from botocore.compat import file_type, six
 
 
+class IgnoreCloseBytesIO(io.BytesIO):
+    def close(self):
+        pass
+
+
 class FakeSocket(object):
-    def __init__(self, read_data, fileclass=io.BytesIO):
+    def __init__(self, read_data, fileclass=IgnoreCloseBytesIO):
         self.sent_data = b''
         self.read_data = read_data
         self.fileclass = fileclass
+        self._fp_object = None
 
     def sendall(self, data):
         self.sent_data += data
 
     def makefile(self, mode, bufsize=None):
-        return self.fileclass(self.read_data)
+        if self._fp_object is None:
+            self._fp_object = self.fileclass(self.read_data)
+        return self._fp_object
 
     def close(self):
         pass
@@ -191,6 +199,27 @@ def test_expect_100_continue_returned(self):
             # Now we should verify that our final response is the 200 OK
             self.assertEqual(response.status, 200)
 
+    def test_expect_100_sends_connection_header(self):
+        # When using squid as an HTTP proxy, it will also send
+        # a Connection: keep-alive header back with the 100 continue
+        # response.  We need to ensure we handle this case.
+        with patch('select.select') as select_mock:
+            # Shows the server first sending a 100 continue response
+            # then a 500 response.  We're picking 500 to confirm we
+            # actually parse the response instead of getting the
+            # default status of 200 which happens when we can't parse
+            # the response.
+            s = FakeSocket(b'HTTP/1.1 100 Continue\r\n'
+                           b'Connection: keep-alive\r\n'
+                           b'\r\n'
+                           b'HTTP/1.1 500 Internal Service Error\r\n')
+            conn = AWSHTTPConnection('s3.amazonaws.com', 443)
+            conn.sock = s
+            select_mock.return_value = ([s], [], [])
+            conn.request('GET', '/bucket/foo', b'body', {'Expect': '100-continue'})
+            response = conn.getresponse()
+            self.assertEqual(response.status, 500)
+
     def test_expect_100_continue_sends_307(self):
         # This is the case where we send a 100 continue and the server
         # immediately sends a 307