New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replaces httplib.request with requests library #728

Merged
merged 32 commits into from Jan 9, 2017
Commits
Jump to file or symbol
Failed to load files and symbols.
+688 −943
Diff settings

Always

Just for now

View
@@ -39,13 +39,11 @@ def enable_debug(fo):
:param fo: Where to append debugging information
:type fo: File like object, only write operations are used.
"""
from libcloud.common.base import (Connection,
LoggingHTTPConnection,
LoggingHTTPSConnection)
LoggingHTTPSConnection.log = fo
LoggingHTTPConnection.log = fo
Connection.conn_classes = (LoggingHTTPConnection,
LoggingHTTPSConnection)
from libcloud.common.base import Connection
from libcloud.utils.loggingconnection import LoggingConnection
LoggingConnection.log = fo
Connection.conn_class = LoggingConnection
def _init_once():
View
@@ -21,37 +21,32 @@
import binascii
import time
import xml.dom.minidom
try:
from lxml import etree as ET
except ImportError:
from xml.etree import ElementTree as ET
from pipes import quote as pquote
try:
import simplejson as json
except:
import json
import requests
import libcloud
from libcloud.utils.py3 import PY3, PY25
from libcloud.utils.py3 import PY25
from libcloud.utils.py3 import httplib
from libcloud.utils.py3 import urlparse
from libcloud.utils.py3 import urlencode
from libcloud.utils.py3 import StringIO
from libcloud.utils.py3 import u
from libcloud.utils.py3 import b
from libcloud.utils.misc import lowercase_keys, retry
from libcloud.utils.compression import decompress_data
from libcloud.common.exceptions import exception_from_message
from libcloud.common.types import LibcloudError, MalformedResponseError
from libcloud.httplib_ssl import LibcloudHTTPConnection
from libcloud.httplib_ssl import LibcloudHTTPSConnection
from libcloud.httplib_ssl import LibcloudConnection
__all__ = [
'RETRY_FAILED_HTTP_REQUESTS',
@@ -63,8 +58,6 @@
'ConnectionKey',
'ConnectionUserAndKey',
'CertificateConnection',
'LoggingHTTPConnection',
'LoggingHTTPSConnection',
'Response',
'HTTPResponse',
@@ -155,28 +148,16 @@ def __init__(self, response, connection):
# http.client In Python 3 doesn't automatically lowercase the header
# names
self.headers = lowercase_keys(dict(response.getheaders()))
self.headers = lowercase_keys(dict(response.headers))
self.error = response.reason
self.status = response.status
self.status = response.status_code
# This attribute is set when using LoggingConnection.
original_data = getattr(response, '_original_data', None)
if original_data:
# LoggingConnection already decompresses data so it can log it
# which means we don't need to decompress it here.
self.body = response._original_data
else:
self.body = self._decompress_response(body=response.read(),
headers=self.headers)
if PY3:
self.body = b(self.body).decode('utf-8')
self.body = response.text.strip() \
if response.text is not None else ''
if not self.success():
raise exception_from_message(code=self.status,
message=self.parse_error(),
headers=self.headers)
message=self.parse_error())
self.object = self.parse_body()
@@ -189,7 +170,7 @@ def parse_body(self):
:return: Parsed body.
:rtype: ``str``
"""
return self.body
return self.body if self.body is not None else ''
def parse_error(self):
"""
@@ -212,7 +193,8 @@ def success(self):
:rtype: ``bool``
:return: ``True`` or ``False``
"""
return self.status in [httplib.OK, httplib.CREATED]
return self.status in [requests.codes.ok, requests.codes.created,
httplib.OK, httplib.CREATED]
def _decompress_response(self, body, headers):
"""
@@ -267,7 +249,7 @@ class XmlResponse(Response):
def parse_body(self):
if len(self.body) == 0 and not self.parse_zero_length_body:
return self.body
return self.body if self.body is not None else ''
try:
body = ET.XML(self.body)
@@ -322,195 +304,12 @@ def reason(self):
return self._reason
# TODO: Move this to a better location/package
class LoggingConnection():
"""
Debug class to log all HTTP(s) requests as they could be made
with the curl command.
:cvar log: file-like object that logs entries are written to.
"""
log = None
http_proxy_used = False
def _log_response(self, r):
rv = "# -------- begin %d:%d response ----------\n" % (id(self), id(r))
ht = ""
v = r.version
if r.version == 10:
v = "HTTP/1.0"
if r.version == 11:
v = "HTTP/1.1"
ht += "%s %s %s\r\n" % (v, r.status, r.reason)
body = r.read()
for h in r.getheaders():
ht += "%s: %s\r\n" % (h[0].title(), h[1])
ht += "\r\n"
# this is evil. laugh with me. ha arharhrhahahaha
class fakesock(object):
def __init__(self, s):
self.s = s
def makefile(self, *args, **kwargs):
if PY3:
from io import BytesIO
cls = BytesIO
else:
cls = StringIO
return cls(b(self.s))
rr = r
headers = lowercase_keys(dict(r.getheaders()))
encoding = headers.get('content-encoding', None)
content_type = headers.get('content-type', None)
if encoding in ['zlib', 'deflate']:
body = decompress_data('zlib', body)
elif encoding in ['gzip', 'x-gzip']:
body = decompress_data('gzip', body)
pretty_print = os.environ.get('LIBCLOUD_DEBUG_PRETTY_PRINT_RESPONSE',
False)
if r.chunked:
ht += "%x\r\n" % (len(body))
ht += body.decode('utf-8')
ht += "\r\n0\r\n"
else:
if pretty_print and content_type == 'application/json':
try:
body = json.loads(body.decode('utf-8'))
body = json.dumps(body, sort_keys=True, indent=4)
except:
# Invalid JSON or server is lying about content-type
pass
elif pretty_print and content_type == 'text/xml':
try:
elem = xml.dom.minidom.parseString(body.decode('utf-8'))
body = elem.toprettyxml()
except Exception:
# Invalid XML
pass
ht += u(body)
if sys.version_info >= (2, 6) and sys.version_info < (2, 7):
cls = HTTPResponse
else:
cls = httplib.HTTPResponse
rr = cls(sock=fakesock(ht), method=r._method,
debuglevel=r.debuglevel)
rr.begin()
rv += ht
rv += ("\n# -------- end %d:%d response ----------\n"
% (id(self), id(r)))
rr._original_data = body
return (rr, rv)
def _log_curl(self, method, url, body, headers):
cmd = ["curl"]
if self.http_proxy_used:
if self.proxy_username and self.proxy_password:
proxy_url = 'http://%s:%s@%s:%s' % (self.proxy_username,
self.proxy_password,
self.proxy_host,
self.proxy_port)
else:
proxy_url = 'http://%s:%s' % (self.proxy_host,
self.proxy_port)
proxy_url = pquote(proxy_url)
cmd.extend(['--proxy', proxy_url])
cmd.extend(['-i'])
if method.lower() == 'head':
# HEAD method need special handling
cmd.extend(["--head"])
else:
cmd.extend(["-X", pquote(method)])
for h in headers:
cmd.extend(["-H", pquote("%s: %s" % (h, headers[h]))])
cert_file = getattr(self, 'cert_file', None)
if cert_file:
cmd.extend(["--cert", pquote(cert_file)])
# TODO: in python 2.6, body can be a file-like object.
if body is not None and len(body) > 0:
cmd.extend(["--data-binary", pquote(body)])
cmd.extend(["--compress"])
cmd.extend([pquote("%s://%s:%d%s" % (self.protocol, self.host,
self.port, url))])
return " ".join(cmd)
class LoggingHTTPSConnection(LoggingConnection, LibcloudHTTPSConnection):
"""
Utility Class for logging HTTPS connections
"""
protocol = 'https'
def getresponse(self):
r = LibcloudHTTPSConnection.getresponse(self)
if self.log is not None:
r, rv = self._log_response(r)
self.log.write(rv + "\n")
self.log.flush()
return r
def request(self, method, url, body=None, headers=None):
headers.update({'X-LC-Request-ID': str(id(self))})
if self.log is not None:
pre = "# -------- begin %d request ----------\n" % id(self)
self.log.write(pre +
self._log_curl(method, url, body, headers) + "\n")
self.log.flush()
return LibcloudHTTPSConnection.request(self, method, url, body,
headers)
class LoggingHTTPConnection(LoggingConnection, LibcloudHTTPConnection):
"""
Utility Class for logging HTTP connections
"""
protocol = 'http'
def getresponse(self):
r = LibcloudHTTPConnection.getresponse(self)
if self.log is not None:
r, rv = self._log_response(r)
self.log.write(rv + "\n")
self.log.flush()
return r
def request(self, method, url, body=None, headers=None):
headers.update({'X-LC-Request-ID': str(id(self))})
if self.log is not None:
pre = '# -------- begin %d request ----------\n' % id(self)
self.log.write(pre +
self._log_curl(method, url, body, headers) + "\n")
self.log.flush()
return LibcloudHTTPConnection.request(self, method, url,
body, headers)
class Connection(object):
"""
A Base Connection class to derive from.
"""
# conn_classes = (LoggingHTTPSConnection)
conn_classes = (LibcloudHTTPConnection, LibcloudHTTPSConnection)
# conn_class = LoggingHTTPSConnection
conn_class = LibcloudConnection
responseCls = Response
rawResponseCls = RawResponse
@@ -660,12 +459,12 @@ def connect(self, host=None, port=None, base_url=None, **kwargs):
if self.proxy_url:
kwargs.update({'proxy_url': self.proxy_url})
connection = self.conn_classes[secure](**kwargs)
connection = self.conn_class(**kwargs)
# You can uncoment this line, if you setup a reverse proxy server
# which proxies to your endpoint, and lets you easily capture
# connections in cleartext when you setup the proxy to do SSL
# for you
# connection = self.conn_classes[False]("127.0.0.1", 8080)
# connection = self.conn_class("127.0.0.1", 8080)
self.connection = connection
@@ -792,9 +591,10 @@ def request(self, action, params=None, data=None, headers=None,
else:
url = action
# Removed terrible hack...this a less-bad hack that doesn't execute a
# request twice, but it's still a hack.
self.connect()
# IF connection has not yet been established
if self.connection is None:
self.connect()
try:
# @TODO: Should we just pass File object as body to request method
# instead of dealing with splitting and sending the file ourselves?
@@ -70,6 +70,5 @@ def exception_from_message(code, message, headers=None):
if headers and 'retry_after' in headers:
kwargs['retry_after'] = headers['retry_after']
cls = _code_map.get(code, BaseHTTPError)
return cls(**kwargs)
@@ -583,7 +583,7 @@ def __init__(self, auth_url, user_id, key, tenant_name=None,
# enable tests to use the same mock connection classes.
if parent_conn:
self.conn_classes = parent_conn.conn_classes
self.conn_class = parent_conn.conn_class
self.driver = parent_conn.driver
else:
self.driver = None
@@ -26,7 +26,7 @@
from libcloud.utils.connection import get_response_object
from libcloud.common.types import InvalidCredsError
from libcloud.common.base import ConnectionUserAndKey, JsonResponse
from libcloud.httplib_ssl import LibcloudHTTPSConnection
from libcloud.httplib_ssl import LibcloudConnection
__all__ = [
'RunAboveResponse',
@@ -100,7 +100,7 @@ def request_consumer_key(self, user_id):
'Content-Type': 'application/json',
'X-Ra-Application': user_id,
}
httpcon = LibcloudHTTPSConnection(self.host)
httpcon = LibcloudConnection(self.host)
httpcon.request(method='POST', url=action, body=data, headers=headers)
response = httpcon.getresponse()
Oops, something went wrong.
ProTip! Use n and p to navigate between commits in a pull request.