Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 636 lines (467 sloc) 19.451 kb
14ef462 @kennethreitz package refactor
authored
1 # -*- coding: utf-8 -*-
2
3 """
59c747b @jgorset Fix typos
jgorset authored
4 requests.models
14ef462 @kennethreitz package refactor
authored
5 ~~~~~~~~~~~~~~~
6
2584c48 @kennethreitz python2.5 cookie support
authored
7 This module contains the primary objects that power Requests.
14ef462 @kennethreitz package refactor
authored
8 """
9
10 import urllib
11 import zlib
12
4504953 @kennethreitz Only send URL path to servers
authored
13 from urlparse import urlparse, urlunparse, urljoin, urlsplit
b4eac4c @kennethreitz redirect fix
authored
14 from datetime import datetime
14ef462 @kennethreitz package refactor
authored
15
c108c53 @kennethreitz No mo' HttpError
authored
16 from .auth import dispatch as auth_dispatch
491a3c0 @kennethreitz Move hooks into internal event loop
authored
17 from .hooks import dispatch_hook
85b8b2b @kennethreitz CaseInsensitiveDict for Headers
authored
18 from .structures import CaseInsensitiveDict
c75eaec @kennethreitz integrate codes into models.py
authored
19 from .status_codes import codes
5309ad0 @kennethreitz cookies are oreos
authored
20 from .packages import oreos
6a51f6b @kennethreitz add 'max_retries' configuration
authored
21 from .packages.urllib3.exceptions import MaxRetryError
7c206c7 @kennethreitz Timeouts for urllib3
authored
22 from .packages.urllib3.exceptions import SSLError as _SSLError
23 from .packages.urllib3.exceptions import HTTPError as _HTTPError
8723980 @kennethreitz proxy pooling!
authored
24 from .packages.urllib3 import connectionpool, poolmanager
c0d8e8e @kennethreitz models cleanup
authored
25 from .packages.urllib3.filepost import encode_multipart_formdata
6a51f6b @kennethreitz add 'max_retries' configuration
authored
26 from .exceptions import (
27 Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError)
c108c53 @kennethreitz No mo' HttpError
authored
28 from .utils import (
296d8cc @kennethreitz new (elegant) request decoding model.
authored
29 get_encoding_from_headers, stream_decode_response_unicode,
f7fdfe8 @rfk Correct handling of URLs with quoted slashes.
rfk authored
30 decode_gzip, stream_decode_gzip, guess_filename, requote_path)
14ef462 @kennethreitz package refactor
authored
31
32
c75eaec @kennethreitz integrate codes into models.py
authored
33 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
34
b4eac4c @kennethreitz redirect fix
authored
35
14ef462 @kennethreitz package refactor
authored
36
37 class Request(object):
596aceb @kennethreitz big docs update
authored
38 """The :class:`Request <Request>` object. It carries out all functionality of
14ef462 @kennethreitz package refactor
authored
39 Requests. Recommended interface is with the Requests functions.
40 """
41
00f066a @kennethreitz PATCH support
authored
42 def __init__(self,
903f96c @kennethreitz use config throughout models
authored
43 url=None,
44 headers=dict(),
45 files=None,
46 method=None,
47 data=dict(),
48 params=dict(),
49 auth=None,
50 cookies=None,
51 timeout=None,
52 redirect=False,
53 allow_redirects=False,
54 proxies=None,
55 hooks=None,
3d3f058 @kennethreitz use new system
authored
56 config=None,
57 _poolmanager=None):
14ef462 @kennethreitz package refactor
authored
58
055f7c1 @tvaught Some Doc and doctoring fixes by the Speling Poleece.
tvaught authored
59 #: Float describes the timeout of the request.
8d2ab68 @kennethreitz space
authored
60 # (Use socket.setdefaulttimeout() as fallback)
647cdd3 @jedie Use socket.setdefaulttimeout(timeout) only if timeout argument doesn'…
jedie authored
61 self.timeout = timeout
14ef462 @kennethreitz package refactor
authored
62
6e13e73 @kennethreitz Big API update
authored
63 #: Request URL.
14ef462 @kennethreitz package refactor
authored
64 self.url = url
17d3e1d @kennethreitz whitespace
authored
65
055f7c1 @tvaught Some Doc and doctoring fixes by the Speling Poleece.
tvaught authored
66 #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
35ed9b5 @kennethreitz dict(o or [])
authored
67 self.headers = dict(headers or [])
17d3e1d @kennethreitz whitespace
authored
68
6e13e73 @kennethreitz Big API update
authored
69 #: Dictionary of files to multipart upload (``{filename: content}``).
14ef462 @kennethreitz package refactor
authored
70 self.files = files
17d3e1d @kennethreitz whitespace
authored
71
d0fdf0e @kennethreitz no longer required
authored
72 #: HTTP Method to use.
14ef462 @kennethreitz package refactor
authored
73 self.method = method
17d3e1d @kennethreitz whitespace
authored
74
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
75 #: Dictionary or byte of request body data to attach to the
596aceb @kennethreitz big docs update
authored
76 #: :class:`Request <Request>`.
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
77 self.data = None
17d3e1d @kennethreitz whitespace
authored
78
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
79 #: Dictionary or byte of querystring data to attach to the
596aceb @kennethreitz big docs update
authored
80 #: :class:`Request <Request>`.
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
81 self.params = None
147c241 @kennethreitz better defaults for "models"
authored
82 self.params = dict(params or [])
17d3e1d @kennethreitz whitespace
authored
83
596aceb @kennethreitz big docs update
authored
84 #: True if :class:`Request <Request>` is part of a redirect chain (disables history
6e13e73 @kennethreitz Big API update
authored
85 #: and HTTPError storage).
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
86 self.redirect = redirect
17d3e1d @kennethreitz whitespace
authored
87
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
88 #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``)
89 self.allow_redirects = allow_redirects
17d3e1d @kennethreitz whitespace
authored
90
3e7c682 @moliware Support for proxies
moliware authored
91 # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
35ed9b5 @kennethreitz dict(o or [])
authored
92 self.proxies = dict(proxies or [])
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
93
023ab75 @rboulton Move the code to encode the request data into a static helper method …
rboulton authored
94 self.data, self._enc_data = self._encode_params(data)
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
95 self.params, self._enc_params = self._encode_params(params)
96
596aceb @kennethreitz big docs update
authored
97 #: :class:`Response <Response>` instance, containing
6e13e73 @kennethreitz Big API update
authored
98 #: content and metadata of HTTP Response, once :attr:`sent <send>`.
14ef462 @kennethreitz package refactor
authored
99 self.response = Response()
100
20e0ee4 @kennethreitz Remove AuthObject entirely.
authored
101 #: Authentication tuple to attach to :class:`Request <Request>`.
3efd5db @davidfischer A cleaner and more complete fix for auth/redirects
davidfischer authored
102 self._auth = auth
20e0ee4 @kennethreitz Remove AuthObject entirely.
authored
103 self.auth = auth_dispatch(auth)
17d3e1d @kennethreitz whitespace
authored
104
596aceb @kennethreitz big docs update
authored
105 #: CookieJar to attach to :class:`Request <Request>`.
35ed9b5 @kennethreitz dict(o or [])
authored
106 self.cookies = dict(cookies or [])
17d3e1d @kennethreitz whitespace
authored
107
903f96c @kennethreitz use config throughout models
authored
108 #: Dictionary of configurations for this request.
35ed9b5 @kennethreitz dict(o or [])
authored
109 self.config = dict(config or [])
903f96c @kennethreitz use config throughout models
authored
110
6e13e73 @kennethreitz Big API update
authored
111 #: True if Request has been sent.
14ef462 @kennethreitz package refactor
authored
112 self.sent = False
113
892fcd6 @kennethreitz store hooks in Request.
authored
114 #: Event-handling hooks.
115 self.hooks = hooks
c3c90bb @kennethreitz default headers and content encoding settings
authored
116
6a23625 @kennethreitz give Request session reference for parameter+redirects
authored
117 #: Session.
118 self.session = None
119
c3c90bb @kennethreitz default headers and content encoding settings
authored
120 if headers:
121 headers = CaseInsensitiveDict(self.headers)
122 else:
123 headers = CaseInsensitiveDict()
124
903f96c @kennethreitz use config throughout models
authored
125 for (k, v) in self.config.get('base_headers', {}).items():
c3c90bb @kennethreitz default headers and content encoding settings
authored
126 if k not in headers:
127 headers[k] = v
128
5fff2c7 @kennethreitz forced_basic default handler.
authored
129 self.headers = headers
3d3f058 @kennethreitz use new system
authored
130 self._poolmanager = _poolmanager
5fff2c7 @kennethreitz forced_basic default handler.
authored
131
491a3c0 @kennethreitz Move hooks into internal event loop
authored
132 # Pre-request hook.
133 r = dispatch_hook('pre_request', hooks, self)
134 self.__dict__.update(r.__dict__)
135
14ef462 @kennethreitz package refactor
authored
136
137 def __repr__(self):
138 return '<Request [%s]>' % (self.method)
139
140
5693314 @kennethreitz response error handling rework
authored
141 def _build_response(self, resp, is_error=False):
596aceb @kennethreitz big docs update
authored
142 """Build internal :class:`Response <Response>` object
882ea76 @kennethreitz cleanups, response.fo
authored
143 from given response.
144 """
145
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
146 def build(resp):
14ef462 @kennethreitz package refactor
authored
147
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
148 response = Response()
3d3f058 @kennethreitz use new system
authored
149
150 # Pass settings over.
903f96c @kennethreitz use config throughout models
authored
151 response.config = self.config
14ef462 @kennethreitz package refactor
authored
152
35ed9b5 @kennethreitz dict(o or [])
authored
153 if resp:
154
155 # Fallback to None if there's no staus_code, for whatever reason.
156 response.status_code = getattr(resp, 'status', None)
7e25517 @kennethreitz get cookies from response
authored
157
35ed9b5 @kennethreitz dict(o or [])
authored
158 # Make headers case-insensitive.
159 response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
7e25517 @kennethreitz get cookies from response
authored
160
296d8cc @kennethreitz new (elegant) request decoding model.
authored
161 # Set encoding.
162 response.encoding = get_encoding_from_headers(response.headers)
163
35ed9b5 @kennethreitz dict(o or [])
authored
164 # Start off with our local cookies.
165 cookies = self.cookies or dict()
7e25517 @kennethreitz get cookies from response
authored
166
35ed9b5 @kennethreitz dict(o or [])
authored
167 # Add new cookies from the server.
168 if 'set-cookie' in response.headers:
169 cookie_header = response.headers['set-cookie']
5309ad0 @kennethreitz cookies are oreos
authored
170 cookies = oreos.dict_from_string(cookie_header)
3d3f058 @kennethreitz use new system
authored
171
35ed9b5 @kennethreitz dict(o or [])
authored
172 # Save cookies in Response.
173 response.cookies = cookies
3d3f058 @kennethreitz use new system
authored
174
175 # Save original resopnse for later.
176 response.raw = resp
14ef462 @kennethreitz package refactor
authored
177
5693314 @kennethreitz response error handling rework
authored
178 if is_error:
179 response.error = resp
180
4504953 @kennethreitz Only send URL path to servers
authored
181 response.url = self.full_url
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
182
183 return response
184
185 history = []
186
187 r = build(resp)
52a95cb @kennethreitz header and cookie fixes for redirects
authored
188 cookies = self.cookies
189 self.cookies.update(r.cookies)
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
190
1e1fb1e @kennethreitz Redirect history fix (Closes #91)
authored
191 if r.status_code in REDIRECT_STATI and not self.redirect:
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
192
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
193 while (
194 ('location' in r.headers) and
aa04cc9 Make get and head requests respect allow_redirects=False.
Daniel Miller authored
195 ((r.status_code is codes.see_other) or (self.allow_redirects))
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
196 ):
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
197
903f96c @kennethreitz use config throughout models
authored
198 if not len(history) < self.config.get('max_redirects'):
e7e3955 @jerem Handle too many redirects.
jerem authored
199 raise TooManyRedirects()
200
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
201 history.append(r)
202
203 url = r.headers['location']
204
c22b71f @jerem Handle redirection without scheme.
jerem authored
205 # Handle redirection without scheme (see: RFC 1808 Section 4)
206 if url.startswith('//'):
207 parsed_rurl = urlparse(r.url)
208 url = '%s:%s' % (parsed_rurl.scheme, url)
209
b4eac4c @kennethreitz redirect fix
authored
210 # Facilitate non-RFC2616-compliant 'location' headers
8aa6fab @jgorset Support relative redirects. Fixes #36
jgorset authored
211 # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
571d808 @jerem Fixed bad merge.
jerem authored
212 if not urlparse(url).netloc:
3900243 @rfk Dont use requote_path when dealing with redirects.
rfk authored
213 url = urljoin(r.url, url)
8aa6fab @jgorset Support relative redirects. Fixes #36
jgorset authored
214
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
215 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
c75eaec @kennethreitz integrate codes into models.py
authored
216 if r.status_code is codes.see_other:
e153b94 @kennethreitz Improved redirection behavior for 301/303
authored
217 method = 'GET'
218 else:
219 method = self.method
220
52a95cb @kennethreitz header and cookie fixes for redirects
authored
221 # Remove the cookie headers that were sent.
222 headers = self.headers
223 try:
224 del headers['Cookie']
225 except KeyError:
226 pass
227
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
228 request = Request(
903f96c @kennethreitz use config throughout models
authored
229 url=url,
52a95cb @kennethreitz header and cookie fixes for redirects
authored
230 headers=headers,
903f96c @kennethreitz use config throughout models
authored
231 files=self.files,
232 method=method,
6a23625 @kennethreitz give Request session reference for parameter+redirects
authored
233 params=self.session.params,
3efd5db @davidfischer A cleaner and more complete fix for auth/redirects
davidfischer authored
234 auth=self._auth,
52a95cb @kennethreitz header and cookie fixes for redirects
authored
235 cookies=cookies,
903f96c @kennethreitz use config throughout models
authored
236 redirect=True,
3d3f058 @kennethreitz use new system
authored
237 config=self.config,
f0edd4a @pennersr Pass along timeout when following redirects (closes #263)
pennersr authored
238 timeout=self.timeout,
7c66e6f @benmao fixed: 301 request does not use proxy.
benmao authored
239 _poolmanager=self._poolmanager,
240 proxies = self.proxies,
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
241 )
52a95cb @kennethreitz header and cookie fixes for redirects
authored
242
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
243 request.send()
52a95cb @kennethreitz header and cookie fixes for redirects
authored
244 cookies.update(request.response.cookies)
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
245 r = request.response
52a95cb @kennethreitz header and cookie fixes for redirects
authored
246 self.cookies.update(r.cookies)
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
247
248 r.history = history
249
250 self.response = r
2d70647 @kennethreitz attach request object to responses
authored
251 self.response.request = self
52a95cb @kennethreitz header and cookie fixes for redirects
authored
252 self.response.cookies.update(self.cookies)
14ef462 @kennethreitz package refactor
authored
253
254
255 @staticmethod
023ab75 @rboulton Move the code to encode the request data into a static helper method …
rboulton authored
256 def _encode_params(data):
257 """Encode parameters in a piece of data.
258
259 If the data supplied is a dictionary, encodes each parameter in it, and
73b414a @rabad We can represent multivalued params with arrays
rabad authored
260 returns a list of tuples containing the encoded parameters, and a urlencoded
261 version of that.
023ab75 @rboulton Move the code to encode the request data into a static helper method …
rboulton authored
262
263 Otherwise, assumes the data is already encoded appropriately, and
264 returns it twice.
265 """
c5d787d @kennethreitz cleanups
authored
266
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
267 if hasattr(data, '__iter__'):
268 data = dict(data)
269
023ab75 @rboulton Move the code to encode the request data into a static helper method …
rboulton authored
270 if hasattr(data, 'items'):
73b414a @rabad We can represent multivalued params with arrays
rabad authored
271 result = []
272 for k, vs in data.items():
273 for v in isinstance(vs, list) and vs or [vs]:
274 result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
275 v.encode('utf-8') if isinstance(v, unicode) else v))
d2ff0f4 add doseq to urlencode in model._encode_params to enable list as dict…
Xu Pan authored
276 return result, urllib.urlencode(result, doseq=True)
023ab75 @rboulton Move the code to encode the request data into a static helper method …
rboulton authored
277 else:
278 return data, data
279
4504953 @kennethreitz Only send URL path to servers
authored
280 @property
281 def full_url(self):
24da410 @kennethreitz hmmm
authored
282 """Build the actual URL to use."""
14ef462 @kennethreitz package refactor
authored
283
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
284 if not self.url:
285 raise URLRequired()
286
0b6dc09 @jerem Added support for URLs with path not encoded.
jerem authored
287 # Support for unicode domain names and paths.
aa0f78b @jerem Better support of utf-8 paths.
jerem authored
288 scheme, netloc, path, params, query, fragment = urlparse(self.url)
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
289
290 if not scheme:
291 raise ValueError()
292
aa0f78b @jerem Better support of utf-8 paths.
jerem authored
293 netloc = netloc.encode('idna')
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
294
aa0f78b @jerem Better support of utf-8 paths.
jerem authored
295 if isinstance(path, unicode):
296 path = path.encode('utf-8')
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
297
f7fdfe8 @rfk Correct handling of URLs with quoted slashes.
rfk authored
298 path = requote_path(path)
4504953 @kennethreitz Only send URL path to servers
authored
299
300 url = str(urlunparse([ scheme, netloc, path, params, query, fragment ]))
868cd3d @jerem Implemented idna (fixes #76).
jerem authored
301
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
302 if self._enc_params:
4504953 @kennethreitz Only send URL path to servers
authored
303 if urlparse(url).query:
304 return '%s&%s' % (url, self._enc_params)
14ef462 @kennethreitz package refactor
authored
305 else:
4504953 @kennethreitz Only send URL path to servers
authored
306 return '%s?%s' % (url, self._enc_params)
80d860d @rboulton Allow POST and PUT requests to take both querystring params and reque…
rboulton authored
307 else:
4504953 @kennethreitz Only send URL path to servers
authored
308 return url
309
310 @property
311 def path_url(self):
312 """Build the path URL to use."""
313
314 url = []
315
316 p = urlsplit(self.full_url)
317
318 # Proxies use full URLs.
319 if p.scheme in self.proxies:
320 return self.full_url
321
322 path = p.path
323 if not path:
324 path = '/'
325 url.append(path)
326
327 query = p.query
328 if query:
329 url.append('?')
330 url.append(query)
331
332 return ''.join(url)
333
14ef462 @kennethreitz package refactor
authored
334
335
bd45bc7 @kennethreitz add prefetch to main request distpatch
authored
336 def send(self, anyway=False, prefetch=False):
14ef462 @kennethreitz package refactor
authored
337 """Sends the request. Returns True of successful, false if not.
338 If there was an HTTPError during transmission,
339 self.response.status_code will contain the HTTPError code.
340
341 Once a request is successfully sent, `sent` will equal True.
342
343 :param anyway: If True, request will be sent, even if it has
344 already been sent.
345 """
24da410 @kennethreitz hmmm
authored
346
b4eac4c @kennethreitz redirect fix
authored
347 # Logging
903f96c @kennethreitz use config throughout models
authored
348 if self.config.get('verbose'):
349 self.config.get('verbose').write('%s %s %s\n' % (
b4eac4c @kennethreitz redirect fix
authored
350 datetime.now().isoformat(), self.method, self.url
351 ))
352
302c4d9 @kennethreitz Don't alter behavior for safe methods anymore.
authored
353 # Build the URL
4504953 @kennethreitz Only send URL path to servers
authored
354 url = self.full_url
14ef462 @kennethreitz package refactor
authored
355
3d3f058 @kennethreitz use new system
authored
356 # Nottin' on you.
357 body = None
358 content_type = None
359
360 # Multi-part file uploads.
302c4d9 @kennethreitz Don't alter behavior for safe methods anymore.
authored
361 if self.files:
3d3f058 @kennethreitz use new system
authored
362 if not isinstance(self.data, basestring):
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
363
364 try:
365 fields = self.data.copy()
366 except AttributeError:
367 fields = dict(self.data)
368
3d3f058 @kennethreitz use new system
authored
369 for (k, v) in self.files.items():
bf499a8 @darkrho added support for explicit filenames in `files` parameter.
darkrho authored
370 # support for explicit filename
371 if isinstance(v, (tuple, list)):
372 fn, fp = v
373 else:
374 fn = guess_filename(v) or k
375 fp = v
376 fields.update({k: (fn, fp.read())})
0086f97 @kennethreitz Guess filename.
authored
377
3d3f058 @kennethreitz use new system
authored
378 (body, content_type) = encode_multipart_formdata(fields)
a9822b0 @kennethreitz fix content uploads
authored
379 else:
380 pass
381 # TODO: Conflict?
382 else:
383 if self.data:
c0d8e8e @kennethreitz models cleanup
authored
384
a9822b0 @kennethreitz fix content uploads
authored
385 body = self._enc_data
c0d8e8e @kennethreitz models cleanup
authored
386 if isinstance(self.data, basestring):
387 content_type = None
388 else:
389 content_type = 'application/x-www-form-urlencoded'
14ef462 @kennethreitz package refactor
authored
390
3d3f058 @kennethreitz use new system
authored
391 # Add content-type if it wasn't explicitly provided.
392 if (content_type) and (not 'content-type' in self.headers):
393 self.headers['Content-Type'] = content_type
302c4d9 @kennethreitz Don't alter behavior for safe methods anymore.
authored
394
14ef462 @kennethreitz package refactor
authored
395
495c3d5 @kennethreitz Use new authentication style in models
authored
396 if self.auth:
397 auth_func, auth_args = self.auth
398
52a95cb @kennethreitz header and cookie fixes for redirects
authored
399 # Allow auth to make its changes.
495c3d5 @kennethreitz Use new authentication style in models
authored
400 r = auth_func(self, *auth_args)
401
52a95cb @kennethreitz header and cookie fixes for redirects
authored
402 # Update self to reflect the auth changes.
495c3d5 @kennethreitz Use new authentication style in models
authored
403 self.__dict__.update(r.__dict__)
404
bd45bc7 @kennethreitz add prefetch to main request distpatch
authored
405 _p = urlparse(url)
406 proxy = self.proxies.get(_p.scheme)
407
408 if proxy:
35b0c5e @benmao fixed proxy
benmao authored
409 conn = poolmanager.proxy_from_url(proxy)
0c4b973 @kennethreitz keep_alive is now disable-able!
authored
410 else:
bd45bc7 @kennethreitz add prefetch to main request distpatch
authored
411 # Check to see if keep_alive is allowed.
412 if self.config.get('keep_alive'):
413 conn = self._poolmanager.connection_from_url(url)
414 else:
415 conn = connectionpool.connection_from_url(url)
14ef462 @kennethreitz package refactor
authored
416
417 if not self.sent or anyway:
418
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
419 if self.cookies:
3e8a14e @kennethreitz pre_request hook
authored
420
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
421 # Skip if 'cookie' header is explicitly set.
422 if 'cookie' not in self.headers:
3e8a14e @kennethreitz pre_request hook
authored
423
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
424 # Simple cookie with our dict.
8ffe17c @kennethreitz use monkey patched cookies
authored
425 c = oreos.monkeys.SimpleCookie()
2584c48 @kennethreitz python2.5 cookie support
authored
426 for (k, v) in self.cookies.items():
427 c[k] = v
6b11cf2 @kennethreitz todo
authored
428
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
429 # Turn it into a header.
89241c5 @darkrho use ";" instead "\r\n" as separator for cookies as is better supported
darkrho authored
430 cookie_header = c.output(header='', sep='; ').strip()
6b11cf2 @kennethreitz todo
authored
431
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
432 # Attach Cookie header to request.
433 self.headers['Cookie'] = cookie_header
6b11cf2 @kennethreitz todo
authored
434
6a51f6b @kennethreitz add 'max_retries' configuration
authored
435 try:
0c4b973 @kennethreitz keep_alive is now disable-able!
authored
436 # Send the request.
6a51f6b @kennethreitz add 'max_retries' configuration
authored
437 r = conn.urlopen(
438 method=self.method,
4504953 @kennethreitz Only send URL path to servers
authored
439 url=self.path_url,
6a51f6b @kennethreitz add 'max_retries' configuration
authored
440 body=body,
441 headers=self.headers,
442 redirect=False,
443 assert_same_host=False,
bd45bc7 @kennethreitz add prefetch to main request distpatch
authored
444 preload_content=prefetch,
6a51f6b @kennethreitz add 'max_retries' configuration
authored
445 decode_content=False,
7c206c7 @kennethreitz Timeouts for urllib3
authored
446 retries=self.config.get('max_retries', 0),
447 timeout=self.timeout,
6a51f6b @kennethreitz add 'max_retries' configuration
authored
448 )
0c4b973 @kennethreitz keep_alive is now disable-able!
authored
449
7c206c7 @kennethreitz Timeouts for urllib3
authored
450
6a51f6b @kennethreitz add 'max_retries' configuration
authored
451 except MaxRetryError, e:
35ed9b5 @kennethreitz dict(o or [])
authored
452 if not self.config.get('safe_mode', False):
453 raise ConnectionError(e)
454 else:
455 r = None
14ef462 @kennethreitz package refactor
authored
456
7c206c7 @kennethreitz Timeouts for urllib3
authored
457 except (_SSLError, _HTTPError), e:
458 if not self.config.get('safe_mode', False):
459 raise Timeout('Request timed out.')
460
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
461 self._build_response(r)
14ef462 @kennethreitz package refactor
authored
462
3d3f058 @kennethreitz use new system
authored
463 # Response manipulation hook.
464 self.response = dispatch_hook('response', self.hooks, self.response)
491a3c0 @kennethreitz Move hooks into internal event loop
authored
465
3d3f058 @kennethreitz use new system
authored
466 # Post-request hook.
467 r = dispatch_hook('post_request', self.hooks, self)
468 self.__dict__.update(r.__dict__)
491a3c0 @kennethreitz Move hooks into internal event loop
authored
469
d1b6db0 @kennethreitz prefetch handling for _content_consumed
authored
470 # If prefetch is True, mark content as consumed.
471 if prefetch:
472 self.response._content_consumed = True
473
3d3f058 @kennethreitz use new system
authored
474 return self.sent
14ef462 @kennethreitz package refactor
authored
475
476
477 class Response(object):
596aceb @kennethreitz big docs update
authored
478 """The core :class:`Response <Response>` object. All
479 :class:`Request <Request>` objects contain a
480 :class:`response <Response>` attribute, which is an instance
6e13e73 @kennethreitz Big API update
authored
481 of this class.
14ef462 @kennethreitz package refactor
authored
482 """
483
484 def __init__(self):
317c5b6 @kennethreitz _content
authored
485
891e52d @jerem Added the possibility to access headers without fetching the whole bo…
jerem authored
486 self._content = None
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
487 self._content_consumed = False
13db806 @kennethreitz docstringin' it up
authored
488
6e13e73 @kennethreitz Big API update
authored
489 #: Integer Code of responded HTTP Status.
14ef462 @kennethreitz package refactor
authored
490 self.status_code = None
1f3e53b @kennethreitz yay
authored
491
6e13e73 @kennethreitz Big API update
authored
492 #: Case-insensitive Dictionary of Response Headers.
493 #: For example, ``headers['content-encoding']`` will return the
494 #: value of a ``'Content-Encoding'`` response header.
85b8b2b @kennethreitz CaseInsensitiveDict for Headers
authored
495 self.headers = CaseInsensitiveDict()
13db806 @kennethreitz docstringin' it up
authored
496
497 #: File-like object representation of response (for advanced usage).
573ab09 @kennethreitz fo => raw
authored
498 self.raw = None
13db806 @kennethreitz docstringin' it up
authored
499
6e13e73 @kennethreitz Big API update
authored
500 #: Final URL location of Response.
14ef462 @kennethreitz package refactor
authored
501 self.url = None
13db806 @kennethreitz docstringin' it up
authored
502
055f7c1 @tvaught Some Doc and doctoring fixes by the Speling Poleece.
tvaught authored
503 #: Resulting :class:`HTTPError` of request, if one occurred.
14ef462 @kennethreitz package refactor
authored
504 self.error = None
13db806 @kennethreitz docstringin' it up
authored
505
296d8cc @kennethreitz new (elegant) request decoding model.
authored
506 #: Encoding to decode with when accessing r.content.
507 self.encoding = None
508
596aceb @kennethreitz big docs update
authored
509 #: A list of :class:`Response <Response>` objects from
6e13e73 @kennethreitz Big API update
authored
510 #: the history of the Request. Any redirect responses will end
511 #: up here.
cd8ce63 @kennethreitz Custom RedirectHandler and .history support.
authored
512 self.history = []
13db806 @kennethreitz docstringin' it up
authored
513
28ee978 @kennethreitz Request
authored
514 #: The :class:`Request <Request>` that created the Response.
2d70647 @kennethreitz attach request object to responses
authored
515 self.request = None
13db806 @kennethreitz docstringin' it up
authored
516
e0ae2b4 @kennethreitz fixes
authored
517 #: A dictionary of Cookies the server sent back.
147c241 @kennethreitz better defaults for "models"
authored
518 self.cookies = {}
2d70647 @kennethreitz attach request object to responses
authored
519
903f96c @kennethreitz use config throughout models
authored
520 #: Dictionary of configurations for this request.
7192b80 @kennethreitz import ConnectionError
authored
521 self.config = {}
903f96c @kennethreitz use config throughout models
authored
522
14ef462 @kennethreitz package refactor
authored
523
524 def __repr__(self):
525 return '<Response [%s]>' % (self.status_code)
526
527 def __nonzero__(self):
6e13e73 @kennethreitz Big API update
authored
528 """Returns true if :attr:`status_code` is 'OK'."""
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
529 return self.ok
882ea76 @kennethreitz cleanups, response.fo
authored
530
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
531 @property
532 def ok(self):
d04f31b @kennethreitz HTTPError
authored
533 try:
534 self.raise_for_status()
535 except HTTPError:
536 return False
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
537 return True
538
14ef462 @kennethreitz package refactor
authored
539
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
540 def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
541 """Iterates over the response data. This avoids reading the content
542 at once into memory for large responses. The chunk size is the number
543 of bytes it should read into memory. This is not necessarily the
544 length of each item returned as decoding can take place.
545 """
546 if self._content_consumed:
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
547 raise RuntimeError(
548 'The content for this response was already consumed'
549 )
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
550
551 def generate():
552 while 1:
573ab09 @kennethreitz fo => raw
authored
553 chunk = self.raw.read(chunk_size)
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
554 if not chunk:
555 break
556 yield chunk
557 self._content_consumed = True
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
558
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
559 gen = generate()
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
560
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
561 if 'gzip' in self.headers.get('content-encoding', ''):
562 gen = stream_decode_gzip(gen)
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
563
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
564 if decode_unicode is None:
903f96c @kennethreitz use config throughout models
authored
565 decode_unicode = self.config.get('decode_unicode')
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
566
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
567 if decode_unicode:
568 gen = stream_decode_response_unicode(gen, self)
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
569
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
570 return gen
14ef462 @kennethreitz package refactor
authored
571
428fc9c @kennethreitz PASS ALL TEH TESTS
authored
572
6a96652 @densh Refactor response.content into a property. This makes content attribu…
densh authored
573 @property
574 def content(self):
575 """Content of the response, in bytes or unicode
317c5b6 @kennethreitz _content
authored
576 (if available).
577 """
14ef462 @kennethreitz package refactor
authored
578
296d8cc @kennethreitz new (elegant) request decoding model.
authored
579 if self._content is None:
580 # Read the contents.
581 try:
582 if self._content_consumed:
583 raise RuntimeError(
584 'The content for this response was already consumed')
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
585
296d8cc @kennethreitz new (elegant) request decoding model.
authored
586 self._content = self.raw.read()
587 except AttributeError:
588 self._content = None
f7968b6 @kennethreitz r.content is None of there's an invalid response.
authored
589
296d8cc @kennethreitz new (elegant) request decoding model.
authored
590 content = self._content
882ea76 @kennethreitz cleanups, response.fo
authored
591
6a96652 @densh Refactor response.content into a property. This makes content attribu…
densh authored
592 # Decode GZip'd content.
593 if 'gzip' in self.headers.get('content-encoding', ''):
594 try:
296d8cc @kennethreitz new (elegant) request decoding model.
authored
595 content = decode_gzip(self._content)
6a96652 @densh Refactor response.content into a property. This makes content attribu…
densh authored
596 except zlib.error:
597 pass
882ea76 @kennethreitz cleanups, response.fo
authored
598
6a96652 @densh Refactor response.content into a property. This makes content attribu…
densh authored
599 # Decode unicode content.
903f96c @kennethreitz use config throughout models
authored
600 if self.config.get('decode_unicode'):
296d8cc @kennethreitz new (elegant) request decoding model.
authored
601
602 # Try charset from content-type
603
604 if self.encoding:
605 try:
606 content = unicode(content, self.encoding)
607 except UnicodeError:
608 pass
609
610 # Fall back:
611 try:
612 content = unicode(content, self.encoding, errors='replace')
613 except TypeError:
614 pass
882ea76 @kennethreitz cleanups, response.fo
authored
615
41876fd @mitsuhiko Implemented content streaming for responses.
mitsuhiko authored
616 self._content_consumed = True
296d8cc @kennethreitz new (elegant) request decoding model.
authored
617 return content
882ea76 @kennethreitz cleanups, response.fo
authored
618
891e52d @jerem Added the possibility to access headers without fetching the whole bo…
jerem authored
619
14ef462 @kennethreitz package refactor
authored
620 def raise_for_status(self):
055f7c1 @tvaught Some Doc and doctoring fixes by the Speling Poleece.
tvaught authored
621 """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
dac050b @kennethreitz New status_code-based errors
authored
622
14ef462 @kennethreitz package refactor
authored
623 if self.error:
624 raise self.error
625
dac050b @kennethreitz New status_code-based errors
authored
626 if (self.status_code >= 300) and (self.status_code < 400):
a548b62 @kennethreitz HTTPError
authored
627 raise HTTPError('%s Redirection' % self.status_code)
dac050b @kennethreitz New status_code-based errors
authored
628
629 elif (self.status_code >= 400) and (self.status_code < 500):
a548b62 @kennethreitz HTTPError
authored
630 raise HTTPError('%s Client Error' % self.status_code)
dac050b @kennethreitz New status_code-based errors
authored
631
632 elif (self.status_code >= 500) and (self.status_code < 600):
a548b62 @kennethreitz HTTPError
authored
633 raise HTTPError('%s Server Error' % self.status_code)
dac050b @kennethreitz New status_code-based errors
authored
634
635
Something went wrong with that request. Please try again.