Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 324 lines (260 sloc) 10.5 kB
7c680db @jschneier add HTTPHeaderDict
jschneier authored
1 from collections import Mapping, MutableMapping
929f158 @shazow Add support for platforms without threading, fixes #289.
authored
2 try:
3 from threading import RLock
f3d4488 @seocam Improved HTTPHeaderDict performance
seocam authored
4 except ImportError: # Platform-specific: No threads available
929f158 @shazow Add support for platforms without threading, fixes #289.
authored
5 class RLock:
6 def __enter__(self):
7 pass
8
9 def __exit__(self, exc_type, exc_value, traceback):
10 pass
11
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
12
f3d4488 @seocam Improved HTTPHeaderDict performance
seocam authored
13 try: # Python 2.7+
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
14 from collections import OrderedDict
15 except ImportError:
16 from .packages.ordered_dict import OrderedDict
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
17 from .packages.six import iterkeys, itervalues, PY3
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
18
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
19
7c680db @jschneier add HTTPHeaderDict
jschneier authored
20 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
21
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
22
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
23 _Null = object()
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
24
25
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
26 class RecentlyUsedContainer(MutableMapping):
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
27 """
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
28 Provides a thread-safe dict-like container which maintains up to
29 ``maxsize`` keys while throwing away the least-recently-used keys beyond
30 ``maxsize``.
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
31
32 :param maxsize:
33 Maximum number of recent elements to retain.
34
35 :param dispose_func:
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
36 Every time an item is evicted from the container,
37 ``dispose_func(value)`` is called. Callback which will get called
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
38 """
39
9fb930f @shazow More pool-closing tests, 99% coverage.
authored
40 ContainerCls = OrderedDict
4a68690 @shazow <Py26 support and more dict-like behaviour for RecentlyUsedContainer …
authored
41
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
42 def __init__(self, maxsize=10, dispose_func=None):
43 self._maxsize = maxsize
44 self.dispose_func = dispose_func
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
45
9fb930f @shazow More pool-closing tests, 99% coverage.
authored
46 self._container = self.ContainerCls()
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
47 self.lock = RLock()
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
48
49 def __getitem__(self, key):
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
50 # Re-insert the item, moving it to the end of the eviction line.
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
51 with self.lock:
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
52 item = self._container.pop(key)
53 self._container[key] = item
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
54 return item
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
55
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
56 def __setitem__(self, key, value):
57 evicted_value = _Null
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
58 with self.lock:
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
59 # Possibly evict the existing value of 'key'
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
60 evicted_value = self._container.get(key, _Null)
61 self._container[key] = value
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
62
63 # If we didn't evict an existing value, we might have to evict the
64 # least recently used item from the beginning of the container.
65 if len(self._container) > self._maxsize:
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
66 _key, evicted_value = self._container.popitem(last=False)
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
67
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
68 if self.dispose_func and evicted_value is not _Null:
69 self.dispose_func(evicted_value)
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
70
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
71 def __delitem__(self, key):
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
72 with self.lock:
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
73 value = self._container.pop(key)
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
74
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
75 if self.dispose_func:
31f2711 @shazow Clarification and documentation for our new RecentlyUsedContainer.
authored
76 self.dispose_func(value)
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
77
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
78 def __len__(self):
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
79 with self.lock:
8fc94d7 @shazow Cleanup new RecentlyUsedContainer.
authored
80 return len(self._container)
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
81
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
82 def __iter__(self):
83 raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
20b85f5 @shazow Refactoring: Moved RecentlyUsedContainer out to its own submodule.
authored
84
ba162b7 @shazow More tests, PoolManager.close is now PoolManager.clear.
authored
85 def clear(self):
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
86 with self.lock:
ba162b7 @shazow More tests, PoolManager.close is now PoolManager.clear.
authored
87 # Copy pointers to all values, then wipe the mapping
2666629 @abacabadabacaba Made RecentlyUsedContainer.keys() thread-safe on Python 3, removed un…
abacabadabacaba authored
88 values = list(itervalues(self._container))
ba162b7 @shazow More tests, PoolManager.close is now PoolManager.clear.
authored
89 self._container.clear()
90
91 if self.dispose_func:
92 for value in values:
93 self.dispose_func(value)
94
9cd973b @slingamn Refactor to support eager release of pooled connections
slingamn authored
95 def keys(self):
29a57df @AudriusButkevicius Fix a race condition between pools.get and pools.set
AudriusButkevicius authored
96 with self.lock:
2666629 @abacabadabacaba Made RecentlyUsedContainer.keys() thread-safe on Python 3, removed un…
abacabadabacaba authored
97 return list(iterkeys(self._container))
7c680db @jschneier add HTTPHeaderDict
jschneier authored
98
99
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
100 class HTTPHeaderDict(MutableMapping):
332a901 @jschneier document and change HTTPHeaderDict, update CHANGES for bugfix
jschneier authored
101 """
102 :param headers:
103 An iterable of field-value pairs. Must not contain multiple field names
104 when compared case-insensitively.
105
106 :param kwargs:
107 Additional field-value pairs to pass in to ``dict.update``.
108
109 A ``dict`` like container for storing HTTP Headers.
110
111 Field names are stored and compared case-insensitively in compliance with
4a968d7 @Lukasa Update references to RFC 2616
Lukasa authored
112 RFC 7230. Iteration provides the first case-sensitive key seen for each
332a901 @jschneier document and change HTTPHeaderDict, update CHANGES for bugfix
jschneier authored
113 case-insensitive pair.
114
115 Using ``__setitem__`` syntax overwrites fields that compare equal
116 case-insensitively in order to maintain ``dict``'s api. For fields that
117 compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
118 in a loop.
119
120 If multiple fields that are equal case-insensitively are passed to the
121 constructor or ``.update``, the behavior is undefined and some will be
122 lost.
123
124 >>> headers = HTTPHeaderDict()
125 >>> headers.add('Set-Cookie', 'foo=bar')
126 >>> headers.add('set-cookie', 'baz=quxx')
127 >>> headers['content-length'] = '7'
128 >>> headers['SET-cookie']
129 'foo=bar, baz=quxx'
130 >>> headers['Content-Length']
131 '7'
132 """
7c680db @jschneier add HTTPHeaderDict
jschneier authored
133
134 def __init__(self, headers=None, **kwargs):
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
135 super(HTTPHeaderDict, self).__init__()
136 self._container = {}
d9640af @ml31415 Faster headers implementation
ml31415 authored
137 if headers is not None:
c496370 @ml31415 Inheritance check modified on init
ml31415 authored
138 if isinstance(headers, HTTPHeaderDict):
1affd18 @ml31415 Tests and fix for #564
ml31415 authored
139 self._copy_from(headers)
140 else:
141 self.extend(headers)
d9640af @ml31415 Faster headers implementation
ml31415 authored
142 if kwargs:
49c4727 @ml31415 from_httplib removed; extend used by default in constructor
ml31415 authored
143 self.extend(kwargs)
d9640af @ml31415 Faster headers implementation
ml31415 authored
144
145 def __setitem__(self, key, val):
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
146 self._container[key.lower()] = (key, val)
147 return self._container[key.lower()]
d9640af @ml31415 Faster headers implementation
ml31415 authored
148
149 def __getitem__(self, key):
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
150 val = self._container[key.lower()]
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
151 return ', '.join(val[1:])
d9640af @ml31415 Faster headers implementation
ml31415 authored
152
153 def __delitem__(self, key):
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
154 del self._container[key.lower()]
7c680db @jschneier add HTTPHeaderDict
jschneier authored
155
d9640af @ml31415 Faster headers implementation
ml31415 authored
156 def __contains__(self, key):
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
157 return key.lower() in self._container
d9640af @ml31415 Faster headers implementation
ml31415 authored
158
159 def __eq__(self, other):
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
160 if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
d9640af @ml31415 Faster headers implementation
ml31415 authored
161 return False
162 if not isinstance(other, type(self)):
163 other = type(self)(other)
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
164 return (dict((k.lower(), v) for k, v in self.itermerged()) ==
165 dict((k.lower(), v) for k, v in other.itermerged()))
d9640af @ml31415 Faster headers implementation
ml31415 authored
166
dc946a0 @ml31415 Fixed non-equal comparison; rename update_add to extend
ml31415 authored
167 def __ne__(self, other):
168 return not self.__eq__(other)
169
8e43f8e @ml31415 .items() returns now list including duplicates identical to PY3 behav…
ml31415 authored
170 if not PY3: # Python 2
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
171 iterkeys = MutableMapping.iterkeys
172 itervalues = MutableMapping.itervalues
328d129 @ml31415 Python3 fixes
ml31415 authored
173
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
174 __marker = object()
175
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
176 def __len__(self):
177 return len(self._container)
178
179 def __iter__(self):
180 # Only provide the originally cased names
181 for vals in self._container.values():
182 yield vals[0]
183
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
184 def pop(self, key, default=__marker):
185 '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
186 If key is not found, d is returned if given, otherwise KeyError is raised.
187 '''
188 # Using the MutableMapping function directly fails due to the private marker.
189 # Using ordinary dict.pop would expose the internal structures.
190 # So let's reinvent the wheel.
191 try:
192 value = self[key]
193 except KeyError:
194 if default is self.__marker:
195 raise
196 return default
197 else:
198 del self[key]
199 return value
200
201 def discard(self, key):
202 try:
203 del self[key]
204 except KeyError:
205 pass
206
d9640af @ml31415 Faster headers implementation
ml31415 authored
207 def add(self, key, val):
332a901 @jschneier document and change HTTPHeaderDict, update CHANGES for bugfix
jschneier authored
208 """Adds a (name, value) pair, doesn't overwrite the value if it already
209 exists.
7c680db @jschneier add HTTPHeaderDict
jschneier authored
210
332a901 @jschneier document and change HTTPHeaderDict, update CHANGES for bugfix
jschneier authored
211 >>> headers = HTTPHeaderDict(foo='bar')
212 >>> headers.add('Foo', 'baz')
213 >>> headers['foo']
214 'bar, baz'
215 """
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
216 key_lower = key.lower()
217 new_vals = key, val
218 # Keep the common case aka no item present as fast as possible
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
219 vals = self._container.setdefault(key_lower, new_vals)
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
220 if new_vals is not vals:
7115efd @ml31415 Checking header name before creating multiheader
ml31415 authored
221 # new_vals was not inserted, as there was a previous one
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
222 if isinstance(vals, list):
223 # If already several items got inserted, we have a list
224 vals.append(val)
d9640af @ml31415 Faster headers implementation
ml31415 authored
225 else:
7115efd @ml31415 Checking header name before creating multiheader
ml31415 authored
226 # vals should be a tuple then, i.e. only one item so far
94e633e @ml31415 Fix #561
ml31415 authored
227 # Need to convert the tuple to list for further extension
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
228 self._container[key_lower] = [vals[0], vals[1], val]
d9640af @ml31415 Faster headers implementation
ml31415 authored
229
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
230 def extend(self, *args, **kwargs):
b493eeb @ml31415 Full test coverage for _collection
ml31415 authored
231 """Generic import function for any type of header-like object.
232 Adapted version of MutableMapping.update in order to insert items
d9640af @ml31415 Faster headers implementation
ml31415 authored
233 with self.add instead of self.__setitem__
234 """
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
235 if len(args) > 1:
236 raise TypeError("extend() takes at most 1 positional "
d9640af @ml31415 Faster headers implementation
ml31415 authored
237 "arguments ({} given)".format(len(args)))
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
238 other = args[0] if len(args) >= 1 else ()
838d23a @shazow Tests for HTTPHeaderDict in requests (re: #632)
authored
239
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
240 if isinstance(other, HTTPHeaderDict):
241 for key, val in other.iteritems():
242 self.add(key, val)
243 elif isinstance(other, Mapping):
d9640af @ml31415 Faster headers implementation
ml31415 authored
244 for key in other:
245 self.add(key, other[key])
246 elif hasattr(other, "keys"):
247 for key in other.keys():
248 self.add(key, other[key])
249 else:
250 for key, value in other:
251 self.add(key, value)
252
49c4727 @ml31415 from_httplib removed; extend used by default in constructor
ml31415 authored
253 for key, value in kwargs.items():
d9640af @ml31415 Faster headers implementation
ml31415 authored
254 self.add(key, value)
7c680db @jschneier add HTTPHeaderDict
jschneier authored
255
332a901 @jschneier document and change HTTPHeaderDict, update CHANGES for bugfix
jschneier authored
256 def getlist(self, key):
257 """Returns a list of all the values for the named field. Returns an
258 empty list if the key doesn't exist."""
d9640af @ml31415 Faster headers implementation
ml31415 authored
259 try:
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
260 vals = self._container[key.lower()]
d9640af @ml31415 Faster headers implementation
ml31415 authored
261 except KeyError:
262 return []
263 else:
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
264 if isinstance(vals, tuple):
265 return [vals[1]]
d9640af @ml31415 Faster headers implementation
ml31415 authored
266 else:
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
267 return vals[1:]
7c680db @jschneier add HTTPHeaderDict
jschneier authored
268
d9640af @ml31415 Faster headers implementation
ml31415 authored
269 # Backwards compatibility for httplib
270 getheaders = getlist
271 getallmatchingheaders = getlist
272 iget = getlist
7c680db @jschneier add HTTPHeaderDict
jschneier authored
273
274 def __repr__(self):
8e43f8e @ml31415 .items() returns now list including duplicates identical to PY3 behav…
ml31415 authored
275 return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
bd258fc @seocam Replaced concat/split by actual collection stuff
seocam authored
276
1affd18 @ml31415 Tests and fix for #564
ml31415 authored
277 def _copy_from(self, other):
278 for key in other:
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
279 val = other.getlist(key)
d9640af @ml31415 Faster headers implementation
ml31415 authored
280 if isinstance(val, list):
5e73e72 @ml31415 Alternative implementation of headers
ml31415 authored
281 # Don't need to convert tuples
d9640af @ml31415 Faster headers implementation
ml31415 authored
282 val = list(val)
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
283 self._container[key.lower()] = [key] + val
1affd18 @ml31415 Tests and fix for #564
ml31415 authored
284
285 def copy(self):
286 clone = type(self)()
287 clone._copy_from(self)
d9640af @ml31415 Faster headers implementation
ml31415 authored
288 return clone
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
289
66a5141 @ml31415 self.items() with preserved headers
ml31415 authored
290 def iteritems(self):
8e43f8e @ml31415 .items() returns now list including duplicates identical to PY3 behav…
ml31415 authored
291 """Iterate over all header lines, including duplicate ones."""
292 for key in self:
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
293 vals = self._container[key.lower()]
8e43f8e @ml31415 .items() returns now list including duplicates identical to PY3 behav…
ml31415 authored
294 for val in vals[1:]:
295 yield vals[0], val
296
297 def itermerged(self):
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
298 """Iterate over all headers, merging duplicate ones together."""
66a5141 @ml31415 self.items() with preserved headers
ml31415 authored
299 for key in self:
64adf9f @sigmavirus24 Change HTTPHeaderDict superclass to MutableMapping
sigmavirus24 authored
300 val = self._container[key.lower()]
66a5141 @ml31415 self.items() with preserved headers
ml31415 authored
301 yield val[0], ', '.join(val[1:])
302
303 def items(self):
304 return list(self.iteritems())
d9640af @ml31415 Faster headers implementation
ml31415 authored
305
cb6372f @ml31415 .pop() fixed; py2 header parsing fixed
ml31415 authored
306 @classmethod
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
307 def from_httplib(cls, message): # Python 2
a5e6206 @ml31415 Fixing py2.6; renaming of import
ml31415 authored
308 """Read headers from a Python 2 httplib message object."""
20a22d1 @ml31415 Comment added
ml31415 authored
309 # python2.7 does not expose a proper API for exporting multiheaders
838d23a @shazow Tests for HTTPHeaderDict in requests (re: #632)
authored
310 # efficiently. This function re-reads raw lines from the message
20a22d1 @ml31415 Comment added
ml31415 authored
311 # object and extracts the multiheaders properly.
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
312 headers = []
838d23a @shazow Tests for HTTPHeaderDict in requests (re: #632)
authored
313
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
314 for line in message.headers:
315 if line.startswith((' ', '\t')):
316 key, value = headers[-1]
317 headers[-1] = (key, value + '\r\n' + line.rstrip())
318 continue
838d23a @shazow Tests for HTTPHeaderDict in requests (re: #632)
authored
319
ed15c4a @ml31415 Added full support for python2.7 multiheaders, borrowed from https://…
ml31415 authored
320 key, value = line.split(':', 1)
321 headers.append((key, value.strip()))
322
323 return cls(headers)
Something went wrong with that request. Please try again.