Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 347 lines (279 sloc) 11.147 kb
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
1 """
2 kombu.transport.SQS
3 ===================
4
5 Amazon SQS transport.
6
7 :copyright: (c) 2010 - 2011 by Ask Solem
8 :license: BSD, see LICENSE for more details.
9
10 """
66ef950 @ask 2.0-devel: No longer supports Python 2.4
authored
11 from __future__ import absolute_import
12
8bae43c @ask Adds connection/channel_errors for SQS transport
authored
13 import socket
6bcedcb @ask SQS transport now replaces dots in queue names with dashes
authored
14 import string
8bae43c @ask Adds connection/channel_errors for SQS transport
authored
15
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
16 from Queue import Empty
17
18 from anyjson import serialize, deserialize
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
19
8bae43c @ask Adds connection/channel_errors for SQS transport
authored
20 from boto import exception
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
21 from boto import sdb as _sdb
22 from boto import sqs as _sqs
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
23 from boto.sdb.domain import Domain
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
24 from boto.sdb.connection import SDBConnection
25 from boto.sqs.connection import SQSConnection
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
26 from boto.sqs.message import Message
27
66ef950 @ask 2.0-devel: No longer supports Python 2.4
authored
28 from ..utils import cached_property, uuid
b25657c @ask Merge branch 'master' into 2.0-devel
authored
29 from ..utils.encoding import safe_str
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
30
66ef950 @ask 2.0-devel: No longer supports Python 2.4
authored
31 from . import virtual
a5816c4 @ask Updates README with SQS transport
authored
32
6bcedcb @ask SQS transport now replaces dots in queue names with dashes
authored
33 # dots are replaced by dash, all other punctuation
34 # replaced by underscore.
e68693f @dahlia SQS Transport: Must have lowercase transport alias.
dahlia authored
35 CHARS_REPLACE_TABLE = dict((ord(c), 0x5f)
c5b4d47 @dahlia unicode.translate cannot take table encoded by string.maketrans funct…
dahlia authored
36 for c in string.punctuation if c not in '-_.')
e68693f @dahlia SQS Transport: Must have lowercase transport alias.
dahlia authored
37 CHARS_REPLACE_TABLE[0x2e] = 0x2d # '.' -> '-'
6bcedcb @ask SQS transport now replaces dots in queue names with dashes
authored
38
39
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
40 class Table(Domain):
41 """Amazon SimpleDB domain describing the message routing table."""
af4d2b0 @ask SQS improvements
authored
42 # caches queues already bound, so we don't have to declare them again.
43 _already_bound = set()
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
44
45 def routes_for(self, exchange):
46 """Iterator giving all routes for an exchange."""
1380ad0 @ask Fix for list bindings
authored
47 return self.select("""WHERE exchange = '%s'""" % exchange)
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
48
49 def get_queue(self, queue):
50 """Get binding for queue."""
51 qid = self._get_queue_id(queue)
52 if qid:
53 return self.get_item(qid)
54
55 def create_binding(self, queue):
56 """Get binding item for queue.
57
58 Creates the item if it doesn't exist.
59
60 """
61 item = self.get_queue(queue)
62 if item:
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
63 return item, item["id"]
e0bd3e5 @ask gen_unique_id() -> uuid() (but both avail. for compat.)
authored
64 id = uuid()
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
65 return self.new_item(id), id
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
66
af4d2b0 @ask SQS improvements
authored
67 def queue_bind(self, exchange, routing_key, pattern, queue):
68 if queue not in self._already_bound:
69 binding, id = self.create_binding(queue)
70 binding.update(exchange=exchange,
71 routing_key=routing_key or "",
72 pattern=pattern or "",
73 queue=queue or "",
74 id=id)
75 binding.save()
76 self._already_bound.add(queue)
77
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
78 def queue_delete(self, queue):
79 """delete queue by name."""
af4d2b0 @ask SQS improvements
authored
80 self._already_bound.discard(queue)
81 item = self._get_queue_item(queue)
82 if item:
83 self.delete_item(item)
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
84
85 def exchange_delete(self, exchange):
86 """Delete all routes for `exchange`."""
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
87 for item in self.routes_for(exchange):
b24869f @ask PEP8ify + pyflakes
authored
88 self.delete_item(item["id"])
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
89
af4d2b0 @ask SQS improvements
authored
90 def get_item(self, item_name):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
91 """Uses `consistent_read` by default."""
92 # Domain is an old-style class, can't use super().
af4d2b0 @ask SQS improvements
authored
93 for consistent_read in (False, True):
94 item = Domain.get_item(self, item_name, consistent_read)
95 if item:
96 return item
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
97
98 def select(self, query='', next_token=None, consistent_read=True,
99 max_items=None):
100 """Uses `consistent_read` by default."""
1380ad0 @ask Fix for list bindings
authored
101 query = """SELECT * FROM `%s` %s""" % (self.name, query)
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
102 return Domain.select(self, query, next_token,
103 consistent_read, max_items)
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
104
af4d2b0 @ask SQS improvements
authored
105 def _try_first(self, query='', **kwargs):
106 for c in (False, True):
107 for item in self.select(query, consistent_read=c, **kwargs):
108 return item
109
1380ad0 @ask Fix for list bindings
authored
110 def get_exchanges(self):
111 return list(set(i["exchange"] for i in self.select()))
112
af4d2b0 @ask SQS improvements
authored
113 def _get_queue_item(self, queue):
1380ad0 @ask Fix for list bindings
authored
114 return self._try_first("""WHERE queue = '%s' limit 1""" % queue)
af4d2b0 @ask SQS improvements
authored
115
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
116 def _get_queue_id(self, queue):
af4d2b0 @ask SQS improvements
authored
117 item = self._get_queue_item(queue)
118 if item:
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
119 return item["id"]
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
120
121
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
122 class Channel(virtual.Channel):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
123 Table = Table
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
124
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
125 default_region = "us-east-1"
126 domain_format = "kombu%(vhost)s"
127 _sdb = None
128 _sqs = None
af4d2b0 @ask SQS improvements
authored
129 _queue_cache = {}
130 _noack_queues = set()
131
4d3b490 @zmsmith Cache existing SQS queues on startup
zmsmith authored
132 def __init__(self, *args, **kwargs):
133 super(Channel, self).__init__(*args, **kwargs)
f5274e4 @ask Adds missing reference for kombu.utils.encoding
authored
134
135 # SQS blows up when you try to create a new queue if one already
136 # exists with a different visibility_timeout, so this prepopulates
137 # the queue_cache to protect us from recreating
138 # queues that are known to already exist.
4d3b490 @zmsmith Cache existing SQS queues on startup
zmsmith authored
139 queues = self.sqs.get_all_queues()
140 for queue in queues:
141 self._queue_cache[queue.name] = queue
142
af4d2b0 @ask SQS improvements
authored
143 def basic_consume(self, queue, no_ack, *args, **kwargs):
144 if no_ack:
145 self._noack_queues.add(queue)
146 return super(Channel, self).basic_consume(queue, no_ack,
147 *args, **kwargs)
148
149 def basic_cancel(self, consumer_tag):
150 if consumer_tag in self._consumers:
151 queue = self._tag_to_queue[consumer_tag]
152 self._noack_queues.discard(queue)
153 return super(Channel, self).basic_cancel(consumer_tag)
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
154
6bcedcb @ask SQS transport now replaces dots in queue names with dashes
authored
155 def entity_name(self, name, table=CHARS_REPLACE_TABLE):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
156 """Format AMQP queue name into a legal SQS queue name."""
419d08b @ask str.encode does not accept kwargs in Py < 2.7. Closes #68. Thanks t…
authored
157 return safe_str(name).translate(table)
6bcedcb @ask SQS transport now replaces dots in queue names with dashes
authored
158
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
159 def _new_queue(self, queue, **kwargs):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
160 """Ensures a queue exists in SQS."""
48ca2cd Added support for queue prefixes in SQS.
Nitzan Miron authored
161 queue = self.queue_name_prefix + queue
af4d2b0 @ask SQS improvements
authored
162 try:
163 return self._queue_cache[queue]
164 except KeyError:
165 q = self._queue_cache[queue] = self.sqs.create_queue(
166 self.entity_name(queue),
a5f54f0 SQS transport now replaces dots in queue names with dashes, and all o…
Ask Solem authored
167 self.visibility_timeout)
af4d2b0 @ask SQS improvements
authored
168 return q
169
170 def _queue_bind(self, *args):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
171 """Bind ``queue`` to ``exchange`` with routing key.
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
172
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
173 Route will be stored in SDB if so enabled.
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
174
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
175 """
af4d2b0 @ask SQS improvements
authored
176 if self.supports_fanout:
177 self.table.queue_bind(*args)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
178
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
179 def get_table(self, exchange):
180 """Get routing table.
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
181
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
182 Retrieved from SDB if :attr:`supports_fanout`.
183
184 """
185 if self.supports_fanout:
186 return [(r["routing_key"], r["pattern"], r["queue"])
187 for r in self.table.routes_for(exchange)]
188 return super(Channel, self).get_table(exchange)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
189
1380ad0 @ask Fix for list bindings
authored
190 def get_exchanges(self):
191 if self.supports_fanout:
192 return self.table.get_exchanges()
193 return super(Channel, self).get_exchanges()
194
9979ea0 @ask Redis: Automatically delete auto_delete queues at basic_cancel
authored
195 def _delete(self, queue, *args):
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
196 """delete queue by name."""
af4d2b0 @ask SQS improvements
authored
197 self._queue_cache.pop(queue, None)
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
198 self.table.queue_delete(queue)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
199 super(Channel, self)._delete(queue)
200
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
201 def exchange_delete(self, exchange, **kwargs):
202 """Delete exchange by name."""
203 if self.supports_fanout:
204 self.table.exchange_delete(exchange)
205 super(Channel, self).exchange_delete(exchange, **kwargs)
206
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
207 def _has_queue(self, queue, **kwargs):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
208 """Returns True if ``queue`` has been previously declared."""
209 if self.supports_fanout:
210 return bool(self.table.get_queue(queue))
211 return super(Channel, self)._has_queue(queue)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
212
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
213 def _put(self, queue, message, **kwargs):
214 """Put message onto queue."""
215 q = self._new_queue(queue)
216 m = Message()
217 m.set_body(serialize(message))
218 q.write(m)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
219
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
220 def _put_fanout(self, exchange, message, **kwargs):
221 """Deliver fanout message to all queues in ``exchange``."""
222 for route in self.table.routes_for(exchange):
223 self._put(route["queue"], message, **kwargs)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
224
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
225 def _get(self, queue):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
226 """Try to retrieve a single message off ``queue``."""
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
227 q = self._new_queue(queue)
228 rs = q.get_messages(1)
229 if rs:
5ca7701 @ask SQS: Needs to delete message after receiving.
authored
230 m = rs[0]
91bc3eb @ask SQS: Now deletes the message on Message.ack()
authored
231 payload = deserialize(rs[0].get_body())
af4d2b0 @ask SQS improvements
authored
232 if queue in self._noack_queues:
233 q.delete_message(m)
234 else:
235 payload["properties"]["delivery_info"].update({
236 "sqs_message": m, "sqs_queue": q, })
91bc3eb @ask SQS: Now deletes the message on Message.ack()
authored
237 return payload
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
238 raise Empty()
239
91bc3eb @ask SQS: Now deletes the message on Message.ack()
authored
240 def basic_ack(self, delivery_tag):
241 delivery_info = self.qos.get(delivery_tag).delivery_info
af4d2b0 @ask SQS improvements
authored
242 try:
243 queue = delivery_info["sqs_queue"]
244 except KeyError:
245 pass
246 else:
247 queue.delete_message(delivery_info["sqs_message"])
91bc3eb @ask SQS: Now deletes the message on Message.ack()
authored
248 super(Channel, self).basic_ack(delivery_tag)
249
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
250 def _size(self, queue):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
251 """Returns the number of messages in a queue."""
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
252 return self._new_queue(queue).count()
253
254 def _purge(self, queue):
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
255 """Deletes all current messages in a queue."""
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
256 q = self._new_queue(queue)
897ee5c @ask Passes functional tests
authored
257 # SQS is slow at registering messages, so run for a few
258 # iterations to ensure messages are deleted.
259 size = 0
260 for i in xrange(10):
261 size += q.count()
262 if not size:
263 break
1ab629c @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
264 q.clear()
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
265 return size
266
267 def close(self):
268 super(Channel, self).close()
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
269 for conn in (self._sqs, self._sdb):
270 if conn:
ebd372a @adamn One of the previous merges changed the indentation of kombu/transport…
adamn authored
271 try:
824d1d9 @adamn Another indentation problem with the merge
adamn authored
272 conn.close()
ebd372a @adamn One of the previous merges changed the indentation of kombu/transport…
adamn authored
273 except AttributeError, exc: # FIXME ???
274 if "can't set attribute" not in str(exc):
275 raise
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
276
5ca7701 @ask SQS: Needs to delete message after receiving.
authored
277 def _get_regioninfo(self, regions):
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
278 if self.region:
279 for _r in regions:
280 if _r.name == self.region:
5ca7701 @ask SQS: Needs to delete message after receiving.
authored
281 return _r
282
283 def _aws_connect_to(self, fun, regions):
284 conninfo = self.conninfo
285 region = self._get_regioninfo(regions)
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
286 return fun(region=region,
287 aws_access_key_id=conninfo.userid,
288 aws_secret_access_key=conninfo.password,
289 port=conninfo.port)
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
290
25993c9 SQS Transport: Fix for KeyError on message acknowledgment-- (#73)
Brian Bernstein authored
291 def _next_delivery_tag(self):
292 return uuid() # See #73
293
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
294 @property
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
295 def sqs(self):
296 if self._sqs is None:
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
297 self._sqs = self._aws_connect_to(SQSConnection, _sqs.regions())
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
298 return self._sqs
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
299
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
300 @property
301 def sdb(self):
302 if self._sdb is None:
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
303 self._sdb = self._aws_connect_to(SDBConnection, _sdb.regions())
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
304 return self._sdb
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
305
306 @property
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
307 def table(self):
e0fc18e @ask SDB for fanout now works, but something is very slow (like declaring …
authored
308 name = self.entity_name(self.domain_format % {
309 "vhost": self.conninfo.virtual_host})
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
310 d = self.sdb.get_object("CreateDomain", {"DomainName": name},
311 self.Table)
312 d.name = name
313 return d
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
314
315 @property
316 def conninfo(self):
317 return self.connection.client
318
319 @property
320 def transport_options(self):
321 return self.connection.client.transport_options
322
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
323 @cached_property
324 def visibility_timeout(self):
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
325 return self.transport_options.get("visibility_timeout")
48ca2cd Added support for queue prefixes in SQS.
Nitzan Miron authored
326
327 @cached_property
328 def queue_name_prefix(self):
329 return self.transport_options.get("queue_name_prefix", '')
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
330
5ca7701 @ask SQS: Needs to delete message after receiving.
authored
331 @cached_property
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
332 def supports_fanout(self):
322ebcb @ask SQS: Disables SimpleDB persistence by default
authored
333 return self.transport_options.get("sdb_persistence", False)
04f104e @ask SQS Backend: Using SDB to store routing tables. Not actually execute…
authored
334
5ca7701 @ask SQS: Needs to delete message after receiving.
authored
335 @cached_property
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
336 def region(self):
337 return self.transport_options.get("region") or self.default_region
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
338
339
340 class Transport(virtual.Transport):
341 Channel = Channel
342
af4d2b0 @ask SQS improvements
authored
343 polling_interval = 1
03da6ed @ask Amazon SQS transport passing functional tests. Sponsored by the good…
authored
344 default_port = None
3e44c38 @ask SQS: SDB related fixes and cleanup
authored
345 connection_errors = (exception.SQSError, socket.error)
8bae43c @ask Adds connection/channel_errors for SQS transport
authored
346 channel_errors = (exception.SQSDecodeError, )
Something went wrong with that request. Please try again.