Permalink
Browse files

zlib compressing data at the broker, decompressing at the processor, …

…re-compressing at the processor. Should save some bandwidth.
  • Loading branch information...
1 parent cd9a07f commit bfeb22486e21f91750bae4898b71383ca8232ee7 Greg Taylor committed Apr 12, 2012
@@ -38,7 +38,6 @@ def relay_worker(message):
:param str message: A JSON string to re-broadcast.
"""
- print message
sender.send(message)
logger.info("Announcer is now listening for order data.")
@@ -4,6 +4,7 @@
gateway WSGI app.
"""
import logging
+import zlib
import simplejson
from gevent.queue import Queue
from gevent_zeromq import zmq
@@ -39,9 +40,9 @@ def worker():
logger.error('Unable to serialize a job dict. Discarding.')
continue
- # Push a JSON representation of the job dict to a processor for
- # a further look.
- sender.send(job_json)
+ # Push a zlib compressed JSON representation of the job dict to a
+ # processor for a further look.
+ sender.send(zlib.compress(job_json))
logger.info('Pushed orders.')
@@ -2,6 +2,7 @@
The worker function in this module performs the order processing.
"""
import logging
+import zlib
import simplejson
from emdr.core import serialization
@@ -16,7 +17,7 @@ def parse_order(job_json):
:rtype: SerializableOrderList
:returns: A serializable list of MarketOrder objects.
"""
- job_dict = simplejson.loads(job_json)
+ job_dict = simplejson.loads(zlib.decompress(job_json))
# The format attrib on the job dict determines which parser to use.
order_format = job_dict.get('format', 'unknown')
@@ -53,4 +54,5 @@ def worker(job_json, sender):
order_list = parse_order(job_json)
print order_list
if order_list:
- sender.send(serialization.unified.encode_to_json(order_list))
+ json_str = serialization.unified.encode_to_json(order_list)
+ sender.send(zlib.compress(json_str))
@@ -39,7 +39,8 @@ def relay_worker(message):
:param str message: A JSON string to re-broadcast.
"""
- print message
+ import zlib
+ print zlib.decompress(message)
sender.send(message)
logger.info("Relay is now listening for order data.")

0 comments on commit bfeb224

Please sign in to comment.