Skip to content
Find file
Fetching contributors…
Cannot retrieve contributors at this time
73 lines (61 sloc) 2.09 KB
#
# tools.py: some convenient wrappers related to mq and db.
import json
import pika
import sqlalchemy
import tools
import sys
#
# 'fetchenv' fetches the dotcloud passed environment (not the wsgi one).
#
def fetchenv():
with open('/home/dotcloud/environment.json') as f:
env = json.load(f)
return env
#
# 'getdbhandle' returns a handle on a sqlalchemy db engine
#
def getdbhandle():
env = fetchenv()
db = sqlalchemy.create_engine(env['DOTCLOUD_DB_MYSQL_URL'] + "/crawldb")
db.echo = False
return db
#
# 'putdbhandle' does nothing effectively. Just to keep symmetric api.
#
def putdbhandle(db):
pass
#
# 'getmqhandle' returns a handle on a rabbitmq connection and channel.
#
def getmqhandle():
env = fetchenv()
conn = pika.BlockingConnection(pika.ConnectionParameters(host=str(env['DOTCLOUD_QUEUE_AMQP_HOST']),
port=int(env['DOTCLOUD_QUEUE_AMQP_PORT']),
credentials=pika.PlainCredentials(str(env['DOTCLOUD_QUEUE_AMQP_LOGIN']),
str(env['DOTCLOUD_QUEUE_AMQP_PASSWORD']))))
channel = conn.channel()
channel.queue_declare(queue=str(env['CRAWLER_AMQP_ROUTING_KEY']), durable=True)
return conn, channel
#
# 'pushtomq' pushes a message to the crawler job queue.
#
def pushtomq(channel, message):
env = tools.fetchenv()
channel.basic_publish(exchange='',
routing_key=str(env['CRAWLER_AMQP_ROUTING_KEY']),
body=message,
properties=pika.BasicProperties(delivery_mode = 2))
#
# 'consumemq' puts the calling context in a blocking listening mode for messages
# coming out of the crawler job queue.
#
def consumemq(channel, callback):
env = tools.fetchenv()
channel.basic_consume(callback, queue=str(env['CRAWLER_AMQP_ROUTING_KEY']))
channel.start_consuming()
#
# 'putmqhandle' does some cleanup on the rabbitmq resources
#
def putmqhandle(conn, channel):
conn.close()
Jump to Line
Something went wrong with that request. Please try again.