Permalink
Browse files

Get the celery updated to 3.0+ and working with in app as well as daemon

  • Loading branch information...
1 parent 9efea26 commit c83205bd609e9477fac879f2be7e30fec925ed9f @mitechie committed Aug 6, 2012
View
@@ -1,7 +1,7 @@
# Makefile to help automate tasks in bookie
WD := $(shell pwd)
PY := bin/python
-CELERY := PYTHONPATH="bookie/bcelery/." bin/celeryd -B --loglevel=INFO
+CELERY := PYTHONPATH="bookie/bcelery/." celery worker -B --app=bookie.bcelery.celeryd:celery --loglevel=DEBUG
PEP8 := bin/pep8
PIP := bin/pip
PIP_MIR = PIP_FIND_LINKS='http://mypi http://simple.crate.io/'
View
@@ -36,6 +36,9 @@ def main(global_config, **settings):
authorization_policy=authz_policy)
config.set_request_factory(RequestWithUserAttribute)
+ import bookie.bcelery.celeryapp
+ bookie.bcelery.celeryapp.load_config(settings)
+
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
config.scan('bookie.views')
@@ -0,0 +1,6 @@
+from celery import Celery
+
+
+# Override these in each implementation for app and celeryd running.
+celery = Celery()
+ini = None
@@ -0,0 +1,11 @@
+from celery import Celery
+from datetime import timedelta
+from os import environ
+from os import path
+
+
+def load_config(settings=None):
+ import bookie.bcelery
+ bookie.bcelery.ini = settings
+ # Only import the tasks after we've setup the ini config.
+ import bookie.bcelery.tasks
@@ -1,80 +1,11 @@
#!/usr/bin/env python
"""Celery config for Bookie Instance"""
-from ConfigParser import ConfigParser
-from datetime import timedelta
-from os import environ
-from os import path
+from bookie.bcelery import load_config
+print 'loading tasks'
-import tasks
+ini = load_config()
+print 'finished loading config'
-def load_config():
- selected_ini = environ.get('BOOKIE_INI', None)
-
- if selected_ini is None:
- msg = "Please set the BOOKIE_INI env variable!"
- raise Exception(msg)
-
- ini = ConfigParser()
- ini_path = path.join(
- path.dirname(
- path.dirname(
- path.dirname(__file__)
- )
- ),
- selected_ini
- )
- ini.readfp(open(ini_path))
- return ini
-
-INI = load_config()
-# we have to go up two dirs to get to the ini file, so any string with
-# {here} needs to be adjusted those two dirs
-HERE = path.join(path.dirname(__file__), '../../')
-
-# List of modules to import when celery starts.
-CELERY_IMPORTS = ("bookie.bcelery.tasks", )
-CELERY_ENABLE_UTC = True
-
-## Result store settings.
-CELERY_RESULT_BACKEND = INI.get('celeryd', 'result_backend')
-CELERY_RESULT_DBURI = INI.get('celeryd', 'result_dburi').format(here=HERE)
-
-## Broker settings.
-BROKER_TRANSPORT = INI.get('celeryd', 'broker_transport')
-BROKER_HOST = INI.get('celeryd', 'broker_host').format(here=HERE)
-# BROKER_URL = "amqp://guest:guest@localhost:5672//"
-
-print BROKER_TRANSPORT
-print CELERY_RESULT_DBURI
-
-## Worker settings
-## If you're doing mostly I/O you can have more processes,
-## but if mostly spending CPU, try to keep it close to the
-## number of CPUs on your machine. If not set, the number of CPUs/cores
-## available will be used.
-CELERYD_CONCURRENCY = INI.get('celeryd', 'concurrency')
-
-# CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}}
-CELERYBEAT_SCHEDULE = {
- "tasks.hourly_stats": {
- "task": "tasks.hourly_stats",
- "schedule": timedelta(seconds=60 * 60),
- },
- "tasks.stats_rrd": {
- "task": "tasks.generate_count_rrd",
- "schedule": timedelta(seconds=60 * 60 * 12),
- },
- "tasks.importer_depth": {
- "task": "tasks.importer_depth",
- "schedule": timedelta(seconds=60 * 5),
- },
- "tasks.importer_depth_rrd": {
- "task": "tasks.generate_importer_depth_rrd",
- "schedule": timedelta(seconds=60 * 5),
- },
- "tasks.importer": {
- "task": "tasks.importer_process",
- "schedule": timedelta(seconds=60 * 3),
- },
-}
+import bookie.bcelery.tasks
+bookie.bcelery.tasks.ini_items = ini
View
@@ -0,0 +1,37 @@
+from celery import Celery
+from ConfigParser import ConfigParser
+from datetime import timedelta
+from os import environ
+from os import path
+
+import bookie.bcelery
+
+
+def load_ini():
+ selected_ini = environ.get('BOOKIE_INI', None)
+ if selected_ini is None:
+ msg = "Please set the BOOKIE_INI env variable!"
+ raise Exception(msg)
+
+ cfg = ConfigParser()
+ ini_path = path.join(
+ path.dirname(
+ path.dirname(
+ path.dirname(__file__)
+ )
+ ),
+ selected_ini
+ )
+ cfg.readfp(open(ini_path))
+
+ # Hold onto the ini config.
+ return dict(cfg.items('app:main'))
+
+bookie.bcelery.ini = load_ini()
+
+print bookie.bcelery.ini
+
+print 'celeryd'
+print 'updating celery'
+from bookie.bcelery import celery
+import bookie.bcelery.tasks
Oops, something went wrong.

0 comments on commit c83205b

Please sign in to comment.