diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..235a1a4 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,3 @@ +Jonas VP +Fred Wenzel +Jeff Balogh diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bb3ec5f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..404b893 --- /dev/null +++ b/README.md @@ -0,0 +1,145 @@ +django-beanstalkd +================= + +*django-beanstalkd* is a convenience wrapper for the [beanstalkd][beanstalkd] +[Python Bindings][beanstalkc]. + +With django-beanstalkd, you can code jobs as well as clients in a Django project +with minimal overhead in your application. Server connections etc. all take +place in django-beanstalkd and don't unnecessarily clog your application code. + +This library is based in large part on Fred Wenzel's [django-gearman][django-gearman]. +If you're looking for synchronous execution of jobs, check out [Gearman][gearman] +and Fred's library! Beanstalkd is useful for background processes only. + +[beanstalkd]: http://kr.github.com/beanstalkd/ +[beanstalkc]: http://github.com/earl/beanstalkc/ +[django-gearman]: http://github.com/fwenzel/django-gearman +[gearman]: http://gearman.org/ + +Installation +------------ +It's the same for both the client and worker instances of your django project: + + pip install -e git://github.com/jonasvp/django-beanstalkd.git#egg=django-beanstalkd + +Add ``django_beanstalkd`` to the `INSTALLED_APPS` section of `settings.py`. + +Specify the following settings in your local settings.py file if your beanstalkd +server isn't accessible on port 11300 of localhost (127.0.0.1): + + # My beanstalkd server + BEANSTALK_SERVER = '127.0.0.1:11300' # the default value + +If necessary, you can specify a pattern to be applied to your beanstalk worker +functions: + + # beanstalk job name pattern. Namespacing etc goes here. This is the pattern + # your jobs will register as with the server, and that you'll need to use + # when calling them from a non-django-beanstalkd client. + # replacement patterns are: + # %(app)s : django app name the job is filed under + # %(job)s : job name + BEANSTALK_JOB_NAME = '%(app)s.%(job)s' + + +Workers +------- +### Registering jobs +Create a file `beanstalk_jobs.py` in any of your django apps, and define as many +jobs as functions as you like. The job must accept a single string argument as +passed by the caller. + +Mark each of these functions as beanstalk jobs by decorating them with +`django_beanstalkd.beanstalk_job`. + +For an example, look at the `beanstalk_example` app's `benstalk_jobs.py` file. + +### Starting a worker +To start a worker, run `python manage.py beanstalk_worker`. It will start +serving all registered jobs. + +To spawn more than one worker (if, e.g., most of your jobs are I/O bound), +use the `-w` option: + + python manage.py beanstalk_worker -w 5 + +will start five workers. + +Since the process will keep running while waiting for and executing jobs, +you probably want to run this in a _screen_ session or similar. + +Clients +------- +To make your workers work, you need a client app passing data to them. Create +and instance of the `django_beanstalkd.BeanstalkClient` class and `call` a +function with it: + + from django_beanstalkd import BeanstalkClient + client = BeanstalkClient() + client.call('beanstalk_example.background_counting', '5') + +For a live example look at the `beanstalk_example` app, in the +`management/commands/beanstalk_example_client.py` file. Arguments to `call` are + + priority: an integer number that specifies the priority. Jobs with a + smaller priority get executed first + delay: how many seconds to wait before the job can be reserved + ttr: how many seconds a worker has to process the job before it gets requeued + + +Example App +----------- +For a full, working, example application, add `beanstalk_example` to your +`INSTALLED_APPS`, then run a worker in one shell: + + python manage.py beanstalk_worker -w 4 + +and execute the example app in another: + + python manage.py beanstalk_example_client + +You can see the client sending data and the worker(s) working on it. + +Licensing +--------- +This software is licensed under the [Mozilla Tri-License][MPL]: + + ***** BEGIN LICENSE BLOCK ***** + Version: MPL 1.1/GPL 2.0/LGPL 2.1 + + The contents of this file are subject to the Mozilla Public License Version + 1.1 (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" basis, + WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + for the specific language governing rights and limitations under the + License. + + The Original Code is django-gearman. + + The Initial Developer of the Original Code is Mozilla. + Portions created by the Initial Developer are Copyright (C) 2010 + the Initial Developer. All Rights Reserved. + + Contributor(s): + Jonas VP + Frederic Wenzel + + Alternatively, the contents of this file may be used under the terms of + either the GNU General Public License Version 2 or later (the "GPL"), or + the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + in which case the provisions of the GPL or the LGPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of either the GPL or the LGPL, and not to allow others to + use your version of this file under the terms of the MPL, indicate your + decision by deleting the provisions above and replace them with the notice + and other provisions required by the GPL or the LGPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the MPL, the GPL or the LGPL. + + ***** END LICENSE BLOCK ***** + +[MPL]: http://www.mozilla.org/MPL/ diff --git a/beanstalk_example/__init__.py b/beanstalk_example/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/beanstalk_example/beanstalk_jobs.py b/beanstalk_example/beanstalk_jobs.py new file mode 100644 index 0000000..66c0bd3 --- /dev/null +++ b/beanstalk_example/beanstalk_jobs.py @@ -0,0 +1,21 @@ +""" +Example Beanstalk Job File. +Needs to be called beanstalk_jobs.py and reside inside a registered Django app. +""" +import os +import time + +from django_beanstalkd import beanstalk_job + + +@beanstalk_job +def background_counting(arg): + """ + Do some incredibly useful counting to the value of arg + """ + value = int(arg) + pid = os.getpid() + print "[%s] Counting from 1 to %d." % (pid, value) + for i in range(1, value+1): + print '[%s] %d' % (pid, i) + time.sleep(1) diff --git a/beanstalk_example/management/__init__.py b/beanstalk_example/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/beanstalk_example/management/commands/__init__.py b/beanstalk_example/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/beanstalk_example/management/commands/beanstalk_example_client.py b/beanstalk_example/management/commands/beanstalk_example_client.py new file mode 100644 index 0000000..03fb77f --- /dev/null +++ b/beanstalk_example/management/commands/beanstalk_example_client.py @@ -0,0 +1,18 @@ +from django.core.management.base import NoArgsCommand +from django_beanstalkd import BeanstalkClient + + +class Command(NoArgsCommand): + help = "Execute an example command with the django_beanstalk_jobs interface" + __doc__ = help + + def handle_noargs(self, **options): + client = BeanstalkClient() + + print "Asynchronous Beanstalk Call" + print "-------------------------" + print "Notice how this app exits, while the workers still work on the tasks." + for i in range(4): + client.call( + 'beanstalk_example.background_counting', '5' + ) diff --git a/django_beanstalkd/__init__.py b/django_beanstalkd/__init__.py new file mode 100644 index 0000000..4ea2216 --- /dev/null +++ b/django_beanstalkd/__init__.py @@ -0,0 +1,46 @@ +""" +Django Beanstalk Interface +""" +from django.conf import settings + +from beanstalkc import Connection, SocketError, DEFAULT_PRIORITY, DEFAULT_TTR + +from decorators import beanstalk_job + +def connect_beanstalkd(): + """Connect to beanstalkd server(s) from settings file""" + + server = getattr(settings, 'BEANSTALK_SERVER', '127.0.0.1') + port = 11300 + if server.find(':') > -1: + server, port = server.split(':', 1) + + try: + port = int(port) + return Connection(server, port) + except (ValueError, SocketError), e: + raise BeanstalkError(e) + + +class BeanstalkError(Exception): + pass + + +class BeanstalkClient(object): + """beanstalk client, automatically connecting to server""" + + def call(self, func, arg='', priority=DEFAULT_PRIORITY, delay=0, ttr=DEFAULT_TTR): + """ + Calls the specified function (in beanstalk terms: put the specified arg + in tube func) + + priority: an integer number that specifies the priority. Jobs with a + smaller priority get executed first + delay: how many seconds to wait before the job can be reserved + ttr: how many seconds a worker has to process the job before it gets requeued + """ + self._beanstalk.use(func) + self._beanstalk.put(str(arg), priority=priority, delay=delay, ttr=ttr) + + def __init__(self, **kwargs): + self._beanstalk = connect_beanstalkd() diff --git a/django_beanstalkd/decorators.py b/django_beanstalkd/decorators.py new file mode 100644 index 0000000..73662b9 --- /dev/null +++ b/django_beanstalkd/decorators.py @@ -0,0 +1,27 @@ +class beanstalk_job(object): + """ + Decorator marking a function inside some_app/beanstalk_jobs.py as a + beanstalk job + """ + + def __init__(self, f): + self.f = f + self.__name__ = f.__name__ + + # determine app name + parts = f.__module__.split('.') + if len(parts) > 1: + self.app = parts[-2] + else: + self.app = '' + + # store function in per-app job list (to be picked up by a worker) + bs_module = __import__(f.__module__) + try: + bs_module.beanstalk_job_list.append(self) + except AttributeError: + bs_module.beanstalk_job_list = [self] + + def __call__(self, arg): + # call function with argument passed by the client only + return self.f(arg) diff --git a/django_beanstalkd/management/__init__.py b/django_beanstalkd/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_beanstalkd/management/commands/__init__.py b/django_beanstalkd/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_beanstalkd/management/commands/beanstalk_worker.py b/django_beanstalkd/management/commands/beanstalk_worker.py new file mode 100644 index 0000000..77a5f87 --- /dev/null +++ b/django_beanstalkd/management/commands/beanstalk_worker.py @@ -0,0 +1,114 @@ +from optparse import make_option +import os +import sys + +from django.conf import settings +from django.core.management.base import NoArgsCommand +from django_beanstalkd import connect_beanstalkd + + +class Command(NoArgsCommand): + help = "Start a Beanstalk worker serving all registered Beanstalk jobs" + __doc__ = help + option_list = NoArgsCommand.option_list + ( + make_option('-w', '--workers', action='store', dest='worker_count', + default='1', help='Number of workers to spawn.'), + ) + children = [] # list of worker processes + jobs = {} + + def handle_noargs(self, **options): + # find beanstalk job modules + bs_modules = [] + for app in settings.INSTALLED_APPS: + try: + bs_modules.append(__import__("%s.beanstalk_jobs" % app)) + except ImportError: + pass + if not bs_modules: + print "No beanstalk_jobs modules found!" + return + + # find all jobs + jobs = [] + for bs_module in bs_modules: + try: + bs_module.beanstalk_job_list + except AttributeError: + continue + jobs += bs_module.beanstalk_job_list + if not jobs: + print "No beanstalk jobs found!" + return + print "Available jobs:" + for job in jobs: + # determine right name to register function with + app = job.app + jobname = job.__name__ + try: + func = settings.BEANSTALK_JOB_NAME % { + 'app': app, + 'job': jobname, + } + except AttributeError: + func = '%s.%s' % (app, jobname) + self.jobs[func] = job + print "* %s" % func + + # spawn all workers and register all jobs + try: + worker_count = int(options['worker_count']) + assert(worker_count > 0) + except (ValueError, AssertionError): + worker_count = 1 + self.spawn_workers(worker_count) + + # start working + print "Starting to work... (press ^C to exit)" + try: + for child in self.children: + os.waitpid(child, 0) + except KeyboardInterrupt: + sys.exit(0) + + def spawn_workers(self, worker_count): + """ + Spawn as many workers as desired (at least 1). + Accepts: + - worker_count, positive int + """ + # no need for forking if there's only one worker + if worker_count == 1: + return self.work() + + print "Spawning %s worker(s)" % worker_count + # spawn children and make them work (hello, 19th century!) + for i in range(worker_count): + child = os.fork() + if child: + self.children.append(child) + continue + else: + self.work() + break + + def work(self): + """children only: watch tubes for all jobs, start working""" + beanstalk = connect_beanstalkd() + for job in self.jobs.keys(): + beanstalk.watch(job) + beanstalk.ignore('default') + + try: + while True: + job = beanstalk.reserve() + job_name = job.stats()['tube'] + if job_name in self.jobs: + #print "Calling %s with arg: %s" % (job_name, job.body) + self.jobs[job_name](job.body) + job.delete() + else: + job.release() + + except KeyboardInterrupt: + sys.exit(0) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..71731bc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +# pip requirements file +pyyaml +beanstalkc + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ead800a --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +from setuptools import setup, find_packages + + +setup( + name='django-beanstalkd', + version='0.1', + description='A convenience wrapper for beanstalkd clients and workers ' + 'in Django using the beanstalkc library for Python', + long_description=open('README.md').read(), + author='Jonas VP', + author_email='jvp@jonasundderwolf.de', + url='http://github.com/jonasvp/django-beanstalkd', + license='MPL', + packages=find_packages(), + include_package_data=True, + zip_safe=False, + install_requires=['pyyaml', 'beanstalkc'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] +)