Skip to content
Browse files

Merge pull request #9 from toudi/task-parameters

Support for task parameters.
  • Loading branch information...
2 parents d3b5975 + c278ba3 commit 1f27a7170afdf6667292301f74ccd148d0d8e740 @fwenzel committed Aug 6, 2012
Showing with 123 additions and 7 deletions.
  1. +77 −0 README.md
  2. +2 −5 django_gearman/decorators.py
  3. +44 −2 django_gearman/models.py
View
77 README.md
@@ -46,6 +46,83 @@ the task name, by specifying `name` parameter of the decorator. Here's how:
def my_task_function(foo):
pass
+### Task parameters
+The gearman specifies, that the job function can accept one parameter (usually
+it is reffered to as ``data`` parameter). Sometimes it may not be enough. For
+example, gearman only allows to pass a string as ``data`` parameter. What if you
+would like to pass an array or a dict? You would need to serialize and deserialize
+them. Fortunately, django-gearman can take care of this, so that you can spend
+all of your time on coding the actual task.
+
+ @gearman_job(name='my-task-name')
+ def my_task_function(foo):
+ pass
+
+ client.submit_job('my-task-name', {'foo':'becomes', 'this':'dict'})
+ client.submit_job('my-task-name', Decimal(1.0))
+
+### Tasks with more than one parameter
+
+You can pass as many arguments as you want, of whatever type you like.
+Here's an example job definition:
+
+ @gearman_job(name='my-task-name')
+ def my_task_function(one, two, three):
+ pass
+
+You can execute this function in two different ways:
+
+ client.submit_job('my-task-name', one=1, two=2, three=3)
+ client.submit_job('my-task-name', args=[1,2,3])
+
+Unfortunetely, executing it like that
+
+ client.submit_job('my-task-name', 1,2,3)
+
+Would produce the error, simply because ``submit_job`` from gearman's python bindings
+contains __a lot__ of arguments and it's much easier to specify them via keyword names
+or special ``args`` keyword than to type something like 7 None's instead:
+
+ client.submit_job('my-task-name', None, None, None, None, None, None, None, 1, 2, 3)
+
+The only limitation that you have are gearman reserved keyword parameters. As of
+gearman 2.0.2 these are:
+
+ * data
+ * unique
+ * priority
+ * background
+ * wait_until_complete
+ * max_retries
+ * poll_timeout
+
+So, when you want your job definition to have, for example, ``unique`` or ``background``
+keyword parameter, you need to execute the job in a special, more verbose way. Here's an
+example of such job, and it's execution.
+
+ @gearman_job(name='my-task-name')
+ def my_task_function(background, unique):
+ pass
+
+ client.submit_job('my-task-name', kwargs={"background": True, "unique": False})
+ client.submit_job('my-task-name', args=[True,False])
+
+Now, for the final stride:
+
+ client.submit_job('my-task-name', background=True, unique=True, kwargs={"background": False, "unique": False})
+
+Don't panic, your task is safe! That's because you're using ``kwargs`` directly. Therefore,
+gearman's bindings would recieve ``True`` for ``submit_job`` function, whether your task would
+recieve ``False``.
+
+However, executing the task like this:
+
+ client.submit_job('my-task-name', background=True, unique=True)
+
+would lead to a disaster of unimaginable proportions! Both ``background`` and ``unique`` arguments
+would be swallowed by submit_job function and wouldn't made it to your task! Therefore always remember
+to double-check your parameter names with the reserved words list.
+
### Starting a worker
To start a worker, run `python manage.py gearman_worker`. It will start
serving all registered jobs.
View
7 django_gearman/decorators.py
@@ -31,10 +31,7 @@ def __init__(self, f):
def __call__(self, worker, job, *args, **kwargs):
# Call function with argument passed by the client only.
- try:
- arg = job.data
- except IndexError:
- arg = None
- return self.f(arg)
+ job_args = job.data
+ return self.f(*job_args["args"], **job_args["kwargs"])
return gearman_job_cls
View
46 django_gearman/models.py
@@ -27,14 +27,56 @@ def __init__(self, **kwargs):
"""instantiate Gearman client with servers from settings file"""
return super(DjangoGearmanClient, self).__init__(
settings.GEARMAN_SERVERS, **kwargs)
+
+ def parse_data(self, arg, args = [], kwargs = {}, *arguments, **karguments):
+ data = {
+ "args": [],
+ "kwargs": {}
+ }
+
+ """
+ The order is significant:
+ - First, use pythonic *args and/or *kwargs
+ - If someone provided explicit declaration of args/kwargs, use those instead
+ """
+ if arg:
+ data["args"] = [arg]
+ elif arguments:
+ data["args"] = arguments
+ elif args:
+ data["args"] = args
+
+ data["kwargs"].update(karguments)
+ data["kwargs"].update(kwargs)
+
+ return data
- def dispatch_background_task(self, func, arg, uniq=None, high_priority=False):
+ """
+ Re-implement the submit_job function, in order to handle *args and **kwargs
+ """
+ def submit_job(self, task, orig_data = None, unique=None, priority=None,
+ background=False, wait_until_complete=True, max_retries=0,
+ poll_timeout=None, args=[], kwargs={}, *arguments, **karguments):
+
+ data = self.parse_data(orig_data, args, kwargs, *arguments, **karguments)
+
+ return super(DjangoGearmanClient, self).submit_job(
+ task, data, unique, priority, background, wait_until_complete, max_retries, poll_timeout
+ )
+
+ def dispatch_background_task(self, func, arg = None, uniq=None, high_priority=False, args=[], kwargs={},
+ *arguments, **karguments):
"""Submit a background task and return its handle."""
priority = None
if high_priority:
priority = gearman.PRIORITY_HIGH
- request = self.submit_job(func, arg, unique=uniq, wait_until_complete=False, priority=priority)
+
+ request = self.submit_job(func, arg, unique=uniq,
+ wait_until_complete=False, priority=priority, args=args, kwargs=kwargs,
+ *arguments, **karguments
+ )
+
return request

0 comments on commit 1f27a71

Please sign in to comment.
Something went wrong with that request. Please try again.