Permalink
Browse files

initial commit - gearman_overview, gearman_submit_job, gearman_worker…

…_footest
  • Loading branch information...
0 parents commit d93134250d548bb76e93e92c041bdc5e593cef64 @jsk jsk committed Apr 5, 2012
@@ -0,0 +1,11 @@
+*.pyc
+*.pyo
+# Emacs backup files & locks
+*~
+\#*
+.\#*
+*.swp
+*.swo
+pip-log.txt*
+*.swn
+logs
24 LICENSE
@@ -0,0 +1,24 @@
+Copyright (c) 2012, Jozef Ševčík
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the tastypie nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL tastypie BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,3 @@
+include README.rst
+include LICENSE
+include tests.py
@@ -0,0 +1,49 @@
+=========================
+ Django Gearman Commands
+=========================
+
+django-gearman-commands is set of Django management commands
+for working with Gearman job server from Django.
+
+What is Gearman
+===============
+
+
+Why django-gearman-commands
+===========================
+
+Getting started with django-gearman-commands
+============================================
+
+Setup
+-----
+
+Writing workers
+---------------
+
+Submitting jobs
+---------------
+
+Gearman administration command
+------------------------------
+
+Practical production deployment with supervisord
+================================================
+
+License
+=======
+
+BSD, see LICENSE for details
+
+Authors and contributors
+========================
+
+Jozef Ševčík, sevcik@codescale.net
+
+TODO
+====
+
+* improve logging
+* gearman_overview - support for multiple gearman servers
+* gearman_overview - output 'workers'
+* tests (foo/reverse worker + job submission, status)
@@ -0,0 +1,3 @@
+__version__ = '0.1'
+
+from .base import *
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+from datetime import datetime
+import time
+import gearman
+import logging
+
+log = logging.getLogger(__name__)
+
+class HookedGearmanWorker(gearman.GearmanWorker):
+
+ def __init__(self, exit_after_job, host_list=None):
+ super(HookedGearmanWorker, self).__init__(host_list=host_list)
+ self.exit_after_job = exit_after_job
+
+ def after_job(self):
+ return (not self.exit_after_job)
+
+class GearmanWorkerBaseCommand(BaseCommand):
+ """Base command for Gearman workers."""
+
+ @property
+ def task_name(self):
+ """Override task_name property in worker to indicate what task should be registered in Gearman."""
+ raise NotImplementedError, 'task_name should be implemented in worker'
+
+ @property
+ def exit_after_job(self):
+ """Return True if worker should exit after processing job. False by default.
+
+ You do not need to override this in standard case, except in case
+ you want to control and terminate worker after processing jobs.
+ Used by test worker 'footest'.
+
+ """
+ return False
+
+ def do_job(self, job_data):
+ """Gearman job execution logic.
+
+ Override this in worker to perform job.
+
+ """
+ raise NotImplementedError, 'do_job() should be implemented in worker'
+
+ def handle(self, *args, **options):
+ try:
+ worker = HookedGearmanWorker(exit_after_job=self.exit_after_job, host_list=settings.GEARMAN_SERVERS)
+ log.info('Registering gearman task: %s', self.task_name)
+ worker.register_task(self.task_name, self._invoke_job)
+ except:
+ log.exception('Problem with registering gearman task')
+ raise
+
+ worker.work()
+
+ def _invoke_job(self, worker, job):
+ """Invoke gearman job.
+
+ Honestly, wrapper for do_job().
+
+ """
+ try:
+ # represent default job data '' as None
+ job_data = job.data if job.data else None
+ self.stdout.write('Invoking gearman job, task: %s.\n' % self.task_name)
+
+ result = self.do_job(job_data)
+
+ log.info('Job finished, task: %s', self.task_name)
+ self.stdout.write('Job finished, task: %s\n' % self.task_name)
+
+ if result is not None:
+ log.info(result)
+ self.stdout.write('%s\n' % result)
+
+ return 'OK'
+ except:
+ log.exception('Error occured when invoking job, task: %s', self.task_name)
+ raise
+
+
+
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+from prettytable import PrettyTable
+import gearman
+import logging
+
+log = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+ """Pretty-print overview of Gearman server status and workers."""
+
+ help = 'Print overview of Gearman server status and workers.'
+
+ def handle(self, *args, **options):
+ client = gearman.GearmanAdminClient(settings.GEARMAN_SERVERS)
+
+ # get server version
+ version = client.get_version()
+
+ table = PrettyTable(['Gearman Server Version'])
+ table.add_row([version])
+
+ self.stdout.write('%s.\n\n' % table)
+
+ # status
+ raw_status = client.get_status()
+ table = PrettyTable(['Task Name', 'Total Workers', 'Running Jobs', 'Queued Jobs'])
+ for r in raw_status:
+ table.add_row([r['task'], r['workers'], r['running'], r['queued']])
+
+ self.stdout.write('%s.\n' % table)
+
+ # TODO: workers
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+import gearman
+import logging
+
+log = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+ """Submit specific gearman job with job data as an arguments."""
+
+ args = '<task_name> [job_data]'
+ help = 'Submit gearman job with specified task, optionally with job data'
+
+ def handle(self, *args, **options):
+ try:
+ task_name = None
+ job_data = ''
+
+ if len(args) == 0:
+ raise CommandError('At least task name must be provided.')
+
+ task_name = args[0]
+ if len(args) > 1:
+ job_data = args[1]
+
+ self.stdout.write('Submitting job: %s, job data: %s.\n' % (task_name, job_data if job_data else '(empty)'))
+
+ client = gearman.GearmanClient(settings.GEARMAN_SERVERS)
+ result = client.submit_job(task_name, job_data, wait_until_complete=False, background=True)
+
+ self.stdout.write('Job submission done, result: %s.\n' % result)
+ except:
+ log.exception('Error when submitting gearman job')
+ raise
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+from django.core.cache import cache
+import django_gearman_commands
+import logging
+
+log = logging.getLogger(__name__)
+
+class Command(django_gearman_commands.GearmanWorkerBaseCommand):
+ """Base command for Gearman workers."""
+
+ @property
+ def task_name(self):
+ return 'footest'
+
+ @property
+ def exit_after_job(self):
+ return True # terminate after job is handled. Do not do this for standard workers !
+
+ def do_job(self, job_data):
+ # set data to cache
+ cache.set('footest', u'I AM FOO !')
+
+
+
+
+
No changes.
@@ -0,0 +1,4 @@
+from django.conf import settings
+
+# gearman jobs servers
+GEARMAN_SERVERS = getattr(settings, 'GEARMAN_SERVERS', ['127.0.0.1:4730'])
@@ -0,0 +1 @@
+from .commands import *
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from django.core.management import call_command
+from django.conf import settings
+from django.test import TestCase
+from django.core.cache import cache
+import django_gearman_commands
+import logging
+
+log = logging.getLogger(__name__)
+
+class GearmanCommandsTest(TestCase):
+
+ def test_overview(self):
+ call_command('gearman_overview')
+
+ def test_worker_simple(self):
+ # submit job
+ call_command('gearman_submit_job', 'footest')
+
+ # let the worker process the job
+ call_command('gearman_worker_footest')
+
+ # verify job was processed
+ self.assertEqual(cache.get('footest'), u'I AM FOO !', 'Unexpected footest worker result')
@@ -0,0 +1,5 @@
+Django==1.4
+distribute==0.6.19
+-e git+https://github.com/Yelp/python-gearman.git@2ed9d88941e31e3358a0b80787254d0c2cfaa78a#egg=gearman-dev
+prettytable==0.5
+wsgiref==0.1.2
@@ -0,0 +1,49 @@
+"""
+This code provides a mechanism for running django_gearman_commands internal
+test suite without having a full Django project. It sets up the
+global configuration, then dispatches out to `call_command` to
+kick off the test suite.
+
+"""
+
+# Setup and configure the minimal settings necessary to
+# run the test suite. Note that Django requires that the
+# `DATABASES` value be present and configured in order to
+# do anything.
+
+from django.conf import settings
+import logging
+
+logging.basicConfig(level=logging.DEBUG, format='%(levelname)s %(asctime)s %(module)s: %(message)s')
+
+settings.configure(
+ DEBUG=True,
+
+ INSTALLED_APPS=[
+ "django_gearman_commands",
+ ],
+
+ DATABASES={
+ "default": {
+ "ENGINE": "django.db.backends.sqlite3",
+ "NAME": ":memory:",
+ }
+ },
+
+ # tests dependency - run gearman server on localhost on default port
+ GEARMAN_SERVERS=[
+ '127.0.0.1:4730'
+ ],
+
+ # LocMemCache used as a verification storage for tests
+ CACHES = {
+ 'default': {
+ 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+ 'LOCATION': 'django-gearman-commands-tests'
+ }
+ },
+ )
+
+# Start the test suite now that the settings are configured.
+from django.core.management import call_command
+call_command("test", "django_gearman_commands")
Oops, something went wrong.

0 comments on commit d931342

Please sign in to comment.