Gearman support

docs/config/gearman.rst

@@ -0,0 +1,32 @@
+Configuring Gearman
+===================
+
+Gearman provides a generic application framework to farm out work to other machines or processes that are better
+suited to do the work. It allows you to do work in parallel, to load balance processing, and to call functions
+between languages. For more information visit http://gearman.org/.
+
+In order to use gearman support in raven, you have to have installed gearmand job server on your machine.
+
+If you're using Django add this to your ``settings``::
+
+    GEARMAN_SERVERS = ['127.0.0.1:4730']
+    SENTRY_CLIENT = 'raven.contrib.django.gearman.GearmanClient'
+    SENTRY_GEARMAN_CLIENT = 'raven.contrib.django.client.DjangoClient'
+
+
+Next you need to run ``raven_gearman`` worker to process your logging events in the background::
+
+    $ python manage.py raven_gearman
+
+
+``raven_gearman`` command is build on top of django-gearman-commands app. For more information about
+setting-up your worker please have a look at this url https://github.com/CodeScaleInc/django-gearman-commands.
+
+
+How does it work ?
+------------------
+
+All logging events as submitted as job to gearmand job server. They are not submitted directly to the sentry server.
+``raven_gearman`` is running in background and downloads jobs from gearmand job server. Every downloaded job is then
+transformed into raven event and sent by ``SENTRY_GEARMAN_CLIENT`` to sentry server. The point is, that all this happens
+asynchronously and don't block your application main thread.

docs/config/index.rst

@@ -11,6 +11,7 @@ This document describes configuration options available to Sentry.
 
    bottle
    celery
+   gearman
    django
    flask
    logbook

raven/contrib/django/gearman/__init__.py

@@ -0,0 +1,60 @@
+import json
+import uuid
+import hashlib
+
+from django_gearman_commands import GearmanWorkerBaseCommand
+
+from django.conf import settings
+
+from raven.contrib.gearman import GearmanMixin, RAVEN_GEARMAN_JOB_NAME
+from raven.contrib.django import DjangoClient
+from raven.contrib.django.models import get_client
+
+__all__ = ('GearmanClient', 'GearmanWorkerCommand')
+
+
+class GearmanClient(GearmanMixin, DjangoClient):
+    """Gearman client implementation for django applications."""
+    def build_msg(self, event_type, data=None, date=None,
+                  time_spent=None, extra=None, stack=None, public_key=None,
+                  tags=None, **kwargs):
+        msg = super(GearmanClient, self).build_msg(event_type, data=data, date=date,
+                                                   time_spen=time_spent, extra=extra, stack=stack,
+                                                   public_key=public_key, tags=None, **kwargs)
+        msg['checksum'] = hashlib.md5(str(uuid.uuid4())).hexdigest()
+        return msg
+
+
+class GearmanWorkerCommand(GearmanWorkerBaseCommand):
+    """Gearman worker implementation.
+
+    This worker is run as django management command. Gearman client send messages to gearman deamon. Next
+    the messages are downloaded from gearman daemon by this worker, and sent to sentry server by standrd
+    django raven client 'raven.contrib.django.client.DjangoClient', if not specified otherwise by
+    SENTRY_GEARMAN_CLIENT django setting.
+
+    This worker is dependent on django-gearman-commands app. For more information how this works, please
+    visit https://github.com/CodeScaleInc/django-gearman-commands.
+
+    """
+
+    _client = None
+
+    @property
+    def task_name(self):
+        return RAVEN_GEARMAN_JOB_NAME
+
+    @property
+    def client(self):
+        if self._client is None:
+            self._client = get_client(getattr(settings, 'SENTRY_GEARMAN_CLIENT',
+                                              'raven.contrib.django.client.DjangoClient'))
+        return self._client
+
+    @property
+    def exit_after_job(self):
+        return getattr(settings, 'SENTRY_GEARMAN_EXIT_AFTER_JOB', False)
+
+    def do_job(self, job_data):
+        payload = json.loads(job_data)
+        return self.client.send_encoded(payload['message'], auth_header=payload['auth_header'])

raven/contrib/django/raven_compat/management/commands/raven_gearman.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import
+
+from raven.contrib.django.gearman import GearmanWorkerCommand
+
+
+class Command(GearmanWorkerCommand):
+    pass

raven/contrib/gearman/__init__.py

@@ -0,0 +1,35 @@
+import json
+
+from django_gearman_commands import submit_job
+
+from raven.base import Client
+
+__all__ = ('RAVEN_GEARMAN_JOB_NAME', 'GearmanMixin', 'GearmanClient')
+
+
+RAVEN_GEARMAN_JOB_NAME = 'raven_gearman'
+
+
+class GearmanMixin(object):
+    """This class servers as a Mixin for client implementations that wants to support gearman async queue."""
+
+    def send_encoded(self, message, auth_header=None, **kwargs):
+        """Encoded data are sent to gearman, instead of directly sent to the sentry server.
+
+        :param message: encoded message
+        :type message: string
+        :param auth_header: auth_header: authentication header for sentry
+        :type auth_header: string
+        :returns: void
+        :rtype: None
+
+        """
+        payload = json.dumps({
+            'message': message,
+            'auth_header': auth_header
+        })
+        submit_job(RAVEN_GEARMAN_JOB_NAME, data=payload)
+
+
+class GearmanClient(GearmanMixin, Client):
+    """Independent implementation of gearman client for raven."""

setup.py

@@ -42,6 +42,11 @@
     'Flask-Login>=0.2.0',
 ]
 
+gearman_requires = [
+    'gearman==dev',
+    'django-gearman-commands==dev'
+]
+
 # If it's python3, remove flask & unittest2
 if sys.version_info[0] == 3:
     flask_requires = []
@@ -67,7 +72,7 @@
     'webob',
     'webtest',
     'anyjson',
-] + flask_requires + flask_tests_requires + unittest2_requires
+] + flask_requires + flask_tests_requires + gearman_requires + unittest2_requires
 
 
 class PyTest(TestCommand):
@@ -95,8 +100,9 @@ def run_tests(self):
     zip_safe=False,
     extras_require={
         'flask': flask_requires,
+        'gearman': gearman_requires,
         'tests': tests_require,
-        'dev': dev_requires,
+        'dev': dev_requires
     },
     license='BSD',
     tests_require=tests_require,
@@ -121,4 +127,8 @@ def run_tests(self):
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.2',
     ],
+    dependency_links = [
+        'git+git://github.com/Yelp/python-gearman.git#egg=gearman-dev',
+        'git+git://github.com/CodeScaleInc/django-gearman-commands.git#egg=django_gearman_commands-dev'
+    ]
 )

tests/contrib/django/tests.py

@@ -21,10 +21,12 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.template import TemplateSyntaxError
 from django.test import TestCase
+from django.core.management import call_command
 
 from raven.base import Client
 from raven.contrib.django.client import DjangoClient
 from raven.contrib.django.celery import CeleryClient
+from raven.contrib.django.gearman import GearmanClient, GearmanWorkerCommand
 from raven.contrib.django.handlers import SentryHandler
 from raven.contrib.django.models import client, get_client, sentry_exception_handler
 from raven.contrib.django.middleware.wsgi import Sentry
@@ -619,6 +621,39 @@ def test_with_eager(self, send_encoded):
         self.assertEquals(send_encoded.call_count, 1)
 
 
+class GearmanClientTest(TestCase):
+    def setUp(self):
+        self.client = GearmanClient(servers=['http://example.com'])
+
+    @mock.patch('raven.contrib.gearman.submit_job')
+    def test_send_encoded(self, submit_job):
+        self.client.send_encoded('foo')
+        submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}')
+
+
+class GearmanWorkerTest(TestCase):
+    def setUp(self):
+        self.worker = GearmanWorkerCommand()
+
+    @mock.patch('django_gearman_commands.GearmanWorkerBaseCommand.handle')
+    @mock.patch('raven.contrib.django.gearman.get_client')
+    def test_worker_handle_job(self, get_client, handle):
+        """With patching handle method, we disabled worker to connect to and listen on gearman daemon."""
+        get_client.return_value = mock.MagicMock()
+        self.worker.execute()
+        self.worker.do_job('{"message":"foo","auth_header":"bar"}')
+        handle.assert_called_once_with()
+        get_client.assert_call_once_with('raven.contrib.django.client.DjangoClient')
+        self.assertEqual([mock.call.send_encoded(u'foo', auth_header=u'bar')], get_client.return_value.mock_calls)
+
+    def test_worker_client_default_settings(self):
+        self.assertTrue(isinstance(self.worker.client, DjangoClient))
+
+    def test_worker_client_custom_settings(self):
+        with Settings(SENTRY_GEARMAN_CLIENT='raven.contrib.django.gearman.GearmanClient'):
+            self.assertTrue(isinstance(self.worker.client, GearmanClient))
+
+
 class IsValidOriginTestCase(TestCase):
     def test_setting_empty(self):
         with Settings(SENTRY_ALLOW_ORIGIN=None):

tests/contrib/gearman/tests.py

@@ -0,0 +1,14 @@
+import mock
+
+from raven.utils.testutils import TestCase
+from raven.contrib.gearman import GearmanClient
+
+
+class ClientTest(TestCase):
+    def setUp(self):
+        self.client = GearmanClient(servers=['http://example.com'])
+
+    @mock.patch('raven.contrib.gearman.submit_job')
+    def test_send_encoded(self, submit_job):
+        self.client.send_encoded('foo')
+        submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}')