diff --git a/docs/config/gearman.rst b/docs/config/gearman.rst new file mode 100644 index 000000000..e1478c4ae --- /dev/null +++ b/docs/config/gearman.rst @@ -0,0 +1,32 @@ +Configuring Gearman +=================== + +Gearman provides a generic application framework to farm out work to other machines or processes that are better +suited to do the work. It allows you to do work in parallel, to load balance processing, and to call functions +between languages. For more information visit http://gearman.org/. + +In order to use gearman support in raven, you have to have installed gearmand job server on your machine. + +If you're using Django add this to your ``settings``:: + + GEARMAN_SERVERS = ['127.0.0.1:4730'] + SENTRY_CLIENT = 'raven.contrib.django.gearman.GearmanClient' + SENTRY_GEARMAN_CLIENT = 'raven.contrib.django.client.DjangoClient' + + +Next you need to run ``raven_gearman`` worker to process your logging events in the background:: + + $ python manage.py raven_gearman + + +``raven_gearman`` command is build on top of django-gearman-commands app. For more information about +setting-up your worker please have a look at this url https://github.com/CodeScaleInc/django-gearman-commands. + + +How does it work ? +------------------ + +All logging events as submitted as job to gearmand job server. They are not submitted directly to the sentry server. +``raven_gearman`` is running in background and downloads jobs from gearmand job server. Every downloaded job is then +transformed into raven event and sent by ``SENTRY_GEARMAN_CLIENT`` to sentry server. The point is, that all this happens +asynchronously and don't block your application main thread. \ No newline at end of file diff --git a/docs/config/index.rst b/docs/config/index.rst index 523a29e66..12643b07f 100644 --- a/docs/config/index.rst +++ b/docs/config/index.rst @@ -11,6 +11,7 @@ This document describes configuration options available to Sentry. bottle celery + gearman django flask logbook diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py new file mode 100644 index 000000000..6e80137c3 --- /dev/null +++ b/raven/contrib/django/gearman/__init__.py @@ -0,0 +1,60 @@ +import json +import uuid +import hashlib + +from django_gearman_commands import GearmanWorkerBaseCommand + +from django.conf import settings + +from raven.contrib.gearman import GearmanMixin, RAVEN_GEARMAN_JOB_NAME +from raven.contrib.django import DjangoClient +from raven.contrib.django.models import get_client + +__all__ = ('GearmanClient', 'GearmanWorkerCommand') + + +class GearmanClient(GearmanMixin, DjangoClient): + """Gearman client implementation for django applications.""" + def build_msg(self, event_type, data=None, date=None, + time_spent=None, extra=None, stack=None, public_key=None, + tags=None, **kwargs): + msg = super(GearmanClient, self).build_msg(event_type, data=data, date=date, + time_spen=time_spent, extra=extra, stack=stack, + public_key=public_key, tags=None, **kwargs) + msg['checksum'] = hashlib.md5(str(uuid.uuid4())).hexdigest() + return msg + + +class GearmanWorkerCommand(GearmanWorkerBaseCommand): + """Gearman worker implementation. + + This worker is run as django management command. Gearman client send messages to gearman deamon. Next + the messages are downloaded from gearman daemon by this worker, and sent to sentry server by standrd + django raven client 'raven.contrib.django.client.DjangoClient', if not specified otherwise by + SENTRY_GEARMAN_CLIENT django setting. + + This worker is dependent on django-gearman-commands app. For more information how this works, please + visit https://github.com/CodeScaleInc/django-gearman-commands. + + """ + + _client = None + + @property + def task_name(self): + return RAVEN_GEARMAN_JOB_NAME + + @property + def client(self): + if self._client is None: + self._client = get_client(getattr(settings, 'SENTRY_GEARMAN_CLIENT', + 'raven.contrib.django.client.DjangoClient')) + return self._client + + @property + def exit_after_job(self): + return getattr(settings, 'SENTRY_GEARMAN_EXIT_AFTER_JOB', False) + + def do_job(self, job_data): + payload = json.loads(job_data) + return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) \ No newline at end of file diff --git a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py new file mode 100644 index 000000000..22c7e94e4 --- /dev/null +++ b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import + +from raven.contrib.django.gearman import GearmanWorkerCommand + + +class Command(GearmanWorkerCommand): + pass diff --git a/raven/contrib/gearman/__init__.py b/raven/contrib/gearman/__init__.py new file mode 100644 index 000000000..5da4ce6d9 --- /dev/null +++ b/raven/contrib/gearman/__init__.py @@ -0,0 +1,35 @@ +import json + +from django_gearman_commands import submit_job + +from raven.base import Client + +__all__ = ('RAVEN_GEARMAN_JOB_NAME', 'GearmanMixin', 'GearmanClient') + + +RAVEN_GEARMAN_JOB_NAME = 'raven_gearman' + + +class GearmanMixin(object): + """This class servers as a Mixin for client implementations that wants to support gearman async queue.""" + + def send_encoded(self, message, auth_header=None, **kwargs): + """Encoded data are sent to gearman, instead of directly sent to the sentry server. + + :param message: encoded message + :type message: string + :param auth_header: auth_header: authentication header for sentry + :type auth_header: string + :returns: void + :rtype: None + + """ + payload = json.dumps({ + 'message': message, + 'auth_header': auth_header + }) + submit_job(RAVEN_GEARMAN_JOB_NAME, data=payload) + + +class GearmanClient(GearmanMixin, Client): + """Independent implementation of gearman client for raven.""" \ No newline at end of file diff --git a/setup.py b/setup.py index ce247c04e..c8e158f17 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,11 @@ 'Flask-Login>=0.2.0', ] +gearman_requires = [ + 'gearman==dev', + 'django-gearman-commands==dev' +] + # If it's python3, remove flask & unittest2 if sys.version_info[0] == 3: flask_requires = [] @@ -67,7 +72,7 @@ 'webob', 'webtest', 'anyjson', -] + flask_requires + flask_tests_requires + unittest2_requires +] + flask_requires + flask_tests_requires + gearman_requires + unittest2_requires class PyTest(TestCommand): @@ -95,8 +100,9 @@ def run_tests(self): zip_safe=False, extras_require={ 'flask': flask_requires, + 'gearman': gearman_requires, 'tests': tests_require, - 'dev': dev_requires, + 'dev': dev_requires }, license='BSD', tests_require=tests_require, @@ -121,4 +127,8 @@ def run_tests(self): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.2', ], + dependency_links = [ + 'git+git://github.com/Yelp/python-gearman.git#egg=gearman-dev', + 'git+git://github.com/CodeScaleInc/django-gearman-commands.git#egg=django_gearman_commands-dev' + ] ) diff --git a/tests/contrib/django/tests.py b/tests/contrib/django/tests.py index 1d1b1c933..9198480d4 100644 --- a/tests/contrib/django/tests.py +++ b/tests/contrib/django/tests.py @@ -21,10 +21,12 @@ from django.core.handlers.wsgi import WSGIRequest from django.template import TemplateSyntaxError from django.test import TestCase +from django.core.management import call_command from raven.base import Client from raven.contrib.django.client import DjangoClient from raven.contrib.django.celery import CeleryClient +from raven.contrib.django.gearman import GearmanClient, GearmanWorkerCommand from raven.contrib.django.handlers import SentryHandler from raven.contrib.django.models import client, get_client, sentry_exception_handler from raven.contrib.django.middleware.wsgi import Sentry @@ -619,6 +621,39 @@ def test_with_eager(self, send_encoded): self.assertEquals(send_encoded.call_count, 1) +class GearmanClientTest(TestCase): + def setUp(self): + self.client = GearmanClient(servers=['http://example.com']) + + @mock.patch('raven.contrib.gearman.submit_job') + def test_send_encoded(self, submit_job): + self.client.send_encoded('foo') + submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}') + + +class GearmanWorkerTest(TestCase): + def setUp(self): + self.worker = GearmanWorkerCommand() + + @mock.patch('django_gearman_commands.GearmanWorkerBaseCommand.handle') + @mock.patch('raven.contrib.django.gearman.get_client') + def test_worker_handle_job(self, get_client, handle): + """With patching handle method, we disabled worker to connect to and listen on gearman daemon.""" + get_client.return_value = mock.MagicMock() + self.worker.execute() + self.worker.do_job('{"message":"foo","auth_header":"bar"}') + handle.assert_called_once_with() + get_client.assert_call_once_with('raven.contrib.django.client.DjangoClient') + self.assertEqual([mock.call.send_encoded(u'foo', auth_header=u'bar')], get_client.return_value.mock_calls) + + def test_worker_client_default_settings(self): + self.assertTrue(isinstance(self.worker.client, DjangoClient)) + + def test_worker_client_custom_settings(self): + with Settings(SENTRY_GEARMAN_CLIENT='raven.contrib.django.gearman.GearmanClient'): + self.assertTrue(isinstance(self.worker.client, GearmanClient)) + + class IsValidOriginTestCase(TestCase): def test_setting_empty(self): with Settings(SENTRY_ALLOW_ORIGIN=None): diff --git a/tests/contrib/gearman/__init__.py b/tests/contrib/gearman/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/contrib/gearman/tests.py b/tests/contrib/gearman/tests.py new file mode 100644 index 000000000..76d76e545 --- /dev/null +++ b/tests/contrib/gearman/tests.py @@ -0,0 +1,14 @@ +import mock + +from raven.utils.testutils import TestCase +from raven.contrib.gearman import GearmanClient + + +class ClientTest(TestCase): + def setUp(self): + self.client = GearmanClient(servers=['http://example.com']) + + @mock.patch('raven.contrib.gearman.submit_job') + def test_send_encoded(self, submit_job): + self.client.send_encoded('foo') + submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}') \ No newline at end of file