From 292920f2a632bbfee72e797c08006f9de66395ba Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Mon, 27 May 2013 19:28:26 +0200 Subject: [PATCH 1/8] Initial support for gearman async queue. --- .../management/commands/raven_gearman.py | 3 +++ raven/contrib/gearman/__init__.py | 1 + raven/contrib/gearman/client.py | 12 +++++++++ raven/contrib/gearman/worker.py | 25 +++++++++++++++++++ 4 files changed, 41 insertions(+) create mode 100644 raven/contrib/django/raven_compat/management/commands/raven_gearman.py create mode 100644 raven/contrib/gearman/__init__.py create mode 100644 raven/contrib/gearman/client.py create mode 100644 raven/contrib/gearman/worker.py diff --git a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py new file mode 100644 index 000000000..f3f210e4a --- /dev/null +++ b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from raven.contrib.gearman.worker import Command \ No newline at end of file diff --git a/raven/contrib/gearman/__init__.py b/raven/contrib/gearman/__init__.py new file mode 100644 index 000000000..c81d103c2 --- /dev/null +++ b/raven/contrib/gearman/__init__.py @@ -0,0 +1 @@ +RAVEN_GEARMAN_JOB_NAME = 'raven_gearman' \ No newline at end of file diff --git a/raven/contrib/gearman/client.py b/raven/contrib/gearman/client.py new file mode 100644 index 000000000..7f5444e46 --- /dev/null +++ b/raven/contrib/gearman/client.py @@ -0,0 +1,12 @@ +from django_gearman_commands import submit_job + +from raven.contrib.django.client import DjangoClient +from raven.contrib.gearman import RAVEN_GEARMAN_JOB_NAME + +__all__ = ('GearmanClient',) + + +class GearmanClient(DjangoClient): + + def send_encoded(self, message, auth_header=None, **kwargs): + return submit_job(RAVEN_GEARMAN_JOB_NAME, data=message) \ No newline at end of file diff --git a/raven/contrib/gearman/worker.py b/raven/contrib/gearman/worker.py new file mode 100644 index 000000000..bb5129077 --- /dev/null +++ b/raven/contrib/gearman/worker.py @@ -0,0 +1,25 @@ +from django_gearman_commands import GearmanWorkerBaseCommand + +from django.conf import settings + +from raven.contrib.django.models import get_client +from raven.contrib.gearman import RAVEN_GEARMAN_JOB_NAME + + +class Command(GearmanWorkerBaseCommand): + + _client = None + + @property + def task_name(self): + return RAVEN_GEARMAN_JOB_NAME + + @property + def client(self): + if self._client is None: + self._client = get_client(getattr(settings, 'SENTRY_GEARMAN_CLIENT', + 'raven.contrib.django.client.DjangoClient')) + return self._client + + def do_job(self, job_data): + return self.client.send_encoded(job_data) From 91519ffa70e32b396f78dad76c66350015650293 Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Tue, 28 May 2013 11:04:58 +0200 Subject: [PATCH 2/8] Stable implementation of gearman support for raven. --- raven/contrib/django/gearman/__init__.py | 46 +++++++++++++++++++ .../management/commands/raven_gearman.py | 2 +- raven/contrib/gearman/__init__.py | 36 ++++++++++++++- raven/contrib/gearman/client.py | 12 ----- raven/contrib/gearman/worker.py | 25 ---------- 5 files changed, 82 insertions(+), 39 deletions(-) create mode 100644 raven/contrib/django/gearman/__init__.py delete mode 100644 raven/contrib/gearman/client.py delete mode 100644 raven/contrib/gearman/worker.py diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py new file mode 100644 index 000000000..aa4d7aea7 --- /dev/null +++ b/raven/contrib/django/gearman/__init__.py @@ -0,0 +1,46 @@ +import json + +from django_gearman_commands import GearmanWorkerBaseCommand + +from django.conf import settings + +from raven.contrib.gearman import GearmanMixin, RAVEN_GEARMAN_JOB_NAME +from raven.contrib.django import DjangoClient +from raven.contrib.django.models import get_client + +__all__ = ('GearmanClient', 'Command') + + +class GearmanClient(GearmanMixin, DjangoClient): + """Gearman client implementation for django applications.""" + + +class Command(GearmanWorkerBaseCommand): + """Gearman worker implementation. + + This worker is run as django management command. Gearman client send messages to gearman deamon. Next + the messages are downloaded from gearman daemon by this worker, and sent to sentry server by standrd + django raven client 'raven.contrib.django.client.DjangoClient', if not specified otherwise by + SENTRY_GEARMAN_CLIENT django setting. + + This worker is dependent on django-gearman-commands app. For more information how this works, please + visit https://github.com/CodeScaleInc/django-gearman-commands. + + """ + + _client = None + + @property + def task_name(self): + return RAVEN_GEARMAN_JOB_NAME + + @property + def client(self): + if self._client is None: + self._client = get_client(getattr(settings, 'SENTRY_GEARMAN_CLIENT', + 'raven.contrib.django.client.DjangoClient')) + return self._client + + def do_job(self, job_data): + payload = json.loads(job_data) + return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) \ No newline at end of file diff --git a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py index f3f210e4a..e21e74062 100644 --- a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py +++ b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py @@ -1,3 +1,3 @@ from __future__ import absolute_import -from raven.contrib.gearman.worker import Command \ No newline at end of file +from raven.contrib.django.gearman import Command \ No newline at end of file diff --git a/raven/contrib/gearman/__init__.py b/raven/contrib/gearman/__init__.py index c81d103c2..5da4ce6d9 100644 --- a/raven/contrib/gearman/__init__.py +++ b/raven/contrib/gearman/__init__.py @@ -1 +1,35 @@ -RAVEN_GEARMAN_JOB_NAME = 'raven_gearman' \ No newline at end of file +import json + +from django_gearman_commands import submit_job + +from raven.base import Client + +__all__ = ('RAVEN_GEARMAN_JOB_NAME', 'GearmanMixin', 'GearmanClient') + + +RAVEN_GEARMAN_JOB_NAME = 'raven_gearman' + + +class GearmanMixin(object): + """This class servers as a Mixin for client implementations that wants to support gearman async queue.""" + + def send_encoded(self, message, auth_header=None, **kwargs): + """Encoded data are sent to gearman, instead of directly sent to the sentry server. + + :param message: encoded message + :type message: string + :param auth_header: auth_header: authentication header for sentry + :type auth_header: string + :returns: void + :rtype: None + + """ + payload = json.dumps({ + 'message': message, + 'auth_header': auth_header + }) + submit_job(RAVEN_GEARMAN_JOB_NAME, data=payload) + + +class GearmanClient(GearmanMixin, Client): + """Independent implementation of gearman client for raven.""" \ No newline at end of file diff --git a/raven/contrib/gearman/client.py b/raven/contrib/gearman/client.py deleted file mode 100644 index 7f5444e46..000000000 --- a/raven/contrib/gearman/client.py +++ /dev/null @@ -1,12 +0,0 @@ -from django_gearman_commands import submit_job - -from raven.contrib.django.client import DjangoClient -from raven.contrib.gearman import RAVEN_GEARMAN_JOB_NAME - -__all__ = ('GearmanClient',) - - -class GearmanClient(DjangoClient): - - def send_encoded(self, message, auth_header=None, **kwargs): - return submit_job(RAVEN_GEARMAN_JOB_NAME, data=message) \ No newline at end of file diff --git a/raven/contrib/gearman/worker.py b/raven/contrib/gearman/worker.py deleted file mode 100644 index bb5129077..000000000 --- a/raven/contrib/gearman/worker.py +++ /dev/null @@ -1,25 +0,0 @@ -from django_gearman_commands import GearmanWorkerBaseCommand - -from django.conf import settings - -from raven.contrib.django.models import get_client -from raven.contrib.gearman import RAVEN_GEARMAN_JOB_NAME - - -class Command(GearmanWorkerBaseCommand): - - _client = None - - @property - def task_name(self): - return RAVEN_GEARMAN_JOB_NAME - - @property - def client(self): - if self._client is None: - self._client = get_client(getattr(settings, 'SENTRY_GEARMAN_CLIENT', - 'raven.contrib.django.client.DjangoClient')) - return self._client - - def do_job(self, job_data): - return self.client.send_encoded(job_data) From 36679087daca286193fcf2b617c6609722a9ca63 Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Wed, 29 May 2013 15:42:04 +0200 Subject: [PATCH 3/8] Added tests for gearman support. --- raven/contrib/django/gearman/__init__.py | 6 ++-- .../management/commands/raven_gearman.py | 6 +++- raven/contrib/gearman/__init__.py | 2 +- setup.py | 14 ++++++-- tests/contrib/django/tests.py | 35 +++++++++++++++++++ tests/contrib/gearman/__init__.py | 0 tests/contrib/gearman/tests.py | 14 ++++++++ 7 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 tests/contrib/gearman/__init__.py create mode 100644 tests/contrib/gearman/tests.py diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py index aa4d7aea7..9b8521571 100644 --- a/raven/contrib/django/gearman/__init__.py +++ b/raven/contrib/django/gearman/__init__.py @@ -8,14 +8,14 @@ from raven.contrib.django import DjangoClient from raven.contrib.django.models import get_client -__all__ = ('GearmanClient', 'Command') +__all__ = ('GearmanClient', 'GearmanWorkerCommand') class GearmanClient(GearmanMixin, DjangoClient): """Gearman client implementation for django applications.""" -class Command(GearmanWorkerBaseCommand): +class GearmanWorkerCommand(GearmanWorkerBaseCommand): """Gearman worker implementation. This worker is run as django management command. Gearman client send messages to gearman deamon. Next @@ -43,4 +43,4 @@ def client(self): def do_job(self, job_data): payload = json.loads(job_data) - return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) \ No newline at end of file + return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) diff --git a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py index e21e74062..22c7e94e4 100644 --- a/raven/contrib/django/raven_compat/management/commands/raven_gearman.py +++ b/raven/contrib/django/raven_compat/management/commands/raven_gearman.py @@ -1,3 +1,7 @@ from __future__ import absolute_import -from raven.contrib.django.gearman import Command \ No newline at end of file +from raven.contrib.django.gearman import GearmanWorkerCommand + + +class Command(GearmanWorkerCommand): + pass diff --git a/raven/contrib/gearman/__init__.py b/raven/contrib/gearman/__init__.py index 5da4ce6d9..0ee9143e6 100644 --- a/raven/contrib/gearman/__init__.py +++ b/raven/contrib/gearman/__init__.py @@ -32,4 +32,4 @@ def send_encoded(self, message, auth_header=None, **kwargs): class GearmanClient(GearmanMixin, Client): - """Independent implementation of gearman client for raven.""" \ No newline at end of file + """Independent implementation of gearman client for raven.""" diff --git a/setup.py b/setup.py index b64e7a9af..650626ee6 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,11 @@ 'Flask-Login>=0.1.3', ] +gearman_requires = [ + 'gearman==dev', + 'django-gearman-commands==dev' +] + # If it's python3, remove flask & unittest2 if sys.version_info[0] == 3: flask_requires = [] @@ -61,7 +66,7 @@ 'tornado', 'webob', 'anyjson', -] + flask_requires + flask_tests_requires + unittest2_requires +] + flask_requires + flask_tests_requires + gearman_requires + unittest2_requires setup( name='raven', @@ -75,8 +80,9 @@ zip_safe=False, extras_require={ 'flask': flask_requires, + 'gearman': gearman_requires, 'tests': tests_require, - 'dev': dev_requires, + 'dev': dev_requires }, test_suite='runtests.runtests', include_package_data=True, @@ -99,4 +105,8 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.2', ], + dependency_links = [ + 'git+git://github.com/Yelp/python-gearman.git#egg=gearman-dev', + 'git+git://github.com/CodeScaleInc/django-gearman-commands.git#egg=django_gearman_commands-dev' + ] ) diff --git a/tests/contrib/django/tests.py b/tests/contrib/django/tests.py index 8a439a1b4..e0a1be1a6 100644 --- a/tests/contrib/django/tests.py +++ b/tests/contrib/django/tests.py @@ -21,10 +21,12 @@ from django.core.handlers.wsgi import WSGIRequest from django.template import TemplateSyntaxError from django.test import TestCase +from django.core.management import call_command from raven.base import Client from raven.contrib.django import DjangoClient from raven.contrib.django.celery import CeleryClient +from raven.contrib.django.gearman import GearmanClient, GearmanWorkerCommand from raven.contrib.django.handlers import SentryHandler from raven.contrib.django.models import client, get_client, sentry_exception_handler from raven.contrib.django.middleware.wsgi import Sentry @@ -600,6 +602,39 @@ def test_with_eager(self, send_encoded): self.assertEquals(send_encoded.call_count, 1) +class GearmanClientTest(TestCase): + def setUp(self): + self.client = GearmanClient(servers=['http://example.com']) + + @mock.patch('raven.contrib.gearman.submit_job') + def test_send_encoded(self, submit_job): + self.client.send_encoded('foo') + submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}') + + +class GearmanWorkerTest(TestCase): + def setUp(self): + self.worker = GearmanWorkerCommand() + + @mock.patch('django_gearman_commands.GearmanWorkerBaseCommand.handle') + @mock.patch('raven.contrib.django.gearman.get_client') + def test_worker_handle_job(self, get_client, handle): + """With patching handle method, we disabled worker to connect to and listen on gearman daemon.""" + get_client.return_value = mock.MagicMock() + self.worker.execute() + self.worker.do_job('{"message":"foo","auth_header":"bar"}') + handle.assert_called_once_with() + get_client.assert_call_once_with('raven.contrib.django.client.DjangoClient') + self.assertEqual([mock.call.send_encoded(u'foo', auth_header=u'bar')], get_client.return_value.mock_calls) + + def test_worker_client_default_settings(self): + self.assertIsInstance(self.worker.client, DjangoClient) + + def test_worker_client_custom_settings(self): + with Settings(SENTRY_GEARMAN_CLIENT='raven.contrib.django.gearman.GearmanClient'): + self.assertIsInstance(self.worker.client, GearmanClient) + + class IsValidOriginTestCase(TestCase): def test_setting_empty(self): with Settings(SENTRY_ALLOW_ORIGIN=None): diff --git a/tests/contrib/gearman/__init__.py b/tests/contrib/gearman/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/contrib/gearman/tests.py b/tests/contrib/gearman/tests.py new file mode 100644 index 000000000..76d76e545 --- /dev/null +++ b/tests/contrib/gearman/tests.py @@ -0,0 +1,14 @@ +import mock + +from raven.utils.testutils import TestCase +from raven.contrib.gearman import GearmanClient + + +class ClientTest(TestCase): + def setUp(self): + self.client = GearmanClient(servers=['http://example.com']) + + @mock.patch('raven.contrib.gearman.submit_job') + def test_send_encoded(self, submit_job): + self.client.send_encoded('foo') + submit_job.assert_called_once_with('raven_gearman', data='{"message": "foo", "auth_header": null}') \ No newline at end of file From 2a788709e514fb40a84905a8c86d731145ffa9ec Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Wed, 29 May 2013 16:02:29 +0200 Subject: [PATCH 4/8] Added gearman support documentation. --- docs/config/gearman.rst | 32 ++++++++++++++++++++++++++++++++ docs/config/index.rst | 1 + 2 files changed, 33 insertions(+) create mode 100644 docs/config/gearman.rst diff --git a/docs/config/gearman.rst b/docs/config/gearman.rst new file mode 100644 index 000000000..e1478c4ae --- /dev/null +++ b/docs/config/gearman.rst @@ -0,0 +1,32 @@ +Configuring Gearman +=================== + +Gearman provides a generic application framework to farm out work to other machines or processes that are better +suited to do the work. It allows you to do work in parallel, to load balance processing, and to call functions +between languages. For more information visit http://gearman.org/. + +In order to use gearman support in raven, you have to have installed gearmand job server on your machine. + +If you're using Django add this to your ``settings``:: + + GEARMAN_SERVERS = ['127.0.0.1:4730'] + SENTRY_CLIENT = 'raven.contrib.django.gearman.GearmanClient' + SENTRY_GEARMAN_CLIENT = 'raven.contrib.django.client.DjangoClient' + + +Next you need to run ``raven_gearman`` worker to process your logging events in the background:: + + $ python manage.py raven_gearman + + +``raven_gearman`` command is build on top of django-gearman-commands app. For more information about +setting-up your worker please have a look at this url https://github.com/CodeScaleInc/django-gearman-commands. + + +How does it work ? +------------------ + +All logging events as submitted as job to gearmand job server. They are not submitted directly to the sentry server. +``raven_gearman`` is running in background and downloads jobs from gearmand job server. Every downloaded job is then +transformed into raven event and sent by ``SENTRY_GEARMAN_CLIENT`` to sentry server. The point is, that all this happens +asynchronously and don't block your application main thread. \ No newline at end of file diff --git a/docs/config/index.rst b/docs/config/index.rst index 4618922a2..29facf5a3 100644 --- a/docs/config/index.rst +++ b/docs/config/index.rst @@ -10,6 +10,7 @@ This document describes configuration options available to Sentry. :maxdepth: 2 celery + gearman django flask logbook From 952bc8739bbab62529e56142051c7caee08dcc08 Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Wed, 29 May 2013 18:06:41 +0200 Subject: [PATCH 5/8] Fixed unittests for gearman supports. --- tests/contrib/django/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/contrib/django/tests.py b/tests/contrib/django/tests.py index e0a1be1a6..a4dfedcf3 100644 --- a/tests/contrib/django/tests.py +++ b/tests/contrib/django/tests.py @@ -628,11 +628,11 @@ def test_worker_handle_job(self, get_client, handle): self.assertEqual([mock.call.send_encoded(u'foo', auth_header=u'bar')], get_client.return_value.mock_calls) def test_worker_client_default_settings(self): - self.assertIsInstance(self.worker.client, DjangoClient) + self.assertTrue(isinstance(self.worker.client, DjangoClient)) def test_worker_client_custom_settings(self): with Settings(SENTRY_GEARMAN_CLIENT='raven.contrib.django.gearman.GearmanClient'): - self.assertIsInstance(self.worker.client, GearmanClient) + self.assertTrue(isinstance(self.worker.client, GearmanClient)) class IsValidOriginTestCase(TestCase): From a78acc6168957990e1794dbd6ce53801c743f21a Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Fri, 9 Aug 2013 19:44:06 +0200 Subject: [PATCH 6/8] Sending custom checksum to disable message grouping. --- raven/contrib/django/gearman/__init__.py | 12 +++++++++++- raven/contrib/gearman/__init__.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py index 9b8521571..70fcc6290 100644 --- a/raven/contrib/django/gearman/__init__.py +++ b/raven/contrib/django/gearman/__init__.py @@ -1,4 +1,6 @@ import json +import uuid +import hashlib from django_gearman_commands import GearmanWorkerBaseCommand @@ -13,6 +15,14 @@ class GearmanClient(GearmanMixin, DjangoClient): """Gearman client implementation for django applications.""" + def build_msg(self, event_type, data=None, date=None, + time_spent=None, extra=None, stack=None, public_key=None, + tags=None, **kwargs): + msg = super(GearmanClient, self).build_msg(event_type, data=data, date=date, + time_spen=time_spent, extra=extra, stack=stack, + public_key=public_key, tags=None, **kwargs) + msg['checksum'] = hashlib.md5(str(uuid.uuid4())).hexdigest() + return msg class GearmanWorkerCommand(GearmanWorkerBaseCommand): @@ -43,4 +53,4 @@ def client(self): def do_job(self, job_data): payload = json.loads(job_data) - return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) + return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) \ No newline at end of file diff --git a/raven/contrib/gearman/__init__.py b/raven/contrib/gearman/__init__.py index 0ee9143e6..5da4ce6d9 100644 --- a/raven/contrib/gearman/__init__.py +++ b/raven/contrib/gearman/__init__.py @@ -32,4 +32,4 @@ def send_encoded(self, message, auth_header=None, **kwargs): class GearmanClient(GearmanMixin, Client): - """Independent implementation of gearman client for raven.""" + """Independent implementation of gearman client for raven.""" \ No newline at end of file From 35e7babb5eeaa9225ca3ea0f2a87bb8c746ba83c Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Fri, 23 Aug 2013 10:44:57 +0200 Subject: [PATCH 7/8] Added exit_after_job configurable hook. --- raven/contrib/django/gearman/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py index 70fcc6290..1e5bd81ed 100644 --- a/raven/contrib/django/gearman/__init__.py +++ b/raven/contrib/django/gearman/__init__.py @@ -51,6 +51,10 @@ def client(self): 'raven.contrib.django.client.DjangoClient')) return self._client + @property + def exit_after_job(self): + return True + def do_job(self, job_data): payload = json.loads(job_data) return self.client.send_encoded(payload['message'], auth_header=payload['auth_header']) \ No newline at end of file From f2573236dc672047ce115b1dfe4a3afcd1a27281 Mon Sep 17 00:00:00 2001 From: Vladimir Gorej Date: Fri, 23 Aug 2013 10:45:05 +0200 Subject: [PATCH 8/8] Added exit_after_job configurable hook. --- raven/contrib/django/gearman/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/contrib/django/gearman/__init__.py b/raven/contrib/django/gearman/__init__.py index 1e5bd81ed..6e80137c3 100644 --- a/raven/contrib/django/gearman/__init__.py +++ b/raven/contrib/django/gearman/__init__.py @@ -53,7 +53,7 @@ def client(self): @property def exit_after_job(self): - return True + return getattr(settings, 'SENTRY_GEARMAN_EXIT_AFTER_JOB', False) def do_job(self, job_data): payload = json.loads(job_data)