Permalink
Browse files

Much better error handling: yield something() will now raise the corr…

…ect exception for all workers.
  • Loading branch information...
1 parent b72160b commit 4d4bb08c394016880019f772bff31365f2029163 @kmike committed May 2, 2011
Showing with 65 additions and 34 deletions.
  1. +5 −14 slacker/django_backend/views.py
  2. +35 −12 slacker/postpone.py
  3. +12 −4 slacker/workers/local.py
  4. +13 −4 test_project/testapp/tests/workers.py
@@ -1,24 +1,15 @@
from __future__ import absolute_import
-
-#from time import sleep
-from django.http import HttpResponse, Http404, HttpResponseBadRequest
+from django.http import HttpResponse, Http404
from django.views.decorators.csrf import csrf_exempt
-from slacker.postpone import proceed_pickled, SlackerException
+from slacker.postpone import proceed_pickled
@csrf_exempt
def slacker_execute(request):
# FIXME: auth?
+
if request.method != 'POST':
raise Http404
- # TODO: move boilerplate to process_pickled
- # TODO: exceptions should be pickled, returned and re-raised on client
- try:
- data = proceed_pickled(request.raw_post_data)
- return HttpResponse(data)
- except SlackerException, e:
- return HttpResponseBadRequest(str(e))
-# except Exception, e:
-# print e
-# raise
+ data = proceed_pickled(request.raw_post_data)
+ return HttpResponse(data)
View
@@ -7,11 +7,13 @@
except ImportError:
import pickle
-from .workers.local import DummyWorker
+from billiard.serialization import get_pickleable_exception
+
class SlackerException(Exception):
pass
+
class _Module(object):
""" Helper class for pickling python modules """
@@ -43,7 +45,11 @@ class Postponed(object):
def __init__(self, obj, worker = None):
self._obj = obj
self._chain = []
- self._worker = worker or DummyWorker()
+ if not worker:
+ from .workers.local import DummyWorker
+ worker = DummyWorker()
+ self._worker = worker
+
def __repr__(self):
return "%s: %s" % (self._obj, pprint.pformat(self._chain))
@@ -62,6 +68,7 @@ def __setstate__(self, state):
self._obj = self._obj.module
# always use local worker after unpickling
+ from .workers.local import DummyWorker
self._worker = DummyWorker()
@property
@@ -135,21 +142,37 @@ def __call__(self, *args, **kwargs):
return Postponed(self._obj, self._worker).__call__(*args, **kwargs)
+def safe_proceed(postponed):
+ """
+ Proceeds postponed object locally and returns the result.
+ If and exception is thrown during execution, this exception
+ is catched and returned as a result. It is also ensured that
+ returned exception can be pickled.
+ """
+ try:
+ return postponed._proceed()
+ except Exception, e:
+ return get_pickleable_exception(e)
+
+
def proceed_pickled(pickled_postponed_obj):
"""
Unpickles postponed object, proceeds it locally, then pickles the result
- and returns it. Raises SlackerException on errors.
+ and returns it (or the pickled exception if the processing fails).
+ On unpickling errors SlackerException is returned.
Useful for worker implementation.
"""
- try:
- postponed = pickle.loads(pickled_postponed_obj)
- except pickle.PicklingError, e:
- raise SlackerException(str(e))
- if not isinstance(postponed, Postponed):
- raise SlackerException('Pickled object is not an instance of Postponed')
+ def get_result():
+ try:
+ postponed = pickle.loads(pickled_postponed_obj)
+ except pickle.PicklingError, e:
+ return SlackerException(str(e))
+
+ if not isinstance(postponed, Postponed):
+ return SlackerException('Pickled object is not an instance of Postponed')
+
+ return safe_proceed(postponed)
- # TODO: better error handling
- result = postponed._proceed()
- return pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
+ return pickle.dumps(get_result(), pickle.HIGHEST_PROTOCOL)
View
@@ -1,11 +1,14 @@
from functools import partial
+from tornado import stack_context
from tornado.ioloop import IOLoop
-
+from slacker.postpone import safe_proceed
class DummyWorker(object):
""" Dummy worker for local immediate execution """
def proceed(self, postponed, callback = None):
- res = postponed._proceed()
+ # safe_proceed is called instead of _proceed
+ # for consistent error handling
+ res = safe_proceed(postponed)
if callback:
callback(res)
@@ -43,11 +46,16 @@ def __init__(self, pool=None, ioloop=None):
def proceed(self, postponed, callback=None):
+ _proceed = partial(safe_proceed, postponed)
+
if callback is None:
- self.pool.apply_async(postponed._proceed)
+ self.pool.apply_async(_proceed)
return
+ # Without stack_context.wrap exceptions will not be propagated,
+ # they'll be catched by tornado. Hours of debugging ;)
+ @stack_context.wrap
def on_response(result):
self.ioloop.add_callback(partial(callback, result))
- self.pool.apply_async(postponed._proceed, callback = on_response)
+ self.pool.apply_async(_proceed, callback = on_response)
@@ -1,12 +1,11 @@
-
-
from tornado.testing import AsyncHTTPTestCase
from tornado.wsgi import WSGIContainer
from tornado.ioloop import IOLoop
from django.core.handlers.wsgi import WSGIHandler
from django.test import TransactionTestCase as DjangoTestCase
from django.contrib.auth.models import User
+from django.core.exceptions import ObjectDoesNotExist
from slacker import Slacker, adisp
from slacker.workers import ThreadWorker, DjangoWorker
@@ -27,15 +26,25 @@ def get_app(self):
return WSGIContainer(WSGIHandler())
@adisp.process
- def get_user(self):
- self.res = yield self.SlackerClass.objects.get(username=self.user.username)
+ def get_user(self, username=None):
+ username = username or self.user.username
+ self.res = yield self.SlackerClass.objects.get(username=username)
self.stop()
def test_get_user(self):
self.get_user()
self.wait()
self.assertEqual(self.res, self.user)
+ def test_error_handling(self):
+
+ def run():
+ self.get_user('vasia')
+ self.wait()
+
+ self.assertRaises(ObjectDoesNotExist, run)
+
+
class DjangoWorkerTest(BaseWorkerTest):
def setUp(self):

0 comments on commit 4d4bb08

Please sign in to comment.