Permalink
Browse files

Initial import

  • Loading branch information...
kmike committed Apr 27, 2011
1 parent a17bacb commit 377a3bcc9d7c1f5b51791758df054b3852441799
View
19 LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2011 Mikhail Korobov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
View
@@ -0,0 +1,2 @@
+include *.txt
+include *.rst
View
@@ -0,0 +1,109 @@
+================
+django-async-orm
+================
+
+This app makes non-blocking django ORM calls possible.
+
+Installation
+============
+
+::
+ pip install "tornado >= 1.2"
+ pip install django-async-orm
+
+FIXME: this is not uploaded to pypi now
+
+Overview
+========
+
+This app can be used for tornado + django integration: run tornado
+as django management command (on a separate port) => all django code will be
+available in tornado process; then use this library instead of
+plain django ORM calls in Tornado handlers to make these calls non-blocking.
+
+::
+
+ from django.contrib.auth.models import User
+ from async_orm.incarnations.http import AsyncWrapper
+
+ AsyncUser = AsyncWrapper(User)
+
+ # ...
+
+ def process_data(self):
+ # all the django orm syntax is supported here
+ qs = AsyncUser.objects.filter(is_staff=True)[:5]
+ qs.execute(self.on_ready)
+
+ def on_ready(self, users):
+ # do something with query result
+ print users
+
+or, with pep-342 syntax and adisp library (it is bundled)::
+
+ from async_orm.vendor import adisp
+
+ @adisp.process
+ def process_data(self):
+ qs = AsyncUser.objects.filter(is_staff=True)[:5]
+ users = yield qs.fetch()
+ print users
+
+You still can't rely on third-party code that uses django ORM
+in Tornado handlers but it is at least easy to reimplement it now
+if necessary.
+
+Currently the only implemented method for offloading query execution
+from the ioloop is to execute the blocking code in a django view and
+fetch results using tornado's AsyncHttpClient. This way it was possible
+to get a simple implementation, easy deployment and a thread pool
+(managed by webserver) for free. HTTP, however, can cause a
+significant overhead.
+
+Please dig into source code for more info, this README is totally
+incomplete.
+
+TODO: proper usage guide: how to configure django for this? how to deploy?
+
+Configuration
+=============
+
+(async.incarnations.http.urls must be included in urls)
+
+TODO: write this
+
+Usage
+=====
+
+TODO
+
+Contributing
+============
+
+If you have any suggestions, bug reports or
+annoyances please report them to the issue tracker
+at https://github.com/kmike/django-async-orm/issues
+
+Both hg and git pull requests are welcome!
+
+* https://bitbucket.org/kmike/django-async-orm/
+* https://github.com/kmike/django-async-orm/
+
+Credits
+=======
+
+Inspiration:
+
+* http://tornadogists.org/654157/
+* https://github.com/satels/django-async-dbslayer/
+* https://bitbucket.org/david/django-roa/
+
+Third-party software: `adisp <https://code.launchpad.net/adisp>`_ (tornado
+implementation is taken from `brukva <https://github.com/evilkost/brukva>`_).
+
+License
+=======
+
+The license is MIT.
+
+Bundled adisp library uses Simplified BSD License.
View
@@ -0,0 +1 @@
+
View
@@ -0,0 +1,148 @@
+import pprint
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+from django.db.models.loading import get_model
+
+
+class AsyncOrmException(Exception):
+ pass
+
+
+class ChainProxy(object):
+ """
+ Stores attribute, call and slice chain without actully
+ calling methods, accessing attributes and performing slicing.
+
+ Collecting the access to private methods and attributes
+ (beginning with __two_underscores) is not supported.
+
+ FIXME: '_obj', '_chain' and 'restore' attributes of original
+ object are replaced with the ones from this proxy.
+ """
+
+ def __init__(self, obj, **kwargs):
+ self._obj = obj
+ self._chain = []
+ self._extra = kwargs
+
+ def __getattr__(self, attr):
+ # pickle.dumps internally checks if __getnewargs__ is defined
+ # and thus returning ChainProxy object instead of
+ # raising AttributeError breaks pickling. Returning self instead
+ # of raising an exception for private attributes can possible
+ # break something else so the access to private methods and attributes
+ # is not overriden at all.
+ if attr.startswith('__'):
+ return self.__getattribute__(attr)
+
+ # attribute access is stored as 1-element tuple
+ self._chain.append((attr,))
+ return self
+
+ def __getitem__(self, slice):
+ # slicing operation is stored as 2-element tuple
+ self._chain.append((slice, None,))
+ return self
+
+ def __call__(self, *args, **kwargs):
+ # method call is stored as 3-element tuple
+ method_name = self._chain[-1][0]
+ self._chain[-1] = (method_name, args, kwargs)
+ return self
+
+ def restore(self):
+ """ Executes and returns the stored chain. """
+ result = self._obj
+ for op in self._chain:
+ if len(op) == 1: # attribute
+ result = getattr(result, op[0])
+ elif len(op) == 2: # slice or index
+ result = result[op[0]]
+ elif len(op) == 3: # method
+ result = getattr(result, op[0])(*op[1], **op[2])
+ return result
+
+ def __repr__(self):
+ return "%s: %s" % (self._obj, pprint.pformat(self._chain))
+
+
+class ModelChainProxy(ChainProxy):
+ """
+ Adds support for pickling when proxy is applied to
+ django.db.models.Model subclass.
+
+ This handles QuerySet method arguments like Q objects,
+ F objects and aggregate functions (e.g. Count) properly,
+ but can break on QuerySets as arguments (queryset will be executed).
+
+ Why not follow the advice from django docs and just pickle queryset.query?
+ http://docs.djangoproject.com/en/dev/ref/models/querysets/#pickling-querysets
+
+ The advice is limited to QuerySets. With ModelChainProxy it is possible
+ to pickle any ORM calls including ones that don't return QuerySets:
+ http://docs.djangoproject.com/en/dev/ref/models/querysets/#methods-that-do-not-return-querysets
+
+ Moreover, using custom managers and model methods, as well as returning model
+ attributes, is fully supported. This allows user to execute any
+ orm-related code (e.g. populating the instance and saving it) in
+ non-blocking manner: just write the code as a model or manager method.
+ """
+ def _model_data(self):
+ meta = self._obj._meta
+ return meta.app_label, meta.object_name
+
+ def __getstate__(self):
+ return dict(
+ chain = self._chain,
+ model_class = self._model_data()
+ )
+
+ def __setstate__(self, dict):
+ self._chain = dict['chain']
+ model_class = get_model(*dict['model_class'])
+ self._obj = model_class
+
+ @property
+ def _pickled(self):
+ return pickle.dumps(self, pickle.HIGHEST_PROTOCOL)
+
+ def __repr__(self):
+ app, model = self._model_data()
+ return "%s.%s: %s" % (app, model, pprint.pformat(self._chain))
+
+
+class ProxyWrapper(object):
+ """
+ Creates a new ChainProxy subclass instance on every attribute access.
+ Useful for wrapping existing classes into chain proxies.
+ """
+ proxy_class = ModelChainProxy
+
+ def __init__(self, cls, **kwargs):
+ self._cls = cls
+ self._extra = kwargs
+
+ def __getattr__(self, item):
+ return getattr(self.proxy_class(self._cls, **self._extra), item)
+
+
+def repickle_chain(pickled_data):
+ """
+ Unpickles and executes pickled chain, then pickles the result
+ and returns it. Raises AsyncOrmException on errors.
+ """
+ try:
+ chain = pickle.loads(pickled_data)
+ except pickle.PicklingError, e:
+ raise AsyncOrmException(str(e))
+
+ if not isinstance(chain, ChainProxy):
+ raise AsyncOrmException('Pickled query is not an instance of ChainProxy')
+
+ # TODO: better error handling
+ restored = chain.restore()
+ data = pickle.dumps(restored, pickle.HIGHEST_PROTOCOL)
+ return data
@@ -0,0 +1 @@
+
@@ -0,0 +1,61 @@
+# coding: utf8
+"""
+
+This async_orm incarnation makes django ORM queries async
+by executing them in django view and fetching the results via
+http using tornado's async http client.
+
+This way we get a simple implementation, easy deployment and a
+thread pool (managed by webserver) for free.
+
+Http, however, can cause a significant overhead.
+
+"""
+
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+from tornado.httpclient import AsyncHTTPClient
+from django.core.urlresolvers import reverse
+
+from async_orm.vendor import adisp
+from async_orm.chains import ModelChainProxy, ProxyWrapper
+from async_orm.settings import HTTP_SERVER
+
+
+class TornadoHttpModelProxy(ModelChainProxy):
+
+ def execute(self, callback):
+ server = self._extra.get('server', HTTP_SERVER)
+ path = self._extra.get('path', None) or reverse('async-orm-execute')
+ url = server + path
+
+ def on_response(response):
+ result = pickle.loads(response.body)
+ callback(result)
+
+ http = AsyncHTTPClient()
+ http.fetch(url, on_response, method='POST', body=self._pickled)
+
+ fetch = adisp.async(execute)
+
+
+
+class AsyncWrapper(ProxyWrapper):
+ """
+ Returns async proxy for passed django.db.models.Model class.
+ Constructor also accepts 'server' and 'path' keyword arguments.
+
+ 'async-orm-execute' view enabled.
+
+ Example::
+
+ from django.contrib.auth.models import User
+ from async_orm.incarnations.http import AsyncWrapper
+
+ AsyncUser = AsyncWrapper(User, server='http://127.0.0.1:8001')
+
+ """
+ proxy_class = TornadoHttpModelProxy
@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import *
+
+from async_orm.incarnations.http.views import orm_execute
+
+urlpatterns = patterns('',
+ url(r'^execute/$', orm_execute, name='async-orm-execute'),
+)
@@ -0,0 +1,17 @@
+#from time import sleep
+from django.http import HttpResponse, Http404, HttpResponseBadRequest
+from django.views.decorators.csrf import csrf_exempt
+
+from async_orm.chains import repickle_chain, AsyncOrmException
+
+@csrf_exempt
+def orm_execute(request):
+ # FIXME: auth?
+ if request.method != 'POST':
+ raise Http404
+
+ try:
+ data = repickle_chain(request.raw_post_data)
+ return HttpResponse(data)
+ except AsyncOrmException, e:
+ return HttpResponseBadRequest(str(e))
View
@@ -0,0 +1 @@
+# hello, testrunner!
View
@@ -0,0 +1,2 @@
+from django.conf import settings
+HTTP_SERVER = getattr(settings, 'ASYNC_ORM_HTTP_SERVER', 'http://127.0.0.1:8000')
Oops, something went wrong.

0 comments on commit 377a3bc

Please sign in to comment.