Permalink
Browse files

added initial Proof of Concept for async reports

  • Loading branch information...
1 parent edec99d commit 7d2f780b0348f84e53ff0819939731f784635534 @nikolajbaer nikolajbaer committed Aug 2, 2011
View
@@ -50,4 +50,11 @@
'django.contrib.admin',
'reportengine',
'example_reports',
+ 'djcelery',
+ 'djkombu',
)
+
+
+ASYNC_REPORTS=True
+BROKER_TRANSPORT = "djkombu.transport.DatabaseTransport"
+
View
@@ -1,11 +1,15 @@
from django.db import models
-
-HELP_TEXT="""Raw SQL to run. Follows standard python string substitution (e.g. %(name)s). Variables passed in are order_by, page, per_page, and all filter parameters. NOTE: This is for skilled users only, and access should be restricted and monitored as severe dataloss and corruption can occur if used improperly."""
-
-# TODO figure out how to make this read-only
-class SQLReport(models.Model):
- row_sql=models.TextField(help_text=HELP_TEXT)
- aggregate_sql=models.TextField()
-
- # TODO add parameters to this that make it useful
-
+import datetime
+
+class ReportRequest(models.Model):
+ """Session based report request. Report request is made, and the token for the request is stored in the session so only that user can access this report. Task system generates the report and drops it into "content". When content is no longer null, user sees full report and their session token is cleared."""
+ # TODO consider cleanup (when should this be happening? after the request is made? What about caching? throttling?)
+ namespace = models.CharField(max_length=255)
+ slug = models.CharField(max_length=255)
+ params = models.TextField() # url encoded GET params
+ request_made = models.DateTimeField(default=datetime.datetime.now)
+ token = models.CharField(max_length=255)
+ content = models.TextField()
+ viewed_on = models.DateTimeField(null=True)
+ mimetype = models.CharField(max_length=255,null=True)
+
View
@@ -0,0 +1,86 @@
+from celery.decorators import task
+from models import ReportRequest
+import reportengine
+from reportengine.outputformats import CSVOutputFormat, XMLOutputFormat
+import StringIO
+from urlparse import parse_qsl
+
+# TODO not-DRY copy/paste from management/commands/generate_report.py, should be combined
+
+class MockRequest(object):
+ def __init__(self, **kwargs):
+ self.REQUEST = kwargs
+
+@task()
+def async_report(token):
+
+ try:
+ repreq = ReportRequest.objects.get(token=token)
+ except ReportRequest.DoesNotExist:
+ # Error?
+ return
+
+
+ kwargs = dict(parse_qsl(repreq.params))
+
+ # THis is like 90% the same
+ reportengine.autodiscover() ## Populate the reportengine registry
+ try:
+ report = reportengine.get_report(repreq.namespace, repreq.slug)()
+ except Exception, err:
+ raise err
+
+ request = MockRequest(**kwargs)
+ filter_form = report.get_filter_form(request)
+ if filter_form.fields:
+ if filter_form.is_valid():
+ filters = filter_form.cleaned_data
+ else:
+ filters = {}
+ else:
+ if report.allow_unspecified_filters:
+ filters = dict(request.REQUEST)
+ else:
+ filters = {}
+
+ # Remove blank filters
+ for k in filters.keys():
+ if filters[k] == '':
+ del filters[k]
+
+ ## Update the mask and run the report!
+ mask = report.get_default_mask()
+ mask.update(filters)
+ rows, aggregates = report.get_rows(mask, order_by=kwargs.get('order_by',None))
+
+ output = StringIO.StringIO()
+
+ ## Get our output format, setting a default if one wasn't set or isn't valid for this report
+ outputformat = None
+ if output:
+ for format in report.output_formats:
+ if format.slug == kwargs.get('format',None):
+ outputformat = format
+ if not outputformat:
+ ## By default, [0] is AdminOutputFormat, so grab the last one instead
+ outputformat = report.output_formats[-1]
+
+ context = {
+ 'report': report,
+ 'title': report.verbose_name,
+ 'rows': rows,
+ 'filter_form': filter_form,
+ 'aggregates': aggregates,
+ 'paginator': None,
+ 'cl': None,
+ 'page': 0,
+ 'urlparams': repreq.params
+ }
+
+ outputformat.generate_output(context, output)
+ #output.close()
+
+ repreq.content = output.getvalue()
+ repreq.save()
+
+
@@ -0,0 +1,12 @@
+{% extends "admin/base_site.html" %}
+
+{# TODO note that meta refreshes are deprecated.. but is there a nice clean way to do this otherwise? #}
+{% block extrahead %}
+<meta http-equiv="refresh" content="5" />
+{% endblock %}
+
+{% block content %}
+<h2>Please wait while your report is generated...</h2>
+{% endblock %}
+
+
View
@@ -1,12 +1,10 @@
from django.conf.urls.defaults import *
+from django.conf import settings
urlpatterns = patterns('reportengine.views',
# Listing of reports
url('^$', 'report_list', name='reports-list'),
- # View report in first output style
- url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/$', 'view_report', name='reports-view'),
- # view report in specified output format
- url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/(?P<output>[-\w]+)/$', 'view_report', name='reports-view-format'),
+
# view report redirected to current date format (requires date_field argument)
url('^current/(?P<daterange>(day|week|month|year))/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/$',
'current_redirect', name='reports-current'),
@@ -27,3 +25,22 @@
url('^calendar/(?P<year>\d+)/(?P<month>\d+)/(?P<day>\d+)/$', 'calendar_day_view', name='reports-calendar-day'),
)
+
+# Add async report view if we are doing async reports, otherwise go to sync report views
+# NOTE i am doing this here in case we need to re-org the urls for future HTML compat. (my meta refresh is deprecated)
+if hasattr(settings,"ASYNC_REPORTS") and settings.ASYNC_REPORTS:
+ urlpatterns += patterns('reportengine.views',
+ # TODO make this an option? to do async all the time?
+ url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/$', 'async_report', name='reports-view'),
+ # view report in specified output format
+ url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/(?P<output>[-\w]+)/$', 'async_report', name='reports-view-format'),
+ )
+else:
+ urlpatterns += patterns('reportengine.views',
+ # View report in first output style
+ url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/$', 'view_report', name='reports-view'),
+ # view report in specified output format
+ url('^view/(?P<namespace>[-\w]+)/(?P<slug>[-\w]+)/(?P<output>[-\w]+)/$', 'view_report', name='reports-view-format'),
+ )
+
+
View
@@ -4,10 +4,11 @@
from django.contrib.admin.views.main import ALL_VAR,ORDER_VAR, PAGE_VAR
from django.contrib.admin.views.decorators import staff_member_required
from django.core.urlresolvers import reverse
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect,HttpResponse
import reportengine
+from reportengine.models import ReportRequest
from urllib import urlencode
-import datetime,calendar
+import datetime,calendar,hashlib
def next_month(d):
"""helper to get next month"""
@@ -179,3 +180,51 @@ def calendar_day_view(request, year, month,day):
cx={"reports":reports,"date":date,"calendar":cal}
return render_to_response("reportengine/calendar_day.html",cx,
context_instance=RequestContext(request))
+
+def async_report(request, namespace, slug, output=None):
+ from tasks import async_report
+ # TODO build report via celery task
+ # CONSIDER this
+ # Request is stored as model object, only accessible by the requester via sesh token (offloads perms responsiblities)
+ #
+ #
+ # If they have a report request token in their session,
+ tk = request.session.get("report_request",None)
+ # try to pull up the report request and show it to them
+ # if it exists, show them the reportrequest and clear
+ # else clear the report request and continue (what about caching it for a while?)
+ # ** When clearing, mark when it was viewed, and maybe the IP address
+ if tk and tk["namespace"] == namespace \
+ and tk["slug"] == slug \
+ and tk["params"] == request.GET.urlencode():
+ try:
+ rr = ReportRequest.objects.get(token=tk["token"])
+ resp = HttpResponse(rr.content)
+ if rr.mimetype:
+ resp["Content-Type"] = rr.mimetype
+ rr.viewed=datetime.datetime.now()
+ rr.save()
+ del request.session["report_request"]
+ return resp
+ except ReportRequest.DoesNotExist:
+ rr = None
+ del request.session["report_request"]
+
+ # Otherwise we have a new report request
+ # create a ReportRequest object, and store the token in their session
+ # fire off task with requested report
+ # return them a wait page with meta refresh
+ token = hashlib.md5(" ".join([str(datetime.datetime.now()),request.session.session_key,namespace,slug,request.GET.urlencode()])).hexdigest()
+ rr = ReportRequest(token=token,namespace=namespace,slug=slug,params=request.GET.urlencode())
+ rr.save()
+
+ # enqueue celery task to build relevant report
+ cx = {"reportrequest":rr}
+
+ # is this necessary? maybe for debug?
+ cx["task"] = async_report.delay(rr.token)
+
+ request.session["report_request"] = dict(token=token,namespace=namespace,slug=slug,params=request.GET.urlencode(),task=cx["task"])
+ return render_to_response("reportengine/async_wait.html",cx,context_instance=RequestContext(request))
+
+

0 comments on commit 7d2f780

Please sign in to comment.