From 1fd8c1c47fc2514dc1843631d9fa68141787ee11 Mon Sep 17 00:00:00 2001 From: Charles Leifer Date: Mon, 16 May 2011 15:07:25 -0500 Subject: [PATCH] Another go at modeling, starting work on the javascript...man, I'm pretty clueless about javascript. --- djutils/dashboard/__init__.py | 5 + djutils/dashboard/admin.py | 9 +- djutils/dashboard/commands.py | 5 +- djutils/dashboard/models.py | 125 +++++++++++------- djutils/dashboard/provider.py | 19 +-- djutils/dashboard/registry.py | 47 +------ .../templates/dashboard/dashboard_index.html | 35 +++-- djutils/dashboard/views.py | 23 ++-- 8 files changed, 144 insertions(+), 124 deletions(-) diff --git a/djutils/dashboard/__init__.py b/djutils/dashboard/__init__.py index e69de29..cff0158 100644 --- a/djutils/dashboard/__init__.py +++ b/djutils/dashboard/__init__.py @@ -0,0 +1,5 @@ +from djutils.utils.helpers import generic_autodiscover + + +def autodiscover(): + return generic_autodiscover('panels') diff --git a/djutils/dashboard/admin.py b/djutils/dashboard/admin.py index 07c679d..4f6d920 100644 --- a/djutils/dashboard/admin.py +++ b/djutils/dashboard/admin.py @@ -1,11 +1,16 @@ from django.contrib import admin -from djutils.dashboard.models import PanelData +from djutils.dashboard.models import Panel, PanelData + + +class PanelAdmin(admin.ModelAdmin): + list_display = ('title', 'panel_type',) class PanelDataAdmin(admin.ModelAdmin): date_hierarchy = 'created_date' - list_filter = ('panel_type', 'panel_title',) + list_filter = ('panel',) +admin.site.register(Panel, PanelAdmin) admin.site.register(PanelData, PanelDataAdmin) diff --git a/djutils/dashboard/commands.py b/djutils/dashboard/commands.py index 3791f6f..1de7141 100644 --- a/djutils/dashboard/commands.py +++ b/djutils/dashboard/commands.py @@ -2,8 +2,7 @@ from django.conf import settings -from djutils.dashboard.models import PanelData -from djutils.dashboard.registry import registry +from djutils.dashboard.models import Panel, PanelData from djutils.queue.decorators import periodic_command, crontab @@ -15,7 +14,7 @@ def update_panels(): """ Simple task which updates the dashboard panels every minute """ - registry.update_panels() + Panel.objects.update_panels() @periodic_command(crontab(hour=0, minute=0)) def remove_old_panel_data(): diff --git a/djutils/dashboard/models.py b/djutils/dashboard/models.py index 7fa6f0d..f3e1c3a 100644 --- a/djutils/dashboard/models.py +++ b/djutils/dashboard/models.py @@ -1,62 +1,88 @@ +import datetime try: import cPickle as pickle except ImportError: import pickle from django.db import models +from django.db.models import Max +from django.template.defaultfilters import slugify -from djutils.dashboard.provider import PANEL_PROVIDER_TYPES +from djutils.dashboard.registry import registry -class PanelDataManager(models.Manager): - def get_distinct_titles(self): - return self.values_list('panel_title', flat=True).order_by().distinct() - - def get_latest(self, max_ids=None): - payload = {} +PANEL_PROVIDER_GRAPH = 0 +PANEL_PROVIDER_TEXT = 1 + +PANEL_PROVIDER_TYPES = ( + (PANEL_PROVIDER_GRAPH, 'Graph'), + (PANEL_PROVIDER_TEXT, 'Text'), +) + + +class PanelManager(models.Manager): + def update_panels(self): + data = [] + shared_now = datetime.datetime.now() + + # function to sort the panels by priority + key = lambda obj: obj.get_priority() - for title in self.get_distinct_titles(): - payload[title] = self.get_latest_by_title( - panel_title=title, - max_id=max_ids and max_ids.get(panel_title) or None + for provider in sorted(registry.get_provider_instances(), key=key): + # pull the data off the panel and store + panel_obj = provider.get_panel_instance() + + panel_data = PanelData.objects.create( + panel=panel_obj, + data=pickle.dumps(provider.get_data()), + created_date=shared_now, ) + data.append(panel_data) - return payload + return data - def get_latest_by_title(self, panel_title, limit=50, max_id=None): - qs = self.filter(panel_title=panel_title) - - if max_id: - qs = qs.filter(id__gt=max_id) - - qs = qs[:limit] - - agg_qs = qs.aggregate(max_id=models.Max('id')) - - return { - 'queryset': qs, - 'max_id': agg_qs['max_id'], - } + def get_panels(self): + """\ + Purpose is to get a queryset of panel models matching the registered + panel providers + """ + return self.filter(title__in=registry.get_titles()) - def get_latest_of_each(self): - # I'm not 100% certain if this is the most efficient way to do this - # query, as I believe it has On**2 complexity -- an alternative would - # simple be to iterate over panel_types and select the latest: - # return [ - # self.filter(panel_title=t)[0] \ - # for t in self.get_distinct_titles() - # ] - return self.raw(""" - SELECT pd.* - FROM ( - SELECT panel_title, max(created_date) AS max_date - FROM %(db_table)s GROUP BY panel_title - ) AS subq - INNER JOIN %(db_table)s AS pd ON ( - pd.panel_title = subq.panel_title AND - pd.created_date = max_date - ) - """ % {'db_table': self.model._meta.db_table}) + def get_latest(self): + """\ + Get the latest panel data for the registered panel providers + """ + return [ + PanelData.objects.filter(panel=panel)[0] \ + for panel in self.get_panels() + ] + + +class Panel(models.Model): + title = models.CharField(max_length=255, unique=True) + slug = models.SlugField(db_index=True) + panel_type = models.IntegerField(choices=PANEL_PROVIDER_TYPES) + + class Meta: + ordering = ('title',) + + objects = PanelManager() + + def save(self, *args, **kwargs): + self.slug = slugify(self.title) + super(Panel, self).save(*args, **kwargs) + + +class PanelDataManager(models.Manager): + def get_most_recent_update(self): + return self.aggregate(max_date=Max('created_date'))['max_date'] + + def get_data(self, panel, limit=None): + queryset = self.filter(panel=panel) + if limit: + queryset = queryset[:limit] + + return queryset, queryset.aggregate(max_id=Max('id'))['max_id'] class PanelData(models.Model): @@ -64,10 +90,9 @@ class PanelData(models.Model): Preserve historical data from dashboard. Automatically deleted by the periodic command :func:`remove_old_panel_data` """ + panel = models.ForeignKey(Panel, related_name='data') created_date = models.DateTimeField(db_index=True) - panel_title = models.CharField(max_length=255) - panel_data = models.TextField() - panel_type = models.IntegerField(choices=PANEL_PROVIDER_TYPES) + data = models.TextField() objects = PanelDataManager() @@ -75,7 +100,7 @@ class Meta: ordering = ('-created_date',) def __unicode__(self): - return '%s: %s' % (self.panel_title, self.created_date) + return '%s: %s' % (self.panel.title, self.created_date) def get_data(self): - return pickle.loads(str(self.panel_data)) + return pickle.loads(str(self.data)) diff --git a/djutils/dashboard/provider.py b/djutils/dashboard/provider.py index f3a0fb3..4c793c4 100644 --- a/djutils/dashboard/provider.py +++ b/djutils/dashboard/provider.py @@ -1,14 +1,8 @@ -PANEL_PROVIDER_GRAPH = 0 -PANEL_PROVIDER_TEXT = 1 - -PANEL_PROVIDER_TYPES = ( - (PANEL_PROVIDER_GRAPH, 'Graph'), - (PANEL_PROVIDER_TEXT, 'Text'), -) +from djutils.dashboard.models import Panel, PANEL_PROVIDER_GRAPH class PanelProvider(object): - """ + """\ Base class from which other panel providers should derive. Much like a munin plugin, there is no input provided and the output conforms to a standard format. @@ -24,7 +18,7 @@ class PanelProvider(object): """ def get_data(self): - """ + """\ This method returns data to be displayed, but depending on the panel type the output of the function may differ. @@ -52,3 +46,10 @@ def get_panel_type(self): def get_priority(self): return 20 + + def get_panel_instance(self): + panel, _ = Panel.objects.get_or_create( + title=self.get_title(), + defaults=dict(panel_type=self.get_panel_type()) + ) + return panel diff --git a/djutils/dashboard/registry.py b/djutils/dashboard/registry.py index 3ad8c96..ff4103a 100644 --- a/djutils/dashboard/registry.py +++ b/djutils/dashboard/registry.py @@ -1,25 +1,18 @@ -import datetime -try: - import cPickle as pickle -except ImportError: - import pickle - from django.conf import settings -from djutils.dashboard.models import PanelData - class PanelRegistryException(Exception): pass class PanelRegistry(object): - """ + """\ A simple Registry used to track subclasses of :class:`PanelProvider` """ _registry = {} def register(self, panel_class): + # register a panel class and cause it to be updated periodically if panel_class in self._registry: raise PanelRegistryException('"%s" is already registered' % panel_class) @@ -35,39 +28,13 @@ def unregister(self, panel_class): def __contains__(self, panel_class): return panel_class in self._registry - def get_panel_instances(self): + def get_provider_instances(self): return self._registry.values() - def _get_data_for_panels(self): - # store any return data in a list - data = [] - - # function to sort the panels by priority - key = lambda obj: obj.get_priority() - - for panel in sorted(self.get_panel_instances(), key=key): - data.append(dict( - panel_title=panel.get_title(), - panel_type=panel.get_panel_type(), - panel_data=panel.get_panel_data(), - created_date=datetime.datetime.now() - )) - - return data - - def _store_panel_data(self, data): - for data_dict in data: - PanelData.objects.create( - panel_title=data_dict['panel_title'], - panel_type=data_dict['panel_type'], - panel_data=pickle.dumps(data_dict['panel_data']), - created_date=data_dict['created_date'], - ) - - def update_panels(self): - raw_data = self._get_data_for_panels() - self._store_panel_data(raw_data) - return raw_data + def get_titles(self): + return [ + provider.get_title() for provider in self.get_provider_instances() + ] registry = PanelRegistry() diff --git a/djutils/dashboard/templates/dashboard/dashboard_index.html b/djutils/dashboard/templates/dashboard/dashboard_index.html index e3da969..0d12df3 100644 --- a/djutils/dashboard/templates/dashboard/dashboard_index.html +++ b/djutils/dashboard/templates/dashboard/dashboard_index.html @@ -3,8 +3,24 @@ {% block extended_scripts %} {% endblock %} @@ -14,13 +30,12 @@ {% block content_header %}Dashboard{% endblock %} {% block content %} - {{ json_data }} - {% for panel_title, data in data.items %} -
-

{{ panel_title }}

-
- {{ data.queryset }} +
+ {% for panel in panel_list %} +
+

{{ panel.title }}

+
-
- {% endfor %} + {% endfor %} +
{% endblock %} diff --git a/djutils/dashboard/views.py b/djutils/dashboard/views.py index 2ab0b63..cae8c88 100644 --- a/djutils/dashboard/views.py +++ b/djutils/dashboard/views.py @@ -8,29 +8,32 @@ from django.template import RequestContext from django.utils.safestring import mark_safe -from djutils.dashboard.models import PanelData +from djutils.dashboard.models import Panel, PanelData -def serialize_panel_data(latest_data): +def serialize_panel_data(panel_list, limit=1): payload = [] - for panel_title, data in latest_data.items(): + for panel in panel_list: + panel_data_qs, max_id = PanelData.objects.get_data(panel, limit) + serialized = dict( - title=panel_title, - data=[obj.get_data() for obj in data['queryset']], - max_id=data['max_id'], + id=panel.pk, + title=panel.title, + data=[{'id': obj.id, 'data': obj.get_data()} for obj in panel_data_qs], + max_id=max_id, ) payload.append(serialized) - return json.dumps(payload) + return mark_safe(json.dumps(payload)) def dashboard(request): if request.is_ajax(): pass else: - latest_data = PanelData.objects.get_latest() - json_data = serialize_panel_data(latest_data) + panels = Panel.objects.get_panels() + panel_data = serialize_panel_data(panels, 50) return render_to_response('dashboard/dashboard_index.html', { - 'data': latest_data, 'json_data': mark_safe(json_data), + 'panel_list': panels, 'panel_data': panel_data, }, context_instance=RequestContext(request))