use django 1.3 aggregation for report counts

mainapp/models.py

@@ -17,6 +17,7 @@
 from contrib.transmeta import TransMeta
 from contrib.stdimage import StdImageField
 from django.utils.encoding import iri_to_uri
+from django.template.defaultfilters import slugify
 from django.contrib.gis.geos import fromstr
 from django.http import Http404
 from django.contrib.auth.models import User,Group,Permission
@@ -108,7 +109,7 @@ def get_categories(self):
         return( categories )
 
     def get_absolute_url(self):
-        return "/cities/" + str(self.id)
+        return "/cities/%d" %( self.id ) 
 
     def get_rule_descriptions(self):
         rules = EmailRule.objects.filter(city=self)
@@ -150,7 +151,7 @@ class Ward(models.Model):
     email = models.EmailField(blank=True, null=True)
 
     def get_absolute_url(self):
-        return "/wards/" + str(self.id)
+        return "/wards/%d" % ( self.id ) 
 
     def __unicode__(self):      
         return self.name + ", " + self.city.name  
@@ -350,18 +351,8 @@ def get_absolute_url(self):
 
     class Meta:
         db_table = u'reports'
+
 
-class ReportCount(object):        
-    def __init__(self, interval):
-        self.interval = interval
-
-    def dict(self):
-        return({ "recent_new": "count( case when age(clock_timestamp(), reports.created_at) < interval '%s' THEN 1 ELSE null end )" % self.interval,
-          "recent_fixed": "count( case when age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end )" % self.interval,
-          "recent_updated": "count( case when age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at THEN 1 ELSE null end )" % self.interval,
-          "old_fixed": "count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end )" % self.interval,
-          "old_unfixed": "count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = False THEN 1 ELSE null end )" % self.interval } )  
-
 class ReportUpdate(models.Model):   
     report = models.ForeignKey(Report)
     desc = models.TextField(blank=True, null=True, verbose_name = ugettext_lazy("Details"))
@@ -590,107 +581,55 @@ def __init__(self,city):
         GoogleMap.__init__(self,center=ward.geom.centroid,zoom=13,key=settings.GMAP_KEY, polygons=polygons,dom_id='map_canvas')
 
 
-
-
-class SqlQuery(object):
-    """
-        This is a workaround: django doesn't support our optimized 
-        direct SQL queries very well.
-    """
-
-    def __init__(self):
-        self.cursor = None
-        self.index = 0
-        self.results = None    
-
-    def next(self):
-        self.index = self.index + 1
-
-    def get_results(self):
-        if not self.cursor:
-            self.cursor = connection.cursor()
-            self.cursor.execute(self.sql)
-            self.results = self.cursor.fetchall()
-        return( self.results )
-
-class ReportCountQuery(SqlQuery):
-
-    def name(self):
-        return self.get_results()[self.index][5]
-
-    def recent_new(self):
-        return self.get_results()[self.index][0]
-
-    def recent_fixed(self):
-        return self.get_results()[self.index][1]
 
-    def recent_updated(self):
-        return self.get_results()[self.index][2]
+class CountIf(models.sql.aggregates.Aggregate):
+    # take advantage of django 1.3 aggregate functionality
+    # from discussion here: http://groups.google.com/group/django-users/browse_thread/thread/bd5a6b329b009cfa
+    sql_function = 'COUNT'
+    sql_template= '%(function)s(CASE %(condition)s WHEN true THEN 1 ELSE NULL END)'
 
-    def old_fixed(self):
-        return self.get_results()[self.index][3]
-
-    def old_unfixed(self):
-        return self.get_results()[self.index][4]
+    def __init__(self, lookup, **extra):
+        self.lookup = lookup
+        self.extra = extra
+
+    def _default_alias(self):
+        return '%s__%s' % (self.lookup, self.__class__.__name__.lower())
+    default_alias = property(_default_alias)
+
+    def add_to_query(self, query, alias, col, source, is_summary):
+        super(CountIf, self).__init__(col, source, is_summary, **self.extra)
+        query.aggregate_select[alias] = self
 
-    def __init__(self, interval = '1 month'):
-        SqlQuery.__init__(self)
-        self.base_query = """select count( case when age(clock_timestamp(), reports.created_at) < interval '%s' and reports.is_confirmed THEN 1 ELSE null end ) as recent_new,\
- count( case when age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end ) as recent_fixed,\
- count( case when age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at THEN 1 ELSE null end ) as recent_updated,\
- count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end ) as old_fixed,\
- count( case when age(clock_timestamp(), reports.created_at) > interval '%s' AND reports.is_confirmed AND reports.is_fixed = False THEN 1 ELSE null end ) as old_unfixed   
- """ % (interval,interval,interval,interval,interval) 
-        self.sql = self.base_query + " from reports where reports.is_confirmed = true" 
-
-class CityTotals(ReportCountQuery):
-
-    def __init__(self, interval, city):
-        ReportCountQuery.__init__(self,interval)
-        self.sql = self.base_query 
-        self.sql += """ from reports left join wards on reports.ward_id = wards.id left join cities on cities.id = wards.city_id 
-        """ 
-        self.sql += ' where reports.is_confirmed = True and wards.city_id = %d ' % city.id
 
-class CityWardsTotals(ReportCountQuery):
-
-    def __init__(self, city):
-        ReportCountQuery.__init__(self,"1 month")
-        self.sql = self.base_query 
-        self.url_prefix = "/wards/"            
-        self.sql +=  ", wards.name, wards.id, wards.number from wards "
-        self.sql += """left join reports on wards.id = reports.ward_id join cities on wards.city_id = cities.id join province on cities.province_id = province.id
-        """
-        self.sql += "and cities.id = " + str(city.id)
-        self.sql += " group by  wards.name, wards.id, wards.number order by wards.number" 
-
-    def number(self):
-         return(self.get_results()[self.index][7])
+class ReportCounter(CountIf):
+    """ initialize an aggregate with one of 5 typical report queries """
+
+    CONDITIONS = {
+        'recent_new' : "age(clock_timestamp(), reports.created_at) < interval '%s' and reports.is_confirmed = True",
+        'recent_fixed' : "age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True",
+        'recent_updated': "age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at",
+        'old_fixed' : "age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True",
+        'old_unfixed' : "age(clock_timestamp(), reports.created_at) > interval '%s' AND reports.is_confirmed = True AND reports.is_fixed = False"  
+                }
+
+    def __init__(self, col, key, interval ):
+        super(ReportCounter,self).__init__( col, condition=self.CONDITIONS[ key ] % ( interval ) )
+
 
-    def get_absolute_url(self):
-        return( self.url_prefix + str(self.get_results()[self.index][6]))
-
-class AllCityTotals(ReportCountQuery):
-
-    def __init__(self):
-        ReportCountQuery.__init__(self,"1 month")
-        self.sql = self.base_query         
-        self.url_prefix = "/cities/"            
-        self.sql +=  ", cities.name, cities.id, province.name from cities "
-        self.sql += """left join wards on wards.city_id = cities.id join province on cities.province_id = province.id left join reports on wards.id = reports.ward_id 
-        """ 
-        self.sql += "group by cities.name, cities.id, province.name order by province.name, cities.name"
-
-    def get_absolute_url(self):
-        return( self.url_prefix + str(self.get_results()[self.index][6]))
-
-    def province(self):
-        return(self.get_results()[self.index][7])
-
-    def province_changed(self):
-        if (self.index ==0 ):
-            return( True )
-        return( self.get_results()[self.index][7] != self.get_results()[self.index-1][7] )
+class ReportCounters(dict):
+    """ create a dict of typical report count aggregators. """    
+    def __init__(self,report_col, interval = '1 Month'):
+        super(ReportCounters,self).__init__()
+        for key in ReportCounter.CONDITIONS.keys():
+            self[key] = ReportCounter(report_col,key,interval)
+
+class OverallReportCount(dict):
+    """ this query needs some intervention """
+    def __init__(self, interval ):
+        super(OverallReportCount,self).__init__()
+        q = Report.objects.annotate(**ReportCounters('id', interval ) ).values('recent_new','recent_fixed','recent_updated')
+        q.query.group_by = []
+        self.update( q[0] )
 
 class FaqEntry(models.Model):
     __metaclass__ = TransMeta

mainapp/views/cities.py

@@ -1,30 +1,32 @@
 from django.shortcuts import render_to_response, get_object_or_404
-from mainapp.models import City, Report, CityTotals, CityWardsTotals, AllCityTotals, CityMap
+from mainapp.models import City, Ward, Report, ReportCounters, CityMap
 from django.template import Context, RequestContext
 from django.db.models import  Count
 
 def index(request):    
     return render_to_response("cities/index.html",
-                {"report_counts": AllCityTotals() },
+                {"report_counts": City.objects.annotate(**ReportCounters('ward__report')).order_by('province__abbrev') },
                 context_instance=RequestContext(request))
 
 
 def show( request, city_id ):
     city = get_object_or_404(City, id=city_id)
 
     #top problems
-    top_problems = Report.objects.filter(ward__city=city,is_fixed=False).annotate(subscriber_count=Count('reportsubscriber' ) ).filter(subscriber_count__gte=1).order_by('-subscriber_count')[:10]
+    top_problems = Report.objects.filter(ward__city=city,is_fixed=False).annotate(subscriber_count=Count('reportsubscriber' ) ).filter(subscriber_count__gte=1).order_by('-subscriber_count')[:5]
     if request.GET.has_key('test'):
         google = CityMap(city)
     else:
         google = None
 
+
     return render_to_response("cities/show.html",
                 {"city":city,
                  "google": google,
                  'top_problems': top_problems,
-                 'city_totals' : CityTotals( '10 years',city),
-                 "report_counts": CityWardsTotals(city) },
+                 'city_totals' : City.objects.filter(id=city.id).annotate(**ReportCounters('ward__report','10 years'))[0],
+                 "report_counts": Ward.objects.filter(city=city).annotate(**ReportCounters('report'))
+                  },
                  context_instance=RequestContext(request))
 
 def home( request, city, error_msg, disambiguate ):
@@ -34,7 +36,7 @@ def home( request, city, error_msg, disambiguate ):
     recent_reports = Report.objects.filter(is_confirmed=True, ward__city=city).order_by("-created_at")[:5]
 
     return render_to_response("cities/home.html",
-                {"report_counts": CityTotals('1 year', city),
+                {"report_counts": City.objects.filter(id=city.id).annotate(ReportTotalCounters('ward__report','10 years'))[0],
                  "cities": City.objects.all(),
                  'city':city,
                  'top_problems': top_problems,

mainapp/views/main.py

@@ -1,6 +1,6 @@
 from django.shortcuts import render_to_response, get_object_or_404
 from django.http import HttpResponseRedirect, Http404
-from mainapp.models import DictToPoint, Report, ReportUpdate, Ward, FixMyStreetMap, ReportCountQuery, City, FaqEntry
+from mainapp.models import DictToPoint, Report, ReportUpdate, Ward, FixMyStreetMap, OverallReportCount, City, FaqEntry
 from mainapp import search
 from django.template import Context, RequestContext
 from django.contrib.gis.measure import D 
@@ -27,7 +27,7 @@ def home(request, location = None, error_msg =None):
         location = request.GET["q"]
 
     return render_to_response("home.html",
-                {"report_counts": ReportCountQuery('1 year'),
+                {"report_counts": OverallReportCount('1 year'),
                  "cities": City.objects.all(),
                  'search_error': error_msg,
                  'location':location,

templates/cities/_report_count_table.html

@@ -9,19 +9,20 @@
 		<th class='table-stat'>{% trans "Recently Fixed" %}*</th>
 		<th class='table-stat'>{% trans "Old Fixed" %}*</th>
 	</tr>
-{% for entry in report_counts.get_results %}
-	{% if report_counts.province_changed %}
-		<tr ><td colspan=7 class='section' >{{report_counts.province}}</td></tr>
+{% for entry in report_counts %}
+	{% if entry.province %}
+		{% ifchanged entry.province %}
+			<tr ><td colspan=7 class='section' >{{entry.province.name}}</td></tr>
+		{% endifchanged %}
 	{% endif %}
 	<tr class="{% cycle 'row-odd' 'row-even' %}">
-	<td><a href='/feeds{{report_counts.get_absolute_url}}'><img src="/media/images/rss25x26.png" /></a></td>
-	<td>{{report_counts.number}}</td>
-	<td><a href='{{report_counts.get_absolute_url}}'>{{report_counts.name}}</a></td>
-	<td>{{report_counts.recent_new}}</td>
-	<td>{{report_counts.old_unfixed}}</td>
-	<td>{{report_counts.recent_fixed}}</td>
-	<td>{{report_counts.old_fixed}}</td>
-	{% if report_counts.next %}{% endif %}
+	<td><a href='/feeds{{entry.get_absolute_url}}'><img src="/media/images/rss25x26.png" /></a></td>
+	<td>{{entry.number}}</td>
+	<td><a href='{{entry.get_absolute_url}}'>{{entry.name}}</a></td>
+	<td>{{entry.recent_new}}</td>
+	<td>{{entry.old_unfixed}}</td>
+	<td>{{entry.recent_fixed}}</td>
+	<td>{{entry.old_fixed}}</td>
 	</tr>
 {% endfor %}
 </table>