Permalink
Browse files

use django 1.3 aggregation for report counts

  • Loading branch information...
1 parent a244614 commit bd1971f101f834aae36f9dd9eec10a8e6d3c1f78 @visiblegovernment committed Jul 20, 2011
Showing with 70 additions and 128 deletions.
  1. +48 −109 mainapp/models.py
  2. +8 −6 mainapp/views/cities.py
  3. +2 −2 mainapp/views/main.py
  4. +12 −11 templates/cities/_report_count_table.html
View
@@ -17,6 +17,7 @@
from contrib.transmeta import TransMeta
from contrib.stdimage import StdImageField
from django.utils.encoding import iri_to_uri
+from django.template.defaultfilters import slugify
from django.contrib.gis.geos import fromstr
from django.http import Http404
from django.contrib.auth.models import User,Group,Permission
@@ -108,7 +109,7 @@ def get_categories(self):
return( categories )
def get_absolute_url(self):
- return "/cities/" + str(self.id)
+ return "/cities/%d" %( self.id )
def get_rule_descriptions(self):
rules = EmailRule.objects.filter(city=self)
@@ -150,7 +151,7 @@ class Ward(models.Model):
email = models.EmailField(blank=True, null=True)
def get_absolute_url(self):
- return "/wards/" + str(self.id)
+ return "/wards/%d" % ( self.id )
def __unicode__(self):
return self.name + ", " + self.city.name
@@ -350,18 +351,8 @@ def get_absolute_url(self):
class Meta:
db_table = u'reports'
+
-class ReportCount(object):
- def __init__(self, interval):
- self.interval = interval
-
- def dict(self):
- return({ "recent_new": "count( case when age(clock_timestamp(), reports.created_at) < interval '%s' THEN 1 ELSE null end )" % self.interval,
- "recent_fixed": "count( case when age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end )" % self.interval,
- "recent_updated": "count( case when age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at THEN 1 ELSE null end )" % self.interval,
- "old_fixed": "count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end )" % self.interval,
- "old_unfixed": "count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = False THEN 1 ELSE null end )" % self.interval } )
-
class ReportUpdate(models.Model):
report = models.ForeignKey(Report)
desc = models.TextField(blank=True, null=True, verbose_name = ugettext_lazy("Details"))
@@ -590,107 +581,55 @@ def __init__(self,city):
GoogleMap.__init__(self,center=ward.geom.centroid,zoom=13,key=settings.GMAP_KEY, polygons=polygons,dom_id='map_canvas')
-
-
-class SqlQuery(object):
- """
- This is a workaround: django doesn't support our optimized
- direct SQL queries very well.
- """
-
- def __init__(self):
- self.cursor = None
- self.index = 0
- self.results = None
-
- def next(self):
- self.index = self.index + 1
-
- def get_results(self):
- if not self.cursor:
- self.cursor = connection.cursor()
- self.cursor.execute(self.sql)
- self.results = self.cursor.fetchall()
- return( self.results )
-
-class ReportCountQuery(SqlQuery):
-
- def name(self):
- return self.get_results()[self.index][5]
-
- def recent_new(self):
- return self.get_results()[self.index][0]
-
- def recent_fixed(self):
- return self.get_results()[self.index][1]
- def recent_updated(self):
- return self.get_results()[self.index][2]
+class CountIf(models.sql.aggregates.Aggregate):
+ # take advantage of django 1.3 aggregate functionality
+ # from discussion here: http://groups.google.com/group/django-users/browse_thread/thread/bd5a6b329b009cfa
+ sql_function = 'COUNT'
+ sql_template= '%(function)s(CASE %(condition)s WHEN true THEN 1 ELSE NULL END)'
- def old_fixed(self):
- return self.get_results()[self.index][3]
-
- def old_unfixed(self):
- return self.get_results()[self.index][4]
+ def __init__(self, lookup, **extra):
+ self.lookup = lookup
+ self.extra = extra
+
+ def _default_alias(self):
+ return '%s__%s' % (self.lookup, self.__class__.__name__.lower())
+ default_alias = property(_default_alias)
+
+ def add_to_query(self, query, alias, col, source, is_summary):
+ super(CountIf, self).__init__(col, source, is_summary, **self.extra)
+ query.aggregate_select[alias] = self
- def __init__(self, interval = '1 month'):
- SqlQuery.__init__(self)
- self.base_query = """select count( case when age(clock_timestamp(), reports.created_at) < interval '%s' and reports.is_confirmed THEN 1 ELSE null end ) as recent_new,\
- count( case when age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end ) as recent_fixed,\
- count( case when age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at THEN 1 ELSE null end ) as recent_updated,\
- count( case when age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True THEN 1 ELSE null end ) as old_fixed,\
- count( case when age(clock_timestamp(), reports.created_at) > interval '%s' AND reports.is_confirmed AND reports.is_fixed = False THEN 1 ELSE null end ) as old_unfixed
- """ % (interval,interval,interval,interval,interval)
- self.sql = self.base_query + " from reports where reports.is_confirmed = true"
-
-class CityTotals(ReportCountQuery):
-
- def __init__(self, interval, city):
- ReportCountQuery.__init__(self,interval)
- self.sql = self.base_query
- self.sql += """ from reports left join wards on reports.ward_id = wards.id left join cities on cities.id = wards.city_id
- """
- self.sql += ' where reports.is_confirmed = True and wards.city_id = %d ' % city.id
-class CityWardsTotals(ReportCountQuery):
-
- def __init__(self, city):
- ReportCountQuery.__init__(self,"1 month")
- self.sql = self.base_query
- self.url_prefix = "/wards/"
- self.sql += ", wards.name, wards.id, wards.number from wards "
- self.sql += """left join reports on wards.id = reports.ward_id join cities on wards.city_id = cities.id join province on cities.province_id = province.id
- """
- self.sql += "and cities.id = " + str(city.id)
- self.sql += " group by wards.name, wards.id, wards.number order by wards.number"
-
- def number(self):
- return(self.get_results()[self.index][7])
+class ReportCounter(CountIf):
+ """ initialize an aggregate with one of 5 typical report queries """
+
+ CONDITIONS = {
+ 'recent_new' : "age(clock_timestamp(), reports.created_at) < interval '%s' and reports.is_confirmed = True",
+ 'recent_fixed' : "age(clock_timestamp(), reports.fixed_at) < interval '%s' AND reports.is_fixed = True",
+ 'recent_updated': "age(clock_timestamp(), reports.updated_at) < interval '%s' AND reports.is_fixed = False and reports.updated_at != reports.created_at",
+ 'old_fixed' : "age(clock_timestamp(), reports.fixed_at) > interval '%s' AND reports.is_fixed = True",
+ 'old_unfixed' : "age(clock_timestamp(), reports.created_at) > interval '%s' AND reports.is_confirmed = True AND reports.is_fixed = False"
+ }
+
+ def __init__(self, col, key, interval ):
+ super(ReportCounter,self).__init__( col, condition=self.CONDITIONS[ key ] % ( interval ) )
+
- def get_absolute_url(self):
- return( self.url_prefix + str(self.get_results()[self.index][6]))
-
-class AllCityTotals(ReportCountQuery):
-
- def __init__(self):
- ReportCountQuery.__init__(self,"1 month")
- self.sql = self.base_query
- self.url_prefix = "/cities/"
- self.sql += ", cities.name, cities.id, province.name from cities "
- self.sql += """left join wards on wards.city_id = cities.id join province on cities.province_id = province.id left join reports on wards.id = reports.ward_id
- """
- self.sql += "group by cities.name, cities.id, province.name order by province.name, cities.name"
-
- def get_absolute_url(self):
- return( self.url_prefix + str(self.get_results()[self.index][6]))
-
- def province(self):
- return(self.get_results()[self.index][7])
-
- def province_changed(self):
- if (self.index ==0 ):
- return( True )
- return( self.get_results()[self.index][7] != self.get_results()[self.index-1][7] )
+class ReportCounters(dict):
+ """ create a dict of typical report count aggregators. """
+ def __init__(self,report_col, interval = '1 Month'):
+ super(ReportCounters,self).__init__()
+ for key in ReportCounter.CONDITIONS.keys():
+ self[key] = ReportCounter(report_col,key,interval)
+
+class OverallReportCount(dict):
+ """ this query needs some intervention """
+ def __init__(self, interval ):
+ super(OverallReportCount,self).__init__()
+ q = Report.objects.annotate(**ReportCounters('id', interval ) ).values('recent_new','recent_fixed','recent_updated')
+ q.query.group_by = []
+ self.update( q[0] )
class FaqEntry(models.Model):
__metaclass__ = TransMeta
View
@@ -1,30 +1,32 @@
from django.shortcuts import render_to_response, get_object_or_404
-from mainapp.models import City, Report, CityTotals, CityWardsTotals, AllCityTotals, CityMap
+from mainapp.models import City, Ward, Report, ReportCounters, CityMap
from django.template import Context, RequestContext
from django.db.models import Count
def index(request):
return render_to_response("cities/index.html",
- {"report_counts": AllCityTotals() },
+ {"report_counts": City.objects.annotate(**ReportCounters('ward__report')).order_by('province__abbrev') },
context_instance=RequestContext(request))
def show( request, city_id ):
city = get_object_or_404(City, id=city_id)
#top problems
- top_problems = Report.objects.filter(ward__city=city,is_fixed=False).annotate(subscriber_count=Count('reportsubscriber' ) ).filter(subscriber_count__gte=1).order_by('-subscriber_count')[:10]
+ top_problems = Report.objects.filter(ward__city=city,is_fixed=False).annotate(subscriber_count=Count('reportsubscriber' ) ).filter(subscriber_count__gte=1).order_by('-subscriber_count')[:5]
if request.GET.has_key('test'):
google = CityMap(city)
else:
google = None
+
return render_to_response("cities/show.html",
{"city":city,
"google": google,
'top_problems': top_problems,
- 'city_totals' : CityTotals( '10 years',city),
- "report_counts": CityWardsTotals(city) },
+ 'city_totals' : City.objects.filter(id=city.id).annotate(**ReportCounters('ward__report','10 years'))[0],
+ "report_counts": Ward.objects.filter(city=city).annotate(**ReportCounters('report'))
+ },
context_instance=RequestContext(request))
def home( request, city, error_msg, disambiguate ):
@@ -34,7 +36,7 @@ def home( request, city, error_msg, disambiguate ):
recent_reports = Report.objects.filter(is_confirmed=True, ward__city=city).order_by("-created_at")[:5]
return render_to_response("cities/home.html",
- {"report_counts": CityTotals('1 year', city),
+ {"report_counts": City.objects.filter(id=city.id).annotate(ReportTotalCounters('ward__report','10 years'))[0],
"cities": City.objects.all(),
'city':city,
'top_problems': top_problems,
View
@@ -1,6 +1,6 @@
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponseRedirect, Http404
-from mainapp.models import DictToPoint, Report, ReportUpdate, Ward, FixMyStreetMap, ReportCountQuery, City, FaqEntry
+from mainapp.models import DictToPoint, Report, ReportUpdate, Ward, FixMyStreetMap, OverallReportCount, City, FaqEntry
from mainapp import search
from django.template import Context, RequestContext
from django.contrib.gis.measure import D
@@ -27,7 +27,7 @@ def home(request, location = None, error_msg =None):
location = request.GET["q"]
return render_to_response("home.html",
- {"report_counts": ReportCountQuery('1 year'),
+ {"report_counts": OverallReportCount('1 year'),
"cities": City.objects.all(),
'search_error': error_msg,
'location':location,
@@ -9,19 +9,20 @@
<th class='table-stat'>{% trans "Recently Fixed" %}*</th>
<th class='table-stat'>{% trans "Old Fixed" %}*</th>
</tr>
-{% for entry in report_counts.get_results %}
- {% if report_counts.province_changed %}
- <tr ><td colspan=7 class='section' >{{report_counts.province}}</td></tr>
+{% for entry in report_counts %}
+ {% if entry.province %}
+ {% ifchanged entry.province %}
+ <tr ><td colspan=7 class='section' >{{entry.province.name}}</td></tr>
+ {% endifchanged %}
{% endif %}
<tr class="{% cycle 'row-odd' 'row-even' %}">
- <td><a href='/feeds{{report_counts.get_absolute_url}}'><img src="/media/images/rss25x26.png" /></a></td>
- <td>{{report_counts.number}}</td>
- <td><a href='{{report_counts.get_absolute_url}}'>{{report_counts.name}}</a></td>
- <td>{{report_counts.recent_new}}</td>
- <td>{{report_counts.old_unfixed}}</td>
- <td>{{report_counts.recent_fixed}}</td>
- <td>{{report_counts.old_fixed}}</td>
- {% if report_counts.next %}{% endif %}
+ <td><a href='/feeds{{entry.get_absolute_url}}'><img src="/media/images/rss25x26.png" /></a></td>
+ <td>{{entry.number}}</td>
+ <td><a href='{{entry.get_absolute_url}}'>{{entry.name}}</a></td>
+ <td>{{entry.recent_new}}</td>
+ <td>{{entry.old_unfixed}}</td>
+ <td>{{entry.recent_fixed}}</td>
+ <td>{{entry.old_fixed}}</td>
</tr>
{% endfor %}
</table>

0 comments on commit bd1971f

Please sign in to comment.