Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

first commit

  • Loading branch information...
commit 03e1335799c04c2a07cf38da535b1ffcfb80a86c 0 parents
Andy McKay authored
2  license.txt
... ... @@ -0,0 +1,2 @@
  1 +# Clearwind Consulting 2010
  2 +# BSD License
88 readme.txt
... ... @@ -0,0 +1,88 @@
  1 +This is a paginator that does not do a count.
  2 +
  3 +** Warning this needs work :)
  4 +
  5 +It started with this:
  6 +
  7 +http://www.agmweb.ca/blog/andy/2226/
  8 +
  9 +Then there was excellent idea from mmalone and an excellent gist:
  10 +
  11 +http://gist.github.com/213702
  12 +
  13 +So then this got updated for the latest Django and became this.
  14 +
  15 +Example:
  16 +
  17 +>>> from django.db import connection
  18 +>>> from listener.models.error import Error
  19 +>>> queryset = Error.objects.filter(account=1, archived=1)
  20 +
  21 +The old way:
  22 +
  23 +>>> from django.core.paginator import Paginator
  24 +>>> p = Paginator(queryset, 10)
  25 +>>> p.page(1)
  26 +<Page 1 of 859>
  27 +
  28 +Causes:
  29 +
  30 +>>> connection.queries
  31 +[ {'time': '21.953', 'sql': 'SELECT COUNT(*) FROM "listener_error" WHERE ("listener_error"."account_id" = 1 AND "listener_error"."archived" = true )'}]
  32 +
  33 +That's one query. To get the object list it does another.
  34 +
  35 +>>> p.page(1).object_list
  36 +[{'time': '0.000', 'sql': 'SELECT "listener_error"."id", ... FROM "listener_error" WHERE ("listener_error"."account_id" = 1 AND "listener_error"."archived" = true ) LIMIT 10'}]
  37 +
  38 +The problem is that count can be hideously expensive.
  39 +
  40 +The new way:
  41 +
  42 +>>> from lazy_paginator.paginator import LazyPaginator
  43 +>>> p = LazyPaginator(queryset, 10)
  44 +>>> p.page(1)
  45 +<Page 1 of 1000>
  46 +
  47 +>>> connection.queries
  48 +[{'time': '0.000', 'sql': 'SELECT "listener_error"."id",... FROM "listener_error" WHERE ("listener_error"."account_id" = 1 AND "listener_error"."archived" = true ) LIMIT 11'}]
  49 +
  50 +>>> p.page(1).object_list
  51 +[{'time': '0.000', 'sql': 'SELECT "listener_error"."id",... FROM "listener_error" WHERE ("listener_error"."account_id" = 1 AND "listener_error"."archived" = true ) LIMIT 10'}]
  52 +
  53 +By doing a query for one more than you need, it figures out if there's a next. The difference is the select vs the count.
  54 +
  55 +What do you lose? You don't know how many records there are, you just know if there is a next and previous (and you can figure out how many came before). But if you are using postgresql, beware of how expensive those counts can be.
  56 +
  57 +The default assumes there's going to be 1000 pages, but we don't really know how many there. There's a max_safe_pages variable that gets updated as information is provided. For example if you set it to 3 pages... when try and access 4, it fail, thinking that there was no data.
  58 +
  59 +>>> p = LazyPaginator(queryset, 10, max_safe_pages=3)
  60 +>>> p.has_next(1)
  61 +True
  62 +>>> p.has_next(2)
  63 +True
  64 +>>> p.has_next(3)
  65 +False
  66 +>>> p.has_next(4)
  67 +False
  68 +>>> p.page(4)
  69 +Traceback (most recent call last):
  70 + File "<console>", line 1, in <module>
  71 + File "/var/arecibo/lazy_paginator/paginator.py", line 27, in page
  72 + number = self.validate_number(number)
  73 + File "/var/arecibo/lazy_paginator/paginator.py", line 20, in validate_number
  74 + return super(LazyPaginator, self).validate_number(number)
  75 + File "/usr/lib/python2.5/site-packages/django/core/paginator.py", line 32, in validate_number
  76 + raise EmptyPage('That page contains no results')
  77 +EmptyPage: That page contains no results
  78 +
  79 +Now if you start at the beginning:
  80 +
  81 +>>> p.page(2)
  82 +<Page 2 of 3>
  83 +>>> p.page(3)
  84 +<Page 3 of 4>
  85 +>>> p.page(4)
  86 +<Page 4 of 5>
  87 +
  88 +That can be a bit confusing, perhaps in the future it should check and if not then try to get it... improvements welcome.
13 setup.py
... ... @@ -0,0 +1,13 @@
  1 +from distutils.core import setup
  2 +setup(name='lazy_paginator',
  3 + version='0.1',
  4 + description='Lazy Paginator for Django',
  5 + author="Andy McKay",
  6 + author_email="andy@clearwind.ca",
  7 + packages=["lazy_paginator",],
  8 + package_dir = {'lazy_paginator':'src'},
  9 + classifiers = [
  10 + "Development Status :: 4 - Beta"
  11 + "Framework :: Django"
  12 + ]
  13 + )
3  src/__init__.py
... ... @@ -0,0 +1,3 @@
  1 +# Clearwind Consulting Ltd, 2010
  2 +# BSD License
  3 +# based on work by mmalone
60 src/paginator.py
... ... @@ -0,0 +1,60 @@
  1 +# Clearwind Consulting Ltd, 2010
  2 +# BSD License
  3 +# based on work by mmalone
  4 +import sys
  5 +from django.core.paginator import Paginator, Page, InvalidPage
  6 +
  7 +class LazyPaginator(Paginator):
  8 + max_safe_pages = 0
  9 +
  10 + def __init__(self, object_list, per_page, orphans=0,
  11 + allow_empty_first_page=True, max_safe_pages=1000):
  12 + self.max_safe_pages = max_safe_pages
  13 + super(LazyPaginator, self).__init__(object_list, per_page,
  14 + orphans=orphans, allow_empty_first_page=allow_empty_first_page)
  15 +
  16 + def validate_number(self, number):
  17 + try:
  18 + number = int(number)
  19 + except:
  20 + raise InvalidPage
  21 + if number <= self.max_safe_pages:
  22 + return number
  23 + return super(LazyPaginator, self).validate_number(number)
  24 +
  25 + def _get_num_pages(self):
  26 + return self.max_safe_pages
  27 + num_pages = property(_get_num_pages)
  28 +
  29 + def page(self, number):
  30 + number = self.validate_number(number)
  31 + bottom = (number - 1) * self.per_page
  32 + top = bottom + self.per_page
  33 +
  34 + # get one extra object to see if there is a next page
  35 + page = list(self.object_list[bottom:top + 1])
  36 + if len(page) > self.per_page:
  37 + # if we got an extra object, update max_safe_pages
  38 + if number + 1 > self.max_safe_pages:
  39 + self.max_safe_pages = number + 1
  40 + page = page[:self.per_page]
  41 + if number > 0 and len(page) == 0:
  42 + raise InvalidPage
  43 +
  44 + return Page(self.object_list[bottom:top], number, self)
  45 +
  46 + def has_next(self, number):
  47 + if number < self.max_safe_pages:
  48 + return True
  49 + return super(LazyPaginator, self).has_next(number)
  50 +
  51 + def last_on_page(self, number):
  52 + """
  53 + Returns the 1-based index of the last object on the given page,
  54 + relative to total objects found (hits).
  55 + """
  56 + number = self.validate_number(number)
  57 + if number >= self.max_safe_pages:
  58 + return super(LazyPaginator, self).last_on_page(number)
  59 + number += 1 # 1-base
  60 + return number * self.per_page

0 comments on commit 03e1335

Please sign in to comment.
Something went wrong with that request. Please try again.