Permalink
Browse files

added initial version

  • Loading branch information...
0 parents commit b4d6f875e345623bb45a2614b67342a84926bb03 @WoLpH committed Oct 16, 2011
Showing with 53 additions and 0 deletions.
  1. 0 django_utils/__init__.py
  2. +28 −0 django_utils/queryset.py
  3. +25 −0 setup.py
No changes.
@@ -0,0 +1,28 @@
+import gc
+
+def queryset_iterator(queryset, chunksize=1000, getfunc=getattr):
+ '''''
+ Iterate over a Django Queryset ordered by the primary key
+
+ This method loads a maximum of chunksize (default: 1000) rows in it's
+ memory at the same time while django normally would load all rows in it's
+ memory. Using the iterator() method only causes it to not preload all the
+ classes.
+
+ Note that the implementation of the iterator does not support ordered query sets.
+ '''
+ pk = 0
+
+ try:
+ '''In the case of an empty list, return'''
+ last_pk = getfunc(queryset.order_by('-pk')[0], 'pk')
+ except IndexError:
+ return
+
+ queryset = queryset.order_by('pk')
+ while pk < last_pk:
+ for row in queryset.filter(pk__gt=pk)[:chunksize]:
+ pk = getfunc(row, 'pk')
+ yield row
+ gc.collect()
+
@@ -0,0 +1,25 @@
+import os
+from setuptools import setup
+
+if os.path.isfile('README.rst'):
+ long_description = open('README.rst').read()
+else:
+ long_description = 'See http://pypi.python.org/pypi/django-utils/'
+
+setup(
+ name = 'django-utils',
+ version = '1.0',
+ author = 'Rick van Hattem',
+ author_email = 'Rick.van.Hattem@Fawo.nl',
+ description = '''Django Utils is a module with some convenient utilities
+ not included with the standard Django install''',
+ url='https://github.com/WoLpH/django-utils',
+ license = 'BSD',
+ packages=['django_utils'],
+ long_description=long_description,
+# test_suite='nose.collector',
+# setup_requires=['nose'],
+ classifiers=[
+ 'License :: OSI Approved :: BSD License',
+ ],
+)

0 comments on commit b4d6f87

Please sign in to comment.