Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #14244: Allow lists of more than 1000 items to be used with the…

… 'in' lookup in Oracle, by breaking them up into groups of 1000 items and ORing them together. Thanks to rlynch for the report and initial patch.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@13859 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 320c46999c44efdc31be9cc80d3a2d1bf5186f18 1 parent 763bcf8
Ian Kelly authored September 16, 2010
7  django/db/backends/__init__.py
@@ -233,6 +233,13 @@ def lookup_cast(self, lookup_type):
233 233
         """
234 234
         return "%s"
235 235
 
  236
+    def max_in_list_size(self):
  237
+        """
  238
+        Returns the maximum number of items that can be passed in a single 'IN'
  239
+        list condition, or None if the backend does not impose a limit.
  240
+        """
  241
+        return None
  242
+
236 243
     def max_name_length(self):
237 244
         """
238 245
         Returns the maximum length of table and column names, or None if there
3  django/db/backends/oracle/base.py
@@ -178,6 +178,9 @@ def lookup_cast(self, lookup_type):
178 178
             return "UPPER(%s)"
179 179
         return "%s"
180 180
 
  181
+    def max_in_list_size(self):
  182
+        return 1000
  183
+
181 184
     def max_name_length(self):
182 185
         return 30
183 186
 
21  django/db/models/sql/where.py
@@ -2,6 +2,7 @@
2 2
 Code to manage the creation and SQL rendering of 'where' constraints.
3 3
 """
4 4
 import datetime
  5
+from itertools import repeat
5 6
 
6 7
 from django.utils import tree
7 8
 from django.db.models.fields import Field
@@ -178,8 +179,24 @@ def make_atom(self, child, qn, connection):
178 179
                 raise EmptyResultSet
179 180
             if extra:
180 181
                 return ('%s IN %s' % (field_sql, extra), params)
181  
-            return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(params))),
182  
-                    params)
  182
+            max_in_list_size = connection.ops.max_in_list_size()
  183
+            if max_in_list_size and len(params) > max_in_list_size:
  184
+                # Break up the params list into an OR of manageable chunks.
  185
+                in_clause_elements = ['(']
  186
+                for offset in xrange(0, len(params), max_in_list_size):
  187
+                    if offset > 0:
  188
+                        in_clause_elements.append(' OR ')
  189
+                    in_clause_elements.append('%s IN (' % field_sql)
  190
+                    group_size = min(len(params) - offset, max_in_list_size)
  191
+                    param_group = ', '.join(repeat('%s', group_size))
  192
+                    in_clause_elements.append(param_group)
  193
+                    in_clause_elements.append(')')
  194
+                in_clause_elements.append(')')
  195
+                return ''.join(in_clause_elements), params
  196
+            else:
  197
+                return ('%s IN (%s)' % (field_sql,
  198
+                                        ', '.join(repeat('%s', len(params)))),
  199
+                        params)
183 200
         elif lookup_type in ('range', 'year'):
184 201
             return ('%s BETWEEN %%s and %%s' % field_sql, params)
185 202
         elif lookup_type in ('month', 'day', 'week_day'):
20  tests/regressiontests/queries/models.py
@@ -1339,3 +1339,23 @@ def __unicode__(self):
1339 1339
 []
1340 1340
 
1341 1341
 """
  1342
+
  1343
+# Sqlite 3 does not support passing in more than 1000 parameters except by
  1344
+# changing a parameter at compilation time.
  1345
+if settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE'] != "django.db.backends.sqlite3":
  1346
+    __test__["API_TESTS"] += """
  1347
+Bug #14244: Test that the "in" lookup works with lists of 1000 items or more.
  1348
+>>> Number.objects.all().delete()
  1349
+>>> numbers = range(2500)
  1350
+>>> for num in numbers:
  1351
+...     _ = Number.objects.create(num=num)
  1352
+>>> Number.objects.filter(num__in=numbers[:1000]).count()
  1353
+1000
  1354
+>>> Number.objects.filter(num__in=numbers[:1001]).count()
  1355
+1001
  1356
+>>> Number.objects.filter(num__in=numbers[:2000]).count()
  1357
+2000
  1358
+>>> Number.objects.filter(num__in=numbers).count()
  1359
+2500
  1360
+
  1361
+"""

0 notes on commit 320c469

Please sign in to comment.
Something went wrong with that request. Please try again.