Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Lazy queryset eval for Pickle #83

Merged
merged 3 commits into from

2 participants

@felliott

The branch updates PickleQuerySet to use Mongo semantics for sort(), offset(), and limit(). I've updated the tests in test_simple_queries to reflect this.

Cheers,
Fitz

felliott added some commits
@felliott felliott fix incorrectly-named query test
69bf703
@felliott felliott fix simple queries tests to match Mongo semantics
 * The Pickle and Mongo backends disagreed on how .sort(), .limit(), and
   .offset() work together.  The tests were originally written to match
   the Pickle semantics, but it has been decided that going forwards,
   the Mongo semantics will hold sway.  Summary:

   * sort(), offset(), and limit() are now setters.  Successive calls to
     a function will overwrite the previous value.  No evaluation will
     be performed by these functions.  The underlying data will only be
     updated when __iter__, __getitem__, or __len__ is called.  E.g.
     Foo.find().limit(10).limit(15) will return 15 items.

   * The constraints are always processed in the same order regardless
     of the order in which they are declared.  sort()s are run first,
     then offset(), then limit(). E.g.
     Foo.find().limit(10).offset(5).sort("_id") will sort first, then
     apply the offset, then limit the returned results to 10.
f94aa5f
@felliott felliott update Pickle QuerySet to use Mongo filter semantics
ec5bc4c
@jmcarp jmcarp merged commit ec5bc4c into from
@felliott felliott deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 12, 2014
  1. @felliott
  2. @felliott

    fix simple queries tests to match Mongo semantics

    felliott authored
     * The Pickle and Mongo backends disagreed on how .sort(), .limit(), and
       .offset() work together.  The tests were originally written to match
       the Pickle semantics, but it has been decided that going forwards,
       the Mongo semantics will hold sway.  Summary:
    
       * sort(), offset(), and limit() are now setters.  Successive calls to
         a function will overwrite the previous value.  No evaluation will
         be performed by these functions.  The underlying data will only be
         updated when __iter__, __getitem__, or __len__ is called.  E.g.
         Foo.find().limit(10).limit(15) will return 15 items.
    
       * The constraints are always processed in the same order regardless
         of the order in which they are declared.  sort()s are run first,
         then offset(), then limit(). E.g.
         Foo.find().limit(10).offset(5).sort("_id") will sort first, then
         apply the offset, then limit the returned results to 10.
  3. @felliott
This page is out of date. Refresh to see the latest.
View
43 modularodm/storage/picklestorage.py
@@ -42,21 +42,46 @@ def __init__(self, schema, data):
super(PickleQuerySet, self).__init__(schema)
self.data = list(data)
+ self._sort = None
+ self._offset = None
+ self._limit = None
+
+ def _eval(self):
+ if (self._sort is not None):
+ for key in self._sort[::-1]:
+ if key.startswith('-'):
+ reverse = True
+ key = key.lstrip('-')
+ else:
+ reverse = False
+
+ self.data = sorted(self.data, key=lambda record: record[key], reverse=reverse)
+
+ if (self._offset is not None):
+ self.data = self.data[self._offset:]
+
+ if (self._limit is not None):
+ self.data = self.data[:self._limit]
+
+ return self
def __getitem__(self, index, raw=False):
super(PickleQuerySet, self).__getitem__(index)
+ self._eval()
key = self.data[index][self.primary]
if raw:
return key
return self.schema.load(key)
def __iter__(self, raw=False):
+ self._eval()
keys = [obj[self.primary] for obj in self.data]
if raw:
return keys
return (self.schema.load(key) for key in keys)
def __len__(self):
+ self._eval()
return len(self.data)
count = __len__
@@ -69,27 +94,15 @@ def get_keys(self):
def sort(self, *keys):
""" Iteratively sort data by keys in reverse order. """
-
- for key in keys[::-1]:
-
- if key.startswith('-'):
- reverse = True
- key = key.lstrip('-')
- else:
- reverse = False
-
- self.data = sorted(self.data, key=lambda record: record[key], reverse=reverse)
-
+ self._sort = keys
return self
def offset(self, n):
-
- self.data = self.data[n:]
+ self._offset = n
return self
def limit(self, n):
-
- self.data = self.data[:n]
+ self._limit = n
return self
class PickleStorage(Storage):
View
36 tests/queries/test_simple_queries.py
@@ -99,14 +99,14 @@ def test_limit_offset(self):
results = self.Foo.find().limit(10).offset(10)
self.assertSetEqual(
set([x._id for x in results]),
- set()
+ set(range(10,20)),
)
def test_limit_sort(self):
results = self.Foo.find().limit(10).sort('-_id')
self.assertListEqual(
[x._id for x in results],
- range(10)[::-1],
+ range(20, 30)[::-1],
)
def test_offset_limit(self):
@@ -120,14 +120,14 @@ def test_offset_offset(self):
results = self.Foo.find().offset(10).offset(17)
self.assertListEqual(
[x._id for x in results],
- [27, 28, 29],
+ range(17,30)
)
def test_offset_sort(self):
results = self.Foo.find().offset(27).sort('-_id')
self.assertListEqual(
[x._id for x in results],
- [29, 28, 27],
+ [2, 1, 0],
)
def test_sort_limit(self):
@@ -160,32 +160,32 @@ def test_limit_offset_limit(self):
{5, 6}
)
- def test_limit_offset_limit(self):
+ def test_limit_offset_offset(self):
results = self.Foo.find().limit(10).offset(5).offset(2)
self.assertSetEqual(
set([x._id for x in results]),
- {7, 8, 9}
+ set(range(2,12)),
)
def test_limit_offset_sort(self):
results = self.Foo.find().limit(10).offset(7).sort('-_id')
self.assertListEqual(
[x._id for x in results],
- [9, 8, 7],
+ range(13,23)[::-1],
)
def test_limit_sort_limit(self):
results = self.Foo.find().limit(10).sort('-_id').limit(3)
self.assertListEqual(
[x._id for x in results],
- [9, 8, 7],
+ [29, 28, 27],
)
def test_limit_sort_offset(self):
results = self.Foo.find().limit(10).sort('-_id').offset(7)
self.assertListEqual(
[x._id for x in results],
- [2, 1, 0],
+ range(13,23)[::-1],
)
def test_limit_sort_sort(self):
@@ -199,56 +199,56 @@ def test_offset_limit_offset(self):
results = self.Foo.find().offset(10).limit(5).offset(1)
self.assertSetEqual(
set([x._id for x in results]),
- {11, 12, 13, 14}
+ set(range(1,6))
)
def test_offset_limit_sort(self):
results = self.Foo.find().offset(10).limit(3).sort('-_id')
self.assertListEqual(
[x._id for x in results],
- [12, 11, 10]
+ [19, 18, 17],
)
def test_offset_offset_limit(self):
results = self.Foo.find().offset(10).offset(10).limit(3)
self.assertListEqual(
[x._id for x in results],
- [20, 21, 22]
+ [10, 11, 12],
)
def test_offset_offset_sort(self):
results = self.Foo.find().offset(10).offset(17).sort('-_id')
self.assertListEqual(
[x._id for x in results],
- [29, 28, 27]
+ range(13)[::-1],
)
def test_offset_sort_limit(self):
results = self.Foo.find().offset(10).sort('-_id').limit(3)
self.assertListEqual(
[x._id for x in results],
- [29, 28, 27]
+ [19, 18, 17],
)
def test_offset_sort_offset(self):
results = self.Foo.find().offset(10).sort('-_id').offset(17)
self.assertListEqual(
[x._id for x in results],
- [12, 11, 10]
+ range(13)[::-1],
)
def test_sort_limit_offset(self):
results = self.Foo.find().sort('-_id').limit(10).offset(7)
self.assertListEqual(
[x._id for x in results],
- [22, 21, 20]
+ range(13,23)[::-1],
)
def test_sort_limit_sort(self):
results = self.Foo.find().sort('-_id').limit(3).sort('_id')
self.assertListEqual(
[x._id for x in results],
- [27, 28, 29],
+ [0, 1, 2],
)
def test_sort_offset_limit(self):
@@ -262,7 +262,7 @@ def test_sort_offset_sort(self):
results = self.Foo.find().sort('-_id').offset(27).sort('_id')
self.assertListEqual(
[x._id for x in results],
- [0, 1, 2]
+ [27, 28, 29]
)
def test_sort_sort_limit(self):
Something went wrong with that request. Please try again.