Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge branch 'mapreduce' of git://github.com/blackbrrr/mongoengine

Conflicts:
	mongoengine/queryset.py
  • Loading branch information...
commit 047cc218a64a747e948c537bd2fc404d45d214ac 2 parents 39fc862 + f47d926
Harry Marr hmarr authored
3  docs/apireference.rst
Source Rendered
@@ -20,6 +20,9 @@ Documents
20 20
21 21 .. autoclass:: mongoengine.EmbeddedDocument
22 22 :members:
  23 +
  24 +.. autoclass:: mongoengine.MapReduceDocument
  25 + :members:
23 26
24 27 Querying
25 28 ========
2  docs/conf.py
@@ -22,7 +22,7 @@
22 22
23 23 # Add any Sphinx extension module names here, as strings. They can be extensions
24 24 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
25   -extensions = ['sphinx.ext.autodoc']
  25 +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
26 26
27 27 # Add any paths that contain templates here, relative to this directory.
28 28 templates_path = ['_templates']
36 mongoengine/document.py
@@ -115,3 +115,39 @@ def drop_collection(cls):
115 115 """
116 116 db = _get_db()
117 117 db.drop_collection(cls._meta['collection'])
  118 +
  119 +
  120 +class MapReduceDocument(object):
  121 + """A document returned from a map/reduce query.
  122 +
  123 + :param collection: An instance of :class:`~pymongo.Collection`
  124 + :param key: Document/result key, often an instance of
  125 + :class:`~pymongo.objectid.ObjectId`. If supplied as
  126 + an ``ObjectId`` found in the given ``collection``,
  127 + the object can be accessed via the ``object`` property.
  128 + :param value: The result(s) for this key.
  129 +
  130 + .. versionadded:: 0.3
  131 +
  132 + """
  133 +
  134 + def __init__(self, document, collection, key, value):
  135 + self._document = document
  136 + self._collection = collection
  137 + self.key = key
  138 + self.value = value
  139 +
  140 + @property
  141 + def object(self):
  142 + """Lazy-load the object referenced by ``self.key``. If ``self.key``
  143 + is not an ``ObjectId``, simply return ``self.key``.
  144 + """
  145 + if not isinstance(self.key, (pymongo.objectid.ObjectId)):
  146 + try:
  147 + self.key = pymongo.objectid.ObjectId(self.key)
  148 + except:
  149 + return self.key
  150 + if not hasattr(self, "_key_object"):
  151 + self._key_object = self._document.objects.with_id(self.key)
  152 + return self._key_object
  153 + return self._key_object
131 mongoengine/queryset.py
@@ -5,7 +5,7 @@
5 5 import copy
6 6
7 7
8   -__all__ = ['queryset_manager', 'Q', 'InvalidQueryError',
  8 +__all__ = ['queryset_manager', 'Q', 'InvalidQueryError',
9 9 'InvalidCollectionError']
10 10
11 11 # The maximum number of items to display in a QuerySet.__repr__
@@ -13,7 +13,7 @@
13 13
14 14
15 15 class DoesNotExist(Exception):
16   - pass
  16 + pass
17 17
18 18
19 19 class MultipleObjectsReturned(Exception):
@@ -30,8 +30,9 @@ class OperationError(Exception):
30 30
31 31 RE_TYPE = type(re.compile(''))
32 32
  33 +
33 34 class Q(object):
34   -
  35 +
35 36 OR = '||'
36 37 AND = '&&'
37 38 OPERATORS = {
@@ -52,7 +53,7 @@ class Q(object):
52 53 'regex_eq': '%(value)s.test(this.%(field)s)',
53 54 'regex_ne': '!%(value)s.test(this.%(field)s)',
54 55 }
55   -
  56 +
56 57 def __init__(self, **query):
57 58 self.query = [query]
58 59
@@ -132,10 +133,10 @@ def _build_op_js(self, op, key, value, value_name):
132 133 return value, operation_js
133 134
134 135 class QuerySet(object):
135   - """A set of results returned from a query. Wraps a MongoDB cursor,
  136 + """A set of results returned from a query. Wraps a MongoDB cursor,
136 137 providing :class:`~mongoengine.Document` objects as the results.
137 138 """
138   -
  139 +
139 140 def __init__(self, document, collection):
140 141 self._document = document
141 142 self._collection_obj = collection
@@ -143,7 +144,8 @@ def __init__(self, document, collection):
143 144 self._query = {}
144 145 self._where_clause = None
145 146 self._loaded_fields = []
146   -
  147 + self._ordering = []
  148 +
147 149 # If inheritance is allowed, only return instances and instances of
148 150 # subclasses of the class being used
149 151 if document._meta.get('allow_inheritance'):
@@ -151,7 +153,7 @@ def __init__(self, document, collection):
151 153 self._cursor_obj = None
152 154 self._limit = None
153 155 self._skip = None
154   -
  156 +
155 157 def ensure_index(self, key_or_list):
156 158 """Ensure that the given indexes are in place.
157 159
@@ -199,7 +201,7 @@ def _build_index_spec(cls, doc_cls, key_or_list):
199 201 return index_list
200 202
201 203 def __call__(self, q_obj=None, **query):
202   - """Filter the selected documents by calling the
  204 + """Filter the selected documents by calling the
203 205 :class:`~mongoengine.queryset.QuerySet` with a query.
204 206
205 207 :param q_obj: a :class:`~mongoengine.queryset.Q` object to be used in
@@ -213,7 +215,7 @@ def __call__(self, q_obj=None, **query):
213 215 query = QuerySet._transform_query(_doc_cls=self._document, **query)
214 216 self._query.update(query)
215 217 return self
216   -
  218 +
217 219 def filter(self, *q_objs, **query):
218 220 """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__`
219 221 """
@@ -253,11 +255,11 @@ def _cursor(self):
253 255 # Apply where clauses to cursor
254 256 if self._where_clause:
255 257 self._cursor_obj.where(self._where_clause)
256   -
  258 +
257 259 # apply default ordering
258 260 if self._document._meta['ordering']:
259 261 self.order_by(*self._document._meta['ordering'])
260   -
  262 +
261 263 return self._cursor_obj
262 264
263 265 @classmethod
@@ -337,8 +339,8 @@ def _transform_query(cls, _doc_cls=None, **query):
337 339 return mongo_query
338 340
339 341 def get(self, *q_objs, **query):
340   - """Retrieve the the matching object raising
341   - :class:`~mongoengine.queryset.MultipleObjectsReturned` or
  342 + """Retrieve the the matching object raising
  343 + :class:`~mongoengine.queryset.MultipleObjectsReturned` or
342 344 :class:`~mongoengine.queryset.DoesNotExist` exceptions if multiple or
343 345 no results are found.
344 346 """
@@ -354,15 +356,15 @@ def get(self, *q_objs, **query):
354 356
355 357 def get_or_create(self, *q_objs, **query):
356 358 """Retreive unique object or create, if it doesn't exist. Raises
357   - :class:`~mongoengine.queryset.MultipleObjectsReturned` if multiple
358   - results are found. A new document will be created if the document
  359 + :class:`~mongoengine.queryset.MultipleObjectsReturned` if multiple
  360 + results are found. A new document will be created if the document
359 361 doesn't exists; a dictionary of default values for the new document
360 362 may be provided as a keyword argument called :attr:`defaults`.
361 363 """
362 364 defaults = query.get('defaults', {})
363   - if query.has_key('defaults'):
  365 + if 'defaults' in query:
364 366 del query['defaults']
365   -
  367 +
366 368 self.__call__(*q_objs, **query)
367 369 count = self.count()
368 370 if count == 0:
@@ -439,6 +441,70 @@ def count(self):
439 441 def __len__(self):
440 442 return self.count()
441 443
  444 + def map_reduce(self, map_f, reduce_f, finalize_f=None, limit=None,
  445 + scope=None, keep_temp=False):
  446 + """Perform a map/reduce query using the current query spec
  447 + and ordering. While ``map_reduce`` respects ``QuerySet`` chaining,
  448 + it must be the last call made, as it does not return a maleable
  449 + ``QuerySet``.
  450 +
  451 + See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce`
  452 + and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced`
  453 + tests in ``tests.queryset.QuerySetTest`` for usage examples.
  454 +
  455 + :param map_f: map function, as :class:`~pymongo.code.Code` or string
  456 + :param reduce_f: reduce function, as
  457 + :class:`~pymongo.code.Code` or string
  458 + :param finalize_f: finalize function, an optional function that
  459 + performs any post-reduction processing.
  460 + :param scope: values to insert into map/reduce global scope. Optional.
  461 + :param limit: number of objects from current query to provide
  462 + to map/reduce method
  463 + :param keep_temp: keep temporary table (boolean, default ``True``)
  464 +
  465 + Returns an iterator yielding
  466 + :class:`~mongoengine.document.MapReduceDocument`.
  467 +
  468 + .. note:: Map/Reduce requires server version **>= 1.1.1**. The PyMongo
  469 + :meth:`~pymongo.collection.Collection.map_reduce` helper requires
  470 + PyMongo version **>= 1.2**.
  471 +
  472 + .. versionadded:: 0.3
  473 +
  474 + """
  475 + from document import MapReduceDocument
  476 +
  477 + if not hasattr(self._collection, "map_reduce"):
  478 + raise NotImplementedError("Requires MongoDB >= 1.1.1")
  479 +
  480 + if not isinstance(map_f, pymongo.code.Code):
  481 + map_f = pymongo.code.Code(map_f)
  482 + if not isinstance(reduce_f, pymongo.code.Code):
  483 + reduce_f = pymongo.code.Code(reduce_f)
  484 +
  485 + mr_args = {'query': self._query, 'keeptemp': keep_temp}
  486 +
  487 + if finalize_f:
  488 + if not isinstance(finalize_f, pymongo.code.Code):
  489 + finalize_f = pymongo.code.Code(finalize_f)
  490 + mr_args['finalize'] = finalize_f
  491 +
  492 + if scope:
  493 + mr_args['scope'] = scope
  494 +
  495 + if limit:
  496 + mr_args['limit'] = limit
  497 +
  498 + results = self._collection.map_reduce(map_f, reduce_f, **mr_args)
  499 + results = results.find()
  500 +
  501 + if self._ordering:
  502 + results = results.sort(self._ordering)
  503 +
  504 + for doc in results:
  505 + yield MapReduceDocument(self._document, self._collection,
  506 + doc['_id'], doc['value'])
  507 +
442 508 def limit(self, n):
443 509 """Limit the number of returned documents to `n`. This may also be
444 510 achieved using array-slicing syntax (e.g. ``User.objects[:5]``).
@@ -450,6 +516,7 @@ def limit(self, n):
450 516 else:
451 517 self._cursor.limit(n)
452 518 self._limit = n
  519 +
453 520 # Return self to allow chaining
454 521 return self
455 522
@@ -523,13 +590,14 @@ def order_by(self, *keys):
523 590 direction = pymongo.DESCENDING
524 591 if key[0] in ('-', '+'):
525 592 key = key[1:]
526   - key_list.append((key, direction))
  593 + key_list.append((key, direction))
527 594
  595 + self._ordering = key_list
528 596 self._cursor.sort(key_list)
529 597 return self
530   -
  598 +
531 599 def explain(self, format=False):
532   - """Return an explain plan record for the
  600 + """Return an explain plan record for the
533 601 :class:`~mongoengine.queryset.QuerySet`\ 's cursor.
534 602
535 603 :param format: format the plan before returning it
@@ -540,7 +608,7 @@ def explain(self, format=False):
540 608 import pprint
541 609 plan = pprint.pformat(plan)
542 610 return plan
543   -
  611 +
544 612 def delete(self, safe=False):
545 613 """Delete the documents matched by the query.
546 614
@@ -552,7 +620,7 @@ def delete(self, safe=False):
552 620 def _transform_update(cls, _doc_cls=None, **update):
553 621 """Transform an update spec from Django-style format to Mongo format.
554 622 """
555   - operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull',
  623 + operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull',
556 624 'pull_all']
557 625
558 626 mongo_update = {}
@@ -661,8 +729,8 @@ def exec_js(self, code, *fields, **options):
661 729 """Execute a Javascript function on the server. A list of fields may be
662 730 provided, which will be translated to their correct names and supplied
663 731 as the arguments to the function. A few extra variables are added to
664   - the function's scope: ``collection``, which is the name of the
665   - collection in use; ``query``, which is an object representing the
  732 + the function's scope: ``collection``, which is the name of the
  733 + collection in use; ``query``, which is an object representing the
666 734 current query; and ``options``, which is an object containing any
667 735 options specified as keyword arguments.
668 736
@@ -676,7 +744,7 @@ def exec_js(self, code, *fields, **options):
676 744 :param code: a string of Javascript code to execute
677 745 :param fields: fields that you will be using in your function, which
678 746 will be passed in to your function as arguments
679   - :param options: options that you want available to the function
  747 + :param options: options that you want available to the function
680 748 (accessed in Javascript through the ``options`` object)
681 749 """
682 750 code = self._sub_js_fields(code)
@@ -693,7 +761,7 @@ def exec_js(self, code, *fields, **options):
693 761 query = self._query
694 762 if self._where_clause:
695 763 query['$where'] = self._where_clause
696   -
  764 +
697 765 scope['query'] = query
698 766 code = pymongo.code.Code(code, scope=scope)
699 767
@@ -741,7 +809,7 @@ def average(self, field):
741 809 def item_frequencies(self, list_field, normalize=False):
742 810 """Returns a dictionary of all items present in a list field across
743 811 the whole queried set of documents, and their corresponding frequency.
744   - This is useful for generating tag clouds, or searching documents.
  812 + This is useful for generating tag clouds, or searching documents.
745 813
746 814 :param list_field: the list field to use
747 815 :param normalize: normalize the results so they add to 1.0
@@ -791,7 +859,7 @@ def __init__(self, manager_func=None):
791 859 self._collection = None
792 860
793 861 def __get__(self, instance, owner):
794   - """Descriptor for instantiating a new QuerySet object when
  862 + """Descriptor for instantiating a new QuerySet object when
795 863 Document.objects is accessed.
796 864 """
797 865 if instance is not None:
@@ -810,7 +878,7 @@ def __get__(self, instance, owner):
810 878
811 879 if collection in db.collection_names():
812 880 self._collection = db[collection]
813   - # The collection already exists, check if its capped
  881 + # The collection already exists, check if its capped
814 882 # options match the specified capped options
815 883 options = self._collection.options()
816 884 if options.get('max') != max_documents or \
@@ -826,7 +894,7 @@ def __get__(self, instance, owner):
826 894 self._collection = db.create_collection(collection, opts)
827 895 else:
828 896 self._collection = db[collection]
829   -
  897 +
830 898 # owner is the document that contains the QuerySetManager
831 899 queryset = QuerySet(owner, self._collection)
832 900 if self._manager_func:
@@ -836,6 +904,7 @@ def __get__(self, instance, owner):
836 904 queryset = self._manager_func(owner, queryset)
837 905 return queryset
838 906
  907 +
839 908 def queryset_manager(func):
840 909 """Decorator that allows you to define custom QuerySet managers on
841 910 :class:`~mongoengine.Document` classes. The manager must be a function that
252 tests/queryset.py
... ... @@ -1,14 +1,17 @@
  1 +# -*- coding: utf-8 -*-
  2 +
  3 +
1 4 import unittest
2 5 import pymongo
3   -from datetime import datetime
  6 +from datetime import datetime, timedelta
4 7
5   -from mongoengine.queryset import (QuerySet, MultipleObjectsReturned,
  8 +from mongoengine.queryset import (QuerySet, MultipleObjectsReturned,
6 9 DoesNotExist)
7 10 from mongoengine import *
8 11
9 12
10 13 class QuerySetTest(unittest.TestCase):
11   -
  14 +
12 15 def setUp(self):
13 16 connect(db='mongoenginetest')
14 17
@@ -16,12 +19,12 @@ class Person(Document):
16 19 name = StringField()
17 20 age = IntField()
18 21 self.Person = Person
19   -
  22 +
20 23 def test_initialisation(self):
21 24 """Ensure that a QuerySet is correctly initialised by QuerySetManager.
22 25 """
23 26 self.assertTrue(isinstance(self.Person.objects, QuerySet))
24   - self.assertEqual(self.Person.objects._collection.name,
  27 + self.assertEqual(self.Person.objects._collection.name,
25 28 self.Person._meta['collection'])
26 29 self.assertTrue(isinstance(self.Person.objects._collection,
27 30 pymongo.collection.Collection))
@@ -31,15 +34,15 @@ def test_transform_query(self):
31 34 """
32 35 self.assertEqual(QuerySet._transform_query(name='test', age=30),
33 36 {'name': 'test', 'age': 30})
34   - self.assertEqual(QuerySet._transform_query(age__lt=30),
  37 + self.assertEqual(QuerySet._transform_query(age__lt=30),
35 38 {'age': {'$lt': 30}})
36 39 self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50),
37 40 {'age': {'$gt': 20, '$lt': 50}})
38 41 self.assertEqual(QuerySet._transform_query(age=20, age__gt=50),
39 42 {'age': 20})
40   - self.assertEqual(QuerySet._transform_query(friend__age__gte=30),
  43 + self.assertEqual(QuerySet._transform_query(friend__age__gte=30),
41 44 {'friend.age': {'$gte': 30}})
42   - self.assertEqual(QuerySet._transform_query(name__exists=True),
  45 + self.assertEqual(QuerySet._transform_query(name__exists=True),
43 46 {'name': {'$exists': True}})
44 47
45 48 def test_find(self):
@@ -134,7 +137,7 @@ def test_find_one(self):
134 137 self.assertEqual(person.name, "User B")
135 138
136 139 self.assertRaises(IndexError, self.Person.objects.__getitem__, 2)
137   -
  140 +
138 141 # Find a document using just the object id
139 142 person = self.Person.objects.with_id(person1.id)
140 143 self.assertEqual(person.name, "User A")
@@ -170,7 +173,7 @@ def test_get_or_create(self):
170 173 person2.save()
171 174
172 175 # Retrieve the first person from the database
173   - self.assertRaises(MultipleObjectsReturned,
  176 + self.assertRaises(MultipleObjectsReturned,
174 177 self.Person.objects.get_or_create)
175 178
176 179 # Use a query to filter the people found to just person2
@@ -256,36 +259,36 @@ def test_filter_chaining(self):
256 259 """Ensure filters can be chained together.
257 260 """
258 261 from datetime import datetime
259   -
  262 +
260 263 class BlogPost(Document):
261 264 title = StringField()
262 265 is_published = BooleanField()
263 266 published_date = DateTimeField()
264   -
  267 +
265 268 @queryset_manager
266 269 def published(doc_cls, queryset):
267 270 return queryset(is_published=True)
268   -
269   - blog_post_1 = BlogPost(title="Blog Post #1",
  271 +
  272 + blog_post_1 = BlogPost(title="Blog Post #1",
270 273 is_published = True,
271 274 published_date=datetime(2010, 1, 5, 0, 0 ,0))
272   - blog_post_2 = BlogPost(title="Blog Post #2",
  275 + blog_post_2 = BlogPost(title="Blog Post #2",
273 276 is_published = True,
274 277 published_date=datetime(2010, 1, 6, 0, 0 ,0))
275   - blog_post_3 = BlogPost(title="Blog Post #3",
  278 + blog_post_3 = BlogPost(title="Blog Post #3",
276 279 is_published = True,
277 280 published_date=datetime(2010, 1, 7, 0, 0 ,0))
278 281
279 282 blog_post_1.save()
280 283 blog_post_2.save()
281 284 blog_post_3.save()
282   -
  285 +
283 286 # find all published blog posts before 2010-01-07
284 287 published_posts = BlogPost.published()
285 288 published_posts = published_posts.filter(
286 289 published_date__lt=datetime(2010, 1, 7, 0, 0 ,0))
287 290 self.assertEqual(published_posts.count(), 2)
288   -
  291 +
289 292 BlogPost.drop_collection()
290 293
291 294 def test_ordering(self):
@@ -301,22 +304,22 @@ class BlogPost(Document):
301 304
302 305 BlogPost.drop_collection()
303 306
304   - blog_post_1 = BlogPost(title="Blog Post #1",
  307 + blog_post_1 = BlogPost(title="Blog Post #1",
305 308 published_date=datetime(2010, 1, 5, 0, 0 ,0))
306   - blog_post_2 = BlogPost(title="Blog Post #2",
  309 + blog_post_2 = BlogPost(title="Blog Post #2",
307 310 published_date=datetime(2010, 1, 6, 0, 0 ,0))
308   - blog_post_3 = BlogPost(title="Blog Post #3",
  311 + blog_post_3 = BlogPost(title="Blog Post #3",
309 312 published_date=datetime(2010, 1, 7, 0, 0 ,0))
310 313
311 314 blog_post_1.save()
312 315 blog_post_2.save()
313 316 blog_post_3.save()
314   -
  317 +
315 318 # get the "first" BlogPost using default ordering
316 319 # from BlogPost.meta.ordering
317   - latest_post = BlogPost.objects.first()
  320 + latest_post = BlogPost.objects.first()
318 321 self.assertEqual(latest_post.title, "Blog Post #3")
319   -
  322 +
320 323 # override default ordering, order BlogPosts by "published_date"
321 324 first_post = BlogPost.objects.order_by("+published_date").first()
322 325 self.assertEqual(first_post.title, "Blog Post #1")
@@ -375,7 +378,7 @@ class BlogPost(Document):
375 378 result = BlogPost.objects.first()
376 379 self.assertTrue(isinstance(result.author, User))
377 380 self.assertEqual(result.author.name, 'Test User')
378   -
  381 +
379 382 BlogPost.drop_collection()
380 383
381 384 def test_find_dict_item(self):
@@ -442,7 +445,7 @@ class BlogPost(Document):
442 445 self.Person(name='user2', age=20).save()
443 446 self.Person(name='user3', age=30).save()
444 447 self.Person(name='user4', age=40).save()
445   -
  448 +
446 449 self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2)
447 450 self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3)
448 451
@@ -545,17 +548,17 @@ class BlogPost(Document):
545 548 return comments;
546 549 }
547 550 """
548   -
  551 +
549 552 sub_code = BlogPost.objects._sub_js_fields(code)
550   - code_chunks = ['doc["cmnts"];', 'doc["doc-name"],',
  553 + code_chunks = ['doc["cmnts"];', 'doc["doc-name"],',
551 554 'doc["cmnts"][i]["body"]']
552 555 for chunk in code_chunks:
553 556 self.assertTrue(chunk in sub_code)
554 557
555 558 results = BlogPost.objects.exec_js(code)
556 559 expected_results = [
557   - {u'comment': u'cool', u'document': u'post1'},
558   - {u'comment': u'yay', u'document': u'post1'},
  560 + {u'comment': u'cool', u'document': u'post1'},
  561 + {u'comment': u'yay', u'document': u'post1'},
559 562 {u'comment': u'nice stuff', u'document': u'post2'},
560 563 ]
561 564 self.assertEqual(results, expected_results)
@@ -627,10 +630,167 @@ def test_order_by(self):
627 630
628 631 names = [p.name for p in self.Person.objects.order_by('age')]
629 632 self.assertEqual(names, ['User A', 'User C', 'User B'])
630   -
  633 +
631 634 ages = [p.age for p in self.Person.objects.order_by('-name')]
632 635 self.assertEqual(ages, [30, 40, 20])
633 636
  637 + def test_map_reduce(self):
  638 + """Ensure map/reduce is both mapping and reducing.
  639 + """
  640 + class BlogPost(Document):
  641 + title = StringField()
  642 + tags = ListField(StringField())
  643 +
  644 + BlogPost.drop_collection()
  645 +
  646 + BlogPost(title="Post #1", tags=['music', 'film', 'print']).save()
  647 + BlogPost(title="Post #2", tags=['music', 'film']).save()
  648 + BlogPost(title="Post #3", tags=['film', 'photography']).save()
  649 +
  650 + map_f = """
  651 + function() {
  652 + this.tags.forEach(function(tag) {
  653 + emit(tag, 1);
  654 + });
  655 + }
  656 + """
  657 +
  658 + reduce_f = """
  659 + function(key, values) {
  660 + var total = 0;
  661 + for(var i=0; i<values.length; i++) {
  662 + total += values[i];
  663 + }
  664 + return total;
  665 + }
  666 + """
  667 +
  668 + # run a map/reduce operation spanning all posts
  669 + results = BlogPost.objects.map_reduce(map_f, reduce_f)
  670 + results = list(results)
  671 + self.assertEqual(len(results), 4)
  672 +
  673 + music = filter(lambda r: r.key == "music", results)[0]
  674 + self.assertEqual(music.value, 2)
  675 +
  676 + film = filter(lambda r: r.key == "film", results)[0]
  677 + self.assertEqual(film.value, 3)
  678 +
  679 + BlogPost.drop_collection()
  680 +
  681 + def test_map_reduce_finalize(self):
  682 + """Ensure that map, reduce, and finalize run and introduce "scope"
  683 + by simulating "hotness" ranking with Reddit algorithm.
  684 + """
  685 + from time import mktime
  686 +
  687 + class Link(Document):
  688 + title = StringField()
  689 + up_votes = IntField()
  690 + down_votes = IntField()
  691 + submitted = DateTimeField()
  692 +
  693 + Link.drop_collection()
  694 +
  695 + now = datetime.utcnow()
  696 +
  697 + # Note: Test data taken from a custom Reddit homepage on
  698 + # Fri, 12 Feb 2010 14:36:00 -0600. Link ordering should
  699 + # reflect order of insertion below, but is not influenced
  700 + # by insertion order.
  701 + Link(title = "Google Buzz auto-followed a woman's abusive ex ...",
  702 + up_votes = 1079,
  703 + down_votes = 553,
  704 + submitted = now-timedelta(hours=4)).save()
  705 + Link(title = "We did it! Barbie is a computer engineer.",
  706 + up_votes = 481,
  707 + down_votes = 124,
  708 + submitted = now-timedelta(hours=2)).save()
  709 + Link(title = "This Is A Mosquito Getting Killed By A Laser",
  710 + up_votes = 1446,
  711 + down_votes = 530,
  712 + submitted=now-timedelta(hours=13)).save()
  713 + Link(title = "Arabic flashcards land physics student in jail.",
  714 + up_votes = 215,
  715 + down_votes = 105,
  716 + submitted = now-timedelta(hours=6)).save()
  717 + Link(title = "The Burger Lab: Presenting, the Flood Burger",
  718 + up_votes = 48,
  719 + down_votes = 17,
  720 + submitted = now-timedelta(hours=5)).save()
  721 + Link(title="How to see polarization with the naked eye",
  722 + up_votes = 74,
  723 + down_votes = 13,
  724 + submitted = now-timedelta(hours=10)).save()
  725 +
  726 + map_f = """
  727 + function() {
  728 + emit(this._id, {up_delta: this.up_votes - this.down_votes,
  729 + sub_date: this.submitted.getTime() / 1000})
  730 + }
  731 + """
  732 +
  733 + reduce_f = """
  734 + function(key, values) {
  735 + data = values[0];
  736 +
  737 + x = data.up_delta;
  738 +
  739 + // calculate time diff between reddit epoch and submission
  740 + sec_since_epoch = data.sub_date - reddit_epoch;
  741 +
  742 + // calculate 'Y'
  743 + if(x > 0) {
  744 + y = 1;
  745 + } else if (x = 0) {
  746 + y = 0;
  747 + } else {
  748 + y = -1;
  749 + }
  750 +
  751 + // calculate 'Z', the maximal value
  752 + if(Math.abs(x) >= 1) {
  753 + z = Math.abs(x);
  754 + } else {
  755 + z = 1;
  756 + }
  757 +
  758 + return {x: x, y: y, z: z, t_s: sec_since_epoch};
  759 + }
  760 + """
  761 +
  762 + finalize_f = """
  763 + function(key, value) {
  764 + // f(sec_since_epoch,y,z) = log10(z) + ((y*sec_since_epoch) / 45000)
  765 + z_10 = Math.log(value.z) / Math.log(10);
  766 + weight = z_10 + ((value.y * value.t_s) / 45000);
  767 + return weight;
  768 + }
  769 + """
  770 +
  771 + # provide the reddit epoch (used for ranking) as a variable available
  772 + # to all phases of the map/reduce operation: map, reduce, and finalize.
  773 + reddit_epoch = mktime(datetime(2005, 12, 8, 7, 46, 43).timetuple())
  774 + scope = {'reddit_epoch': reddit_epoch}
  775 +
  776 + # run a map/reduce operation across all links. ordering is set
  777 + # to "-value", which orders the "weight" value returned from
  778 + # "finalize_f" in descending order.
  779 + results = Link.objects.order_by("-value")
  780 + results = results.map_reduce(map_f,
  781 + reduce_f,
  782 + finalize_f=finalize_f,
  783 + scope=scope)
  784 + results = list(results)
  785 +
  786 + # assert troublesome Buzz article is ranked 1st
  787 + self.assertTrue(results[0].object.title.startswith("Google Buzz"))
  788 +
  789 + # assert laser vision is ranked last
  790 + self.assertTrue(results[-1].object.title.startswith("How to see"))
  791 +
  792 + Link.drop_collection()
  793 +
634 794 def test_item_frequencies(self):
635 795 """Ensure that item frequencies are properly generated from lists.
636 796 """
@@ -727,20 +887,20 @@ class BlogPost(Document):
727 887 title = StringField(name='postTitle')
728 888 comments = ListField(EmbeddedDocumentField(Comment),
729 889 name='postComments')
730   -
  890 +
731 891
732 892 BlogPost.drop_collection()
733 893
734 894 data = {'title': 'Post 1', 'comments': [Comment(content='test')]}
735 895 BlogPost(**data).save()
736 896
737   - self.assertTrue('postTitle' in
  897 + self.assertTrue('postTitle' in
738 898 BlogPost.objects(title=data['title'])._query)
739   - self.assertFalse('title' in
  899 + self.assertFalse('title' in
740 900 BlogPost.objects(title=data['title'])._query)
741 901 self.assertEqual(len(BlogPost.objects(title=data['title'])), 1)
742 902
743   - self.assertTrue('postComments.commentContent' in
  903 + self.assertTrue('postComments.commentContent' in
744 904 BlogPost.objects(comments__content='test')._query)
745 905 self.assertEqual(len(BlogPost.objects(comments__content='test')), 1)
746 906
@@ -761,7 +921,7 @@ class BlogPost(Document):
761 921 post.save()
762 922
763 923 # Test that query may be performed by providing a document as a value
764   - # while using a ReferenceField's name - the document should be
  924 + # while using a ReferenceField's name - the document should be
765 925 # converted to an DBRef, which is legal, unlike a Document object
766 926 post_obj = BlogPost.objects(author=person).first()
767 927 self.assertEqual(post.id, post_obj.id)
@@ -823,13 +983,13 @@ class BlogPost(Document):
823 983 self.assertFalse([('_types', 1)] in info.values())
824 984
825 985 BlogPost.drop_collection()
826   -
  986 +
827 987 def test_bulk(self):
828 988 """Ensure bulk querying by object id returns a proper dict.
829 989 """
830 990 class BlogPost(Document):
831 991 title = StringField()
832   -
  992 +
833 993 BlogPost.drop_collection()
834 994
835 995 post_1 = BlogPost(title="Post #1")
@@ -843,20 +1003,20 @@ class BlogPost(Document):
843 1003 post_3.save()
844 1004 post_4.save()
845 1005 post_5.save()
846   -
  1006 +
847 1007 ids = [post_1.id, post_2.id, post_5.id]
848 1008 objects = BlogPost.objects.in_bulk(ids)
849   -
  1009 +
850 1010 self.assertEqual(len(objects), 3)
851 1011
852 1012 self.assertTrue(post_1.id in objects)
853 1013 self.assertTrue(post_2.id in objects)
854 1014 self.assertTrue(post_5.id in objects)
855   -
  1015 +
856 1016 self.assertTrue(objects[post_1.id].title == post_1.title)
857 1017 self.assertTrue(objects[post_2.id].title == post_2.title)
858   - self.assertTrue(objects[post_5.id].title == post_5.title)
859   -
  1018 + self.assertTrue(objects[post_5.id].title == post_5.title)
  1019 +
860 1020 BlogPost.drop_collection()
861 1021
862 1022 def tearDown(self):
@@ -864,7 +1024,7 @@ def tearDown(self):
864 1024
865 1025
866 1026 class QTest(unittest.TestCase):
867   -
  1027 +
868 1028 def test_or_and(self):
869 1029 """Ensure that Q objects may be combined correctly.
870 1030 """
@@ -888,8 +1048,8 @@ def test_item_query_as_js(self):
888 1048 examples = [
889 1049 ({'name': 'test'}, 'this.name == i0f0', {'i0f0': 'test'}),
890 1050 ({'age': {'$gt': 18}}, 'this.age > i0f0o0', {'i0f0o0': 18}),
891   - ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}},
892   - 'this.age <= i0f0o0 && this.age > i0f0o1 && this.name == i0f1',
  1051 + ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}},
  1052 + 'this.age <= i0f0o0 && this.age > i0f0o1 && this.name == i0f1',
893 1053 {'i0f0o0': 65, 'i0f0o1': 18, 'i0f1': 'test'}),
894 1054 ]
895 1055 for item, js, scope in examples:

0 comments on commit 047cc21

Please sign in to comment.
Something went wrong with that request. Please try again.