Merge pull request #15 from bluedynamics/fix-paspluginsldap-issues-29

Fix missing paging in UGM group mapping method member_ids.

CHANGES.rst

@@ -4,7 +4,15 @@ History
 1.0b3 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Add a ``batched_search`` generator function, which do the actual batching for us.
+  Use this function internally too.
+  [jensens, rnix]
+
+- In testing set size_limit to 3 in ``slapd.conf`` in order to catch problems with batching.
+  [jensens, rnix]
+
+- Fix missing paging in UGM group mapping method ``member_ids``.
+  [jensens]
 
 
 1.0b2 (2016-09-09)
@@ -14,7 +22,7 @@ History
   [jensens]
 
 - Paginate LDAP node ``__iter__``.
-  [jensens, rnix] 
+  [jensens, rnix]
 
 1.0b1 (31.12.2015)
 ------------------

src/node/ext/ldap/_node.py

@@ -530,6 +530,28 @@ def search(self, queryFilter=None, criteria=None, attrlist=None,
             return (res, cookie)
         return res
 
+    @default
+    def batched_search(self, page_size=None, search_func=None, **kw):
+        """Search generator function which does paging for us.
+        """
+        if page_size is None:
+            page_size = self.ldap_session._props.page_size
+        if search_func is None:
+            search_func = self.search
+        matches = []
+        cookie = None
+        kw['page_size'] = page_size
+        while True:
+            try:
+                kw['cookie'] = cookie
+                matches, cookie = search_func(**kw)
+                for item in matches:
+                    yield item
+            except ValueError:
+                break
+            if not cookie:
+                break
+
     @default
     def invalidate(self, key=None):
         """Invalidate LDAP node.

src/node/ext/ldap/testing/__init__.py

@@ -49,21 +49,27 @@ def read_env(layer):
 slapdconf_template = """\
 %(schema)s
 
-logfile         %(confdir)s/log
-loglevel        256
+logfile     %(confdir)s/log
+loglevel    256
 
-pidfile		%(confdir)s/slapd.pid
-argsfile	%(confdir)s/slapd.args
+pidfile     %(confdir)s/slapd.pid
+argsfile    %(confdir)s/slapd.args
+
+# XXX: sizelimit has no effect in our test slapd right now.
+#      needed for better coverage.
+#      figure out.
+sizelimit   3
+
+database    bdb
+suffix      "%(suffix)s"
+rootdn      "%(binddn)s"
+rootpw      %(bindpw)s
+directory   %(dbdir)s
 
-database	bdb
-suffix		"%(suffix)s"
-rootdn		"%(binddn)s"
-rootpw		%(bindpw)s
-directory	%(dbdir)s
 # Indices to maintain
-index	objectClass	eq
+index       objectClass eq
 
-overlay memberof
+overlay     memberof
 """
 
 
@@ -91,7 +97,7 @@ def setUp(self, args=None):
         slapdconf = self['slapdconf']
         schema = '\n'.join(
             ["include %s" % (schema,) for schema in self.schema]
-            )
+        )
         # generate config file
         with open(slapdconf, 'w') as slapdconf:
             slapdconf.write(

src/node/ext/ldap/ugm/_api.py

@@ -57,6 +57,7 @@ def __repr__(self):
 
     __str__ = __repr__
 
+
 ACCOUNT_EXPIRED = AccountExpired()
 
 
@@ -227,11 +228,11 @@ def group_ids(self):
             # specified by the same criteria as roles, the search returns the
             # role id's as well.
             # XXX: such edge cases should be resolved at UGM init time
-            matches = groups.context.search(
+            matches_generator = groups.context.batched_search(
                 criteria=criteria,
                 attrlist=attrlist
             )
-            res = [att[groups._key_attr][0] for _, att in matches]
+            res = [att[groups._key_attr][0] for _, att in matches_generator]
         return res
 
     @default
@@ -316,11 +317,13 @@ def member_ids(self):
                 users = ugm.users
                 criteria = {'memberOf': self.context.DN}
                 attrlist = [users._key_attr]
-                res = users.context.search(
+                matches_generator = users.context.batched_search(
                     criteria=criteria,
-                    attrlist=attrlist
+                    attrlist=attrlist,
                 )
-                return [att[users._key_attr][0] for _, att in res]
+                return [
+                    att[users._key_attr][0] for _, att in matches_generator
+                ]
         ret = list()
         members = self.context.attrs.get(self._member_attribute, list())
         for member in members:
@@ -496,8 +499,7 @@ def __getitem__(self, key):
     @locktree
     def __iter__(self):
         attrlist = ['rdn', self._key_attr]
-        res = self.context.search(attrlist=attrlist)
-        for principal in res:
+        for principal in self.context.batched_search(attrlist=attrlist):
             prdn = principal[1]['rdn']
             if prdn in self.context._deleted_children:
                 continue