Another optimization for ctable.whereblocks().

bcolz/ctable.py

@@ -18,10 +18,9 @@
 import os
 import re
 import shutil
-from .py2help import _inttypes, _strtypes, imap, izip, xrange
+from .py2help import _inttypes, _strtypes, imap, izip, islice, xrange
 
 _inttypes += (np.integer,)
-islice = itertools.islice
 
 ROOTDIRS = '__rootdirs__'
 
@@ -937,7 +936,7 @@ def whereblocks(self, expression, blen=None, outfields=None, limit=None,
 
         See Also
         --------
-        iterblocks
+        :func:`~toplevel.iterblocks`
 
         """
 
@@ -954,19 +953,17 @@ def whereblocks(self, expression, blen=None, outfields=None, limit=None,
                 dtype = [(name, self[name].dtype) for name in outfields]
             except IndexError:
                 raise ValueError(
-                    "Some names in `outfields` are not real fields")
-
-        buf = np.empty(blen, dtype=dtype)
-        nrow = 0
-        for row in self.where(expression, outfields, limit, skip,
-                              out_flavor=tuple):
-            buf[nrow] = row
-            nrow += 1
-            if nrow == blen:
-                yield buf
-                buf = np.empty(blen, dtype=dtype)
-                nrow = 0
-        yield buf[:nrow]
+                    "Some names in `outfields` are not actual fields")
+
+        it = self.where(expression, outfields, limit, skip, out_flavor=tuple)
+        return self._iterwb(it, blen, dtype)
+
+    def _iterwb(self, it, blen, dtype):
+        while True:
+            ra = np.fromiter(islice(it, blen), dtype)
+            if len(ra) == 0:
+                break
+            yield ra
 
     def __iter__(self):
         return self.iter(0, self.len, 1)

bcolz/py2help.py

@@ -31,15 +31,14 @@
     def dict_iteritems(d):
         return d.iteritems()
     xrange = __builtin__.xrange
-    from itertools import izip
+    from itertools import izip, imap, islice
     unicode = __builtin__.unicode
     basestring = __builtin__.basestring
     reduce = __builtin__.reduce
 
     _strtypes = (str, unicode)
 
     _inttypes = (int, long)
-    imap = itertools.imap
     import urlparse
     def exec_(_code_, _globs_=None, _locs_=None):
         """Execute code in a namespace."""
@@ -57,10 +56,10 @@ def dict_iteritems(d):
         return d.items().__iter__()
     xrange = range
     izip = zip
+    imap = map
     _inttypes = (int,)
     _strtypes = (bytes, str)
     unicode = str
-    imap = map
     basestring = str
     import urllib.parse as urlparse
     from functools import reduce