Merge remote-tracking branch 'upstream/master'

docs/source/api/menpo/visualize/bytes_str.rst

@@ -0,0 +1,7 @@
+.. _menpo-visualize-bytes_str:
+
+.. currentmodule:: menpo.visualize
+
+bytes_str
+=========
+.. autofunction:: bytes_str

docs/source/api/menpo/visualize/index.rst

@@ -39,6 +39,7 @@ Print Utilities
 .. toctree::
   :maxdepth: 1
 
+  print_progress
   print_dynamic
   progress_bar_str
-  print_bytes
+  bytes_str

docs/source/api/menpo/visualize/print_progress.rst

@@ -0,0 +1,7 @@
+.. _menpo-visualize-print_progress:
+
+.. currentmodule:: menpo.visualize
+
+print_progress
+==============
+.. autofunction:: print_progress

menpo/image/base.py

@@ -1937,6 +1937,39 @@ def _normalize_inplace(self, scale_func, mode='all'):
         else:
             self.from_vector_inplace(centered_pixels / scale_factor)
 
+    def rescale_pixels(self, minimum, maximum, per_channel=True):
+        r"""A copy of this image with pixels linearly rescaled to fit a range.
+
+        Note that the only pixels that will considered and rescaled are those
+        that feature in the vectorized form of this image. If you want to use
+        this routine on all the pixels in a :map:`MaskedImage`, consider
+        using `as_unmasked()` prior to this call.
+
+        Parameters
+        ----------
+        minimum: `float`
+            The minimal value of the rescaled pixels
+        maximum: `float`
+            The maximal value of the rescaled pixels
+        per_channel: `boolean`, optional
+            If ``True``, each channel will be rescaled independently. If
+            ``False``, the scaling will be over all channels.
+
+        Returns
+        -------
+        rescaled_image: ``type(self)``
+            A copy of this image with pixels linearly rescaled to fit in the
+            range provided.
+        """
+        v = self.as_vector(keep_channels=True).T
+        if per_channel:
+            min_, max_ = v.min(axis=0), v.max(axis=0)
+        else:
+            min_, max_ = v.min(), v.max()
+        sf = ((maximum - minimum) * 1.0) / (max_ - min_)
+        v_new = ((v - min_) * sf) + minimum
+        return self.from_vector(v_new.T.ravel())
+
 
 def round_image_shape(shape, round):
     if round not in ['ceil', 'round', 'floor']:

menpo/image/test/image_pixel_recale_test.py

@@ -0,0 +1,50 @@
+from menpo.image import Image, MaskedImage
+import numpy as np
+
+
+def test_rescale_pixels():
+    img = Image.init_blank((10, 10), n_channels=1)
+    img.pixels[:, 6:, 6:] = 2
+
+    img_rescaled = img.rescale_pixels(0, 255)
+    assert np.min(img_rescaled.pixels) == 0
+    assert np.max(img_rescaled.pixels) == 255
+    assert np.all(img_rescaled.pixels[:, 6:, 6:] == 255)
+
+
+def test_rescale_pixels_all_channels():
+    img = Image.init_blank((10, 10), n_channels=2)
+    img.pixels[0, 6:, 6:] = 2
+    img.pixels[1, 6:, 6:] = 4
+
+    img_rescaled = img.rescale_pixels(0, 100, per_channel=False)
+    assert np.min(img_rescaled.pixels) == 0
+    assert np.max(img_rescaled.pixels) == 100
+    assert np.all(img_rescaled.pixels[0, 6:, 6:] == 50)
+    assert np.all(img_rescaled.pixels[1, 6:, 6:] == 100)
+
+
+def test_rescale_pixels_per_channel():
+    img = Image.init_blank((10, 10), n_channels=2)
+    img.pixels[0, 6:, 6:] = 2
+    img.pixels[1, 6:, 6:] = 4
+
+    img_rescaled = img.rescale_pixels(0, 100, per_channel=True)
+    assert np.min(img_rescaled.pixels) == 0
+    assert np.max(img_rescaled.pixels) == 100
+    assert np.all(img_rescaled.pixels[0, 6:, 6:] == 100)
+    assert np.all(img_rescaled.pixels[1, 6:, 6:] == 100)
+
+
+def test_rescale_pixels_only_masked():
+    img = MaskedImage.init_blank((10, 10), n_channels=1, fill=1)
+    img.pixels[0, 0, 0] = 0
+    img.pixels[0, 6:, 6:] = 2
+    img.mask.pixels[:, 6:, 6:] = False
+
+    img_rescaled = img.rescale_pixels(0, 100)
+    assert np.min(img_rescaled.pixels) == 0
+    assert np.max(img_rescaled.pixels) == 100
+    assert img_rescaled.pixels[0, 0, 0] == 0
+    assert img_rescaled.pixels[0, 1, 1] == 100
+    assert np.all(img_rescaled.mask.pixels == img.mask.pixels)

menpo/io/input/base.py

@@ -3,7 +3,7 @@
 
 from ..utils import _norm_path
 from menpo.base import menpo_src_dir_path
-from menpo.visualize import progress_bar_str, print_dynamic
+from menpo.visualize import print_progress
 
 
 def data_dir_path():
@@ -13,7 +13,6 @@ def data_dir_path():
     -------
     ``pathlib.Path``
         The path to the local Menpo ./data folder
-
     """
     return menpo_src_dir_path() / 'data'
 
@@ -37,7 +36,6 @@ def data_path_to(asset_filename):
     ------
     ValueError
         If the asset_filename doesn't exist in the `data` folder.
-
     """
     asset_path = data_dir_path() / asset_filename
     if not asset_path.is_file():
@@ -372,15 +370,17 @@ def _import_glob_generator(pattern, extension_map, max_assets=None,
     n_files = len(filepaths)
     if n_files == 0:
         raise ValueError('The glob {} yields no assets'.format(pattern))
-    for i, asset in enumerate(_multi_import_generator(filepaths, extension_map,
-                              landmark_resolver=landmark_resolver,
-                              landmark_ext_map=landmark_ext_map,
-                              importer_kwargs=importer_kwargs)):
-        if verbose:
-            print_dynamic('- Loading {} assets: {}'.format(
-                n_files, progress_bar_str(float(i + 1) / n_files,
-                                          show_bar=True)))
-        yield asset
+
+    generator = _multi_import_generator(
+        filepaths, extension_map,  landmark_resolver=landmark_resolver,
+        landmark_ext_map=landmark_ext_map, importer_kwargs=importer_kwargs)
+
+    if verbose:
+        # wrap the generator with the progress reporter
+        generator = print_progress(generator, prefix='Importing assets',
+                                   n_items=n_files)
+
+    return generator
 
 
 def _import(filepath, extensions_map, keep_importer=False,
@@ -602,7 +602,9 @@ def glob_with_suffix(pattern, extensions_map):
         # we want to extract '.pkl.gz' as an extension - for this we need to
         # use suffixes and join.
         # .suffix only takes
-        if ''.join(path.suffixes) in extensions_map:
+        # However, the filename might have a '.' in it, e.g. '1.1.png'.
+        # In this case, try again with just the suffix.
+        if ''.join(path.suffixes[-2:]) in extensions_map or path.suffix in extensions_map:
             yield path
 
 

menpo/io/input/extensions.py

@@ -1,6 +1,6 @@
 # A list of extensions that different importers support.
 from .landmark import LM2Importer, LJSONImporter
-from .image import PILImporter, PILGIFImporter
+from .image import PILImporter, PILGIFImporter, ABSImporter, FLOImporter
 from .landmark_image import ImageASFImporter, ImagePTSImporter
 from .pickle import PickleImporter, GZipPickleImporter
 
@@ -27,7 +27,9 @@
                '.tiff': PILImporter,
                '.xbm': PILImporter,
                # '.pdf': PILImporter,
-               '.xpm': PILImporter}
+               '.xpm': PILImporter,
+               '.abs': ABSImporter,
+               '.flo': FLOImporter}
 
 image_landmark_types = {'.asf': ImageASFImporter,
                         '.lm2': LM2Importer,

menpo/io/input/image.py

@@ -126,3 +126,80 @@ def build(self):
                 self._pil_image.mode))
 
         return images
+
+
+class ABSImporter(Importer):
+    r"""
+    Allows importing the ABS file format from the FRGC dataset.
+
+    The z-min value is stripped from the mesh to make it renderable.
+
+    Parameters
+    ----------
+    filepath : string
+        Absolute filepath of the mesh.
+    """
+
+    def __init__(self, filepath, **kwargs):
+        # Setup class before super class call
+        super(ABSImporter, self).__init__(filepath)
+
+    def build(self):
+        import re
+
+        with open(self.filepath, 'r') as f:
+            # Currently these are unused, but they are in the format
+            # Could possibly store as metadata?
+            # Assume first result for regexes
+            re_rows = re.compile(u'([0-9]+) rows')
+            n_rows = int(re_rows.findall(f.readline())[0])
+            re_cols = re.compile(u'([0-9]+) columns')
+            n_cols = int(re_cols.findall(f.readline())[0])
+
+        # This also loads the mask
+        #   >>> image_data[:, 0]
+        image_data = np.loadtxt(self.filepath, skiprows=3, unpack=True)
+
+        # Replace the lowest value with nan so that we can render properly
+        data_view = image_data[:, 1:]
+        corrupt_value = np.min(data_view)
+        data_view[np.any(np.isclose(data_view, corrupt_value), axis=1)] = np.nan
+
+        return MaskedImage(
+            np.rollaxis(np.reshape(data_view, [n_rows, n_cols, 3]), -1),
+            np.reshape(image_data[:, 0], [n_rows, n_cols]).astype(np.bool),
+            copy=False)
+
+
+class FLOImporter(Importer):
+    r"""
+    Allows importing the Middlebury FLO file format.
+
+    Parameters
+    ----------
+    filepath : string
+        Absolute filepath of the mesh.
+    """
+
+    def __init__(self, filepath, **kwargs):
+        # Setup class before super class call
+        super(FLOImporter, self).__init__(filepath)
+
+    def build(self):
+        with open(self.filepath, 'rb') as f:
+            fingerprint = f.read(4)
+            if fingerprint != 'PIEH':
+                raise ValueError('Invalid FLO file.')
+
+            width, height = np.fromfile(f, dtype=np.uint32, count=2)
+            # read the raw flow data (u0, v0, u1, v1, u2, v2,...)
+            rawData = np.fromfile(f, dtype=np.float32,
+                                  count=width * height * 2)
+
+        shape = (height, width)
+        u_raw = rawData[::2].reshape(shape)
+        v_raw = rawData[1::2].reshape(shape)
+        uv = np.vstack([u_raw[None, ...], v_raw[None, ...]])
+
+        return Image(uv, copy=False)
+

menpo/io/output/base.py

@@ -101,6 +101,10 @@ def export_pickle(obj, fp, overwrite=False):
     compression. If `.pkl.gz` the object will be pickled using Pickle protocol
     2 with gzip compression (at a fixed compression level of 3).
 
+    Note that a special exception is made for `pathlib.Path` objects - they
+    are pickled down as a `pathlib.PurePath` so that pickles can be easily
+    moved between different platforms.
+
     Parameters
     ----------
     obj : ``object``

menpo/io/output/pickle.py

@@ -1,8 +1,55 @@
+from contextlib import contextmanager
+from pathlib import Path, PurePath
 try:
-    import cPickle as pickle
-except ImportError:
+    import cPickle as pickle  # request cPickle manually on Py2
+except ImportError:  # Py3
     import pickle
 
 
+# -------------- Custom pickle behavior for pathlib.Path objects ------------ #
+#
+# We make heavy use of pathlib.Path throughout Menpo - most obviously, any
+# imported item has it's path automatically duck-typed on at self.path.
+#
+# This causes issues with serialization however. pathlib has two families of
+# paths - PurePaths, which you can perform manipulation on but have no
+# connection to the file system, and concrete paths - paths which additionally
+# have functional methods on them like .is_dir() and .mkdir(). You are not
+# allowed to instantiate a concrete path for one platform (e.g. a WindowsPath)
+# on a different platform (e.g. Linux - which is a PosixPath) as the additional
+# filesystem methods will not make sense.
+#
+# As we attach concrete paths liberally in Menpo, many Menpo objects
+# serialized on one platform (e.g. OS X) will not be unpickable on another
+# (e.g. Windows).
+#
+# To alleviate this issue, we override the pickling behavior of concrete paths
+# so that they pickle out as PurePaths. This ensures you can always open
+# pickled objects on other platforms. We do this using a context manager, so
+# don't effect the pickle behavior of these objects globally.
+#
+def _pure_path_reduce(self):
+    # Pickled paths should go to pure paths so pickles are
+    # useful across different OSes
+    return PurePath(self).__class__, tuple(self.parts)
+
+
+@contextmanager
+def pickle_paths_as_pure():
+    r"""
+    Pickle pathlib.Path subclasses as their corresponding pathlib.PurePath
+    """
+    # save out the original method
+    default_reduce = Path.__reduce__
+    # switch out to our PurePath varient
+    Path.__reduce__ = _pure_path_reduce
+    try:
+        yield
+    finally:
+        # always clean up - restore to default behavior
+        Path.__reduce__ = default_reduce
+
+
 def pickle_export(obj, file_handle):
-    pickle.dump(obj, file_handle, protocol=2)
+    with pickle_paths_as_pure():
+        pickle.dump(obj, file_handle, protocol=2)