Make everything work on pypy. Especially freeimage needed work

imageio/core/__init__.py

@@ -6,7 +6,7 @@
 (everything but the plugins).
 """
 
-from .util import Image, Dict, appdata_dir, urlopen  # noqa
+from .util import Image, Dict, asarray, appdata_dir, urlopen  # noqa
 from .util import BaseProgressIndicator, StdoutProgressIndicator  # noqa
 from .util import string_types, text_type, binary_type, IS_PYPY  # noqa
 from .util import get_platform  # noqa

imageio/core/format.py

@@ -42,7 +42,7 @@
 
 import numpy as np
 
-from . import Image
+from . import Image, asarray
 from . import string_types, text_type, binary_type  # noqa
 
 
@@ -463,7 +463,7 @@ def append_data(self, im, meta=None):
                 total_meta.update(meta)        
 
             # Decouple meta info
-            im = np.asarray(im)
+            im = asarray(im)
             # Call
             return self._append_data(im, total_meta)
 

imageio/core/request.py

@@ -33,7 +33,7 @@
 EXAMPLE_IMAGES = {
     'astronaut.png': 'Image of the astronaut Eileen Collins',
     'camera.png': 'Classic grayscale image of a photographer',
-    'checkerboard.png' : 'Black and white image of a chekerboard',
+    'checkerboard.png': 'Black and white image of a chekerboard',
     'clock.png': 'Photo of a clock with motion blur (Stefan van der Walt)',
     'coffee.png': 'Image of a cup of coffee (Rachel Michetti)',
 

imageio/core/util.py

@@ -117,11 +117,24 @@ def __array_wrap__(self, out, context=None):
         if not out.shape:
             return out.dtype.type(out)  # Scalar
         elif out.shape != self.shape:
-            return np.asarray(out)
+            return out.view(type=np.ndarray)
         else:
             return out  # Type Image
 
 
+def asarray(a):
+    """ Pypy-safe version of np.asarray. Pypy's np.asarray consumes a
+    *lot* of memory if the given array is an ndarray subclass. This
+    function does not.
+    """
+    if isinstance(a, np.ndarray):
+        if IS_PYPY:
+            a = a.copy()  # pypy has issues with base views
+        plain = a.view(type=np.ndarray)
+        return plain
+    return np.asarray(a)
+
+
 try:
     from collections import OrderedDict as _dict
 except ImportError:

imageio/plugins/_freeimage.py

@@ -24,6 +24,8 @@
 from imageio.core import get_remote_file, load_lib, Dict, appdata_dir
 from imageio.core import string_types, binary_type, IS_PYPY, get_platform
 
+TEST_NUMPY_NO_STRIDES = False  # To test pypy fallback
+
 
 def get_freeimage_lib():
     """ Ensure we have our version of the binary freeimage lib.
@@ -612,7 +614,7 @@ def get_meta_data(self):
                         elif tag_type in METADATA_DATATYPE.dtypes:
                             dtype = METADATA_DATATYPE.dtypes[tag_type]
                             if IS_PYPY and isinstance(dtype, (list, tuple)):
-                                pass  # pragma: nocover - or we get a segfault
+                                pass  # pragma: no cover - or we get a segfault
                             else:
                                 try:
                                     tag_val = numpy.fromstring(tag_bytes, 
@@ -667,9 +669,15 @@ def get_tag_type_number(dtype):
 
                     try:
                         # Convert Python value to FI type, val
+                        is_ascii = False
                         if isinstance(tag_val, string_types):
+                            try:
+                                tag_bytes = tag_val.encode('ascii')
+                                is_ascii = True
+                            except UnicodeError:
+                                pass
+                        if is_ascii:
                             tag_type = METADATA_DATATYPE.FIDT_ASCII
-                            tag_bytes = tag_val.encode('utf-8')
                             tag_count = len(tag_bytes)
                         else:
                             if not hasattr(tag_val, 'dtype'):
@@ -712,7 +720,7 @@ class FIBitmap(FIBaseBitmap):
     def allocate(self, array):
 
         # Prepare array
-        array = numpy.asarray(array)
+        assert isinstance(array, numpy.ndarray)
         shape = array.shape
         dtype = array.dtype
 
@@ -846,7 +854,7 @@ def save_to_filename(self, filename=None):
 
     def get_image_data(self):
         dtype, shape, bpp = self._get_type_and_shape()
-        array = self._wrap_bitmap_bits_in_array(shape, dtype, bpp)
+        array = self._wrap_bitmap_bits_in_array(shape, dtype, False)
         with self._fi as lib:
             isle = lib.FreeImage_IsLittleEndian()
 
@@ -884,7 +892,7 @@ def n(arr):
     def set_image_data(self, array):
 
         # Prepare array
-        array = numpy.asarray(array)
+        assert isinstance(array, numpy.ndarray)
         shape = array.shape
         dtype = array.dtype
         with self._fi as lib:
@@ -903,7 +911,7 @@ def set_image_data(self, array):
 
         def n(arr):  # normalise to freeimage's in-memory format
             return arr.T[:, ::-1]
-        wrapped_array = self._wrap_bitmap_bits_in_array(w_shape, dtype)
+        wrapped_array = self._wrap_bitmap_bits_in_array(w_shape, dtype, True)
         # swizzle the color components and flip the scanlines to go to
         # FreeImage's BGR[A] and upside-down internal memory format
         if len(shape) == 3:
@@ -926,24 +934,26 @@ def n(arr):  # normalise to freeimage's in-memory format
                     wrapped_array[3] = n(A)
         else:
             wrapped_array[:] = n(array)
+        if self._need_finish:
+            self._finish_wrapped_array(wrapped_array)
+
         if len(shape) == 2 and dtype.type == numpy.uint8:
             with self._fi as lib:
                 palette = lib.FreeImage_GetPalette(self._bitmap)
             palette = ctypes.c_void_p(palette)
             if not palette:
                 raise RuntimeError('Could not get image palette')
             try:
-                ctypes.memmove(palette, GREY_PALETTE.ctypes.data, 1024)
-            except Exception:  # pragma: no cover
-                if IS_PYPY:
-                    print('WARNING: Cannot set image pallette on Pypy')
-                else:
-                    raise
+                palette_data = GREY_PALETTE.ctypes.data
+            except Exception:  # pragma: no cover - IS_PYPY
+                palette_data = GREY_PALETTE.__array_interface__['data'][0]
+            ctypes.memmove(palette, palette_data, 1024)
 
-    def _wrap_bitmap_bits_in_array(self, shape, dtype, bpp=None):
+    def _wrap_bitmap_bits_in_array(self, shape, dtype, save):
         """Return an ndarray view on the data in a FreeImage bitmap. Only
         valid for as long as the bitmap is loaded (if single page) / locked
-        in memory (if multipage).
+        in memory (if multipage). This is used in loading data, but
+        also during saving, to prepare a strided numpy array buffer.
         
         """
         # Get bitmap info
@@ -965,21 +975,59 @@ def _wrap_bitmap_bits_in_array(self, shape, dtype, bpp=None):
         # Create numpy array and return
         data = (ctypes.c_char*byte_size).from_address(bits)
         try:
-            array = numpy.ndarray(shape, dtype=dtype, buffer=data, 
-                                  strides=strides)
-        except NotImplementedError:  # pragma: no cover
-            # Pypy compatibility. 
-            # (Note that e.g. b''.join(data) consumes vast amounts of memory)
-            bytes = binary_type(bytearray(data))
-            array = numpy.fromstring(bytes, dtype=dtype)
-            # Deal with strides
-            if len(shape) == 3:
-                array.shape = shape[0], strides[-1]/shape[0], shape[2]
-                array = array[:shape[0], :shape[1], :shape[2]] 
+            self._need_finish = False
+            if TEST_NUMPY_NO_STRIDES:
+                raise NotImplementedError()
+            return numpy.ndarray(shape, dtype=dtype, buffer=data, 
+                                 strides=strides)
+        except NotImplementedError:
+            # IS_PYPY - not very efficient. We create a C-contiguous
+            # numpy array (because pypy does not support Fortran-order)
+            # and shape it such that the rest of the code can remain.
+            if save:
+                self._need_finish = True  # Flag to use _finish_wrapped_array
+                return numpy.zeros(shape, dtype=dtype)
             else:
-                array.shape = strides[-1], shape[1]
-                array = array[:shape[0], :shape[1]]
-        return array
+                bytes = binary_type(bytearray(data))
+                array = numpy.fromstring(bytes, dtype=dtype)
+                # Deal with strides
+                if len(shape) == 3:
+                    array.shape = shape[2], strides[-1]/shape[0], shape[0]
+                    array2 = array[:shape[2], :shape[1], :shape[0]]
+                    array = numpy.zeros(shape, dtype=array.dtype)
+                    for i in range(shape[0]):
+                        array[i] = array2[:, :, i].T
+                else:
+                    array.shape = shape[1], strides[-1]
+                    array = array[:shape[1], :shape[0]].T
+                return array
+
+    def _finish_wrapped_array(self, array):  # IS_PYPY
+        """ Hardcore way to inject numpy array in bitmap.
+        """
+        # Get bitmap info
+        with self._fi as lib:
+            pitch = lib.FreeImage_GetPitch(self._bitmap)
+            bits = lib.FreeImage_GetBits(self._bitmap)
+            bpp = lib.FreeImage_GetBPP(self._bitmap)
+        # Get channels and realwidth
+        nchannels = bpp // 8 // array.itemsize
+        realwidth = pitch // nchannels
+        # Apply padding for pitch if necessary
+        extra = realwidth - array.shape[-2]
+        assert extra >= 0 and extra < 10
+        # Make sort of Fortran, also take padding (i.e. pitch) into account
+        newshape = array.shape[-1], realwidth, nchannels
+        array2 = numpy.zeros(newshape, array.dtype)
+        if nchannels == 1:
+            array2[:, :array.shape[-2], 0] = array.T
+        else:
+            for i in range(nchannels):
+                array2[:, :array.shape[-2], i] = array[i, :, :].T
+        # copy data
+        data_ptr = array2.__array_interface__['data'][0]
+        ctypes.memmove(bits, data_ptr, array2.nbytes)
+        del array2
 
     def _get_type_and_shape(self):
         bitmap = self._bitmap

imageio/plugins/_swf.py

@@ -105,7 +105,7 @@ def append(self, bits):
     def reverse(self):
         """ In-place reverse. """
         tmp = self.data[:self._len].copy()
-        self.data[:self._len] = np.flipud(tmp)
+        self.data[:self._len] = tmp[::-1]
 
     def tobytes(self):
         """ Convert to bytes. If necessary,

imageio/plugins/avbin.py

@@ -410,10 +410,14 @@ def _get_data(self, index, out=None):
                     continue
 
                 # Decode the image, storing data in the out array
+                try:
+                    ptr = out.ctypes.data
+                except Exception:  # pragma: no cover - IS_PYPY
+                    ptr = out.__array_interface__['data'][0]
                 result = avbin.avbin_decode_video(self._stream, 
                                                   self._packet.data, 
                                                   self._packet.size, 
-                                                  out.ctypes.data)
+                                                  ptr)
 
                 # Check for success. If not, continue reading the file stream
                 # AK: disabled for now, because this will make the file

imageio/plugins/ffmpeg.py

@@ -462,7 +462,8 @@ def _append_data(self, im, meta):
             depth = 1 if im.ndim == 2 else im.shape[2]
 
             # Ensure that image is in uint8
-            im = im.astype(np.uint8, copy=False)
+            if im.dtype != np.uint8:
+                im = im.astype(np.uint8)  # pypy: no support copy=False
 
             # Set size and initialize if not initialized yet
             if self._size is None:

imageio/plugins/freeimage.py

@@ -118,7 +118,7 @@ def _append_data(self, im, meta):
                                    'can only append image data once.')
             # Pop unit dimension for grayscale images
             if im.ndim == 3 and im.shape[-1] == 1:
-                im = im.reshape(im.shape[:2])
+                im = im[:, :, 0]
             # Lazy instantaion of the bitmap, we need image data
             if self._bm is None:
                 self._bm = fi.create_bitmap(self.request.filename, 

imageio/plugins/freeimagemulti.py

@@ -72,7 +72,7 @@ def _close(self):
         def _append_data(self, im, meta):
             # Prepare data
             if im.ndim == 3 and im.shape[-1] == 1:
-                im = im.reshape(im.shape[:2])
+                im = im[:, :, 0]
             if im.dtype in (np.float32, np.float64):
                 im = (im * 255).astype(np.uint8)
             # Create sub bitmap

imageio/plugins/swf.py

@@ -289,7 +289,7 @@ def _complete(self):
         def _append_data(self, im, meta):
             # Correct shape and type
             if im.ndim == 3 and im.shape[-1] == 1:
-                im = im.reshape(im.shape[:2])
+                im = im[:, :, 0]
             if im.dtype in (np.float32, np.float64):
                 im = (im * 255).astype(np.uint8)
             # Get frame size

tests/test_avbin.py

@@ -8,11 +8,12 @@
 from imageio import core
 from imageio.core import get_remote_file
 
+# if IS_PYPY:
+#     skip('AVBIn not supported on pypy')
 
 test_dir = get_test_dir()
 
-
-_prepared = None
+mean = lambda x: x.sum() / x.size  # pypy-compat mean
 
 
 def test_read():
@@ -26,7 +27,7 @@ def test_read():
         im = reader.get_next_data()
         assert im.shape == (720, 1280, 3)
         # todo: fix this
-        #assert im.mean() > 100 and im.mean() < 115  KNOWN FAIL
+        #assert mean(im) > 100 and mean(im) < 115  KNOWN FAIL
 
     # We can rewind
     reader.get_data(0)
@@ -100,7 +101,7 @@ def test_read_format():
     for i in range(10):
         im = reader.get_next_data()
         assert im.shape == (720, 1280, 3)
-        assert im.mean() > 100 and im.mean() < 115
+        assert mean(im) > 100 and mean(im) < 115
 
 
 def test_stream():
@@ -134,7 +135,7 @@ def test_format_selection():
     assert imageio.formats['.mp4'] is F
 
 
-def show():
+def show_in_mpl():
     reader = imageio.read('cockatoo.mp4')
     for i in range(10):
         reader.get_next_data()

tests/test_core.py

@@ -613,6 +613,17 @@ def test_util():
     # Test get_platform
     platforms = 'win32', 'win64', 'linux32', 'linux64', 'osx32', 'osx32'
     assert core.get_platform() in platforms
+
+    # Test asarray
+    im1 = core.asarray([[1, 2, 3], [4, 5, 6]])
+    im2 = im1.view(type=core.Image)
+    im3 = core.asarray(im2)
+    assert type(im2) != np.ndarray
+    assert type(im3) == np.ndarray
+    for i in (1, 2, 3):
+        im1[0, 0] = i
+        assert im2[0, 0] == i
+        assert im3[0, 0] == i
 
 
 def test_progres_bar(sleep=0):