Fixed Issue #72, where python collections of ndarrays that have the s…

…ame shape are incorrectly converted to multideminsional object arrays of the subelements instead of 1D object arrays of the elements.

(cherry picked from commit 82e4268)

README.rst

@@ -408,6 +408,9 @@ Versions
         * Fixed an assertion in the tests to handle field re-ordering when
           no metadata is used for structured dtypes that only worked on
           older versions of numpy.
+        * Issue #72. Fixed bug where python collections filled with ndarrays
+          that all have the same shape were converted to multi-dimensional
+          object ndarrays instead of a 1D object ndarray of the elements.
 
 0.1.14. Bugfix release that also added a couple features.
         * Issue #45. Fixed syntax errors in unicode strings for Python

hdf5storage/Marshallers.py

@@ -1838,8 +1838,10 @@ def write(self, f, grp, name, data, type_string, options):
         # to be grabbed now as the parent function will have a modified
         # form of data to guess from if not given the right one
         # explicitly.
+        out = np.zeros(dtype='object', shape=(len(data), ))
+        out[:] = data
         NumpyScalarArrayMarshaller.write(self, f, grp, name,
-                                         np.object_(data),
+                                         out,
                                          self.get_type_string(data,
                                          type_string), options)
 

tests/asserts.py

@@ -156,7 +156,10 @@ def assert_equal_none_format(a, b, options=None):
             assert_equal_none_format(keys, tuple(b.keys()), options)
             assert_equal_none_format(values, tuple(b.values()), options)
     elif type(b) in (list, tuple, set, frozenset, collections.deque):
-        assert_equal_none_format(a, np.object_(list(b)), options)
+        b_conv = np.zeros(dtype='object', shape=(len(b), ))
+        for i, v in enumerate(b):
+            b_conv[i] = v
+        assert_equal_none_format(a, b_conv, options)
     elif not isinstance(b, (np.generic, np.ndarray)):
         if b is None:
             # It should be np.float64([])
@@ -323,7 +326,10 @@ def assert_equal_matlab_format(a, b, options=None):
             assert_equal_matlab_format(values, tuple(b.values()),
                                        options)
     elif type(b) in (list, tuple, set, frozenset, collections.deque):
-        assert_equal_matlab_format(a, np.object_(list(b)), options)
+        b_conv = np.zeros(dtype='object', shape=(len(b), ))
+        for i, v in enumerate(b):
+            b_conv[i] = v
+        assert_equal_matlab_format(a, b_conv, options)
     elif not isinstance(b, (np.generic, np.ndarray)):
         if b is None:
             # It should be np.zeros(shape=(0, 1), dtype='float64'))

tests/test_write_readback.py

@@ -242,15 +242,29 @@ def check_numpy_chararray_empty(self, num_chars):
                                   self.options)
         self.assert_equal(out, data)
 
-    def check_python_collection(self, tp):
+    def check_python_collection(self, tp, same_dims):
         # Makes a random collection of the specified type, writes it and
         # reads it back, and then compares it.
         if tp in (set, frozenset):
             data = tp(random_list(max_list_length,
                       python_or_numpy='python'))
         else:
-            data = tp(random_list(max_list_length,
-                      python_or_numpy='numpy'))
+            if same_dims == 'same-dims':
+                shape = random_numpy_shape(random.randrange(2, 4),
+                                           random.randrange(1, 4))
+                dtypes = ('uint8', 'uint16', 'uint32', 'uint64',
+                          'int8', 'int16', 'int32', 'int64',
+                          'float32', 'float64',
+                          'complex64', 'complex128')
+                data = tp([random_numpy(shape, random.choice(dtypes),
+                                        allow_nan=True)
+                           for i in range(random.randrange(2, 7))])
+
+            elif same_dims == 'diff-dims':
+                data = tp(random_list(max_list_length,
+                                      python_or_numpy='numpy'))
+            else:
+                raise ValueError('invalid value of same_dims')
         out = self.write_readback(data, random_name(),
                                   self.options)
         self.assert_equal(out, data)
@@ -629,7 +643,8 @@ def test_numpy_chararray_empty(self):
 
     def test_python_collection(self):
         for tp in (list, tuple, set, frozenset, collections.deque):
-            yield self.check_python_collection, tp
+            yield self.check_python_collection, tp, 'same-dims'
+            yield self.check_python_collection, tp, 'diff-dims'
 
     def test_dict_like(self):
         for tp in self.dict_like: