Merge pull request #161 from martindurant/ordered_dicts

Fail on ordering dict statistics

fastparquet/api.py

@@ -603,10 +603,14 @@ def sorted_partitioned_columns(pf):
         min, max = s['min'][c], s['max'][c]
         if any(x is None for x in min + max):
             continue
-        if (sorted(min) == min and
-            sorted(max) == max and
-            all(mx < mn for mx, mn in zip(max[:-1], min[1:]))):
-            out[c] = {'min': min, 'max': max}
+        try:
+            if (sorted(min) == min and
+                sorted(max) == max and
+                all(mx < mn for mx, mn in zip(max[:-1], min[1:]))):
+                out[c] = {'min': min, 'max': max}
+        except TypeError:
+            # because some types, e.g., dicts cannot be sorted/compared
+            continue
     return out
 
 

fastparquet/test/test_api.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 
 import os
 
@@ -70,18 +71,21 @@ def test_empty_statistics(tempdir):
 
 def test_sorted_row_group_columns(tempdir):
     df = pd.DataFrame({'x': [1, 2, 3, 4],
+                       'v': [{'a': 0}, {'b': -1}, {'c': 5}, {'a': 0}],
                        'y': [1.0, 2.0, 1.0, 2.0],
                        'z': ['a', 'b', 'c', 'd']})
 
     fn = os.path.join(tempdir, 'foo.parquet')
-    write(fn, df, row_group_offsets=[0, 2])
+    write(fn, df, row_group_offsets=[0, 2], object_encoding={'v': 'json',
+                                                             'z': 'utf8'})
 
     pf = ParquetFile(fn)
 
     result = sorted_partitioned_columns(pf)
     expected = {'x': {'min': [1, 3], 'max': [2, 4]},
                 'z': {'min': ['a', 'c'], 'max': ['b', 'd']}}
 
+    # NB column v should not feature, as dict are unorderable
     assert result == expected
 
 

fastparquet/test/test_output.py

@@ -174,13 +174,16 @@ def test_roundtrip_complex(tempdir, scheme,):
     ])
 def test_datetime_roundtrip(tempdir, df, capsys):
     fname = os.path.join(tempdir, 'test.parquet')
-    write(fname, df)
-
-    r = ParquetFile(fname)
-    out, err = capsys.readouterr()
+    w = False
     if 'x' in df and str(df.x.dtype.tz) == 'Europe/London':
-        # warning happens first time only
-        assert "UTC" in err
+        with pytest.warns(UserWarning) as w:
+            write(fname, df)
+    else:
+        write(fname, df)
+    r = ParquetFile(fname)
+
+    if w:
+        assert "UTC" in str(w.list[0].message)
 
     df2 = r.to_pandas()
     if 'x' in df: