More stats (#25)

* plain stats encoding

* More explicit dealing with categories and stats derrived therefrom

fastparquet/test/test_read.py

@@ -255,3 +255,18 @@ def test_cat_filters():
 
     filters = [('cat', '==', 'freda'), ('catnum', '!=', 2.5)]
     assert len(pf.to_pandas(filters=filters)) == 1000
+
+
+def test_statistics(tempdir):
+    s = pd.Series([b'a', b'b', b'c']*20)
+    df = pd.DataFrame({'a': s, 'b': s.astype('category'),
+                       'c': s.astype('category').cat.as_ordered()})
+    fastparquet.write(tempdir, df, file_scheme='hive')
+    pf = fastparquet.ParquetFile(tempdir)
+    stat = pf.statistics
+    assert stat['max']['a'] == [b'c']
+    assert stat['min']['a'] == [b'a']
+    assert stat['max']['b'] == [None]
+    assert stat['min']['b'] == [None]
+    assert stat['max']['c'] == [b'c']
+    assert stat['min']['c'] == [b'a']

fastparquet/writer.py

@@ -389,16 +389,23 @@ def write_column(f, data, selement, encoding='PLAIN', compression=None):
         dict_start = f.tell()
         write_thrift(f, ph)
         f.write(bdata)
+        try:
+            max, min = data.max(), data.min()
+            max = encode['PLAIN'](pd.Series([max]), selement)
+            min = encode['PLAIN'](pd.Series([min]), selement)
+        except TypeError:
+            max, min = None, None
         data = data.cat.codes.astype(np.int32)
         cats = True
         encoding = "PLAIN_DICTIONARY"
 
     start = f.tell()
     bdata = definition_data + repetition_data + encode[encoding](data, selement)
     try:
-        max, min = data.max(), data.min()
-        max = encode['PLAIN'](pd.Series([max], dtype=data.dtype), selement)
-        min = encode['PLAIN'](pd.Series([min], dtype=data.dtype), selement)
+        if encoding != 'PLAIN_DICTIONARY':
+            max, min = data.max(), data.min()
+            max = encode['PLAIN'](pd.Series([max], dtype=data.dtype), selement)
+            min = encode['PLAIN'](pd.Series([min], dtype=data.dtype), selement)
     except TypeError:
         max, min = None, None