Remove thrift_copy

fastparquet/test/test_util.py

@@ -1,8 +1,6 @@
 import pytest
 
-from fastparquet.util import (thrift_copy, analyse_paths, get_file_scheme,
-                              val_to_num)
-from fastparquet import parquet_thrift
+from fastparquet.util import analyse_paths, get_file_scheme, val_to_num
 
 
 def test_analyse_paths():
@@ -27,23 +25,6 @@ def test_analyse_paths():
     assert (base, out) == ('c', ['cat=1\\a', 'cat=2\\b', 'cat=1\\c'])
 
 
-def test_thrift_copy():
-    fmd = parquet_thrift.FileMetaData()
-    rg0 = parquet_thrift.RowGroup()
-    rg0.num_rows = 5
-    rg1 = parquet_thrift.RowGroup()
-    rg1.num_rows = 15
-    fmd.row_groups = [rg0, rg1]
-
-    fmd2 = thrift_copy(fmd)
-
-    assert fmd is not fmd2
-    assert fmd == fmd2
-    assert fmd2.row_groups[0] is not rg0
-    rg0.num_rows = 25
-    assert fmd2.row_groups[0].num_rows == 5
-
-
 def test_file_scheme():
     paths = [None, None]
     assert get_file_scheme(paths) == 'simple'

fastparquet/thrift_structures.py

@@ -73,27 +73,6 @@ def is_thrift_item(item):
     return hasattr(item, 'thrift_spec') and hasattr(item, 'read')
 
 
-def thrift_copy(structure):
-    """
-    Recursively copy a thriftpy structure
-    """
-    base = structure.__class__()
-    for key in dir(structure):
-        if key.startswith('_') or key in ['thrift_spec', 'read', 'write',
-                                          'default_spec', 'validate']:
-            continue
-        val = getattr(structure, key)
-        if isinstance(val, list):
-            setattr(base, key, [thrift_copy(item)
-                                if is_thrift_item(item)
-                                else item for item in val])
-        elif is_thrift_item(val):
-            setattr(base, key, thrift_copy(val))
-        else:
-            setattr(base, key, val)
-    return base
-
-
 def thrift_print(structure, offset=0):
     """
     Handy recursive text ouput for thrift structures
@@ -143,7 +122,9 @@ def setstate_method(self, state):
     self.__dict__ = out.__dict__
 
 
-for cls in [parquet_thrift.FileMetaData, parquet_thrift.RowGroup]:
+for cls in [parquet_thrift.FileMetaData,
+            parquet_thrift.RowGroup,
+            parquet_thrift.ColumnChunk]:
     bind_method(cls, '__getstate__', getstate_method)
     bind_method(cls, '__setstate__', setstate_method)
     bind_method(cls, '__copy__', copy_method)

fastparquet/util.py

@@ -1,11 +1,11 @@
 import ast
+import copy
 import os
 import os.path
 import pandas as pd
 import re
 import six
 
-from .thrift_structures import thrift_copy
 try:
     from pandas.api.types import is_categorical_dtype
 except ImportError:
@@ -133,7 +133,7 @@ def metadata_from_many(file_list, verify_schema=False, open_with=default_open,
             if pf._schema != pfs[0]._schema:
                 raise ValueError('Incompatible schemas')
 
-    fmd = thrift_copy(pfs[0].fmd)  # we inherit "created by" field
+    fmd = copy.copy(pfs[0].fmd)  # we inherit "created by" field
     fmd.row_groups = []
 
     for pf, fn in zip(pfs, file_list):
@@ -142,7 +142,8 @@ def metadata_from_many(file_list, verify_schema=False, open_with=default_open,
             # anyway.
             raise ValueError('Cannot merge multi-file input', fn)
         for rg in pf.row_groups:
-            rg = thrift_copy(rg)
+            rg = copy.copy(rg)
+            rg.columns = [copy.copy(c) for c in rg.columns]
             for chunk in rg.columns:
                 chunk.file_path = fn
             fmd.row_groups.append(rg)

fastparquet/writer.py

@@ -20,7 +20,7 @@
 from .converted_types import tobson
 from . import encoding, api
 from .util import (default_open, default_mkdirs, sep_from_open,
-                   ParquetException, thrift_copy, index_like, PY2, STR_TYPE,
+                   ParquetException, index_like, PY2, STR_TYPE,
                    check_column_names, metadata_from_many, created_by,
                    get_column_metadata)
 from .speedups import array_encode_utf8, pack_byte_array