forked from pandas-dev/pandas
/
feather_format.py
127 lines (94 loc) · 3.88 KB
/
feather_format.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
""" feather-format compat """
from distutils.version import LooseVersion
from pandas.compat import range
from pandas.util._decorators import deprecate_kwarg
from pandas import DataFrame, Int64Index, RangeIndex
from pandas.io.common import _stringify_path
def _try_import():
# since pandas is a dependency of pyarrow
# we need to import on first use
try:
import pyarrow
from pyarrow import feather
except ImportError:
# give a nice error message
raise ImportError("pyarrow is not installed\n\n"
"you can install via conda\n"
"conda install pyarrow -c conda-forge\n"
"or via pip\n"
"pip install -U pyarrow\n")
if LooseVersion(pyarrow.__version__) < LooseVersion('0.9.0'):
raise ImportError("pyarrow >= 0.9.0 required for feather support\n\n"
"you can install via conda\n"
"conda install pyarrow -c conda-forge"
"or via pip\n"
"pip install -U pyarrow\n")
return feather, pyarrow
def to_feather(df, path):
"""
Write a DataFrame to the feather-format
Parameters
----------
df : DataFrame
path : string file path, or file-like object
"""
path = _stringify_path(path)
if not isinstance(df, DataFrame):
raise ValueError("feather only support IO with DataFrames")
feather = _try_import()[0]
valid_types = {'string', 'unicode'}
# validate index
# --------------
# validate that we have only a default index
# raise on anything else as we don't serialize the index
if not isinstance(df.index, Int64Index):
raise ValueError("feather does not support serializing {} "
"for the index; you can .reset_index()"
"to make the index into column(s)".format(
type(df.index)))
if not df.index.equals(RangeIndex.from_range(range(len(df)))):
raise ValueError("feather does not support serializing a "
"non-default index for the index; you "
"can .reset_index() to make the index "
"into column(s)")
if df.index.name is not None:
raise ValueError("feather does not serialize index meta-data on a "
"default index")
# validate columns
# ----------------
# must have value column names (strings only)
if df.columns.inferred_type not in valid_types:
raise ValueError("feather must have string column names")
feather.write_feather(df, path)
@deprecate_kwarg(old_arg_name='nthreads', new_arg_name='use_threads')
def read_feather(path, columns=None, use_threads=True):
"""
Load a feather-format object from the file path
.. versionadded 0.20.0
Parameters
----------
path : string file path, or file-like object
columns : sequence, default None
If not provided, all columns are read
.. versionadded 0.24.0
nthreads : int, default 1
Number of CPU threads to use when reading to pandas.DataFrame
.. versionadded 0.21.0
.. deprecated 0.24.0
use_threads : bool, default True
Whether to parallelize reading using multiple threads
.. versionadded 0.24.0
Returns
-------
type of object stored in file
"""
feather, pyarrow = _try_import()
path = _stringify_path(path)
if LooseVersion(pyarrow.__version__) < LooseVersion('0.11.0'):
int_use_threads = int(use_threads)
if int_use_threads < 1:
int_use_threads = 1
return feather.read_feather(path, columns=columns,
nthreads=int_use_threads)
return feather.read_feather(path, columns=columns,
use_threads=bool(use_threads))