Skip to content

Commit

Permalink
shuffle stuff around
Browse files Browse the repository at this point in the history
  • Loading branch information
pcmoritz committed Aug 18, 2017
1 parent bcebdfe commit adcc8f7
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 33 deletions.
30 changes: 30 additions & 0 deletions python/pyarrow/includes/libarrow.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -775,6 +775,36 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil:
c_bool strings_to_categorical


cdef extern from "arrow/python/api.h" namespace 'arrow::py':

CStatus SerializePythonSequence(
PyObject* sequence,
shared_ptr[CRecordBatch]* batch_out,
vector[shared_ptr[CTensor]]* tensors_out)

CStatus DeserializePythonSequence(
shared_ptr[CRecordBatch] batch,
vector[shared_ptr[CTensor]] tensors,
PyObject* base,
PyObject** out)


cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:

cdef CStatus WriteSerializedPythonSequence(
shared_ptr[CRecordBatch] batch,
vector[shared_ptr[CTensor]] tensors,
OutputStream* dst)

cdef CStatus ReadSerializedPythonSequence(
shared_ptr[RandomAccessFile] src,
shared_ptr[CRecordBatch]* batch_out,
vector[shared_ptr[CTensor]]* tensors_out)

void set_serialization_callbacks(PyObject* serialize_callback,
PyObject* deserialize_callback);


cdef extern from 'arrow/python/init.h':
int arrow_init_numpy() except -1

Expand Down
35 changes: 2 additions & 33 deletions python/pyarrow/serialization.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,10 @@
# specific language governing permissions and limitations
# under the License.

from libcpp cimport bool as c_bool, nullptr
from libcpp.vector cimport vector as c_vector
from cpython.ref cimport PyObject
from cython.operator cimport dereference as deref

from pyarrow.compat import pickle

cdef extern from "arrow/python/api.h" namespace 'arrow::py':

CStatus SerializePythonSequence(
PyObject* sequence,
shared_ptr[CRecordBatch]* batch_out,
c_vector[shared_ptr[CTensor]]* tensors_out)

CStatus DeserializePythonSequence(
shared_ptr[CRecordBatch] batch,
c_vector[shared_ptr[CTensor]] tensors,
PyObject* base,
PyObject** out)

cdef extern from "arrow/python/api.h" namespace 'arrow::py' nogil:

cdef CStatus WriteSerializedPythonSequence(
shared_ptr[CRecordBatch] batch,
c_vector[shared_ptr[CTensor]] tensors,
OutputStream* dst)

cdef CStatus ReadSerializedPythonSequence(
shared_ptr[RandomAccessFile] src,
shared_ptr[CRecordBatch]* batch_out,
c_vector[shared_ptr[CTensor]]* tensors_out)

void set_serialization_callbacks(PyObject* serialize_callback,
PyObject* deserialize_callback);

def is_named_tuple(cls):
"""Return True if cls is a namedtuple and False otherwise."""
b = cls.__bases__
Expand Down Expand Up @@ -170,7 +139,7 @@ def serialize_sequence(object value, NativeFile sink):
sink.write_handle(&stream)

cdef shared_ptr[CRecordBatch] batch
cdef c_vector[shared_ptr[CTensor]] tensors
cdef vector[shared_ptr[CTensor]] tensors

check_status(SerializePythonSequence(<PyObject*> value, &batch, &tensors))

Expand All @@ -197,7 +166,7 @@ def deserialize_sequence(NativeFile source, object base):
source.read_handle(&stream)

cdef shared_ptr[CRecordBatch] batch
cdef c_vector[shared_ptr[CTensor]] tensors
cdef vector[shared_ptr[CTensor]] tensors

with nogil:
check_status(ReadSerializedPythonSequence(stream, &batch, &tensors))
Expand Down

0 comments on commit adcc8f7

Please sign in to comment.