Moved Bytes to own crate

Cargo.toml

@@ -15,6 +15,7 @@ name = "arrow2"
 bench = false
 
 [dependencies]
+foreign_vec = "0.1.0"
 either = "1.6"
 num-traits = "0.2"
 dyn-clone = "1"

src/array/binary/ffi.rs

@@ -9,7 +9,7 @@ use crate::error::Result;
 use super::BinaryArray;
 
 unsafe impl<O: Offset> ToFfi for BinaryArray<O> {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.offsets.as_ptr().cast::<u8>()),

src/array/boolean/ffi.rs

@@ -9,7 +9,7 @@ use crate::error::Result;
 use super::BooleanArray;
 
 unsafe impl ToFfi for BooleanArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.values.as_ptr()),

src/array/dictionary/ffi.rs

@@ -7,7 +7,7 @@ use crate::{
 use super::{DictionaryArray, DictionaryKey};
 
 unsafe impl<K: DictionaryKey> ToFfi for DictionaryArray<K> {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         self.keys.buffers()
     }
 

src/array/ffi.rs

@@ -9,7 +9,7 @@ use crate::error::Result;
 /// Implementing this trait incorrect will lead to UB
 pub(crate) unsafe trait ToFfi {
     /// The pointers to the buffers.
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>>;
+    fn buffers(&self) -> Vec<Option<*const u8>>;
 
     /// The children
     fn children(&self) -> Vec<Box<dyn Array>> {
@@ -47,7 +47,7 @@ macro_rules! ffi_dyn {
 
 type BuffersChildren = (
     usize,
-    Vec<Option<std::ptr::NonNull<u8>>>,
+    Vec<Option<*const u8>>,
     Vec<Box<dyn Array>>,
     Option<Box<dyn Array>>,
 );

src/array/fixed_size_binary/ffi.rs

@@ -8,7 +8,7 @@ use crate::{
 use super::FixedSizeBinaryArray;
 
 unsafe impl ToFfi for FixedSizeBinaryArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.values.as_ptr().cast::<u8>()),

src/array/fixed_size_list/ffi.rs

@@ -9,7 +9,7 @@ use crate::{
 };
 
 unsafe impl ToFfi for FixedSizeListArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![self.validity.as_ref().map(|x| x.as_ptr())]
     }
 

src/array/list/ffi.rs

@@ -4,7 +4,7 @@ use super::super::{ffi::ToFfi, Array, Offset};
 use super::ListArray;
 
 unsafe impl<O: Offset> ToFfi for ListArray<O> {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.offsets.as_ptr().cast::<u8>()),

src/array/map/ffi.rs

@@ -4,7 +4,7 @@ use super::super::{ffi::ToFfi, Array};
 use super::MapArray;
 
 unsafe impl ToFfi for MapArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.offsets.as_ptr().cast::<u8>()),

src/array/null.rs

@@ -127,7 +127,7 @@ impl std::fmt::Debug for NullArray {
 }
 
 unsafe impl ToFfi for NullArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         // `None` is technically not required by the specification, but older C++ implementations require it, so leaving
         // it here for backward compatibility
         vec![None]

src/array/primitive/ffi.rs

@@ -10,7 +10,7 @@ use crate::error::Result;
 use super::PrimitiveArray;
 
 unsafe impl<T: NativeType> ToFfi for PrimitiveArray<T> {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.values.as_ptr().cast::<u8>()),

src/array/struct_/ffi.rs

@@ -3,7 +3,7 @@ use super::StructArray;
 use crate::{error::Result, ffi};
 
 unsafe impl ToFfi for StructArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![self.validity.as_ref().map(|x| x.as_ptr())]
     }
 

src/array/union/ffi.rs

@@ -4,7 +4,7 @@ use super::super::{ffi::ToFfi, Array};
 use super::UnionArray;
 
 unsafe impl ToFfi for UnionArray {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         if let Some(offsets) = &self.offsets {
             vec![
                 Some(self.types.as_ptr().cast::<u8>()),

src/array/utf8/ffi.rs

@@ -8,7 +8,7 @@ use crate::{
 use super::Utf8Array;
 
 unsafe impl<O: Offset> ToFfi for Utf8Array<O> {
-    fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
+    fn buffers(&self) -> Vec<Option<*const u8>> {
         vec![
             self.validity.as_ref().map(|x| x.as_ptr()),
             Some(self.offsets.as_ptr().cast::<u8>()),

src/bitmap/immutable.rs

@@ -1,8 +1,8 @@
+use std::{iter::FromIterator, ops::Deref, sync::Arc};
+
 use either::Either;
-use std::iter::FromIterator;
-use std::sync::Arc;
 
-use crate::{buffer::bytes::Bytes, error::Error, trusted_len::TrustedLen};
+use crate::{buffer::Bytes, error::Error, trusted_len::TrustedLen};
 
 use super::{
     chunk_iter_to_vec,
@@ -216,8 +216,8 @@ impl Bitmap {
 
     /// Returns a pointer to the start of this [`Bitmap`] (ignores `offsets`)
     /// This pointer is allocated iff `self.len() > 0`.
-    pub(crate) fn as_ptr(&self) -> std::ptr::NonNull<u8> {
-        self.bytes.ptr()
+    pub(crate) fn as_ptr(&self) -> *const u8 {
+        self.bytes.deref().as_ptr()
     }
 
     /// Returns a pointer to the start of this [`Bitmap`] (ignores `offsets`)

src/buffer/immutable.rs

@@ -1,8 +1,8 @@
-use std::{iter::FromIterator, sync::Arc, usize};
+use std::{iter::FromIterator, ops::Deref, sync::Arc, usize};
 
 use crate::types::NativeType;
 
-use super::bytes::Bytes;
+use super::Bytes;
 
 /// [`Buffer`] is a contiguous memory region of plain old data types
 /// that can be shared across thread boundaries.
@@ -33,7 +33,7 @@ use super::bytes::Bytes;
 /// // but cloning forbids getting mut since `slice` and `buffer` now share data
 /// assert_eq!(buffer.get_mut(), None);
 /// ```
-#[derive(Clone, PartialEq)]
+#[derive(Clone)]
 pub struct Buffer<T: NativeType> {
     /// the internal byte buffer.
     data: Arc<Bytes<T>>,
@@ -46,6 +46,13 @@ pub struct Buffer<T: NativeType> {
     length: usize,
 }
 
+impl<T: NativeType> PartialEq for Buffer<T> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        self.deref() == other.deref()
+    }
+}
+
 impl<T: NativeType> std::fmt::Debug for Buffer<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         std::fmt::Debug::fmt(&**self, f)
@@ -127,8 +134,8 @@ impl<T: NativeType> Buffer<T> {
 
     /// Returns a pointer to the start of this buffer.
     #[inline]
-    pub(crate) fn as_ptr(&self) -> std::ptr::NonNull<T> {
-        self.data.ptr()
+    pub(crate) fn as_ptr(&self) -> *const T {
+        self.data.deref().as_ptr()
     }
 
     /// Returns the offset of this buffer.

src/buffer/mod.rs

@@ -2,6 +2,8 @@
 
 mod immutable;
 
-pub(crate) mod bytes;
+use crate::ffi::InternalArrowArray;
+
+pub(crate) type Bytes<T> = foreign_vec::ForeignVec<Box<InternalArrowArray>, T>;
 
 pub use immutable::Buffer;

src/ffi/array.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use crate::{
     array::*,
     bitmap::{utils::bytes_for, Bitmap},
-    buffer::{bytes::Bytes, Buffer},
+    buffer::{Buffer, Bytes},
     datatypes::{DataType, PhysicalType},
     error::{Error, Result},
     ffi::schema::get_child,
@@ -101,7 +101,7 @@ impl ArrowArray {
         let buffers_ptr = buffers
             .iter()
             .map(|maybe_buffer| match maybe_buffer {
-                Some(b) => b.as_ptr() as *const std::os::raw::c_void,
+                Some(b) => *b as *const std::os::raw::c_void,
                 None => std::ptr::null(),
             })
             .collect::<Box<[_]>>();
@@ -195,7 +195,7 @@ unsafe fn create_buffer<T: NativeType>(
     let len = buffer_len(array, data_type, index)?;
     let offset = buffer_offset(array, data_type, index);
     let bytes = ptr
-        .map(|ptr| Bytes::from_owned(ptr, len, owner))
+        .map(|ptr| Bytes::from_foreign(ptr.as_ptr(), len, owner))
         .ok_or_else(|| Error::OutOfSpec(format!("The buffer at position {} is null", index)))?;
 
     Ok(Buffer::from_bytes(bytes).slice(offset, len - offset))
@@ -226,7 +226,7 @@ unsafe fn create_bitmap(
     let bytes_len = bytes_for(offset + len);
     let ptr = NonNull::new(ptr as *mut u8);
     let bytes = ptr
-        .map(|ptr| Bytes::from_owned(ptr, bytes_len, owner))
+        .map(|ptr| Bytes::from_foreign(ptr.as_ptr(), bytes_len, owner))
         .ok_or_else(|| {
             Error::OutOfSpec(format!(
                 "The buffer {} is a null pointer and cannot be interpreted as a bitmap",