chore: use arrow instead of arrow_array and arrow_schema (#131)

Cargo.toml

@@ -10,8 +10,7 @@ members = [
 anyhow = "1"
 async-trait = "0.1"
 apache-avro = { version = "0.15", features = ["derive"] }
-arrow-array = { version = ">=40, <45" }
-arrow-schema = { version = ">=40, <45" }
+arrow = { version = ">=40, <45" }
 bytes = "1"
 futures = "0.3"
 opendal = ">=0.37, <0.40"

icelake/Cargo.toml

@@ -12,8 +12,7 @@ all-features = true
 anyhow = {workspace = true}
 async-trait = {workspace = true}
 apache-avro = {workspace = true}
-arrow-array = {workspace = true}
-arrow-schema = {workspace = true}
+arrow = {workspace = true}
 bytes = {workspace = true}
 futures = {workspace = true}
 opendal = {workspace = true}

icelake/src/io/data_file_writer.rs

@@ -6,8 +6,8 @@ use crate::{
     types::{DataFile, StructValue},
     Result,
 };
-use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
 use opendal::Operator;
 use parquet::format::FileMetaData;
 
@@ -209,7 +209,8 @@ impl DataFileWriter {
 mod test {
     use std::{env, fs, sync::Arc};
 
-    use arrow_array::{ArrayRef, Int64Array, RecordBatch};
+    use arrow::array::{ArrayRef, Int64Array};
+    use arrow::record_batch::RecordBatch;
     use bytes::Bytes;
     use opendal::{services::Memory, Operator};
 

icelake/src/io/parquet/stream.rs

@@ -2,7 +2,7 @@ use std::pin::Pin;
 use std::task::Context;
 use std::task::Poll;
 
-use arrow_array::RecordBatch;
+use arrow::record_batch::RecordBatch;
 use futures::Stream;
 use futures::StreamExt;
 use opendal::Reader;
@@ -72,9 +72,9 @@ mod tests {
     use std::sync::Arc;
 
     use anyhow::Result;
-    use arrow_array::ArrayRef;
-    use arrow_array::Int64Array;
-    use arrow_array::RecordBatch;
+    use arrow::array::ArrayRef;
+    use arrow::array::Int64Array;
+    use arrow::record_batch::RecordBatch;
     use opendal::services::Memory;
     use opendal::Operator;
     use parquet::arrow::AsyncArrowWriter;

icelake/src/io/parquet/write.rs

@@ -1,8 +1,8 @@
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
 
-use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
 use opendal::Writer;
 use parquet::arrow::AsyncArrowWriter;
 use parquet::file::properties::WriterProperties;
@@ -111,9 +111,9 @@ mod tests {
     use std::sync::Arc;
 
     use anyhow::Result;
-    use arrow_array::ArrayRef;
-    use arrow_array::Int64Array;
-    use arrow_array::RecordBatch;
+    use arrow::array::ArrayRef;
+    use arrow::array::Int64Array;
+    use arrow::record_batch::RecordBatch;
     use bytes::Bytes;
     use opendal::services::Memory;
     use opendal::Operator;

icelake/src/io/task_writer.rs

@@ -1,8 +1,8 @@
 //! task_writer module provide a task writer for writing data in a table.
 //! table writer used directly by the compute engine.
 
-use arrow_array::RecordBatch;
-use arrow_schema::Schema as ArrowSchema;
+use arrow::datatypes::Schema as ArrowSchema;
+use arrow::record_batch::RecordBatch;
 use opendal::Operator;
 
 use super::data_file_writer::DataFileWriter;

icelake/src/types/to_arrow.rs

@@ -5,10 +5,10 @@ use std::convert::TryFrom;
 use std::sync::Arc;
 
 use crate::error::Error;
-use arrow_schema::DataType as ArrowDataType;
-use arrow_schema::Field as ArrowField;
-use arrow_schema::Schema as ArrowSchema;
-use arrow_schema::TimeUnit;
+use arrow::datatypes::DataType as ArrowDataType;
+use arrow::datatypes::Field as ArrowField;
+use arrow::datatypes::Schema as ArrowSchema;
+use arrow::datatypes::TimeUnit;
 
 use super::in_memory as types;
 

icelake/src/types/transform/identity.rs

@@ -1,9 +1,9 @@
 use crate::types::TransformFunction;
-
+use arrow::array::ArrayRef;
 pub struct Identity {}
 
 impl TransformFunction for Identity {
-    fn transform(&self, input: arrow_array::ArrayRef) -> arrow_array::ArrayRef {
+    fn transform(&self, input: ArrayRef) -> ArrayRef {
         input
     }
 }

icelake/src/types/transform/mod.rs

@@ -1,13 +1,13 @@
 use super::Transform;
-
+use arrow::array::ArrayRef;
 mod identity;
 
 /// TransformFunction is a trait that defines the interface of a transform function.
 pub trait TransformFunction {
     /// transform will take an input array and transform it into a new array.
     /// The implementation of this function will need to check and downcast the input to specific
     /// type.
-    fn transform(&self, input: arrow_array::ArrayRef) -> arrow_array::ArrayRef;
+    fn transform(&self, input: ArrayRef) -> ArrayRef;
 }
 
 /// BoxedTransformFunction is a boxed trait object of TransformFunction.