support compression for IPC

arrow/Cargo.toml

@@ -38,6 +38,8 @@ path = "src/lib.rs"
 bench = false
 
 [dependencies]
+lz4 = { version = "1.23", default-features = false, optional = true }
+zstd = { version = "0.11.1", default-features = false, optional = true }
 ahash = { version = "0.7", default-features = false }
 serde = { version = "1.0", default-features = false }
 serde_derive = { version = "1.0", default-features = false }
@@ -63,6 +65,7 @@ bitflags = { version = "1.2.1", default-features = false }
 
 [features]
 default = ["csv", "ipc", "test_utils"]
+ipc_compression = ["zstd", "lz4"]
 csv = ["csv_crate"]
 ipc = ["flatbuffers"]
 simd = ["packed_simd"]
@@ -83,6 +86,8 @@ rand = { version = "0.8", default-features = false, features =  ["std", "std_rng
 criterion = { version = "0.3", default-features = false }
 flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 tempfile = { version = "3", default-features = false }
+lz4 = { version = "1.23", default-features = false }
+zstd = { version = "0.11", default-features = false }
 
 [build-dependencies]
 

arrow/src/ipc/compression/ipc_compression.rs

@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::ipc::CompressionType;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum CompressionCodecType {
+    Lz4Frame,
+    Zstd,
+}
+
+impl From<CompressionType> for CompressionCodecType {
+    fn from(compression_type: CompressionType) -> Self {
+        match compression_type {
+            CompressionType::ZSTD => CompressionCodecType::Zstd,
+            CompressionType::LZ4_FRAME => CompressionCodecType::Lz4Frame,
+            other_type => {
+                unimplemented!("Not support compression type: {:?}", other_type)
+            }
+        }
+    }
+}
+
+impl From<CompressionCodecType> for CompressionType {
+    fn from(codec: CompressionCodecType) -> Self {
+        match codec {
+            CompressionCodecType::Lz4Frame => CompressionType::LZ4_FRAME,
+            CompressionCodecType::Zstd => CompressionType::ZSTD,
+        }
+    }
+}
+
+#[cfg(any(feature = "ipc_compression", test))]
+mod compression_function {
+    use crate::error::Result;
+    use crate::ipc::compression::ipc_compression::CompressionCodecType;
+    use std::io::{Read, Write};
+
+    impl CompressionCodecType {
+        pub fn compress(&self, input: &[u8], output: &mut Vec<u8>) -> Result<()> {
+            match self {
+                CompressionCodecType::Lz4Frame => {
+                    let mut encoder = lz4::EncoderBuilder::new().build(output)?;
+                    encoder.write_all(input)?;
+                    match encoder.finish().1 {
+                        Ok(_) => Ok(()),
+                        Err(e) => Err(e.into()),
+                    }
+                }
+                CompressionCodecType::Zstd => {
+                    let mut encoder = zstd::Encoder::new(output, 0)?;
+                    encoder.write_all(input)?;
+                    match encoder.finish() {
+                        Ok(_) => Ok(()),
+                        Err(e) => Err(e.into()),
+                    }
+                }
+            }
+        }
+
+        pub fn decompress(&self, input: &[u8], output: &mut Vec<u8>) -> Result<usize> {
+            let result: Result<usize> = match self {
+                CompressionCodecType::Lz4Frame => {
+                    let mut decoder = lz4::Decoder::new(input)?;
+                    match decoder.read_to_end(output) {
+                        Ok(size) => Ok(size),
+                        Err(e) => Err(e.into()),
+                    }
+                }
+                CompressionCodecType::Zstd => {
+                    let mut decoder = zstd::Decoder::new(input)?;
+                    match decoder.read_to_end(output) {
+                        Ok(size) => Ok(size),
+                        Err(e) => Err(e.into()),
+                    }
+                }
+            };
+            result
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::ipc::compression::ipc_compression::CompressionCodecType;
+
+    #[test]
+    fn test_lz4_compression() {
+        let input_bytes = "hello lz4".as_bytes();
+        let codec: CompressionCodecType = CompressionCodecType::Lz4Frame;
+        let mut output_bytes: Vec<u8> = Vec::new();
+        codec.compress(input_bytes, &mut output_bytes).unwrap();
+        let mut result_output_bytes: Vec<u8> = Vec::new();
+        codec
+            .decompress(output_bytes.as_slice(), &mut result_output_bytes)
+            .unwrap();
+        assert_eq!(input_bytes, result_output_bytes.as_slice());
+    }
+
+    #[test]
+    fn test_zstd_compression() {
+        let input_bytes = "hello zstd".as_bytes();
+        let codec: CompressionCodecType = CompressionCodecType::Zstd;
+        let mut output_bytes: Vec<u8> = Vec::new();
+        codec.compress(input_bytes, &mut output_bytes).unwrap();
+        let mut result_output_bytes: Vec<u8> = Vec::new();
+        codec
+            .decompress(output_bytes.as_slice(), &mut result_output_bytes)
+            .unwrap();
+        assert_eq!(input_bytes, result_output_bytes.as_slice());
+    }
+}

arrow/src/ipc/compression/mod.rs

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pub(crate) mod ipc_compression;
+pub(crate) const LENGTH_EMPTY_COMPRESSED_DATA: i64 = 0;
+pub(crate) const LENGTH_NO_COMPRESSED_DATA: i64 = -1;
+pub(crate) const LENGTH_OF_PREFIX_DATA: i64 = 8;

arrow/src/ipc/mod.rs

@@ -22,6 +22,8 @@ pub mod convert;
 pub mod reader;
 pub mod writer;
 
+mod compression;
+
 #[allow(clippy::redundant_closure)]
 #[allow(clippy::needless_lifetimes)]
 #[allow(clippy::extra_unused_lifetimes)]