Skip to content

Commit

Permalink
fix: Lz4Raw compression
Browse files Browse the repository at this point in the history
Using `lz4::block::compress_bound` to determine the size of output buffer
  • Loading branch information
dantengsky committed Apr 6, 2022
1 parent f8eb243 commit ea7820d
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/compression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ pub fn compress(
#[cfg(feature = "lz4")]
Compression::Lz4Raw => {
let output_buf_len = output_buf.len();
let required_len = input_buf.len();
output_buf.resize(output_buf_len + required_len, 0);
let required_len = lz4::block::compress_bound(input_buf.len())?;
output_buf.resize(required_len, 0);
let size = lz4::block::compress_to_buffer(
input_buf,
None,
Expand Down
12 changes: 12 additions & 0 deletions tests/it/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ pub fn alltypes_plain(column: &str) -> Array {
let expected = expected.into_iter().map(Some).collect::<Vec<_>>();
Array::Int32(expected)
}
"id-short-array" => {
let expected = vec![4];
let expected = expected.into_iter().map(Some).collect::<Vec<_>>();
Array::Int32(expected)
}
"bool_col" => {
let expected = vec![true, false, true, false, true, false, true, false];
let expected = expected.into_iter().map(Some).collect::<Vec<_>>();
Expand Down Expand Up @@ -156,6 +161,13 @@ pub fn alltypes_statistics(column: &str) -> Arc<dyn Statistics> {
min_value: Some(0),
max_value: Some(7),
}),
"id-short-array" => Arc::new(PrimitiveStatistics::<i32> {
primitive_type: PrimitiveType::from_physical("col".to_string(), PhysicalType::Int32),
null_count: Some(0),
distinct_count: None,
min_value: Some(4),
max_value: Some(4),
}),
"bool_col" => Arc::new(BooleanStatistics {
null_count: Some(0),
distinct_count: None,
Expand Down
5 changes: 5 additions & 0 deletions tests/it/write/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,11 @@ fn int32_lz4() -> Result<()> {
test_column("id", Compression::Lz4Raw)
}

#[test]
fn int32_lz4_short_i32_array() -> Result<()> {
test_column("id-short-array", Compression::Lz4Raw)
}

#[test]
fn int32_brotli() -> Result<()> {
test_column("id", Compression::Brotli)
Expand Down

0 comments on commit ea7820d

Please sign in to comment.