Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed error in writing compressed arrow
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Feb 18, 2022
1 parent 6ccb154 commit 7124086
Showing 1 changed file with 14 additions and 18 deletions.
32 changes: 14 additions & 18 deletions src/io/ipc/write/serialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -657,15 +657,9 @@ fn write_bytes(
}
} else {
arrow_data.extend_from_slice(bytes);
pad_buffer_to_8(arrow_data, arrow_data.len() - start);
};

let total_len = (arrow_data.len() - start) as i64;
buffers.push(ipc::Buffer {
offset: *offset,
length: total_len,
});
*offset += total_len;
buffers.push(finish_buffer(arrow_data, start, offset));
}

fn write_bitmap(
Expand Down Expand Up @@ -712,15 +706,9 @@ fn write_buffer<T: NativeType>(
_write_compressed_buffer(buffer, arrow_data, is_little_endian, compression);
} else {
_write_buffer(buffer, arrow_data, is_little_endian);
pad_buffer_to_8(arrow_data, arrow_data.len() - start);
};

let total_len = (arrow_data.len() - start) as i64;
buffers.push(ipc::Buffer {
offset: *offset,
length: total_len,
});
*offset += total_len;
buffers.push(finish_buffer(arrow_data, start, offset));
}

#[inline]
Expand Down Expand Up @@ -819,13 +807,21 @@ fn write_buffer_from_iter<T: NativeType, I: TrustedLen<Item = T>>(
_write_compressed_buffer_from_iter(buffer, arrow_data, is_little_endian, compression);
} else {
_write_buffer_from_iter(buffer, arrow_data, is_little_endian);
pad_buffer_to_8(arrow_data, arrow_data.len() - start);
}

buffers.push(finish_buffer(arrow_data, start, offset));
}

fn finish_buffer(arrow_data: &mut Vec<u8>, start: usize, offset: &mut i64) -> ipc::Buffer {
let buffer_len = (arrow_data.len() - start) as i64;

pad_buffer_to_8(arrow_data, arrow_data.len() - start);
let total_len = (arrow_data.len() - start) as i64;
buffers.push(ipc::Buffer {

let buffer = ipc::Buffer {
offset: *offset,
length: total_len,
});
length: buffer_len,
};
*offset += total_len;
buffer
}

0 comments on commit 7124086

Please sign in to comment.