Skip to content
This repository has been archived by the owner on May 10, 2024. It is now read-only.

Commit

Permalink
Cleanup iteration a bit
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Nov 19, 2017
1 parent 920832a commit 32a4abe
Showing 1 changed file with 10 additions and 7 deletions.
17 changes: 10 additions & 7 deletions src/parquet/arrow/writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,7 @@ Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::DecimalType>(
const auto& data = static_cast<const Decimal128Array&>(*array);
const int64_t length = data.length();

// TODO(phillipc): This is potentially very wasteful if we have a lot of nulls
std::vector<uint64_t> big_endian_values(static_cast<size_t>(length) * 2);

RETURN_NOT_OK(data_buffer_.Resize(length * sizeof(FLBA), false));
Expand All @@ -808,7 +809,13 @@ Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::DecimalType>(
const int32_t offset =
decimal_type.byte_width() - DecimalSize(decimal_type.precision());

if (writer->descr()->schema_node()->is_required() || data.null_count() == 0) {
const bool does_not_have_nulls =
writer->descr()->schema_node()->is_required() || data.null_count() == 0;

// TODO(phillipc): Look into whether our compilers will perform loop unswitching so we
// don't have to keep writing two loops to handle the case where we know there are no
// nulls
if (does_not_have_nulls) {
// no nulls, just dump the data
// todo(advancedxy): use a writeBatch to avoid this step
for (int64_t i = 0, j = 0; i < length; ++i, j += 2) {
Expand All @@ -819,17 +826,13 @@ Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::DecimalType>(
reinterpret_cast<const uint8_t*>(&big_endian_values[j]) + offset);
}
} else {
int32_t buffer_idx = 0;
int32_t j = 0;

for (int64_t i = 0; i < length; ++i) {
for (int64_t i = 0, buffer_idx = 0, j = 0; i < length; ++i) {
if (!data.IsNull(i)) {
auto unsigned_64_bit = reinterpret_cast<const uint64_t*>(data.GetValue(i));
big_endian_values[j] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[1]);
big_endian_values[j + 1] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[0]);
buffer_ptr[buffer_idx] = FixedLenByteArray(
buffer_ptr[buffer_idx++] = FixedLenByteArray(
reinterpret_cast<const uint8_t*>(&big_endian_values[j]) + offset);
++buffer_idx;
j += 2;
}
}
Expand Down

0 comments on commit 32a4abe

Please sign in to comment.