Skip to content
This repository has been archived by the owner on May 10, 2024. It is now read-only.

Commit

Permalink
ARROW-1811
Browse files Browse the repository at this point in the history
  • Loading branch information
@cpcloud
cpcloud committed Nov 19, 2017
1 parent c5c4294 commit 6036ca5
Show file tree
Hide file tree
Showing 5 changed files with 21 additions and 20 deletions.
4 changes: 2 additions & 2 deletions src/parquet/arrow/arrow-reader-writer-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ static std::shared_ptr<GroupNode> MakeSimpleSchema(const ::arrow::DataType& type
break;
case ::arrow::Type::DECIMAL: {
const auto& decimal_type =
static_cast<const ::arrow::DecimalType&>(values_type);
static_cast<const ::arrow::Decimal128Type&>(values_type);
precision = decimal_type.precision();
scale = decimal_type.scale();
byte_width = DecimalSize(precision);
Expand All @@ -375,7 +375,7 @@ static std::shared_ptr<GroupNode> MakeSimpleSchema(const ::arrow::DataType& type
byte_width = static_cast<const ::arrow::FixedSizeBinaryType&>(type).byte_width();
break;
case ::arrow::Type::DECIMAL: {
const auto& decimal_type = static_cast<const ::arrow::DecimalType&>(type);
const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(type);
precision = decimal_type.precision();
scale = decimal_type.scale();
byte_width = DecimalSize(precision);
Expand Down
2 changes: 1 addition & 1 deletion src/parquet/arrow/arrow-schema-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ const auto TIMESTAMP_MS = ::arrow::timestamp(TimeUnit::MILLI);
const auto TIMESTAMP_US = ::arrow::timestamp(TimeUnit::MICRO);
const auto TIMESTAMP_NS = ::arrow::timestamp(TimeUnit::NANO);
const auto BINARY = ::arrow::binary();
const auto DECIMAL_8_4 = std::make_shared<::arrow::DecimalType>(8, 4);
const auto DECIMAL_8_4 = std::make_shared<::arrow::Decimal128Type>(8, 4);

class TestConvertParquetSchema : public ::testing::Test {
public:
Expand Down
16 changes: 8 additions & 8 deletions src/parquet/arrow/reader.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -970,7 +970,7 @@ static inline void RawBytesToDecimalBytes(const uint8_t* value, int32_t byte_wid
/// 3. Converting the big-endian bytes in the FixedSizeBinaryArray to two integers
/// representing the high and low bits of each decimal value.
template <>
struct TransferFunctor<::arrow::DecimalType, FLBAType> {
struct TransferFunctor<::arrow::Decimal128Type, FLBAType> {
Status operator()(RecordReader* reader, MemoryPool* pool,
const std::shared_ptr<::arrow::DataType>& type,
std::shared_ptr<Array>* out) {
Expand All @@ -991,7 +991,7 @@ struct TransferFunctor<::arrow::DecimalType, FLBAType> {

// The byte width of each decimal value
const int32_t type_length =
static_cast<const ::arrow::DecimalType&>(*type).byte_width();
static_cast<const ::arrow::Decimal128Type&>(*type).byte_width();

// number of elements in the entire array
const int64_t length = fixed_size_binary_array.length();
Expand Down Expand Up @@ -1046,7 +1046,7 @@ static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool,

const auto values = reinterpret_cast<const ElementType*>(reader->values());

const auto& decimal_type = static_cast<const ::arrow::DecimalType&>(*type);
const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(*type);
const int64_t type_length = decimal_type.byte_width();

std::shared_ptr<Buffer> data;
Expand Down Expand Up @@ -1079,7 +1079,7 @@ static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool,
}

template <>
struct TransferFunctor<::arrow::DecimalType, Int32Type> {
struct TransferFunctor<::arrow::Decimal128Type, Int32Type> {
Status operator()(RecordReader* reader, MemoryPool* pool,
const std::shared_ptr<::arrow::DataType>& type,
std::shared_ptr<Array>* out) {
Expand All @@ -1088,7 +1088,7 @@ struct TransferFunctor<::arrow::DecimalType, Int32Type> {
};

template <>
struct TransferFunctor<::arrow::DecimalType, Int64Type> {
struct TransferFunctor<::arrow::Decimal128Type, Int64Type> {
Status operator()(RecordReader* reader, MemoryPool* pool,
const std::shared_ptr<::arrow::DataType>& type,
std::shared_ptr<Array>* out) {
Expand Down Expand Up @@ -1157,13 +1157,13 @@ Status PrimitiveImpl::NextBatch(int64_t records_to_read, std::shared_ptr<Array>*
case ::arrow::Type::DECIMAL: {
switch (descr_->physical_type()) {
case ::parquet::Type::INT32: {
TRANSFER_DATA(::arrow::DecimalType, Int32Type);
TRANSFER_DATA(::arrow::Decimal128Type, Int32Type);
} break;
case ::parquet::Type::INT64: {
TRANSFER_DATA(::arrow::DecimalType, Int64Type);
TRANSFER_DATA(::arrow::Decimal128Type, Int64Type);
} break;
case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: {
TRANSFER_DATA(::arrow::DecimalType, FLBAType);
TRANSFER_DATA(::arrow::Decimal128Type, FLBAType);
} break;
default:
return Status::Invalid(
Expand Down
13 changes: 7 additions & 6 deletions src/parquet/arrow/schema.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ const auto TIMESTAMP_MS = ::arrow::timestamp(::arrow::TimeUnit::MILLI);
const auto TIMESTAMP_US = ::arrow::timestamp(::arrow::TimeUnit::MICRO);
const auto TIMESTAMP_NS = ::arrow::timestamp(::arrow::TimeUnit::NANO);

TypePtr MakeDecimalType(const PrimitiveNode& node) {
TypePtr MakeDecimal128Type(const PrimitiveNode& node) {
const auto& metadata = node.decimal_metadata();
return ::arrow::decimal(metadata.precision, metadata.scale);
}
Expand All @@ -61,7 +61,7 @@ static Status FromByteArray(const PrimitiveNode& node, TypePtr* out) {
*out = ::arrow::utf8();
break;
case LogicalType::DECIMAL:
*out = MakeDecimalType(node);
*out = MakeDecimal128Type(node);
break;
default:
// BINARY
Expand All @@ -77,7 +77,7 @@ static Status FromFLBA(const PrimitiveNode& node, TypePtr* out) {
*out = ::arrow::fixed_size_binary(node.type_length());
break;
case LogicalType::DECIMAL:
*out = MakeDecimalType(node);
*out = MakeDecimal128Type(node);
break;
default:
std::stringstream ss;
Expand Down Expand Up @@ -120,7 +120,7 @@ static Status FromInt32(const PrimitiveNode& node, TypePtr* out) {
*out = ::arrow::time32(::arrow::TimeUnit::MILLI);
break;
case LogicalType::DECIMAL:
*out = MakeDecimalType(node);
*out = MakeDecimal128Type(node);
break;
default:
std::stringstream ss;
Expand All @@ -144,7 +144,7 @@ static Status FromInt64(const PrimitiveNode& node, TypePtr* out) {
*out = ::arrow::uint64();
break;
case LogicalType::DECIMAL:
*out = MakeDecimalType(node);
*out = MakeDecimal128Type(node);
break;
case LogicalType::TIMESTAMP_MILLIS:
*out = TIMESTAMP_MS;
Expand Down Expand Up @@ -542,7 +542,8 @@ Status FieldToNode(const std::shared_ptr<Field>& field,
case ArrowType::DECIMAL: {
type = ParquetType::FIXED_LEN_BYTE_ARRAY;
logical_type = LogicalType::DECIMAL;
const auto& decimal_type = static_cast<const ::arrow::DecimalType&>(*field->type());
const auto& decimal_type =
static_cast<const ::arrow::Decimal128Type&>(*field->type());
precision = decimal_type.precision();
scale = decimal_type.scale();
length = DecimalSize(precision);
Expand Down
6 changes: 3 additions & 3 deletions src/parquet/arrow/writer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -791,7 +791,7 @@ Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::FixedSizeBinaryType>
}

template <>
Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::DecimalType>(
Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::Decimal128Type>(
ColumnWriter* column_writer, const std::shared_ptr<Array>& array, int64_t num_levels,
const int16_t* def_levels, const int16_t* rep_levels) {
const auto& data = static_cast<const Decimal128Array&>(*array);
Expand All @@ -805,7 +805,7 @@ Status FileWriter::Impl::TypedWriteBatch<FLBAType, ::arrow::DecimalType>(

auto writer = reinterpret_cast<TypedColumnWriter<FLBAType>*>(column_writer);

const auto& decimal_type = static_cast<const ::arrow::DecimalType&>(*data.type());
const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(*data.type());
const int32_t offset =
decimal_type.byte_width() - DecimalSize(decimal_type.precision());

Expand Down Expand Up @@ -947,7 +947,7 @@ Status FileWriter::Impl::WriteColumnChunk(const Array& data) {
WRITE_BATCH_CASE(BINARY, BinaryType, ByteArrayType)
WRITE_BATCH_CASE(STRING, BinaryType, ByteArrayType)
WRITE_BATCH_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryType, FLBAType)
WRITE_BATCH_CASE(DECIMAL, DecimalType, FLBAType)
WRITE_BATCH_CASE(DECIMAL, Decimal128Type, FLBAType)
WRITE_BATCH_CASE(DATE32, Date32Type, Int32Type)
WRITE_BATCH_CASE(DATE64, Date64Type, Int32Type)
WRITE_BATCH_CASE(TIME32, Time32Type, Int32Type)
Expand Down

0 comments on commit 6036ca5

Please sign in to comment.