Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Added support for decimal 32 and 64
Browse files Browse the repository at this point in the history
  • Loading branch information
@jorgecarleitao
jorgecarleitao committed Mar 12, 2022
1 parent 1431b96 commit cfb7ff5
Show file tree
Hide file tree
Showing 42 changed files with 912 additions and 416 deletions.
4 changes: 2 additions & 2 deletions guide/src/high_level.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ The following arrays are supported:

* `NullArray` (just holds nulls)
* `BooleanArray` (booleans)
* `PrimitiveArray<T>` (for ints, floats)
* `PrimitiveArray<T>` (for ints, floats, decimal)
* `Utf8Array<i32>` and `Utf8Array<i64>` (for strings)
* `BinaryArray<i32>` and `BinaryArray<i64>` (for opaque binaries)
* `FixedSizeBinaryArray` (like `BinaryArray`, but fixed size)
Expand Down Expand Up @@ -124,7 +124,7 @@ There is a one to one relationship between each variant of `PhysicalType` (an en
an each implementation of `Array` (a struct):

| `PhysicalType` | `Array` |
|-------------------|------------------------|
| ----------------- | ---------------------- |
| `Primitive(_)` | `PrimitiveArray<_>` |
| `Binary` | `BinaryArray<i32>` |
| `LargeBinary` | `BinaryArray<i64>` |
Expand Down
25 changes: 23 additions & 2 deletions src/array/primitive/fmt.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use std::fmt::{Debug, Formatter, Result, Write};

use crate::array::Array;
use crate::datatypes::{IntervalUnit, TimeUnit};
use crate::datatypes::{DecimalType, IntervalUnit, TimeUnit};
use crate::types::{days_ms, months_days_ns};

use super::super::super::temporal_conversions;
Expand Down Expand Up @@ -104,7 +104,27 @@ pub fn get_write_value<'a, T: NativeType, F: Write>(
Duration(TimeUnit::Millisecond) => dyn_primitive!(array, i64, |x| format!("{}ms", x)),
Duration(TimeUnit::Microsecond) => dyn_primitive!(array, i64, |x| format!("{}us", x)),
Duration(TimeUnit::Nanosecond) => dyn_primitive!(array, i64, |x| format!("{}ns", x)),
Decimal(_, scale) => {
Decimal(DecimalType::Int32, _, scale) => {
// The number 999.99 has a precision of 5 and scale of 2
let scale = *scale as u32;
let display = move |x| {
let base = x / 10i32.pow(scale);
let decimals = x - base * 10i32.pow(scale);
format!("{}.{}", base, decimals)
};
dyn_primitive!(array, i32, display)
}
Decimal(DecimalType::Int64, _, scale) => {
// The number 999.99 has a precision of 5 and scale of 2
let scale = *scale as u32;
let display = move |x| {
let base = x / 10i64.pow(scale);
let decimals = x - base * 10i64.pow(scale);
format!("{}.{}", base, decimals)
};
dyn_primitive!(array, i64, display)
}
Decimal(DecimalType::Int128, _, scale) => {
// The number 999.99 has a precision of 5 and scale of 2
let scale = *scale as u32;
let display = move |x| {
Expand All @@ -114,6 +134,7 @@ pub fn get_write_value<'a, T: NativeType, F: Write>(
};
dyn_primitive!(array, i128, display)
}

_ => unreachable!(),
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/compute/aggregate/min_max.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::bitmap::utils::{BitChunkIterExact, BitChunksExact};
use crate::datatypes::{DataType, IntervalUnit};
use crate::datatypes::{DataType, DecimalType, IntervalUnit};
use crate::error::{ArrowError, Result};
use crate::scalar::*;
use crate::types::simd::*;
Expand Down Expand Up @@ -394,7 +394,7 @@ pub fn max(array: &dyn Array) -> Result<Box<dyn Scalar>> {
DataType::Float16 => unreachable!(),
DataType::Float32 => dyn_primitive!(f32, array, max_primitive),
DataType::Float64 => dyn_primitive!(f64, array, max_primitive),
DataType::Decimal(_, _) => dyn_primitive!(i128, array, max_primitive),
DataType::Decimal(DecimalType::Int128, _, _) => dyn_primitive!(i128, array, max_primitive),
DataType::Utf8 => dyn_generic!(Utf8Array<i32>, Utf8Scalar<i32>, array, max_string),
DataType::LargeUtf8 => dyn_generic!(Utf8Array<i64>, Utf8Scalar<i64>, array, max_string),
DataType::Binary => dyn_generic!(BinaryArray<i32>, BinaryScalar<i32>, array, max_binary),
Expand Down Expand Up @@ -436,7 +436,7 @@ pub fn min(array: &dyn Array) -> Result<Box<dyn Scalar>> {
DataType::Float16 => unreachable!(),
DataType::Float32 => dyn_primitive!(f32, array, min_primitive),
DataType::Float64 => dyn_primitive!(f64, array, min_primitive),
DataType::Decimal(_, _) => dyn_primitive!(i128, array, min_primitive),
DataType::Decimal(DecimalType::Int128, _, _) => dyn_primitive!(i128, array, min_primitive),
DataType::Utf8 => dyn_generic!(Utf8Array<i32>, Utf8Scalar<i32>, array, min_string),
DataType::LargeUtf8 => dyn_generic!(Utf8Array<i64>, Utf8Scalar<i64>, array, min_string),
DataType::Binary => dyn_generic!(BinaryArray<i32>, BinaryScalar<i32>, array, min_binary),
Expand Down
37 changes: 19 additions & 18 deletions src/compute/arithmetics/decimal/add.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use crate::{
arity::{binary, binary_checked},
utils::{check_same_len, combine_validities},
},
datatypes::DecimalType,
};
use crate::{
datatypes::DataType,
Expand All @@ -25,13 +26,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits};
/// ```
/// use arrow2::compute::arithmetics::decimal::add;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = add(&a, &b);
/// let expected = PrimitiveArray::from([Some(2i128), Some(3i128), None, Some(4i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([Some(2i128), Some(3i128), None, Some(4i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -64,13 +65,13 @@ pub fn add(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> PrimitiveA
/// ```
/// use arrow2::compute::arithmetics::decimal::saturating_add;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = saturating_add(&a, &b);
/// let expected = PrimitiveArray::from([Some(99999i128), Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([Some(99999i128), Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -108,13 +109,13 @@ pub fn saturating_add(
/// ```
/// use arrow2::compute::arithmetics::decimal::checked_add;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = checked_add(&a, &b);
/// let expected = PrimitiveArray::from([None, Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([None, Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -172,12 +173,12 @@ impl ArraySaturatingAdd<PrimitiveArray<i128>> for PrimitiveArray<i128> {
/// ```
/// use arrow2::compute::arithmetics::decimal::adaptive_add;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2));
/// let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(8, 3));
/// let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2));
/// let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3));
/// let result = adaptive_add(&a, &b).unwrap();
/// let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal(8, 3));
/// let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3));
///
/// assert_eq!(result, expected);
/// ```
Expand All @@ -188,7 +189,7 @@ pub fn adaptive_add(
check_same_len(lhs, rhs)?;

let (lhs_p, lhs_s, rhs_p, rhs_s) =
if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) =
if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) =
(lhs.data_type(), rhs.data_type())
{
(*lhs_p, *lhs_s, *rhs_p, *rhs_s)
Expand Down Expand Up @@ -237,7 +238,7 @@ pub fn adaptive_add(
let validity = combine_validities(lhs.validity(), rhs.validity());

Ok(PrimitiveArray::<i128>::new(
DataType::Decimal(res_p, res_s),
DataType::Decimal(DecimalType::Int128, res_p, res_s),
values.into(),
validity,
))
Expand Down
38 changes: 19 additions & 19 deletions src/compute/arithmetics/decimal/div.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use crate::{
arity::{binary, binary_checked, unary},
utils::{check_same_len, combine_validities},
},
datatypes::DataType,
datatypes::{DataType, DecimalType},
error::{ArrowError, Result},
scalar::{PrimitiveScalar, Scalar},
};
Expand All @@ -25,13 +25,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits};
/// ```
/// use arrow2::compute::arithmetics::decimal::div;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = div(&a, &b);
/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -121,13 +121,13 @@ pub fn div_scalar(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveScalar<i128>) -> Pr
/// ```
/// use arrow2::compute::arithmetics::decimal::saturating_div;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(999_99i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(000_01i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(999_99i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(000_01i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = saturating_div(&a, &b);
/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -170,13 +170,13 @@ pub fn saturating_div(
/// ```
/// use arrow2::compute::arithmetics::decimal::checked_div;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2));
/// let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2));
/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
/// let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// let result = checked_div(&a, &b);
/// let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal(5, 2));
/// let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2));
///
/// assert_eq!(result, expected);
/// ```
Expand Down Expand Up @@ -232,12 +232,12 @@ impl ArrayCheckedDiv<PrimitiveArray<i128>> for PrimitiveArray<i128> {
/// ```
/// use arrow2::compute::arithmetics::decimal::adaptive_div;
/// use arrow2::array::PrimitiveArray;
/// use arrow2::datatypes::DataType;
/// use arrow2::datatypes::{DataType, DecimalType};
///
/// let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(7, 2));
/// let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(6, 4));
/// let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2));
/// let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4));
/// let result = adaptive_div(&a, &b).unwrap();
/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(9, 4));
/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 9, 4));
///
/// assert_eq!(result, expected);
/// ```
Expand All @@ -248,7 +248,7 @@ pub fn adaptive_div(
check_same_len(lhs, rhs)?;

let (lhs_p, lhs_s, rhs_p, rhs_s) =
if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) =
if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) =
(lhs.data_type(), rhs.data_type())
{
(*lhs_p, *lhs_s, *rhs_p, *rhs_s)
Expand Down Expand Up @@ -302,7 +302,7 @@ pub fn adaptive_div(
let validity = combine_validities(lhs.validity(), rhs.validity());

Ok(PrimitiveArray::<i128>::new(
DataType::Decimal(res_p, res_s),
DataType::Decimal(DecimalType::Int128, res_p, res_s),
values.into(),
validity,
))
Expand Down
2 changes: 1 addition & 1 deletion src/compute/arithmetics/decimal/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ fn number_digits(num: i128) -> usize {
}

fn get_parameters(lhs: &DataType, rhs: &DataType) -> Result<(usize, usize)> {
if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) =
if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) =
(lhs.to_logical_type(), rhs.to_logical_type())
{
if lhs_p == rhs_p && lhs_s == rhs_s {
Expand Down
Loading

0 comments on commit cfb7ff5

Please sign in to comment.