Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into decimal_round_#31…
Browse files Browse the repository at this point in the history 
…37_with_false
  • Loading branch information
@liukun4515
liukun4515 committed Dec 1, 2022
2 parents d2fa48b + 961e114 commit 31586e9
Show file tree
Hide file tree
Showing 12 changed files with 1,175 additions and 858 deletions.
28 changes: 19 additions & 9 deletions arrow-array/src/array/primitive_array.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,10 @@ pub type DurationMicrosecondArray = PrimitiveArray<DurationMicrosecondType>;
pub type DurationNanosecondArray = PrimitiveArray<DurationNanosecondType>;

/// An array where each element is a 128-bits decimal with precision in [1, 38] and
/// scale in [-38, 38].
/// scale less or equal to 38.
pub type Decimal128Array = PrimitiveArray<Decimal128Type>;
/// An array where each element is a 256-bits decimal with precision in [1, 76] and
/// scale in [-76, 76].
/// scale less or equal to 76.
pub type Decimal256Array = PrimitiveArray<Decimal256Type>;

/// Trait bridging the dynamic-typed nature of Arrow (via [`DataType`]) with the
Expand Down Expand Up @@ -1121,13 +1121,6 @@ impl<T: DecimalType + ArrowPrimitiveType> PrimitiveArray<T> {
T::MAX_SCALE
)));
}
if scale < -T::MAX_SCALE {
return Err(ArrowError::InvalidArgumentError(format!(
"scale {} is smaller than min {}",
scale,
-Decimal128Type::MAX_SCALE
)));
}
if scale > 0 && scale as u8 > precision {
return Err(ArrowError::InvalidArgumentError(format!(
"scale {} is greater than precision {}",
Expand All @@ -1151,6 +1144,14 @@ impl<T: DecimalType + ArrowPrimitiveType> PrimitiveArray<T> {
})
}

/// Validates the Decimal Array, if the value of slot is overflow for the specified precision, and
/// will be casted to Null
pub fn null_if_overflow_precision(&self, precision: u8) -> Self {
self.unary_opt::<_, T>(|v| {
(T::validate_decimal_precision(v, precision).is_ok()).then_some(v)
})
}

/// Returns [`Self::value`] formatted as a string
pub fn value_as_string(&self, row: usize) -> String {
T::format_decimal(self.value(row), self.precision(), self.scale())
Expand Down Expand Up @@ -2055,6 +2056,15 @@ mod tests {
.unwrap();
}

#[test]
fn test_decimal_array_set_null_if_overflow_with_precision() {
let array =
Decimal128Array::from(vec![Some(123456), Some(123), None, Some(123456)]);
let result = array.null_if_overflow_precision(5);
let expected = Decimal128Array::from(vec![None, Some(123), None, None]);
assert_eq!(result, expected);
}

#[test]
fn test_decimal256_iter() {
let mut builder = Decimal256Builder::with_capacity(30);
Expand Down
105 changes: 16 additions & 89 deletions arrow-array/src/builder/map_builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,32 @@ use std::sync::Arc;

/// Creates a new `MapBuilder`
/// ```
/// use arrow_array::builder::{MapBuilder, Int32Builder, StringBuilder};
/// use arrow_array::{StringArray, Int32Array};
/// use std::sync::Arc;
/// # use arrow_array::builder::{Int32Builder, MapBuilder, StringBuilder};
/// # use arrow_array::{Int32Array, StringArray};
///
/// let string_builder = StringBuilder::new();
/// let int_builder = Int32Builder::with_capacity(4);
///
/// // Construct `[{"joe": 1}, {"blogs": 2, "foo": 4}, {}, null]`
/// let mut builder = MapBuilder::new(None, string_builder, int_builder);
///
/// let string_builder = builder.keys();
/// string_builder.append_value("joe");
/// string_builder.append_value("n1");
/// string_builder.append_value("n2");
/// string_builder.append_value("mark");
///
/// let int_builder = builder.values();
/// int_builder.append_value(1);
/// int_builder.append_value(2);
/// int_builder.append_null();
/// int_builder.append_value(4);
/// builder.keys().append_value("joe");
/// builder.values().append_value(1);
/// builder.append(true).unwrap();
///
/// builder.keys().append_value("blogs");
/// builder.values().append_value(2);
/// builder.keys().append_value("foo");
/// builder.values().append_value(4);
/// builder.append(true).unwrap();
/// builder.append(false).unwrap();
/// builder.append(true).unwrap();
/// builder.append(false).unwrap();
///
/// let array = builder.finish();
/// assert_eq!(array.value_offsets(), &[0, 1, 3, 3, 3]);
/// assert_eq!(*array.values(), Int32Array::from(vec![1, 2, 4]));
/// assert_eq!(*array.keys(), StringArray::from(vec!["joe", "blogs", "foo"]));
///
/// let arr = builder.finish();
/// assert_eq!(
/// *arr.values(),
/// Int32Array::from(vec![Some(1), Some(2), None, Some(4)])
/// );
/// assert_eq!(
/// *arr.keys(),
/// StringArray::from(vec![Some("joe"), Some("n1"), Some("n2"), Some("mark")])
/// );
/// ```
#[derive(Debug)]
pub struct MapBuilder<K: ArrayBuilder, V: ArrayBuilder> {
Expand Down Expand Up @@ -91,7 +83,6 @@ impl Default for MapFieldNames {
}
}

#[allow(dead_code)]
impl<K: ArrayBuilder, V: ArrayBuilder> MapBuilder<K, V> {
/// Creates a new `MapBuilder`
pub fn new(
Expand Down Expand Up @@ -264,67 +255,3 @@ impl<K: ArrayBuilder, V: ArrayBuilder> ArrayBuilder for MapBuilder<K, V> {
self
}
}

#[cfg(test)]
mod tests {
use super::*;
use arrow_buffer::Buffer;
use arrow_data::Bitmap;

use crate::builder::{Int32Builder, StringBuilder};

// TODO: add a test that finishes building, after designing a spec-compliant
// way of inserting values to the map.
// A map's values shouldn't be repeated within a slot

#[test]
fn test_map_array_builder() {
let string_builder = StringBuilder::new();
let int_builder = Int32Builder::with_capacity(4);

let mut builder = MapBuilder::new(None, string_builder, int_builder);

let string_builder = builder.keys();
string_builder.append_value("joe");
string_builder.append_value("n1");
string_builder.append_value("n2");
string_builder.append_value("mark");

let int_builder = builder.values();
int_builder.append_value(1);
int_builder.append_value(2);
int_builder.append_null();
int_builder.append_value(4);

builder.append(true).unwrap();
builder.append(false).unwrap();
builder.append(true).unwrap();

let arr = builder.finish();

let map_data = arr.data();
assert_eq!(3, map_data.len());
assert_eq!(1, map_data.null_count());
assert_eq!(
Some(&Bitmap::from(Buffer::from(&[5_u8]))),
map_data.null_bitmap()
);

let expected_string_data = ArrayData::builder(DataType::Utf8)
.len(4)
.add_buffer(Buffer::from_slice_ref([0, 3, 5, 7, 11]))
.add_buffer(Buffer::from_slice_ref(b"joen1n2mark"))
.build()
.unwrap();

let expected_int_data = ArrayData::builder(DataType::Int32)
.len(4)
.null_bit_buffer(Some(Buffer::from_slice_ref([11_u8])))
.add_buffer(Buffer::from_slice_ref([1, 2, 0, 4]))
.build()
.unwrap();

assert_eq!(&expected_string_data, arr.keys().data());
assert_eq!(&expected_int_data, arr.values().data());
}
}
Loading

0 comments on commit 31586e9

Please sign in to comment.