Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Fixed clippy and fmt #521

Merged
merged 2 commits into from
Oct 10, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/array/binary/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ impl<O: Offset> MutableBinaryArray<O> {
extend_from_trusted_len_iter(
&mut self.offsets,
&mut self.values,
&mut self.validity.as_mut().unwrap(),
self.validity.as_mut().unwrap(),
iterator,
);

Expand Down
6 changes: 3 additions & 3 deletions src/array/display.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,23 +149,23 @@ pub fn get_value_display<'a>(array: &'a dyn Array) -> Box<dyn Fn(usize) -> Strin
List(_) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, ListArray<i32>, f)
}
FixedSizeList(_, _) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, FixedSizeListArray, f)
}
LargeList(_) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, ListArray<i64>, f)
Expand Down
2 changes: 2 additions & 0 deletions src/array/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ use crate::error::Result;

/// Trait describing how a struct presents itself to the
/// [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI).
/// Safety:
/// Implementing this trait incorrect will lead to UB
pub unsafe trait ToFfi {
/// The pointers to the buffers.
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>>;
Expand Down
4 changes: 3 additions & 1 deletion src/array/utf8/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ impl<O: Offset, A: ffi::ArrowArrayRef> FromFfi<A> for Utf8Array<O> {
validity = validity.map(|x| x.slice(offset, length))
}
let data_type = Self::default_data_type();
Ok(Self::from_data_unchecked(data_type, offsets, values, validity))
Ok(Self::from_data_unchecked(
data_type, offsets, values, validity,
))
}
}
2 changes: 1 addition & 1 deletion src/array/utf8/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ impl<O: Offset> MutableUtf8Array<O> {
extend_from_trusted_len_iter(
&mut self.offsets,
&mut self.values,
&mut self.validity.as_mut().unwrap(),
self.validity.as_mut().unwrap(),
iterator,
);

Expand Down
2 changes: 1 addition & 1 deletion src/bitmap/bitmap_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ where
{
let rem = op(iter.remainder());

let iterator = iter.map(|left| op(left)).chain(std::iter::once(rem));
let iterator = iter.map(op).chain(std::iter::once(rem));

let buffer = MutableBuffer::from_chunk_iter(iterator);

Expand Down
2 changes: 1 addition & 1 deletion src/bitmap/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ impl MutableBitmap {
/// Panics iff `index >= self.len()`.
#[inline]
pub fn set(&mut self, index: usize, value: bool) {
set_bit(&mut self.buffer.as_mut_slice(), index, value)
set_bit(self.buffer.as_mut_slice(), index, value)
}

/// Shrinks the capacity of the [`MutableBitmap`] to fit its current length.
Expand Down
8 changes: 5 additions & 3 deletions src/compute/arithmetics/decimal/add.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,11 @@ pub fn add(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
let op = move |a, b| {
let res: i128 = a + b;

if res.abs() > max_value(*lhs_p) {
panic!("Overflow in addition presented for precision {}", lhs_p);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in addition presented for precision {}",
lhs_p
);

res
};
Expand Down
11 changes: 5 additions & 6 deletions src/compute/arithmetics/decimal/div.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,11 @@ pub fn div(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
// by zero.
let res: i128 = numeral.checked_div(b).expect("Found division by zero");

if res.abs() > max_value(*lhs_p) {
panic!(
"Overflow in multiplication presented for precision {}",
lhs_p
);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in multiplication presented for precision {}",
lhs_p
);

res
};
Expand Down
11 changes: 5 additions & 6 deletions src/compute/arithmetics/decimal/mul.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,11 @@ pub fn mul(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
// 24691.308 <-- 24691308642
let res = res / 10i128.pow(*lhs_s as u32);

if res.abs() > max_value(*lhs_p) {
panic!(
"Overflow in multiplication presented for precision {}",
lhs_p
);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in multiplication presented for precision {}",
lhs_p
);

res
};
Expand Down
8 changes: 5 additions & 3 deletions src/compute/arithmetics/decimal/sub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ pub fn sub(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
let op = move |a, b| {
let res: i128 = a - b;

if res.abs() > max_value(*lhs_p) {
panic!("Overflow in subtract presented for precision {}", lhs_p);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in subtract presented for precision {}",
lhs_p
);

res
};
Expand Down
8 changes: 1 addition & 7 deletions src/compute/cast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,14 @@ pub use primitive_to::*;
pub use utf8_to::*;

/// options defining how Cast kernels behave
#[derive(Clone, Copy, Debug)]
#[derive(Clone, Copy, Debug, Default)]
struct CastOptions {
/// default to false
/// whether an overflowing cast should be converted to `None` (default), or be wrapped (i.e. `256i16 as u8 = 0` vectorized).
/// Settings this to `true` is 5-6x faster for numeric types.
wrapped: bool,
}

impl Default for CastOptions {
fn default() -> Self {
Self { wrapped: false }
}
}

impl CastOptions {
fn with_wrapped(&self, v: bool) -> Self {
let mut option = *self;
Expand Down
4 changes: 2 additions & 2 deletions src/compute/sort/primitive/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ where

// sort all non-null values
sort_values(
&mut buffer.as_mut_slice(),
buffer.as_mut_slice(),
cmp,
options.descending,
limit - validity.null_count(),
Expand Down Expand Up @@ -153,7 +153,7 @@ where
let mut buffer = MutableBuffer::<T>::new();
buffer.extend_from_slice(values);

sort_values(&mut buffer.as_mut_slice(), cmp, options.descending, limit);
sort_values(buffer.as_mut_slice(), cmp, options.descending, limit);
buffer.truncate(limit);
buffer.shrink_to_fit();

Expand Down
4 changes: 1 addition & 3 deletions src/io/avro/read/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,7 @@ fn read_block<R: Read>(reader: &mut R, buf: &mut Vec<u8>, file_marker: [u8; 16])
let mut marker = [0u8; 16];
reader.read_exact(&mut marker)?;

if marker != file_marker {
panic!();
}
assert!(!(marker != file_marker));
Ok(rows)
}

Expand Down
72 changes: 41 additions & 31 deletions src/io/parquet/read/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
//! APIs to read from Parquet format.
use std::{convert::TryInto, io::{Read, Seek}, sync::Arc};
use std::{
convert::TryInto,
io::{Read, Seek},
sync::Arc,
};

use futures::{AsyncRead, AsyncSeek, Stream};
pub use parquet2::{
Expand All @@ -18,7 +22,11 @@ pub use parquet2::{
types::int96_to_i64_ns,
};

use crate::{array::{Array, DictionaryKey, PrimitiveArray}, datatypes::{DataType, IntervalUnit, TimeUnit}, error::{ArrowError, Result}};
use crate::{
array::{Array, DictionaryKey, PrimitiveArray},
datatypes::{DataType, IntervalUnit, TimeUnit},
error::{ArrowError, Result},
};

mod binary;
mod boolean;
Expand Down Expand Up @@ -205,45 +213,47 @@ pub fn page_iter_to_array<
iter, data_type, metadata,
)?)),
Decimal(_, _) => match metadata.descriptor().type_() {
ParquetType::PrimitiveType { physical_type, ..} => match physical_type{
PhysicalType::Int32 => primitive::iter_to_array(
iter,
metadata,
data_type,
|x: i32| x as i128,
),
PhysicalType::Int64 => primitive::iter_to_array(
iter,
metadata,
data_type,
|x: i64| x as i128,
),
ParquetType::PrimitiveType { physical_type, .. } => match physical_type {
PhysicalType::Int32 => {
primitive::iter_to_array(iter, metadata, data_type, |x: i32| x as i128)
}
PhysicalType::Int64 => {
primitive::iter_to_array(iter, metadata, data_type, |x: i64| x as i128)
}
PhysicalType::FixedLenByteArray(n) => {
if *n > 16 {
Err(ArrowError::NotYetImplemented(format!(
"Can't decode Decimal128 type from Fixed Size Byte Array of len {:?}",
n
)))
} else {
let paddings = (0..(16-*n)).map(|_| 0u8).collect::<Vec<_>>();
fixed_size_binary::iter_to_array(iter, DataType::FixedSizeBinary(*n), metadata)
.map(|e|{
let a = e.into_iter().map(|v|
v.and_then(|v1| {
[&paddings, v1].concat().try_into().map(
|pad16| i128::from_be_bytes(pad16)
).ok()
}
)
).collect::<Vec<_>>();
Box::new(PrimitiveArray::<i128>::from(a).to(data_type)) as Box<dyn Array>
}
let paddings = (0..(16 - *n)).map(|_| 0u8).collect::<Vec<_>>();
fixed_size_binary::iter_to_array(
iter,
DataType::FixedSizeBinary(*n),
metadata,
)
.map(|e| {
let a = e
.into_iter()
.map(|v| {
v.and_then(|v1| {
[&paddings, v1]
.concat()
.try_into()
.map(i128::from_be_bytes)
.ok()
})
})
.collect::<Vec<_>>();
Box::new(PrimitiveArray::<i128>::from(a).to(data_type))
as Box<dyn Array>
})
}
},
_ => unreachable!()
}
_ => unreachable!(),
},
_ => unreachable!()
_ => unreachable!(),
},
List(ref inner) => match inner.data_type() {
UInt8 => primitive::iter_to_array_nested(iter, metadata, data_type, |x: i32| x as u8),
Expand Down
2 changes: 1 addition & 1 deletion src/io/parquet/read/primitive/basic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ fn read_required<T, A, F>(
assert_eq!(values_buffer.len(), additional * std::mem::size_of::<T>());
let iterator = ExactChunksIter::<T>::new(values_buffer);

let iterator = iterator.map(|value| op(value));
let iterator = iterator.map(op);

values.extend_from_trusted_len_iter(iterator);
}
Expand Down
2 changes: 1 addition & 1 deletion src/io/parquet/read/primitive/nested.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ where
A: ArrowNativeType,
F: Fn(T) -> A,
{
let iterator = new_values.map(|v| op(v));
let iterator = new_values.map(op);
values.extend_from_trusted_len_iter(iterator);
}

Expand Down
6 changes: 3 additions & 3 deletions src/io/parquet/read/schema/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub fn parquet_to_arrow_schema(
schema
.fields()
.iter()
.map(|t| to_field(t))
.map(to_field)
.filter_map(|x| x.transpose())
.collect::<Result<Vec<_>>>()
.map(|fields| Schema::new_from(fields, metadata))
Expand Down Expand Up @@ -167,7 +167,7 @@ pub fn from_int64(
ParquetTimeUnit::MICROS(_) => DataType::Time64(TimeUnit::Microsecond),
ParquetTimeUnit::NANOS(_) => DataType::Time64(TimeUnit::Nanosecond),
},
(Some(PrimitiveConvertedType::Decimal(precision,scale)), _) => {
(Some(PrimitiveConvertedType::Decimal(precision, scale)), _) => {
DataType::Decimal(*precision as usize, *scale as usize)
}
(c, l) => {
Expand Down Expand Up @@ -284,7 +284,7 @@ fn to_group_type_inner(
fn to_struct(fields: &[ParquetType]) -> Result<Option<DataType>> {
fields
.iter()
.map(|field| to_field(field))
.map(to_field)
.collect::<Result<Vec<Option<Field>>>>()
.map(|result| result.into_iter().flatten().collect::<Vec<Field>>())
.map(|fields| {
Expand Down
4 changes: 2 additions & 2 deletions src/io/parquet/read/statistics/fixlen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,15 @@ impl TryFrom<(&ParquetFixedLenStatistics, DataType)> for PrimitiveStatistics<i12
[paddings.as_slice(), value]
.concat()
.try_into()
.map(|v| i128::from_be_bytes(v))
.map(i128::from_be_bytes)
.ok()
});

let min_value = stats.min_value.as_ref().and_then(|value| {
[paddings.as_slice(), value]
.concat()
.try_into()
.map(|v| i128::from_be_bytes(v))
.map(i128::from_be_bytes)
.ok()
});
Ok(Self {
Expand Down
8 changes: 4 additions & 4 deletions src/io/parquet/write/fixed_len_bytes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use parquet2::{
encoding::Encoding,
metadata::ColumnDescriptor,
page::CompressedDataPage,
statistics::{serialize_statistics, deserialize_statistics, ParquetStatistics},
statistics::{deserialize_statistics, serialize_statistics, ParquetStatistics},
write::WriteOptions,
};

Expand Down Expand Up @@ -98,7 +98,7 @@ pub(super) fn build_statistics(
.min_by(|x, y| ord_binary(x, y))
.map(|x| x.to_vec()),
};
deserialize_statistics(pq_statistics,descriptor).map(
|e| serialize_statistics(&*e)
).ok()
deserialize_statistics(pq_statistics, descriptor)
.map(|e| serialize_statistics(&*e))
.ok()
}
2 changes: 1 addition & 1 deletion src/io/parquet/write/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ pub fn to_parquet_type(field: &Field) -> Result<ParquetType> {
// recursively convert children to types/nodes
let fields = fields
.iter()
.map(|f| to_parquet_type(f))
.map(to_parquet_type)
.collect::<Result<Vec<_>>>()?;
Ok(ParquetType::try_from_group(
name, repetition, None, None, fields, None,
Expand Down
Loading