Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed clippy and fmt (#521)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Oct 10, 2021
1 parent 6462c83 commit bfa1e5c
Show file tree
Hide file tree
Showing 23 changed files with 94 additions and 82 deletions.
2 changes: 1 addition & 1 deletion src/array/binary/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ impl<O: Offset> MutableBinaryArray<O> {
extend_from_trusted_len_iter(
&mut self.offsets,
&mut self.values,
&mut self.validity.as_mut().unwrap(),
self.validity.as_mut().unwrap(),
iterator,
);

Expand Down
6 changes: 3 additions & 3 deletions src/array/display.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,23 +149,23 @@ pub fn get_value_display<'a>(array: &'a dyn Array) -> Box<dyn Fn(usize) -> Strin
List(_) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, ListArray<i32>, f)
}
FixedSizeList(_, _) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, FixedSizeListArray, f)
}
LargeList(_) => {
let f = |x: Box<dyn Array>| {
let display = get_value_display(x.as_ref());
let string_values = (0..x.len()).map(|i| display(i)).collect::<Vec<String>>();
let string_values = (0..x.len()).map(display).collect::<Vec<String>>();
format!("[{}]", string_values.join(", "))
};
dyn_display!(array, ListArray<i64>, f)
Expand Down
2 changes: 2 additions & 0 deletions src/array/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ use crate::error::Result;

/// Trait describing how a struct presents itself to the
/// [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) (FFI).
/// Safety:
/// Implementing this trait incorrect will lead to UB
pub unsafe trait ToFfi {
/// The pointers to the buffers.
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>>;
Expand Down
4 changes: 3 additions & 1 deletion src/array/utf8/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ impl<O: Offset, A: ffi::ArrowArrayRef> FromFfi<A> for Utf8Array<O> {
validity = validity.map(|x| x.slice(offset, length))
}
let data_type = Self::default_data_type();
Ok(Self::from_data_unchecked(data_type, offsets, values, validity))
Ok(Self::from_data_unchecked(
data_type, offsets, values, validity,
))
}
}
2 changes: 1 addition & 1 deletion src/array/utf8/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ impl<O: Offset> MutableUtf8Array<O> {
extend_from_trusted_len_iter(
&mut self.offsets,
&mut self.values,
&mut self.validity.as_mut().unwrap(),
self.validity.as_mut().unwrap(),
iterator,
);

Expand Down
2 changes: 1 addition & 1 deletion src/bitmap/bitmap_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ where
{
let rem = op(iter.remainder());

let iterator = iter.map(|left| op(left)).chain(std::iter::once(rem));
let iterator = iter.map(op).chain(std::iter::once(rem));

let buffer = MutableBuffer::from_chunk_iter(iterator);

Expand Down
2 changes: 1 addition & 1 deletion src/bitmap/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ impl MutableBitmap {
/// Panics iff `index >= self.len()`.
#[inline]
pub fn set(&mut self, index: usize, value: bool) {
set_bit(&mut self.buffer.as_mut_slice(), index, value)
set_bit(self.buffer.as_mut_slice(), index, value)
}

/// Shrinks the capacity of the [`MutableBitmap`] to fit its current length.
Expand Down
8 changes: 5 additions & 3 deletions src/compute/arithmetics/decimal/add.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,11 @@ pub fn add(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
let op = move |a, b| {
let res: i128 = a + b;

if res.abs() > max_value(*lhs_p) {
panic!("Overflow in addition presented for precision {}", lhs_p);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in addition presented for precision {}",
lhs_p
);

res
};
Expand Down
11 changes: 5 additions & 6 deletions src/compute/arithmetics/decimal/div.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,11 @@ pub fn div(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
// by zero.
let res: i128 = numeral.checked_div(b).expect("Found division by zero");

if res.abs() > max_value(*lhs_p) {
panic!(
"Overflow in multiplication presented for precision {}",
lhs_p
);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in multiplication presented for precision {}",
lhs_p
);

res
};
Expand Down
11 changes: 5 additions & 6 deletions src/compute/arithmetics/decimal/mul.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,11 @@ pub fn mul(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
// 24691.308 <-- 24691308642
let res = res / 10i128.pow(*lhs_s as u32);

if res.abs() > max_value(*lhs_p) {
panic!(
"Overflow in multiplication presented for precision {}",
lhs_p
);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in multiplication presented for precision {}",
lhs_p
);

res
};
Expand Down
8 changes: 5 additions & 3 deletions src/compute/arithmetics/decimal/sub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ pub fn sub(lhs: &PrimitiveArray<i128>, rhs: &PrimitiveArray<i128>) -> Result<Pri
let op = move |a, b| {
let res: i128 = a - b;

if res.abs() > max_value(*lhs_p) {
panic!("Overflow in subtract presented for precision {}", lhs_p);
}
assert!(
!(res.abs() > max_value(*lhs_p)),
"Overflow in subtract presented for precision {}",
lhs_p
);

res
};
Expand Down
8 changes: 1 addition & 7 deletions src/compute/cast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,14 @@ pub use primitive_to::*;
pub use utf8_to::*;

/// options defining how Cast kernels behave
#[derive(Clone, Copy, Debug)]
#[derive(Clone, Copy, Debug, Default)]
struct CastOptions {
/// default to false
/// whether an overflowing cast should be converted to `None` (default), or be wrapped (i.e. `256i16 as u8 = 0` vectorized).
/// Settings this to `true` is 5-6x faster for numeric types.
wrapped: bool,
}

impl Default for CastOptions {
fn default() -> Self {
Self { wrapped: false }
}
}

impl CastOptions {
fn with_wrapped(&self, v: bool) -> Self {
let mut option = *self;
Expand Down
4 changes: 2 additions & 2 deletions src/compute/sort/primitive/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ where

// sort all non-null values
sort_values(
&mut buffer.as_mut_slice(),
buffer.as_mut_slice(),
cmp,
options.descending,
limit - validity.null_count(),
Expand Down Expand Up @@ -153,7 +153,7 @@ where
let mut buffer = MutableBuffer::<T>::new();
buffer.extend_from_slice(values);

sort_values(&mut buffer.as_mut_slice(), cmp, options.descending, limit);
sort_values(buffer.as_mut_slice(), cmp, options.descending, limit);
buffer.truncate(limit);
buffer.shrink_to_fit();

Expand Down
4 changes: 1 addition & 3 deletions src/io/avro/read/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,7 @@ fn read_block<R: Read>(reader: &mut R, buf: &mut Vec<u8>, file_marker: [u8; 16])
let mut marker = [0u8; 16];
reader.read_exact(&mut marker)?;

if marker != file_marker {
panic!();
}
assert!(!(marker != file_marker));
Ok(rows)
}

Expand Down
72 changes: 41 additions & 31 deletions src/io/parquet/read/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
//! APIs to read from Parquet format.
use std::{convert::TryInto, io::{Read, Seek}, sync::Arc};
use std::{
convert::TryInto,
io::{Read, Seek},
sync::Arc,
};

use futures::{AsyncRead, AsyncSeek, Stream};
pub use parquet2::{
Expand All @@ -18,7 +22,11 @@ pub use parquet2::{
types::int96_to_i64_ns,
};

use crate::{array::{Array, DictionaryKey, PrimitiveArray}, datatypes::{DataType, IntervalUnit, TimeUnit}, error::{ArrowError, Result}};
use crate::{
array::{Array, DictionaryKey, PrimitiveArray},
datatypes::{DataType, IntervalUnit, TimeUnit},
error::{ArrowError, Result},
};

mod binary;
mod boolean;
Expand Down Expand Up @@ -205,45 +213,47 @@ pub fn page_iter_to_array<
iter, data_type, metadata,
)?)),
Decimal(_, _) => match metadata.descriptor().type_() {
ParquetType::PrimitiveType { physical_type, ..} => match physical_type{
PhysicalType::Int32 => primitive::iter_to_array(
iter,
metadata,
data_type,
|x: i32| x as i128,
),
PhysicalType::Int64 => primitive::iter_to_array(
iter,
metadata,
data_type,
|x: i64| x as i128,
),
ParquetType::PrimitiveType { physical_type, .. } => match physical_type {
PhysicalType::Int32 => {
primitive::iter_to_array(iter, metadata, data_type, |x: i32| x as i128)
}
PhysicalType::Int64 => {
primitive::iter_to_array(iter, metadata, data_type, |x: i64| x as i128)
}
PhysicalType::FixedLenByteArray(n) => {
if *n > 16 {
Err(ArrowError::NotYetImplemented(format!(
"Can't decode Decimal128 type from Fixed Size Byte Array of len {:?}",
n
)))
} else {
let paddings = (0..(16-*n)).map(|_| 0u8).collect::<Vec<_>>();
fixed_size_binary::iter_to_array(iter, DataType::FixedSizeBinary(*n), metadata)
.map(|e|{
let a = e.into_iter().map(|v|
v.and_then(|v1| {
[&paddings, v1].concat().try_into().map(
|pad16| i128::from_be_bytes(pad16)
).ok()
}
)
).collect::<Vec<_>>();
Box::new(PrimitiveArray::<i128>::from(a).to(data_type)) as Box<dyn Array>
}
let paddings = (0..(16 - *n)).map(|_| 0u8).collect::<Vec<_>>();
fixed_size_binary::iter_to_array(
iter,
DataType::FixedSizeBinary(*n),
metadata,
)
.map(|e| {
let a = e
.into_iter()
.map(|v| {
v.and_then(|v1| {
[&paddings, v1]
.concat()
.try_into()
.map(i128::from_be_bytes)
.ok()
})
})
.collect::<Vec<_>>();
Box::new(PrimitiveArray::<i128>::from(a).to(data_type))
as Box<dyn Array>
})
}
},
_ => unreachable!()
}
_ => unreachable!(),
},
_ => unreachable!()
_ => unreachable!(),
},
List(ref inner) => match inner.data_type() {
UInt8 => primitive::iter_to_array_nested(iter, metadata, data_type, |x: i32| x as u8),
Expand Down
2 changes: 1 addition & 1 deletion src/io/parquet/read/primitive/basic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ fn read_required<T, A, F>(
assert_eq!(values_buffer.len(), additional * std::mem::size_of::<T>());
let iterator = ExactChunksIter::<T>::new(values_buffer);

let iterator = iterator.map(|value| op(value));
let iterator = iterator.map(op);

values.extend_from_trusted_len_iter(iterator);
}
Expand Down
2 changes: 1 addition & 1 deletion src/io/parquet/read/primitive/nested.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ where
A: ArrowNativeType,
F: Fn(T) -> A,
{
let iterator = new_values.map(|v| op(v));
let iterator = new_values.map(op);
values.extend_from_trusted_len_iter(iterator);
}

Expand Down
6 changes: 3 additions & 3 deletions src/io/parquet/read/schema/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub fn parquet_to_arrow_schema(
schema
.fields()
.iter()
.map(|t| to_field(t))
.map(to_field)
.filter_map(|x| x.transpose())
.collect::<Result<Vec<_>>>()
.map(|fields| Schema::new_from(fields, metadata))
Expand Down Expand Up @@ -167,7 +167,7 @@ pub fn from_int64(
ParquetTimeUnit::MICROS(_) => DataType::Time64(TimeUnit::Microsecond),
ParquetTimeUnit::NANOS(_) => DataType::Time64(TimeUnit::Nanosecond),
},
(Some(PrimitiveConvertedType::Decimal(precision,scale)), _) => {
(Some(PrimitiveConvertedType::Decimal(precision, scale)), _) => {
DataType::Decimal(*precision as usize, *scale as usize)
}
(c, l) => {
Expand Down Expand Up @@ -284,7 +284,7 @@ fn to_group_type_inner(
fn to_struct(fields: &[ParquetType]) -> Result<Option<DataType>> {
fields
.iter()
.map(|field| to_field(field))
.map(to_field)
.collect::<Result<Vec<Option<Field>>>>()
.map(|result| result.into_iter().flatten().collect::<Vec<Field>>())
.map(|fields| {
Expand Down
4 changes: 2 additions & 2 deletions src/io/parquet/read/statistics/fixlen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,15 @@ impl TryFrom<(&ParquetFixedLenStatistics, DataType)> for PrimitiveStatistics<i12
[paddings.as_slice(), value]
.concat()
.try_into()
.map(|v| i128::from_be_bytes(v))
.map(i128::from_be_bytes)
.ok()
});

let min_value = stats.min_value.as_ref().and_then(|value| {
[paddings.as_slice(), value]
.concat()
.try_into()
.map(|v| i128::from_be_bytes(v))
.map(i128::from_be_bytes)
.ok()
});
Ok(Self {
Expand Down
8 changes: 4 additions & 4 deletions src/io/parquet/write/fixed_len_bytes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use parquet2::{
encoding::Encoding,
metadata::ColumnDescriptor,
page::CompressedDataPage,
statistics::{serialize_statistics, deserialize_statistics, ParquetStatistics},
statistics::{deserialize_statistics, serialize_statistics, ParquetStatistics},
write::WriteOptions,
};

Expand Down Expand Up @@ -98,7 +98,7 @@ pub(super) fn build_statistics(
.min_by(|x, y| ord_binary(x, y))
.map(|x| x.to_vec()),
};
deserialize_statistics(pq_statistics,descriptor).map(
|e| serialize_statistics(&*e)
).ok()
deserialize_statistics(pq_statistics, descriptor)
.map(|e| serialize_statistics(&*e))
.ok()
}
2 changes: 1 addition & 1 deletion src/io/parquet/write/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ pub fn to_parquet_type(field: &Field) -> Result<ParquetType> {
// recursively convert children to types/nodes
let fields = fields
.iter()
.map(|f| to_parquet_type(f))
.map(to_parquet_type)
.collect::<Result<Vec<_>>>()?;
Ok(ParquetType::try_from_group(
name, repetition, None, None, fields, None,
Expand Down
Loading

0 comments on commit bfa1e5c

Please sign in to comment.