Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed stats from int96 parquet (#1181)
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Jul 24, 2022
1 parent c720eb2 commit f9562e0
Showing 1 changed file with 36 additions and 7 deletions.
43 changes: 36 additions & 7 deletions src/io/parquet/read/statistics/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use parquet2::statistics::{
BinaryStatistics, BooleanStatistics, FixedLenStatistics, PrimitiveStatistics,
Statistics as ParquetStatistics,
};
use parquet2::types::int96_to_i64_ns;

use crate::array::*;
use crate::datatypes::IntervalUnit;
Expand Down Expand Up @@ -487,13 +488,41 @@ fn push(
UInt64 => primitive::push(from, min, max, |x: i64| Ok(x as u64)),
Timestamp(time_unit, _) => {
let time_unit = *time_unit;
primitive::push(from, min, max, |x: i64| {
Ok(primitive::timestamp(
type_.logical_type.as_ref(),
time_unit,
x,
))
})
if physical_type == &ParquetPhysicalType::Int96 {
let from = from.map(|from| {
let from = from
.as_any()
.downcast_ref::<PrimitiveStatistics<[u32; 3]>>()
.unwrap();
PrimitiveStatistics::<i64> {
primitive_type: from.primitive_type.clone(),
null_count: from.null_count,
distinct_count: from.distinct_count,
min_value: from.min_value.map(int96_to_i64_ns),
max_value: from.max_value.map(int96_to_i64_ns),
}
});
primitive::push(
from.as_ref().map(|x| x as &dyn ParquetStatistics),
min,
max,
|x: i64| {
Ok(primitive::timestamp(
type_.logical_type.as_ref(),
time_unit,
x,
))
},
)
} else {
primitive::push(from, min, max, |x: i64| {
Ok(primitive::timestamp(
type_.logical_type.as_ref(),
time_unit,
x,
))
})
}
}
Float32 => primitive::push(from, min, max, |x: f32| Ok(x as f32)),
Float64 => primitive::push(from, min, max, |x: f64| Ok(x as f64)),
Expand Down

0 comments on commit f9562e0

Please sign in to comment.