Skip to content

Commit

Permalink
Refactor: move most temporal related code to polars-time. (#2918)
Browse files Browse the repository at this point in the history 
* move most temporal related code to polars time
  • Loading branch information
@ritchie46
ritchie46 committed Mar 16, 2022
1 parent b90bcfa commit d349899
Show file tree
Hide file tree
Showing 38 changed files with 1,596 additions and 1,083 deletions.
4 changes: 2 additions & 2 deletions polars/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,8 @@ dtype-datetime = [
"polars-io/dtype-datetime",
"polars-time/dtype-datetime",
]
dtype-duration = ["polars-core/dtype-duration", "polars-lazy/dtype-duration"]
dtype-time = ["polars-core/dtype-time", "polars-io/dtype-time"]
dtype-duration = ["polars-core/dtype-duration", "polars-lazy/dtype-duration", "polars-time/dtype-duration"]
dtype-time = ["polars-core/dtype-time", "polars-io/dtype-time", "polars-time/dtype-time"]
dtype-i8 = ["polars-core/dtype-i8", "polars-lazy/dtype-i8"]
dtype-i16 = ["polars-core/dtype-i16", "polars-lazy/dtype-i16"]
dtype-u8 = ["polars-core/dtype-u8", "polars-lazy/dtype-u8"]
Expand Down
7 changes: 0 additions & 7 deletions polars/polars-core/src/chunked_array/kernels/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1 @@
pub(crate) mod take;
#[cfg(any(
feature = "temporal",
feature = "dtype-datetime",
feature = "dtype-date"
))]
#[cfg_attr(docsrs, doc(cfg(feature = "temporal")))]
pub mod temporal;
6 changes: 0 additions & 6 deletions polars/polars-core/src/chunked_array/temporal/conversion.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use super::*;
use crate::prelude::{AnyValue, TimeUnit};
#[cfg(feature = "dtype-time")]
use arrow::temporal_conversions::time64ns_to_time;
Expand Down Expand Up @@ -59,11 +58,6 @@ pub(crate) fn naive_datetime_to_date(v: NaiveDateTime) -> i32 {
(datetime_to_timestamp_ms(v) / (MILLISECONDS * SECONDS_IN_DAY)) as i32
}

pub(crate) fn naive_date_to_date(nd: NaiveDate) -> i32 {
let nt = NaiveTime::from_hms(0, 0, 0);
let ndt = NaiveDateTime::new(nd, nt);
naive_datetime_to_date(ndt)
}
pub(crate) const NS_IN_DAY: i64 = 86_400_000_000_000;
pub(crate) const US_IN_DAY: i64 = 86_400_000_000;
pub(crate) const MS_IN_DAY: i64 = 86_400_000;
84 changes: 0 additions & 84 deletions polars/polars-core/src/chunked_array/temporal/date.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use super::*;
use crate::prelude::*;
use arrow::temporal_conversions::date32_to_date;
use std::fmt::Write;

pub(crate) fn naive_date_to_date(nd: NaiveDate) -> i32 {
let nt = NaiveTime::from_hms(0, 0, 0);
Expand All @@ -22,76 +21,6 @@ impl DateChunked {
}
}

/// Extract month from underlying NaiveDate representation.
/// Returns the year number in the calendar date.
pub fn year(&self) -> Int32Chunked {
self.apply_kernel_cast::<Int32Type>(&date_to_year)
}

/// Extract month from underlying NaiveDateTime representation.
/// Returns the month number starting from 1.
///
/// The return value ranges from 1 to 12.
pub fn month(&self) -> UInt32Chunked {
self.apply_kernel_cast::<UInt32Type>(&date_to_month)
}

/// Extract weekday from underlying NaiveDate representation.
/// Returns the weekday number where monday = 0 and sunday = 6
pub fn weekday(&self) -> UInt32Chunked {
self.apply_kernel_cast::<UInt32Type>(&date_to_weekday)
}

/// Returns the ISO week number starting from 1.
/// The return value ranges from 1 to 53. (The last week of year differs by years.)
pub fn week(&self) -> UInt32Chunked {
self.apply_kernel_cast::<UInt32Type>(&date_to_week)
}

/// Extract day from underlying NaiveDate representation.
/// Returns the day of month starting from 1.
///
/// The return value ranges from 1 to 31. (The last day of month differs by months.)
pub fn day(&self) -> UInt32Chunked {
self.apply_kernel_cast::<UInt32Type>(&date_to_day)
}

/// Returns the day of year starting from 1.
///
/// The return value ranges from 1 to 366. (The last day of year differs by years.)
pub fn ordinal(&self) -> UInt32Chunked {
self.apply_kernel_cast::<UInt32Type>(&date_to_ordinal)
}

/// Format Date with a `fmt` rule. See [chrono strftime/strptime](https://docs.rs/chrono/0.4.19/chrono/format/strftime/index.html).
pub fn strftime(&self, fmt: &str) -> Utf8Chunked {
let date = NaiveDate::from_ymd(2001, 1, 1);
let fmted = format!("{}", date.format(fmt));

let mut ca: Utf8Chunked = self.apply_kernel_cast(&|arr| {
let mut buf = String::new();
let mut mutarr =
MutableUtf8Array::with_capacities(arr.len(), arr.len() * fmted.len() + 1);

for opt in arr.into_iter() {
match opt {
None => mutarr.push_null(),
Some(v) => {
buf.clear();
let datefmt = date32_to_date(*v).format(fmt);
write!(buf, "{}", datefmt).unwrap();
mutarr.push(Some(&buf))
}
}
}

let arr: Utf8Array<i64> = mutarr.into();
Arc::new(arr)
});
ca.rename(self.name());
ca
}

/// Construct a new [`DateChunked`] from an iterator over [`NaiveDate`].
pub fn from_naive_date<I: IntoIterator<Item = NaiveDate>>(name: &str, v: I) -> Self {
let unit = v.into_iter().map(naive_date_to_date).collect::<Vec<_>>();
Expand All @@ -106,17 +35,4 @@ impl DateChunked {
let unit = v.into_iter().map(|opt| opt.map(naive_date_to_date));
Int32Chunked::from_iter_options(name, unit).into()
}

pub fn parse_from_str_slice(name: &str, v: &[&str], fmt: &str) -> Self {
Int32Chunked::from_iter_options(
name,
v.iter().map(|s| {
NaiveDate::parse_from_str(s, fmt)
.ok()
.as_ref()
.map(|v| naive_date_to_date(*v))
}),
)
.into()
}
}
167 changes: 0 additions & 167 deletions polars/polars-core/src/chunked_array/temporal/datetime.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use crate::prelude::*;
use arrow::temporal_conversions::{
timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_us_to_datetime,
};
use std::fmt::Write;

impl DatetimeChunked {
pub fn as_datetime_iter(
Expand Down Expand Up @@ -38,157 +37,6 @@ impl DatetimeChunked {
}
}

/// Extract month from underlying NaiveDateTime representation.
/// Returns the year number in the calendar date.
pub fn year(&self) -> Int32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_year_ns,
TimeUnit::Microseconds => datetime_to_year_us,
TimeUnit::Milliseconds => datetime_to_year_ms,
};
self.apply_kernel_cast::<Int32Type>(&f)
}

/// Extract month from underlying NaiveDateTime representation.
/// Returns the month number starting from 1.
///
/// The return value ranges from 1 to 12.
pub fn month(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_month_ns,
TimeUnit::Microseconds => datetime_to_month_us,
TimeUnit::Milliseconds => datetime_to_month_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract weekday from underlying NaiveDateTime representation.
/// Returns the weekday number where monday = 0 and sunday = 6
pub fn weekday(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_weekday_ns,
TimeUnit::Microseconds => datetime_to_weekday_us,
TimeUnit::Milliseconds => datetime_to_weekday_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Returns the ISO week number starting from 1.
/// The return value ranges from 1 to 53. (The last week of year differs by years.)
pub fn week(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_week_ns,
TimeUnit::Microseconds => datetime_to_week_us,
TimeUnit::Milliseconds => datetime_to_week_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract day from underlying NaiveDateTime representation.
/// Returns the day of month starting from 1.
///
/// The return value ranges from 1 to 31. (The last day of month differs by months.)
pub fn day(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_day_ns,
TimeUnit::Microseconds => datetime_to_day_us,
TimeUnit::Milliseconds => datetime_to_day_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract hour from underlying NaiveDateTime representation.
/// Returns the hour number from 0 to 23.
pub fn hour(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_hour_ns,
TimeUnit::Microseconds => datetime_to_hour_us,
TimeUnit::Milliseconds => datetime_to_hour_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract minute from underlying NaiveDateTime representation.
/// Returns the minute number from 0 to 59.
pub fn minute(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_minute_ns,
TimeUnit::Microseconds => datetime_to_minute_us,
TimeUnit::Milliseconds => datetime_to_minute_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract second from underlying NaiveDateTime representation.
/// Returns the second number from 0 to 59.
pub fn second(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_second_ns,
TimeUnit::Microseconds => datetime_to_second_us,
TimeUnit::Milliseconds => datetime_to_second_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Extract second from underlying NaiveDateTime representation.
/// Returns the number of nanoseconds since the whole non-leap second.
/// The range from 1,000,000,000 to 1,999,999,999 represents the leap second.
pub fn nanosecond(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_nanosecond_ns,
TimeUnit::Microseconds => datetime_to_nanosecond_us,
TimeUnit::Milliseconds => datetime_to_nanosecond_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Returns the day of year starting from 1.
///
/// The return value ranges from 1 to 366. (The last day of year differs by years.)
pub fn ordinal(&self) -> UInt32Chunked {
let f = match self.time_unit() {
TimeUnit::Nanoseconds => datetime_to_ordinal_ns,
TimeUnit::Microseconds => datetime_to_ordinal_us,
TimeUnit::Milliseconds => datetime_to_ordinal_ms,
};
self.apply_kernel_cast::<UInt32Type>(&f)
}

/// Format Datetime with a `fmt` rule. See [chrono strftime/strptime](https://docs.rs/chrono/0.4.19/chrono/format/strftime/index.html).
pub fn strftime(&self, fmt: &str) -> Utf8Chunked {
let conversion_f = match self.time_unit() {
TimeUnit::Nanoseconds => timestamp_ns_to_datetime,
TimeUnit::Microseconds => timestamp_us_to_datetime,
TimeUnit::Milliseconds => timestamp_ms_to_datetime,
};

let dt = NaiveDate::from_ymd(2001, 1, 1).and_hms(0, 0, 0);
let fmted = format!("{}", dt.format(fmt));

let mut ca: Utf8Chunked = self.apply_kernel_cast(&|arr| {
let mut buf = String::new();
let mut mutarr =
MutableUtf8Array::with_capacities(arr.len(), arr.len() * fmted.len() + 1);

for opt in arr.into_iter() {
match opt {
None => mutarr.push_null(),
Some(v) => {
buf.clear();
let datefmt = conversion_f(*v).format(fmt);
write!(buf, "{}", datefmt).unwrap();
mutarr.push(Some(&buf))
}
}
}

let arr: Utf8Array<i64> = mutarr.into();
Arc::new(arr)
});
ca.rename(self.name());
ca
}

/// Construct a new [`DatetimeChunked`] from an iterator over [`NaiveDateTime`].
pub fn from_naive_datetime<I: IntoIterator<Item = NaiveDateTime>>(
name: &str,
Expand Down Expand Up @@ -218,21 +66,6 @@ impl DatetimeChunked {
Int64Chunked::from_iter_options(name, vals).into_datetime(tu, None)
}

pub fn parse_from_str_slice(name: &str, v: &[&str], fmt: &str, tu: TimeUnit) -> Self {
let func = match tu {
TimeUnit::Nanoseconds => datetime_to_timestamp_ns,
TimeUnit::Microseconds => datetime_to_timestamp_us,
TimeUnit::Milliseconds => datetime_to_timestamp_ms,
};

Int64Chunked::from_iter_options(
name,
v.iter()
.map(|s| NaiveDateTime::parse_from_str(s, fmt).ok().map(func)),
)
.into_datetime(tu, None)
}

/// Change the underlying [`TimeUnit`]. And update the data accordingly.
#[must_use]
pub fn cast_time_unit(&self, tu: TimeUnit) -> Self {
Expand Down
49 changes: 0 additions & 49 deletions polars/polars-core/src/chunked_array/temporal/duration.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
use crate::export::chrono::Duration as ChronoDuration;
use crate::prelude::DataType::Duration;
use crate::prelude::*;
use arrow::temporal_conversions::{MICROSECONDS, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS};

const NANOSECONDS_IN_MILLISECOND: i64 = 1_000_000;
const SECONDS_IN_HOUR: i64 = 3600;

impl DurationChunked {
pub fn time_unit(&self) -> TimeUnit {
Expand Down Expand Up @@ -93,49 +89,4 @@ impl DurationChunked {
let vals = v.into_iter().map(|opt| opt.map(func));
Int64Chunked::from_iter_options(name, vals).into_duration(tu)
}

/// Extract the hours from a `Duration`
pub fn hours(&self) -> Int64Chunked {
match self.time_unit() {
TimeUnit::Milliseconds => &self.0 / (MILLISECONDS * SECONDS_IN_HOUR),
TimeUnit::Microseconds => &self.0 / (MICROSECONDS * SECONDS_IN_HOUR),
TimeUnit::Nanoseconds => &self.0 / (NANOSECONDS * SECONDS_IN_HOUR),
}
}

/// Extract the days from a `Duration`
pub fn days(&self) -> Int64Chunked {
match self.time_unit() {
TimeUnit::Milliseconds => &self.0 / MILLISECONDS_IN_DAY,
TimeUnit::Microseconds => &self.0 / (MICROSECONDS * SECONDS_IN_DAY),
TimeUnit::Nanoseconds => &self.0 / (NANOSECONDS * SECONDS_IN_DAY),
}
}

/// Extract the milliseconds from a `Duration`
pub fn milliseconds(&self) -> Int64Chunked {
match self.time_unit() {
TimeUnit::Milliseconds => self.0.clone(),
TimeUnit::Microseconds => self.0.clone() / 1000,
TimeUnit::Nanoseconds => &self.0 / 1_000_000,
}
}

/// Extract the nanoseconds from a `Duration`
pub fn nanoseconds(&self) -> Int64Chunked {
match self.time_unit() {
TimeUnit::Milliseconds => &self.0 * NANOSECONDS_IN_MILLISECOND,
TimeUnit::Microseconds => &self.0 * 1000,
TimeUnit::Nanoseconds => self.0.clone(),
}
}

/// Extract the seconds from a `Duration`
pub fn seconds(&self) -> Int64Chunked {
match self.time_unit() {
TimeUnit::Milliseconds => &self.0 / MILLISECONDS,
TimeUnit::Microseconds => &self.0 / MICROSECONDS,
TimeUnit::Nanoseconds => &self.0 / NANOSECONDS,
}
}
}
1 change: 0 additions & 1 deletion polars/polars-core/src/chunked_array/temporal/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ mod time;
mod utf8;

pub use self::conversion::*;
use crate::chunked_array::kernels::temporal::*;
use chrono::{NaiveDate, NaiveDateTime, NaiveTime};

pub fn unix_time() -> NaiveDateTime {
Expand Down

0 comments on commit d349899

Please sign in to comment.