-
-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(rust): POC metadata reading and writing (#17112)
- Loading branch information
1 parent
42ba1b0
commit b60788d
Showing
12 changed files
with
280 additions
and
127 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| use std::ops::Deref; | ||
| use std::sync::RwLockReadGuard; | ||
|
|
||
| use super::Metadata; | ||
| use crate::chunked_array::PolarsDataType; | ||
|
|
||
| /// A read guard for [`Metadata`] | ||
| pub enum MetadataReadGuard<'a, T: PolarsDataType + 'a> { | ||
| Unlocked(RwLockReadGuard<'a, Metadata<T>>), | ||
| Locked(&'a Metadata<T>), | ||
| } | ||
|
|
||
| impl<'a, T: PolarsDataType + 'a> Deref for MetadataReadGuard<'a, T> { | ||
| type Target = Metadata<T>; | ||
|
|
||
| #[inline] | ||
| fn deref(&self) -> &Self::Target { | ||
| match self { | ||
| Self::Unlocked(v) => v.deref(), | ||
| Self::Locked(v) => v, | ||
| } | ||
| } | ||
| } |
75 changes: 75 additions & 0 deletions
75
crates/polars-core/src/chunked_array/metadata/interior_mutable.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; | ||
|
|
||
| use super::{Metadata, MetadataTrait}; | ||
| use crate::chunked_array::PolarsDataType; | ||
|
|
||
| // I have attempted multiple times to move this interior mutability to a per metadata field basis. | ||
| // While this might allow the use of Atomics instead of RwLocks, it suffers several problems: | ||
| // | ||
| // 1. The amount of boilerplate explodes. For example, you want read, read_blocking, write, | ||
| // write_blocking, get_mut, set for each field. | ||
| // 2. It is also very difficult to combine with the dynamic dispatch. | ||
| // 3. It is difficult to combine with types that do not allow for atomics (e.g. Box<[u8]>). | ||
| // 4. You actually have 2 fields per field: the Option and the Value. You run into critical section | ||
| // problems if you try to separate these. | ||
|
|
||
| /// An interiorally mutable [`Metadata`] | ||
| /// | ||
| /// This is essentially a more convenient API around `RwLock<Metadata>`. This also allows it to be | ||
| /// `Clone`. | ||
| pub struct IMMetadata<T: PolarsDataType>(RwLock<Metadata<T>>); | ||
|
|
||
| impl<'a, T: PolarsDataType + 'a> IMMetadata<T> | ||
| where | ||
| Metadata<T>: MetadataTrait + 'a, | ||
| { | ||
| /// Cast the [`IMMetadata`] to a trait object of [`MetadataTrait`] | ||
| pub fn upcast(&'a self) -> &'a RwLock<dyn MetadataTrait + 'a> { | ||
| &self.0 as &RwLock<dyn MetadataTrait + 'a> | ||
| } | ||
| } | ||
|
|
||
| impl<T: PolarsDataType> IMMetadata<T> { | ||
| pub const fn new(md: Metadata<T>) -> Self { | ||
| Self(RwLock::new(md)) | ||
| } | ||
|
|
||
| /// Try to grab a read guard to the [`Metadata`], this fails if this blocks. | ||
| pub fn try_read(&self) -> Option<RwLockReadGuard<Metadata<T>>> { | ||
| self.0.try_read().ok() | ||
| } | ||
| /// Block to grab a read guard the [`Metadata`] | ||
| pub fn read(&self) -> RwLockReadGuard<Metadata<T>> { | ||
| self.0.read().unwrap() | ||
| } | ||
|
|
||
| /// Try to grab a write guard to the [`Metadata`], this fails if this blocks. | ||
| pub fn try_write(&self) -> Option<RwLockWriteGuard<Metadata<T>>> { | ||
| self.0.try_write().ok() | ||
| } | ||
| /// Block to grab a write guard the [`Metadata`] | ||
| pub fn write(&self) -> RwLockWriteGuard<Metadata<T>> { | ||
| self.0.write().unwrap() | ||
| } | ||
|
|
||
| /// Take the internal [`Metadata`] | ||
| pub fn take(self) -> Metadata<T> { | ||
| self.0.into_inner().unwrap() | ||
| } | ||
| /// Get the mutable to the internal [`Metadata`] | ||
| pub fn get_mut(&mut self) -> &mut Metadata<T> { | ||
| self.0.get_mut().unwrap() | ||
| } | ||
| } | ||
|
|
||
| impl<T: PolarsDataType> Clone for IMMetadata<T> { | ||
| fn clone(&self) -> Self { | ||
| Self::new(self.read().clone()) | ||
| } | ||
| } | ||
|
|
||
| impl<T: PolarsDataType> Default for IMMetadata<T> { | ||
| fn default() -> Self { | ||
| Self::new(Default::default()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| use polars_utils::IdxSize; | ||
|
|
||
| use super::{Metadata, MetadataFlags}; | ||
| use crate::chunked_array::{IntoScalar, PolarsDataType, Scalar}; | ||
|
|
||
| pub trait MetadataTrait { | ||
| fn get_flags(&self) -> MetadataFlags; | ||
| fn min_value(&self) -> Option<Scalar>; | ||
| fn max_value(&self) -> Option<Scalar>; | ||
|
|
||
| /// Number of unique non-null values | ||
| fn distinct_count(&self) -> Option<IdxSize>; | ||
| } | ||
|
|
||
| impl<T: PolarsDataType> MetadataTrait for Metadata<T> | ||
| where | ||
| T::OwnedPhysical: IntoScalar + Clone, | ||
| { | ||
| fn get_flags(&self) -> MetadataFlags { | ||
| self.get_flags() | ||
| } | ||
|
|
||
| fn min_value(&self) -> Option<Scalar> { | ||
| self.get_min_value() | ||
| .map(|v| v.clone().into_scalar(T::get_dtype()).unwrap()) | ||
| } | ||
|
|
||
| fn max_value(&self) -> Option<Scalar> { | ||
| self.get_max_value() | ||
| .map(|v| v.clone().into_scalar(T::get_dtype()).unwrap()) | ||
| } | ||
|
|
||
| fn distinct_count(&self) -> Option<IdxSize> { | ||
| self.get_distinct_count() | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.