Skip to content

Commit

Permalink
chore: fix features (#14977)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Mar 11, 2024
1 parent 295751f commit 9ee7c9e
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 2 deletions.
1 change: 1 addition & 0 deletions crates/polars-ops/src/chunked_array/list/sets.rs
Expand Up @@ -421,6 +421,7 @@ pub fn list_set_operation(
b.prune_empty_chunks();

// Make categoricals compatible
#[cfg(feature = "dtype-categorical")]
if let (DataType::Categorical(_, _), DataType::Categorical(_, _)) =
(&a.inner_dtype(), &b.inner_dtype())
{
Expand Down
8 changes: 7 additions & 1 deletion crates/polars-pipe/src/executors/sinks/memory.rs
Expand Up @@ -41,7 +41,7 @@ impl MemTracker {
}

/// This shouldn't be called often as this is expensive.
fn refresh_memory(&self) {
pub fn refresh_memory(&self) {
self.available_mem
.store(MEMINFO.free() as usize, Ordering::Relaxed);
}
Expand All @@ -57,6 +57,12 @@ impl MemTracker {
self.available_mem.load(Ordering::Relaxed)
}

pub(super) fn get_available_latest(&self) -> usize {
self.refresh_memory();
self.fetch_count.store(0, Ordering::Relaxed);
self.available_mem.load(Ordering::Relaxed)
}

pub(super) fn free_memory_fraction_since_start(&self) -> f64 {
// We divide first to reduce the precision loss in floats.
// We also add 1.0 to available_at_start to prevent division by zero.
Expand Down
5 changes: 4 additions & 1 deletion crates/polars-pipe/src/executors/sinks/sort/ooc.rs
Expand Up @@ -117,7 +117,10 @@ pub(super) fn sort_ooc(
) -> PolarsResult<FinalizedSink> {
let samples = samples.to_physical_repr().into_owned();
// Try to use available memory. At least 32MB per spill.
let spill_size = std::cmp::max(memtrack.get_available() / (samples.len() * 2), 1 << 25) as u64;
let spill_size = std::cmp::max(
memtrack.get_available_latest() / (samples.len() * 3),
1 << 25,
) as u64;

// we collect as I am not sure that if we write to the same directory the
// iterator will read those also.
Expand Down

0 comments on commit 9ee7c9e

Please sign in to comment.