from parallel iterator consumed too much memory

pola-rs · Oct 29, 2021 · 8e88408 · 8e88408
1 parent b883e53
commit 8e88408
Showing 1 changed file with 4 additions and 11 deletions.
diff --git a/polars/polars-core/src/chunked_array/upstream_traits.rs b/polars/polars-core/src/chunked_array/upstream_traits.rs
@@ -272,20 +272,13 @@ fn collect_into_linked_list<I>(par_iter: I) -> LinkedList<Vec<I::Item>>
 where
     I: IntoParallelIterator,
 {
-    use crate::POOL;
     let it = par_iter.into_par_iter();
-    if let Some(len) = it.opt_len() {
-        it.fold(
-            || Vec::with_capacity(len / POOL.current_num_threads()),
-            vec_push,
-        )
+    // be careful optimizing allocations. Its hard to figure out the size
+    // needed
+    // https://github.com/pola-rs/polars/issues/1562
+    it.fold(Vec::new, vec_push)
         .map(as_list)
         .reduce(LinkedList::new, list_append)
-    } else {
-        it.fold(Vec::new, vec_push)
-            .map(as_list)
-            .reduce(LinkedList::new, list_append)
-    }
 }
 
 fn get_capacity_from_par_results<T>(ll: &LinkedList<Vec<T>>) -> usize {