Handle small datasets with Memory File System

nalepae · Apr 5, 2020 · 10fa6ec · 10fa6ec
1 parent 5c67fa0
commit 10fa6ec
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/pandarallel/__init__.py b/pandarallel/__init__.py
@@ -1,3 +1,3 @@
-__version__ = "1.4.7"
+__version__ = "1.4.8"
 
 from .pandarallel import pandarallel
diff --git a/pandarallel/pandarallel.py b/pandarallel/pandarallel.py
@@ -251,13 +251,16 @@ def dump_and_get_lenght(chunk, input_file):
 
     if use_memory_fs:
         input_files = create_temp_files(nb_workers)
-        output_files = create_temp_files(nb_workers)
 
         try:
             chunk_lengths = [
                 dump_and_get_lenght(chunk, input_file)
                 for chunk, input_file in zip(chunks, input_files)
             ]
+
+            nb_chunks = len(chunk_lengths)
+            output_files = create_temp_files(nb_chunks)
+
         except OSError:
             link = "https://stackoverflow.com/questions/58804022/how-to-resize-dev-shm"
             msg = " ".join(

diff --git a/setup.py b/setup.py
@@ -4,13 +4,13 @@
 
 setup(
     name="pandarallel",
-    version="1.4.7",
+    version="1.4.8",
     python_requires=">=3.5",
     packages=find_packages(),
     author="Manu NALEPA",
     author_email="nalepae@gmail.com",
     description="An easy to use library to speed up computation (by parallelizing on multi CPUs) with pandas.",
-    long_description="See https://github.com/nalepae/pandarallel/tree/v1.4.7 for complete user guide.",
+    long_description="See https://github.com/nalepae/pandarallel/tree/v1.4.8 for complete user guide.",
     url="https://github.com/nalepae/pandarallel",
     install_requires=install_requires,
     license="BSD",