progress commit

AsyncioDemo/parallel_movement/app.py

@@ -7,7 +7,8 @@
 import os
 import shutil
 import sys
-
+import time
+import random
 
 def list_files(path: str, files: list = None)->list:
     """List of files recursively.
@@ -67,6 +68,20 @@ def make_dir(path: str, dir_name: str):
     os.makedirs(dir_path, exist_ok=True)
 
 
+def move_content_sync(source: str, target: str):
+    """Move content.
+
+    Args:
+        source (str): source path.
+        target (str): target path.
+    """
+    shutil.copy(source, target)
+
+    # adding this random sleep for real time scenario
+    # that this copy will take certain time
+    time.sleep(random.randint(1,5)) # it can take upto 5 seconds
+
+
 def move_files_sync(file_obj: dict, dest_path: str):
     """Move files synchronously.
 
@@ -80,9 +95,9 @@ def move_files_sync(file_obj: dict, dest_path: str):
     for ext in file_obj:
         ext_path = os.path.join(dest_path, ext)
         for file in file_obj[ext]:
-            shutil.copy(file, ext_path)
+            move_content_sync(file, ext_path)
 
-async def move_content(source: str, target: str):
+async def move_content_async(source: str, target: str):
     """Move content.
 
     Args:
@@ -91,6 +106,10 @@ async def move_content(source: str, target: str):
     """
     shutil.copy(source, target)
 
+    # adding this random sleep for real time scenario
+    # that this copy will take certain time
+    await asyncio.sleep(random.randint(1,5))
+
 async def move_files_async(file_obj: dict, dest_path: str):
     """Move files asynchronously.
 
@@ -105,20 +124,29 @@ async def move_files_async(file_obj: dict, dest_path: str):
     for ext in file_obj:
         ext_path = os.path.join(dest_path, ext)
         for file in file_obj[ext]:
-            task_list.append(move_content(file, ext_path))
+            task_list.append(move_content_async(file, ext_path))
 
     await asyncio.gather(*task_list)
 
 if __name__ == '__main__':
     source_path = sys.argv[1]
     target_path = sys.argv[2]
 
+
+    start = time.time()
     files = list_files(path=source_path)
     file_obj = extension_segregation(files_list=files)
 
 
-    # syncronous
+    # for scenario of moving 50 files
+    # where every move takes from 1 to 5 seconds
+
+
+    # syncronous total time taken : 146.13251948356628
     # move_files_sync(file_obj, target_path)
 
-    # asyncronous
+    # asyncronous total time taken : 5.008042812347412
     asyncio.run(move_files_async(file_obj, target_path))
+
+    end = time.time()
+    print("total time taken :",end - start)

AsyncioDemo/parallel_movement/create_files.py

@@ -0,0 +1,11 @@
+import os 
+
+
+n_files = 50
+path = './storage/source'
+ext = 'csv'
+for i in range(n_files):
+    f_name = f'a_{i}.{ext}'
+    with open(os.path.join(path,f_name),'w') as file:
+
+        file.write('Hello')