Add download progress bar (#34)

* Introduction of Download Progress Bar

Co-authored-by: Kevin Rohan Vaz <36708857+excalibur-kvrv@users.noreply.github.com>
Co-authored-by: Kevin Vaz <excalibur.krv@gmail.com>

core/download_process.py

@@ -20,14 +20,14 @@
 
 
 def download_process(links, total_links, session, http2, max_retries,
-                     convert, file_link_maps, path_prefix, debug) -> None:
+                     convert, file_link_maps, path_prefix, debug, progress_bar_queue) -> None:
     print(f"Starting Download process {current_process().name}")
     start_time = time()
     try:
         download_manager = DownloadProcess(links, total_links, session, http2,
                                            max_retries, convert, debug)
 
-        start_processes(download_manager, file_link_maps, path_prefix)
+        start_processes(download_manager, file_link_maps, path_prefix, progress_bar_queue)
         try:
             client = Client(IP, PORT)
             client.send_data(f"{'STOP_QUEUE':<{HEADER_SIZE}}{download_manager.get_total_downloaded_links_count()}")
@@ -37,7 +37,7 @@ def download_process(links, total_links, session, http2, max_retries,
     except (KeyboardInterrupt, Exception):
         print_exc()
 
-    print(f"Download took {time() - start_time} seconds")
+    print(f"\nDownload took {time() - start_time} seconds")
     print(f"Stopped Download process {current_process().name}")
 
 
@@ -72,7 +72,7 @@ def __init__(self, links: List[str], total_links: int, session: requests.Session
         self.convert = convert
         self.__sent = 0
         self.__process_num = len(os.sched_getaffinity(os.getpid())) if platform.system() == "Linux" else 4
-        self.__thread_num = int(ceil((total_links - self.__sent) / self.__process_num))
+        self.__thread_num = int(ceil((total_links - self.__sent) / (self.__process_num * 4)))
         self.debug = debug
         self.done_retries = 0
         self.error_links = []
@@ -102,20 +102,22 @@ def set_total_downloaded_links_count(self, val: int) -> None:
         self.__sent = val
 
 
-def start_processes(download_manager: DownloadProcess, file_link_maps: Dict[str, str], path_prefix: str) -> None:
+def start_processes(download_manager: DownloadProcess, file_link_maps: Dict[str, str], path_prefix: str
+                    , progress_bar_queue) -> None:
     process_num: int = download_manager.get_process_num()
     if download_manager.debug:
         print(f"starting {process_num} processes for {len(download_manager.get_download_links())} links")
 
     with ProcessPoolExecutor(max_workers=process_num) as process_pool_executor:
         try:
-            process_pool_executor_handler(process_pool_executor, download_manager, file_link_maps, path_prefix)
+            process_pool_executor_handler(process_pool_executor, download_manager, file_link_maps, path_prefix
+                                          , progress_bar_queue)
         except (KeyboardInterrupt, Exception):
             sys.exit()
 
 
 def process_pool_executor_handler(executor: ProcessPoolExecutor, manager: DownloadProcess,
-                                  file_maps: Dict[str, str], directory: str) -> None:
+                                  file_maps: Dict[str, str], directory: str, progress_bar_queue) -> None:
     done_queue = JoinableQueue()
 
     def update_hook(future: Future):
@@ -149,7 +151,7 @@ def update_hook(future: Future):
             cpu_num = available_cpus[temp_num % len(available_cpus)]
             process_futures.append(executor.submit(start_threads, download_links[start:end],
                                                    file_maps, manager.get_session(), directory,
-                                                   manager.http2, manager.debug, cpu_num))
+                                                   manager.http2, progress_bar_queue, manager.debug, cpu_num))
             process_futures[-1].add_done_callback(update_hook)
             start = end
 
@@ -181,7 +183,7 @@ def update_hook(future: Future):
 
 
 def start_threads(links: List[str], maps: Dict[str, str], session: requests.Session,
-                  file_path_prefix: str, http2: bool, debug: bool = False,
+                  file_path_prefix: str, http2: bool, progress_bar_queue, debug: bool = False,
                   cpu_num: int = 0) -> List[Optional[str]]:
     failed_links = Queue()
     sessions_queue = Queue()
@@ -233,6 +235,7 @@ def update_hook(future: Future):
     if debug:
         print(f"Sending data to server of size {len(send_data)} bytes")
 
+    progress_bar_queue.put(len(sent_links))
     return failed
 
 

core/m3u8dl.py

@@ -1,6 +1,6 @@
 from requests.adapters import HTTPAdapter
 from hyper.contrib import HTTP20Adapter
-from multiprocessing import Process
+from multiprocessing import Process, Manager
 from traceback import print_exc
 from shutil import rmtree
 
@@ -9,6 +9,7 @@
 from .video_handling_process import video_handling
 from .download_process import download_process
 from .weblib.parse import construct_headers
+from .progressbar import update_progress_bar
 
 import platform
 import requests
@@ -96,8 +97,16 @@ def main():
                         name="video_handling_process")
         video.start()
 
+        queue = Manager().Queue()
+
+        progress_bar_process = Process(target=update_progress_bar, args=(queue, len(links)),
+                                       name="progress_bar_process_going_on")
+
+        progress_bar_process.daemon = True
+        progress_bar_process.start()
+
         download_process(links, len(links), sess, http2, MAX_RETRIES, cli_args.convert,
-                         file_link_maps, path_prefix, debug)
+                         file_link_maps, path_prefix, debug, queue)
 
         server.join()
         video.join()

core/progressbar.py

@@ -0,0 +1,11 @@
+from progress.bar import ChargingBar
+
+
+def update_progress_bar(queue, length):
+    bar = ChargingBar('Downloading ------------>', max=length)
+    while True:
+        try:
+            data = queue.get()
+            bar.next(data)
+        except EOFError:
+            pass

requirements.txt

@@ -9,4 +9,7 @@ idna==2.8
 PyInstaller==3.6
 requests==2.24.0
 urllib3==1.25.8
-Cython
+Cython~=0.29.21
+
+progress~=1.5
+setuptools~=49.6.0