bring back progressbar for multipart upload and download

gdc_client/download/client.py

@@ -10,7 +10,7 @@
 
 from gdc_client.parcel import HTTPClient, utils
 from gdc_client.parcel.download_stream import DownloadStream
-from gdc_client.parcel.utils import tqdm, tqdm_file
+from gdc_client.parcel.utils import get_percentage_pbar
 
 from gdc_client.defaults import SUPERSEDED_INFO_FILENAME_TEMPLATE
 from gdc_client.utils import build_url
@@ -274,9 +274,10 @@ def download_small_groups(self, smalls):
         errors = []
         groupings_len = len(smalls)
 
-        for i, small_group in tqdm(
-            iterable=enumerate(smalls), total=len(smalls), unit="group"
-        ):
+        pbar = get_percentage_pbar(len(smalls))
+
+        for i, small_group in enumerate(smalls):
+            pbar.update(i + 1)
 
             if not small_group:
                 log.error("There are no files to download")
@@ -301,6 +302,8 @@ def download_small_groups(self, smalls):
             if self.md5_check:
                 errors += self._md5_members(members)
 
+        pbar.finish()
+
         return errors, successful_count
 
     def parallel_download(self, stream):

gdc_client/parcel/segment.py

@@ -19,7 +19,7 @@
 
 from gdc_client.parcel.portability import OS_WINDOWS
 from gdc_client.parcel.utils import (
-    get_pbar,
+    get_file_transfer_pbar,
     md5sum,
     mmap_open,
     STRIP,
@@ -64,7 +64,7 @@ def __init__(self, download, n_procs):
         self.schedule()
 
     def _setup_pbar(self):
-        self.pbar = get_pbar(self.download.url, self.download.size)
+        self.pbar = get_file_transfer_pbar(self.download.url, self.download.size)
 
     def _setup_work(self):
         if self.is_complete():
@@ -303,7 +303,7 @@ def finish_download(self):
             time.sleep(0.1)
 
         # Finish the progressbar
-        self.pbar.close()
+        self.pbar.finish()
 
     def wait_for_completion(self):
         try:
@@ -317,7 +317,7 @@ def wait_for_completion(self):
                     this_size = interval.end - interval.begin
                     self.size_complete += this_size
                     since_save += this_size
-                    self.pbar.update(this_size)
+                    self.pbar.update(self.size_complete)
 
                     if self.is_complete():
                         break

gdc_client/parcel/utils.py

@@ -14,7 +14,15 @@
 import os
 import requests
 import stat
-
+import sys
+
+from progressbar import (
+    Bar,
+    ETA,
+    FileTransferSpeed,
+    Percentage,
+    ProgressBar,
+)
 from tqdm import tqdm as std_tqdm
 
 # Logging
@@ -41,7 +49,12 @@ def check_transfer_size(actual, expected):
     return actual == expected
 
 
-def get_pbar(file_id, maxval, start_val=0, desc="Downloading"):
+def get_file_transfer_pbar(
+    file_id: str,
+    maxval: int,
+    start_val: int = 0,
+    desc: str = "Downloading",
+) -> ProgressBar:
     """Create and initialize a custom progressbar
 
     Args:
@@ -51,15 +64,43 @@ def get_pbar(file_id, maxval, start_val=0, desc="Downloading"):
         desc: display message next to the file_id
 
     Returns:
-        tqdm: progress bar instance
+        ProgressBar: progress bar instance
     """
-    log.debug("Downloading {0}:".format(file_id))
-    pbar = tqdm_file(
-        total=maxval, initial=start_val, desc="{} {}".format(desc, file_id),
+    log.debug("{} {}:".format(desc, file_id))
+
+    pbar = ProgressBar(
+        widgets=[
+            Percentage(),
+            " ",
+            Bar(marker="#", left="[", right="]"),
+            " ",
+            ETA(),
+            " ",
+            FileTransferSpeed(),
+            " ",
+        ],
+        max_value=maxval,
+        initial_value=start_val,
+        fd=sys.stdout,
     )
     return pbar
 
 
+def get_percentage_pbar(maxval: int):
+    """Create and initialize a simple percentage progressbar"""
+    return ProgressBar(
+        widgets=[
+            Percentage(),
+            " ",
+            Bar(marker="#", left="[", right="]"),
+            " ",
+            ETA(),
+            " ",
+        ],
+        max_value=maxval,
+    )
+
+
 def print_opening_header(file_id):
     log.debug("")
     log.debug(

gdc_client/upload/client.py

@@ -16,7 +16,7 @@
 from lxml import etree
 from urllib import parse as urlparse
 
-from gdc_client.parcel.utils import tqdm, tqdm_file
+from gdc_client.parcel.utils import get_file_transfer_pbar, get_percentage_pbar
 from gdc_client.upload import manifest
 
 log = logging.getLogger("upload")
@@ -51,7 +51,7 @@ def __init__(self):
 
 
 class Stream(object):
-    def __init__(self, file, pbar: tqdm, filesize: int):
+    def __init__(self, file, pbar, filesize: int):
         self._file = file
         self.pbar = pbar
         self.filesize = filesize
@@ -63,7 +63,8 @@ def read(self, num):
         chunk = self._file.read(num)
 
         if self.pbar:
-            self.pbar.update(len(chunk))
+            pbar_value = min(self.pbar.value + len(chunk), self.filesize)
+            self.pbar.update(pbar_value)
 
         return chunk
 
@@ -431,7 +432,7 @@ def _upload(self):
                     log.error("Can't upload: {}".format(r.content))
                     return
 
-                pbar = tqdm_file(total=self.file_size)
+                pbar = get_file_transfer_pbar(self.file_path, self.file_size, desc="Uploading")
 
                 stream = Stream(f, pbar, self.file_size)
 
@@ -442,7 +443,8 @@ def _upload(self):
                 if r.status_code != 200:
                     log.error("Upload failed {}".format(r.content))
                     return
-                pbar.close()
+
+                pbar.finish()
 
                 self.cleanup()
                 log.info("Upload finished for file {}".format(self.node_id))
@@ -563,9 +565,8 @@ def upload_parts(self):
         if self.total_parts == 0:
             return
 
-        with ThreadPoolExecutor(max_workers=self.processes) as executor, tqdm(
-            total=len(args_list), unit="part", desc="Multipart Upload"
-        ) as pbar:
+        pbar = get_percentage_pbar(len(args_list))
+        with ThreadPoolExecutor(max_workers=self.processes) as executor:
 
             future_to_part_number = {
                 executor.submit(upload_multipart, *payload): payload[5]
@@ -575,7 +576,8 @@ def upload_parts(self):
             for future in as_completed(future_to_part_number):
                 part_number = future_to_part_number[future]
                 log.debug("Part: {} is done".format(part_number))
-                pbar.update()
+                pbar.update(self.ns.completed)
+        pbar.finish()
 
     def list_parts(self):
         r = requests.get(

requirements.txt

@@ -9,3 +9,4 @@ intervaltree==3.0.2
 termcolor==1.1.0
 requests==2.22.0
 tqdm==4.46.1
+progressbar2==3.43.1