feat(upload): ask for user input about resume

gdc_client/repl/__init__.py

@@ -72,7 +72,6 @@ class GDCREPL(Cmd):
     def __init__(self, *args, **kwargs):
         self.file_ids = set()
         self.token = None
-        self.resume = None
         Cmd.__init__(self, *args, **kwargs)
         print(self.HEADER)
         print(self.BASIC_COMMANDS)
@@ -349,13 +348,7 @@ def _get_upload_client(self, manifest):
 
     def do_upload(self, manifest):
     	'''upload files given a manifest path'''
-    	if self.resume:
-    		print "Resume previous upload"
-    		client = self._get_upload_client(self.resume)
-    	else:
-	        client = self._get_upload_client(manifest)
-        client.upload()
-        self.resume = client.resume_path
+        client = self._get_upload_client(manifest)
 
     def do_abort(self, manifest):
         '''abort a partially uploaded file'''

gdc_client/upload/__init__.py

@@ -4,7 +4,7 @@
 import yaml
 import logging
 import requests
-from client import GDCUploadClient
+from client import GDCUploadClient, read_manifest
 from ..argparser import subparsers
 
 command = 'upload'
@@ -65,7 +65,7 @@ def main():
         [{"id": args.identifier, "project_id": args.project_id,
           "path": args.path, "upload_id": args.upload_id}]
 
-    manifest_name = args.manifest.name if args.manifest else None
+    manifest_name = args.manifest.name if args.manifest else args.identifier
 
     client = GDCUploadClient(
         token=args.token.read(), processes=args.n_processes,
@@ -80,13 +80,4 @@ def main():
         client.upload()
 
 
-def read_manifest(manifest):
-    manifest = yaml.load(manifest)
-    def _read_manifest(manifest):
-      if type(manifest) == list:
-          return sum([_read_manifest(item) for item in manifest], [])
-      if "files" in manifest:
-          return manifest['files']
-      else:
-          return sum([_read_manifest(item) for item in manifest.values()], [])
-    return _read_manifest(manifest)
+

gdc_client/upload/client.py

@@ -1,4 +1,5 @@
 from urlparse import urljoin
+import random
 from multiprocessing import Pool, Manager
 import requests
 from exceptions import KeyError
@@ -20,6 +21,7 @@
 
 MAX_RETRIES = 10
 MAX_TIMEOUT = 60
+MIN_PARTSIZE = 5242880
 
 OS_WINDOWS = platform.system() == 'Windows'
 
@@ -50,6 +52,17 @@ def upload_multipart_wrapper(args):
     return upload_multipart(*args)
 
 
+def read_manifest(manifest):
+    manifest = yaml.load(manifest)
+    def _read_manifest(manifest):
+      if type(manifest) == list:
+          return sum([_read_manifest(item) for item in manifest], [])
+      if "files" in manifest:
+          return manifest['files']
+      else:
+          return sum([_read_manifest(item) for item in manifest.values()], [])
+    return _read_manifest(manifest)
+
 class Stream(object):
 
     def __init__(self, file, pbar, filesize):
@@ -139,9 +152,9 @@ def __init__(self, token, processes, server, part_size,
         self.upload_id = None
         self.debug = debug
         self.processes = processes
-        self.part_size = (part_size/PAGESIZE+1)*PAGESIZE
+        self.part_size = (max(part_size, MIN_PARTSIZE)/PAGESIZE+1)*PAGESIZE
         self._metadata = None
-        self.resume_path = None
+        self.resume_path = "resume_{}".format(self.manifest_name)
 
     @property
     def metadata(self):
@@ -210,6 +223,12 @@ def called(self, arg):
 
     def upload(self):
         '''Upload files to object storage'''
+        if os.path.isfile(self.resume_path):
+            use_resume = raw_input("Found an {}. Press Y to resume last upload and n to start a new upload [Y/n]: ".format(self.resume_path))
+            if use_resume.lower() not in ['n','no']:
+                with open(self.resume_path,'r') as f:
+                    self.files = read_manifest(f)
+
         for f in self.files:
             self.get_file(f)
             print("Attempting to upload to {}".format(self.url))
@@ -290,17 +309,18 @@ def handle_multipart(self):
         try:
             yield
             self.upload_id = None
+            if os.path.isfile(self.resume_path):
+                os.remove(self.resume_path)
         except Exception as e:
             print "Saving unfinished upload file"
             if self.upload_id:
                 self.incompleted[0]['upload_id'] = self.upload_id
-            path = "resume_{}".format(self.manifest_name or self.node_id)
+            path = self.resume_path
             with open(path, 'w') as f:
                 f.write(
                     yaml.dump({"files": list(self.incompleted)},
                               default_flow_style=False))
             print 'Saved to', path
-            self.resume_path = path
             if self.debug:
                 raise
             else: