Allow multiple concurrent uploads,

and multiple concurrent downloads of the same archive.
sz3 · Feb 22, 2020 · de68154 · de68154
1 parent ebaa8ee
commit de68154
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 12 deletions.
diff --git a/pogui/pogui.py b/pogui/pogui.py
@@ -43,6 +43,7 @@ def __init__(self, config):
         self.config = config
         self.cli.set_keyfiles(*self.config.get('keyfiles', []))
         self._refresh_list_manifests()
+        self._progress_count = 0
 
     def _refresh_list_manifests(self):
         locations = self.config.get('fs', []) + ['local']
@@ -109,12 +110,12 @@ def scanManifest(self, mfn):
         paths = backfill_parent_dirs(blobs.keys())
         return [{'path': p, **blob_details(p, blobs.get(p))} for p in paths]
 
-    def _backgroundProgress(self, status_iter, mfn, dest_path=None):
+    def _backgroundProgress(self, status_iter, progress_id, dest_path=None):
         for info in status_iter:
             # need to snapshot these better
             percent = info['current'] * 100 / info['total']
             print('giving {} to window'.format(info))
-            window.evaluate_js("ProgressBar.update('{}', '{:.2f}%');".format(mfn, percent))
+            window.evaluate_js("ProgressBar.update('{}', '{:.2f}%');".format(progress_id, percent))
 
         if dest_path:
             system_open_folder(dest_path)
@@ -132,10 +133,15 @@ def downloadArchive(self, mfn):
 
         status_iter = self.cli.decrypt(real_mfn_path, cwd=path)
         first_chunk = next(status_iter)
+
+        self._progress_count += 1
+        progress_id = f'{mfn}.{self._progress_count}'
         Thread(
             target=self._backgroundProgress,
-            kwargs={'status_iter': status_iter, 'mfn': mfn, 'dest_path': path}
+            kwargs={'status_iter': status_iter, 'progress_id': progress_id, 'dest_path': path}
         ).start()
+
+        first_chunk['progress_id'] = progress_id
         return first_chunk
 
     def downloadFile(self, params):
@@ -147,10 +153,15 @@ def createArchive(self, params):
         paths, destinations = params
         status_iter = self.cli.encrypt(paths, destinations)
         first_chunk = next(status_iter)
+
+        self._progress_count += 1
+        progress_id = f'NewArchive.{self._progress_count}'
         Thread(
             target=self._backgroundProgress,
-            kwargs={'status_iter': status_iter, 'mfn': 'create-archive'}
+            kwargs={'status_iter': status_iter, 'progress_id': progress_id}
         ).start()
+
+        first_chunk['progress_id'] = progress_id
         return first_chunk
 
     def getLocalFolders(self, __=None):

diff --git a/pogui/web/create-archive.js b/pogui/web/create-archive.js
@@ -66,8 +66,9 @@ return {
   {
     var paths = CheckList.get('create-archive-list').items();
     var destinations = CreateArchive.getDestinations();
-    Api.createArchive(paths, destinations).then(function() {
-      ProgressBar.add('create-archive');
+    Api.createArchive(paths, destinations).then(function(res) {
+      if (res)
+        ProgressBar.add(res['progress_id']);
     });
   }
 }

diff --git a/pogui/web/filebrowser/fb.js b/pogui/web/filebrowser/fb.js
@@ -393,7 +393,7 @@ return {
         download.click(function() {
           Api.downloadArchive(id).then(function(res) {
             if (res)
-              ProgressBar.add(id);
+              ProgressBar.add(res['progress_id']);
           });
         });
         act.append(download);

diff --git a/setup.py b/setup.py
@@ -27,7 +27,7 @@
     install_requires=[
         'b2',
         'boto3',
-        'pogcli==0.1.2',
+        'pogcli==0.1.3',
         'pywebview',
         'pyyaml',
     ],

diff --git a/tests/test_api.py b/tests/test_api.py
@@ -179,10 +179,10 @@ def test_downloadArchive(self, mock_thread, mock_window):
         ])
 
         res = self.api.downloadArchive('s3:bucket/dir/file.mfn')
-        self.assertEqual(res, {'current': 1, 'total': 2, 'filename': 'foo'})
+        self.assertEqual(res, {'current': 1, 'total': 2, 'filename': 'foo', 'progress_id': 's3:bucket/dir/file.mfn.1'})
         mock_thread.assert_called_once_with(target=self.api._backgroundProgress, kwargs={
             'status_iter': self.api.cli.decrypt.return_value,
-            'mfn': 's3:bucket/dir/file.mfn',
+            'progress_id': 's3:bucket/dir/file.mfn.1',
             'dest_path': '/home/user/',
         })
 
@@ -208,11 +208,11 @@ def test_createArchive(self, mock_thread, mock_window):
         ])
 
         res = self.api.createArchive((['foo', 'bar'], ['s3:bucket1', 'b2:bucket2']))
-        self.assertEqual(res, {'current': 1, 'total': 2, 'filename': 'foo'})
+        self.assertEqual(res, {'current': 1, 'total': 2, 'filename': 'foo', 'progress_id': 'NewArchive.1'})
 
         mock_thread.assert_called_once_with(target=self.api._backgroundProgress, kwargs={
             'status_iter': self.api.cli.encrypt.return_value,
-            'mfn': 'create-archive',
+            'progress_id': 'NewArchive.1',
         })
 
     def test_getLocalFolders(self, mock_window):