fix: allow to run async create_todump_list in CLI

biothings/hub/dataload/dumper.py

@@ -2022,11 +2022,22 @@ async def create_todump_list(self, force=False, job_manager=None, **kwargs):
             self.prepare_client()
         self.prepare_remote_container()
         # unprepare unpicklable objects so we can use multiprocessing
-        self.unprepare()
+        state = self.unprepare()
         # set up job to generate remote file
         pinfo = self.get_pinfo()
         pinfo["step"] = "check"
-        job = await job_manager.defer_to_process(pinfo, partial(self.generate_remote_file))
+        if job_manager:
+            job = job_manager.defer_to_process(pinfo, partial(self.generate_remote_file))
+        else:
+            # otherwise, just run it with asyncio loop directly
+            async def run(fut):
+                res = self.generate_remote_file()
+                fut.set_result(res)
+
+            loop = asyncio.get_event_loop()
+            job = loop.create_future()
+            loop.create_task(run(job))
+
         remote_error = False
 
         def remote_done(f):
@@ -2039,6 +2050,8 @@ def remote_done(f):
         if remote_error:
             raise remote_error
         # Need to reinit _state b/c of unprepare
+        self.prepare(state)  # reverse of unpreare after async job is done
+        # TODO: test if the following two lines can be removed after we call self.prepare(state) above
         if self.need_prepare():
             self.prepare_client()
         self.setup_log()

biothings/management/dataplugin_localhub.py

@@ -1,4 +1,6 @@
 # flake8: noqa: B008
+import asyncio
+import inspect
 import os
 import pathlib
 from shutil import copytree
@@ -121,7 +123,11 @@ def dump_and_upload(
     uploader_classes = uploader_manager[plugin_name]
     dumper = dumper_class()
     dumper.prepare()
-    dumper.create_todump_list(force=True)
+    if inspect.iscoroutinefunction(dumper.create_todump_list):
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(dumper.create_todump_list(force=True))
+    else:
+        dumper.create_todump_list(force=True)
     for item in dumper.to_dump:
         dumper.download(item["remote"], item["local"])
     dumper.steps = ["post"]