change upload_python_package to upload_python_packages. (#93)

* fix type errors * change upload_python_package to upload_python_packages. correct type required by dflow * convert element of upload_python_packages to Path * fix the bug of iterate over None * ignore type errors maybe a bug of pyright Co-authored-by: Jinzhe Zeng <jinzhe.zeng@rutgers.edu> Co-authored-by: Han Wang <wang_han@iapcm.ac.cn>
deepmodeling · Oct 26, 2022 · 91d8798 · 91d8798
1 parent de21500
commit 91d8798
Show file tree

Hide file tree

Showing 16 changed files with 84 additions and 79 deletions.
diff --git a/dpgen2/entrypoint/submit.py b/dpgen2/entrypoint/submit.py
@@ -111,7 +111,7 @@ def make_concurrent_learning_op (
         select_confs_config : dict = default_config,
         collect_data_config : dict = default_config,
         cl_step_config : dict = default_config,
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):
     if train_style == 'dp':
         prep_run_train_op = PrepRunDPTrain(
@@ -120,7 +120,7 @@ def make_concurrent_learning_op (
             RunDPTrain,
             prep_config = prep_train_config,
             run_config = run_train_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
     else:
         raise RuntimeError(f'unknown train_style {train_style}')
@@ -131,7 +131,7 @@ def make_concurrent_learning_op (
             RunLmp,
             prep_config = prep_explore_config,
             run_config = run_explore_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
     else:
         raise RuntimeError(f'unknown explore_style {explore_style}')
@@ -142,7 +142,7 @@ def make_concurrent_learning_op (
             RunVasp,
             prep_config = prep_fp_config,
             run_config = run_fp_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
     else:
         raise RuntimeError(f'unknown fp_style {fp_style}')
@@ -157,13 +157,13 @@ def make_concurrent_learning_op (
         CollectData,
         select_confs_config = select_confs_config,
         collect_data_config = collect_data_config,
-        upload_python_package = upload_python_package,
+        upload_python_packages = upload_python_packages,
     )    
     # dpgen
     dpgen_op = ConcurrentLearning(
         "concurrent-learning",
         block_cl_op,
-        upload_python_package = upload_python_package,
+        upload_python_packages = upload_python_packages,
         step_config = cl_step_config,
     )
 
@@ -311,8 +311,13 @@ def workflow_concurrent_learning(
     select_confs_config = normalize_step_dict(config.get('select_confs_config', default_config)) if old_style else config['step_configs']['select_confs_config']
     collect_data_config = normalize_step_dict(config.get('collect_data_config', default_config)) if old_style else config['step_configs']['collect_data_config']
     cl_step_config = normalize_step_dict(config.get('cl_step_config', default_config)) if old_style else config['step_configs']['cl_step_config']
-    upload_python_package = config.get('upload_python_package', None)
+    upload_python_packages = config.get('upload_python_packages', None)
     init_models_paths = config.get('training_iter0_model_path', None) if old_style else config['train'].get('training_iter0_model_path', None)
+    if upload_python_packages is not None and isinstance(upload_python_packages, str):
+        upload_python_packages = [upload_python_packages]
+    if upload_python_packages is not None:
+        _upload_python_packages: List[os.PathLike] = [Path(ii) for ii in upload_python_packages]
+        upload_python_packages = _upload_python_packages
 
     concurrent_learning_op = make_concurrent_learning_op(
         train_style,
@@ -327,7 +332,7 @@ def workflow_concurrent_learning(
         select_confs_config = select_confs_config,
         collect_data_config = collect_data_config,
         cl_step_config = cl_step_config,
-        upload_python_package = upload_python_package,
+        upload_python_packages = upload_python_packages,
     )
     scheduler = make_naive_exploration_scheduler(config, old_style=old_style)
 

diff --git a/dpgen2/entrypoint/submit_args.py b/dpgen2/entrypoint/submit_args.py
@@ -146,7 +146,7 @@ def dpgen_step_config_args(default_config):
 
 def submit_args(default_step_config = normalize_step_dict({})):
     doc_step_configs = "Configurations for executing dflow steps"
-    doc_upload_python_package = "Upload python package, for debug purpose"
+    doc_upload_python_packages = "Upload python package, for debug purpose"
     doc_inputs = "The input parameter and artifacts for dpgen2"
     doc_train = "The configuration for training"
     doc_explore = "The configuration for exploration"
@@ -157,7 +157,7 @@ def submit_args(default_step_config = normalize_step_dict({})):
         lebesgue_conf_args() + \
         default_step_config_args() + [
         Argument("step_configs", dict, dpgen_step_config_args(default_step_config), optional=True, default={}, doc=doc_step_configs),
-        Argument("upload_python_package", str, optional=True, default=None, doc=doc_upload_python_package),
+        Argument("upload_python_packages", [list,str], optional=True, default=None, doc=doc_upload_python_packages, alias=["upload_python_package"]),
         Argument("inputs", dict, input_args(), optional=False, doc=doc_inputs),
         Argument("train", dict, [], [variant_train()], optional=False, doc=doc_train),
         Argument("explore", dict, [], [variant_explore()], optional=False, doc=doc_explore),

diff --git a/dpgen2/flow/dpgen_loop.py b/dpgen2/flow/dpgen_loop.py
@@ -118,7 +118,7 @@ def __init__(
             name : str,
             block_op : OPTemplate,
             step_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self._input_parameters={
             "block_id" : InputParameter(),
@@ -174,7 +174,7 @@ def __init__(
             name,
             block_op,
             step_config = step_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
 
     @property
@@ -204,13 +204,13 @@ def __init__(
             name : str,
             block_op : OPTemplate,
             step_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self.loop = ConcurrentLearningLoop(
             name+'-loop',
             block_op,
             step_config = step_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
 
         self._input_parameters={
@@ -260,7 +260,7 @@ def __init__(
             self.loop,
             self.loop_key,
             step_config = step_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
 
     @property
@@ -294,7 +294,7 @@ def _loop (
         name : str,
         block_op : OPTemplate,
         step_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):    
     step_config = deepcopy(step_config)
     step_template_config = step_config.pop('template_config')
@@ -327,7 +327,7 @@ def _loop (
         name = name + '-scheduler',
         template=PythonOPTemplate(
             SchedulerWrapper,
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **step_template_config,
         ),
         parameters={
@@ -347,7 +347,7 @@ def _loop (
         name = name + '-make-block-id',
         template=PythonOPTemplate(
             MakeBlockId,
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **step_template_config,
         ),
         parameters={
@@ -414,7 +414,7 @@ def _dpgen(
         loop_op,
         loop_key,
         step_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None
+        upload_python_packages : Optional[List[os.PathLike]] = None
 ):    
     step_config = deepcopy(step_config)
     step_template_config = step_config.pop('template_config')
@@ -424,7 +424,7 @@ def _dpgen(
         name = name + '-scheduler',
         template=PythonOPTemplate(
             SchedulerWrapper,
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **step_template_config,
         ),
         parameters={
@@ -444,7 +444,7 @@ def _dpgen(
         name = name + '-make-block-id',
         template=PythonOPTemplate(
             MakeBlockId,
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **step_template_config,
         ),
         parameters={

diff --git a/dpgen2/superop/block.py b/dpgen2/superop/block.py
@@ -42,7 +42,7 @@ def __init__(
             collect_data_op : OP,
             select_confs_config : dict = normalize_step_dict({}),
             collect_data_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self._input_parameters={
             "block_id" : InputParameter(),
@@ -105,7 +105,7 @@ def __init__(
             collect_data_op,
             select_confs_config = select_confs_config,
             collect_data_config = collect_data_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )
 
     @property
@@ -140,7 +140,7 @@ def _block_cl(
         collect_data_op : OP,
         select_confs_config : dict = normalize_step_dict({}),
         collect_data_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):
     select_confs_config = deepcopy(select_confs_config)
     collect_data_config = deepcopy(collect_data_config)
@@ -189,7 +189,7 @@ def _block_cl(
             output_artifact_archive={
                 "confs": None
             },
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **select_confs_template_config,
         ),
         parameters={
@@ -229,7 +229,7 @@ def _block_cl(
             output_artifact_archive={
                 "iter_data": None
             },
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **collect_data_template_config,
         ),
         parameters={

diff --git a/dpgen2/superop/prep_run_dp_train.py b/dpgen2/superop/prep_run_dp_train.py
@@ -41,7 +41,7 @@ def __init__(
             run_train_op : OP,
             prep_config : dict = normalize_step_dict({}),
             run_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self._input_parameters = {
             "block_id" : InputParameter(type=str, value=""),
@@ -91,7 +91,7 @@ def __init__(
             run_train_op,
             prep_config = prep_config,
             run_config = run_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )            
 
     @property
@@ -122,7 +122,7 @@ def _prep_run_dp_train(
         run_train_op : OP,
         prep_config : dict = normalize_step_dict({}),
         run_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):
     prep_config = deepcopy(prep_config)
     run_config = deepcopy(run_config)
@@ -138,7 +138,7 @@ def _prep_run_dp_train(
             output_artifact_archive={
                 "task_paths": None
             },
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **prep_template_config,
         ),
         parameters={
@@ -163,7 +163,7 @@ def _prep_run_dp_train(
                 input_artifact = ["task_path", "init_model"],
                 output_artifact = ["model", "lcurve", "log", "script"],
             ),
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **run_template_config,
         ),
         parameters={

diff --git a/dpgen2/superop/prep_run_fp.py b/dpgen2/superop/prep_run_fp.py
@@ -41,7 +41,7 @@ def __init__(
             run_op : OP,
             prep_config : dict = normalize_step_dict({}),
             run_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self._input_parameters = {
             "block_id" : InputParameter(type=str, value=""),
@@ -89,7 +89,7 @@ def __init__(
             run_op,
             prep_config = prep_config,
             run_config = run_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )            
 
     @property
@@ -121,7 +121,7 @@ def _prep_run_fp(
         run_op : OP,
         prep_config : dict = normalize_step_dict({}),
         run_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):
     prep_config = deepcopy(prep_config)
     run_config = deepcopy(run_config)
@@ -137,7 +137,7 @@ def _prep_run_fp(
             output_artifact_archive={
                 "task_paths": None
             },
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **prep_template_config,
         ),
         parameters={
@@ -163,7 +163,7 @@ def _prep_run_fp(
                 input_artifact = ["task_path"],
                 output_artifact = ["log", "labeled_data"],
             ),
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **run_template_config,
         ),
         parameters={

diff --git a/dpgen2/superop/prep_run_lmp.py b/dpgen2/superop/prep_run_lmp.py
@@ -41,7 +41,7 @@ def __init__(
             run_op : OP,
             prep_config : dict = normalize_step_dict({}),
             run_config : dict = normalize_step_dict({}),
-            upload_python_package : Optional[List[os.PathLike]] = None,
+            upload_python_packages : Optional[List[os.PathLike]] = None,
     ):
         self._input_parameters = {
             "block_id" : InputParameter(type=str, value=""),
@@ -91,7 +91,7 @@ def __init__(
             run_op,
             prep_config = prep_config,
             run_config = run_config,
-            upload_python_package = upload_python_package,
+            upload_python_packages = upload_python_packages,
         )            
 
     @property
@@ -122,7 +122,7 @@ def _prep_run_lmp(
         run_op : OP,
         prep_config : dict = normalize_step_dict({}),
         run_config : dict = normalize_step_dict({}),
-        upload_python_package : Optional[List[os.PathLike]] = None,
+        upload_python_packages : Optional[List[os.PathLike]] = None,
 ):
     prep_config = deepcopy(prep_config)
     run_config = deepcopy(run_config)
@@ -138,7 +138,7 @@ def _prep_run_lmp(
             output_artifact_archive={
                 "task_paths": None
             },
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **prep_template_config,
         ),
         parameters={
@@ -162,7 +162,7 @@ def _prep_run_lmp(
                 input_artifact = ["task_path"],
                 output_artifact = ["log", "traj", "model_devi", "plm_output"],
             ),
-            python_packages = upload_python_package,
+            python_packages = upload_python_packages,
             **run_template_config,
         ),
         parameters={

diff --git a/examples/almg/input.json b/examples/almg/input.json
@@ -88,7 +88,7 @@
 	"_comment" : "all"
     },
 
-    "upload_python_package" : "/path/to/dpgen2",
+    "upload_python_packages" : "/path/to/dpgen2",
 
     "inputs": {
 	"type_map":		["Al", "Mg"],

diff --git a/tests/context.py b/tests/context.py
@@ -9,7 +9,7 @@
 else:
     skip_ut_with_dflow = False
     skip_ut_with_dflow_reason = ''
-upload_python_package = os.path.join(dpgen_path, 'dpgen2')
+upload_python_packages = [os.path.join(dpgen_path, 'dpgen2')]
 # one needs to set proper values for the following variable.
 default_image = 'dptechnology/dpgen2:latest'
 default_host = None