Dflow debug mode for dpgen2 (#125)

Adding dflow debug mode for dpgen2, which is more convenient for
`knowledge distillation` in many cases. One can run the program without
argo as long as `DeePMD-kit` and `lammps` are installed in local
environment.

---------

Co-authored-by: Sikai Yao <yaosk@dp.tech>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

dpgen2/entrypoint/common.py

@@ -1,5 +1,6 @@
 import dflow
 from pathlib import Path
+import os
 from dpgen2.utils import (
     dump_object_to_file,
     load_object_from_file,
@@ -25,6 +26,10 @@ def global_config_workflow(
     # dflow_config, dflow_s3_config
     workflow_config_from_dict(wf_config)
 
+    if os.getenv("DFLOW_DEBUG"):
+        dflow.config["mode"] = "debug"
+        return None
+
     # lebesgue context
     lebesgue_context = None
     if do_lebesgue:

dpgen2/utils/bohrium_config.py

@@ -1,6 +1,7 @@
 import dflow, importlib
 from dflow import config, s3_config
 from dflow.plugins import bohrium
+import os
 
 
 def bohrium_config_from_dict(

dpgen2/utils/run_command.py

@@ -1,13 +1,14 @@
 from dflow.utils import run_command as dflow_run_command
+from dflow import config
 from typing import Tuple, Union, List
+import os
 
 
 def run_command(
     cmd: Union[str, List[str]],
     shell: bool = False,
 ) -> Tuple[int, str, str]:
+    interactive = False if config["mode"] == "debug" else True
     return dflow_run_command(
-        cmd,
-        raise_error=False,
-        try_bash=shell,
+        cmd, raise_error=False, try_bash=shell, interactive=interactive
     )

dpgen2/utils/step_config.py

@@ -6,6 +6,8 @@
 from dpgen2.constants import default_image
 from dflow.plugins.lebesgue import LebesgueExecutor
 from dflow.plugins.dispatcher import DispatcherExecutor
+from dflow import config
+import os
 
 
 def lebesgue_extra_args():
@@ -160,7 +162,7 @@ def gen_doc(*, make_anchor=True, make_link=True, **kwargs):
 def init_executor(
     executor_dict,
 ):
-    if executor_dict is None:
+    if executor_dict is None or config["mode"] == "debug":
         return None
     etype = executor_dict.pop("type")
     if etype == "lebesgue_v2":

pyproject.toml

@@ -17,7 +17,7 @@ classifiers = [
 dependencies = [
 	     'numpy',
 	     'dpdata',
-	     'pydflow>=1.6.30',
+	     'pydflow>=1.6.33',
 	     'dargs>=0.3.1',
 	     'scipy',
 	     'lbg',

tests/utils/test_step_config.py

@@ -12,6 +12,23 @@
 )
 import dflow
 
+from contextlib import contextmanager
+from pathlib import Path
+from dflow.python import (
+    OPIO,
+)
+from copy import deepcopy
+
+
+@contextmanager
+def dflow_mode(mode: str = "default"):
+    _mode = dflow.config["mode"]
+    dflow.config["mode"] = mode
+    try:
+        yield
+    finally:
+        dflow.config["mode"] = _mode
+
 
 class TestStepConfig(unittest.TestCase):
     def test_success(self):
@@ -85,8 +102,13 @@ def test_init_executor(self):
             },
         }
         odict = normalize(idict)
-        ret = init_executor(odict.pop("executor"))
-        self.assertTrue(isinstance(ret, dflow.plugins.lebesgue.LebesgueExecutor))
+        with dflow_mode("debug"):
+            ret = init_executor(deepcopy(odict).pop("executor"))
+            self.assertTrue(ret is None)
+
+        with dflow_mode("default"):
+            ret = init_executor(deepcopy(odict).pop("executor"))
+            self.assertTrue(isinstance(ret, dflow.plugins.lebesgue.LebesgueExecutor))
 
     def test_init_executor_notype(self):
         idict = {
@@ -107,5 +129,12 @@ def test_init_executor_dispatcher(self):
         }
         odict = normalize(idict)
         self.assertEqual(odict["executor"], idict["executor"])
-        ret = init_executor(odict.pop("executor"))
-        self.assertTrue(isinstance(ret, dflow.plugins.dispatcher.DispatcherExecutor))
+        with dflow_mode("debug"):
+            ret = init_executor(deepcopy(odict).pop("executor"))
+            self.assertTrue(ret is None)
+
+        with dflow_mode("default"):
+            ret = init_executor(deepcopy(odict).pop("executor"))
+            self.assertTrue(
+                isinstance(ret, dflow.plugins.dispatcher.DispatcherExecutor)
+            )