Merge pull request #39 from y1xiaoc/master

reorganize package init, setup and dargs doc

.github/workflows/test.yml

@@ -10,7 +10,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - 3.6
           - 3.7
           - 3.8
           - 3.9

.gitignore

@@ -16,8 +16,7 @@ tests/graph_real.pb
 *.pyc
 \#*
 iter.*
-tests/test_shell_trival_dir/parent_dir/dir?/out.txt
-# tests/jsons/compute.dp_cloud_server.json ## git update-index --assume-unchanged tests/jsons/compute.dp_cloud_server.json
+
 # out.txt
 topol.tpr
 mdout.mdp
@@ -45,3 +44,4 @@ dbconfig.json
 */_date.py
 *.egg
 *.egg-info
+venv/*

doc/machine-auto.rst

@@ -24,55 +24,52 @@ machine:
         | type: ``str``, optional
         | argument path: ``machine/remote_root``
 
-        The dir where the tasks are executed on the remote machine.
+        The dir where the tasks are executed on the remote machine. Only needed when context is not lazy-local.
 
     remote_profile: 
-        | type: ``dict``, optional
+        | type: ``dict``
         | argument path: ``machine/remote_profile``
 
-        The information used to maintain the connection with remote machine. see subclass introduction
-SSHContext.remote_profile: 
-    | type: ``dict``
-    | argument path: ``SSHContext.remote_profile``
+        The information used to maintain the connection with remote machine. Only needed when context is ssh.
 
-    hostname: 
-        | type: ``str``
-        | argument path: ``SSHContext.remote_profile/hostname``
+        hostname: 
+            | type: ``str``
+            | argument path: ``machine/remote_profile/hostname``
 
-        hostname or ip of ssh connection.
+            hostname or ip of ssh connection.
 
-    username: 
-        | type: ``str``
-        | argument path: ``SSHContext.remote_profile/username``
+        username: 
+            | type: ``str``
+            | argument path: ``machine/remote_profile/username``
 
-        username of target linux system
+            username of target linux system
 
-    password: 
-        | type: ``str``, optional
-        | argument path: ``SSHContext.remote_profile/password``
+        password: 
+            | type: ``str``, optional
+            | argument path: ``machine/remote_profile/password``
 
-        password of linux system
+            password of linux system
 
-    port: 
-        | type: ``int``, optional, default: ``22``
-        | argument path: ``SSHContext.remote_profile/port``
+        port: 
+            | type: ``int``, optional, default: ``22``
+            | argument path: ``machine/remote_profile/port``
 
-        ssh connection port.
+            ssh connection port.
 
-    key_filename: 
-        | type: ``NoneType`` | ``str``, optional, default: ``None``
-        | argument path: ``SSHContext.remote_profile/key_filename``
+        key_filename: 
+            | type: ``str`` | ``NoneType``, optional, default: ``None``
+            | argument path: ``machine/remote_profile/key_filename``
 
-        key_filename used by ssh connection
+            key_filename used by ssh connection
 
-    passphrase: 
-        | type: ``NoneType`` | ``str``, optional, default: ``None``
-        | argument path: ``SSHContext.remote_profile/passphrase``
+        passphrase: 
+            | type: ``str`` | ``NoneType``, optional, default: ``None``
+            | argument path: ``machine/remote_profile/passphrase``
 
-        passphrase used by ssh connection
+            passphrase used by ssh connection
 
-    timeout: 
-        | type: ``int``, optional, default: ``10``
-        | argument path: ``SSHContext.remote_profile/timeout``
+        timeout: 
+            | type: ``int``, optional, default: ``10``
+            | argument path: ``machine/remote_profile/timeout``
 
-        timeout of ssh connection
+            timeout of ssh connection

doc/resources-auto.rst

@@ -1,63 +1,61 @@
-resources_dict: 
+resources: 
     | type: ``dict``
-    | argument path: ``resources_dict``
+    | argument path: ``resources``
 
     number_node: 
         | type: ``int``
-        | argument path: ``resources_dict/number_node``
+        | argument path: ``resources/number_node``
 
         The number of node need for each `job`
 
     cpu_per_node: 
         | type: ``int``
-        | argument path: ``resources_dict/cpu_per_node``
+        | argument path: ``resources/cpu_per_node``
 
-        cpu numbers of each node.
+        cpu numbers of each node assigned to each job.
 
     gpu_per_node: 
         | type: ``int``
-        | argument path: ``resources_dict/gpu_per_node``
+        | argument path: ``resources/gpu_per_node``
 
-        gpu numbers of each node.
+        gpu numbers of each node assigned to each job.
 
     queue_name: 
         | type: ``str``
-        | argument path: ``resources_dict/queue_name``
+        | argument path: ``resources/queue_name``
 
         The queue name of batch job scheduler system.
 
     group_size: 
         | type: ``int``
-        | argument path: ``resources_dict/group_size``
+        | argument path: ``resources/group_size``
 
         The number of `tasks` in a `job`.
 
     custom_flags: 
         | type: ``str``, optional
-        | argument path: ``resources_dict/custom_flags``
+        | argument path: ``resources/custom_flags``
 
         The extra lines pass to job submitting script header
 
     strategy: 
         | type: ``dict``, optional
-        | argument path: ``resources_dict/strategy``
+        | argument path: ``resources/strategy``
 
         strategies we use to generation job submitting scripts.
 
+        if_cuda_multi_devices: 
+            | type: ``bool``, optional, default: ``True``
+            | argument path: ``resources/strategy/if_cuda_multi_devices``
+
     para_deg: 
-        | type: ``int``, optional
-        | argument path: ``resources_dict/para_deg``
+        | type: ``int``, optional, default: ``1``
+        | argument path: ``resources/para_deg``
 
         Decide how many tasks will be run in parallel.
 
     source_list: 
-        | type: ``list``, optional
-        | argument path: ``resources_dict/source_list``
+        | type: ``list``, optional, default: ``[]``
+        | argument path: ``resources/source_list``
 
         The env file to be sourced before the command execution.
-
-    kwargs: 
-        | type: ``dict``, optional
-        | argument path: ``resources_dict/kwargs``
-
-        extra key-value pair

dpdispatcher/__init__.py

@@ -38,11 +38,15 @@
     from .pbs import PBS
     from .shell import Shell
     from .lsf import LSF
-    from .dp_cloud_server import DpCloudServer
 
     from .lazy_local_context import LazyLocalContext
     from .local_context import LocalContext
     from .ssh_context import SSHContext
+except:
+    print("Dependency not satisfied. Unable to import main classes.")
+
+try:
+    from .dp_cloud_server import DpCloudServer
     from .dp_cloud_server_context import DpCloudServerContext
 except ImportError:
     pass
@@ -60,7 +64,7 @@ def info():
     print('')
     print('Dependency')
     print('------------')
-    for modui in ['monty', 'paramiko' ]:
+    for modui in ['psutil', 'paramiko', 'dargs', 'oss2']:
         try:
             mm = __import__(modui)
             print('%10s %10s   %s' % (modui, mm.__version__, mm.__path__[0]))

dpdispatcher/dpcloudserver/temp_test.py

@@ -42,11 +42,11 @@ class DPTest(unittest.TestCase):
     BUCKET_NAME = 'dpcloudserver'
 
     @classmethod
-    def setUpClass(self):
+    def setUpClass(cls):
         print('execute', sys._getframe().f_code.co_name)
 
     @classmethod
-    def tearDownClass(self):
+    def tearDownClass(cls):
         print('execute', sys._getframe().f_code.co_name)
 
     def setUp(self):

dpdispatcher/machine.py

@@ -86,39 +86,6 @@ def load_from_json(cls, json_path):
         machine = cls.load_from_dict(machine_dict=machine_dict)
         return machine
 
-    @classmethod
-    def get_arginfo(cls):
-        doc_batch_type = 'The batch job system type. Option: Slurm, PBS, LSF, Shell, DpCloudServer'
-        doc_context_type = 'The connection used to remote machine. Option: LocalContext, LazyLocalContext, SSHContext， DpCloudServerContext'
-        doc_local_root = 'The dir where the tasks and relating files locate. Typically the project dir.'
-        doc_remote_root = 'The dir where the tasks are executed on the remote machine.'
-        doc_remote_profile = 'The information used to maintain the connection with remote machine. see subclass introduction'
-
-        machine_args = [
-            Argument("batch_type", str, optional=False, doc=doc_batch_type),
-            Argument("context_type", str, optional=False, doc=doc_context_type),
-            Argument("local_root", str, optional=False, doc=doc_local_root, default='./'),
-            Argument("remote_root", str, optional=True, doc=doc_remote_root),
-            Argument("remote_profile", dict, optional=True, doc=doc_remote_profile, default={}),
-        ]
-
-        machine_format = Argument("machine", dict, machine_args)
-        return machine_format
-
-    @classmethod
-    def dargs_check(cls, machine_dict={}):
-        machine_format = cls.get_arginfo()
-        check_return = machine_format.check_value(machine_dict)
-        return check_return
-
-    @classmethod
-    def dargs_gen_doc(cls):
-        machine_format = cls.get_arginfo()
-        ptr = machine_format.gen_doc()
-        ssh_remote_profile_format = BaseContext.subclasses_dict['SSHContext'].get_remote_profile_arginfo()
-        ptr += ssh_remote_profile_format.gen_doc()
-        return ptr
-
     @classmethod
     def load_from_dict(cls, machine_dict):
         batch_type = machine_dict['batch_type']
@@ -259,3 +226,26 @@ def gen_command_env_cuda_devices(self, resources):
             #     command_env+="{ii},".format(ii=ii) 
         return command_env
 
+    @staticmethod
+    def arginfo():
+        # TODO: change the possible value of batch and context types after we refactor the code
+        doc_batch_type = 'The batch job system type. Option: Slurm, PBS, LSF, Shell, DpCloudServer'
+        doc_context_type = 'The connection used to remote machine. Option: LocalContext, LazyLocalContext, SSHContext， DpCloudServerContext'
+        doc_local_root = 'The dir where the tasks and relating files locate. Typically the project dir.'
+        doc_remote_root = 'The dir where the tasks are executed on the remote machine. Only needed when context is not lazy-local.'
+        doc_remote_profile = 'The information used to maintain the connection with remote machine. Only needed when context is ssh.'
+
+        remote_profile_format = SSHSession.arginfo()
+        remote_profile_format.name = "remote_profile"
+        remote_profile_format.doc = doc_remote_profile
+        machine_args = [
+            Argument("batch_type", str, optional=False, doc=doc_batch_type),
+            Argument("context_type", str, optional=False, doc=doc_context_type),
+            # TODO: add default to local_root and remote_root after refactor the code
+            Argument("local_root", str, optional=False, doc=doc_local_root),
+            Argument("remote_root", str, optional=True, doc=doc_remote_root),
+            remote_profile_format,
+        ]
+
+        machine_format = Argument("machine", dict, machine_args)
+        return machine_format

dpdispatcher/ssh_context.py

@@ -138,6 +138,28 @@ def sftp(self):
             self._sftp = self.ssh.open_sftp()
         return self._sftp
 
+    @staticmethod
+    def arginfo():
+        doc_hostname = 'hostname or ip of ssh connection.'
+        doc_username = 'username of target linux system'
+        doc_password = 'password of linux system'
+        doc_port = 'ssh connection port.'
+        doc_key_filename = 'key_filename used by ssh connection'
+        doc_passphrase = 'passphrase used by ssh connection'
+        doc_timeout = 'timeout of ssh connection'
+
+        ssh_remote_profile_args = [
+            Argument("hostname", str, optional=False, doc=doc_hostname),
+            Argument("username", str, optional=False, doc=doc_username),
+            Argument("password", str, optional=True, doc=doc_password),
+            Argument("port", int, optional=True, default=22, doc=doc_port),
+            Argument("key_filename", [str, None], optional=True, default=None, doc=doc_key_filename),
+            Argument("passphrase", [str, None], optional=True, default=None, doc=doc_passphrase),
+            Argument("timeout", int, optional=True, default=10, doc=doc_timeout)
+        ]
+        ssh_remote_profile_format = Argument("ssh_session", dict, ssh_remote_profile_args)
+        return ssh_remote_profile_format
+
 
 class SSHContext(BaseContext):
     def __init__ (self,
@@ -197,28 +219,6 @@ def load_from_dict(cls, context_dict):
         #     )
         return ssh_context
 
-    @staticmethod
-    def get_remote_profile_arginfo():
-        doc_hostname = 'hostname or ip of ssh connection.'
-        doc_username = 'username of target linux system'
-        doc_password = 'password of linux system'
-        doc_port = 'ssh connection port.'
-        doc_key_filename = 'key_filename used by ssh connection'
-        doc_passphrase = 'passphrase used by ssh connection'
-        doc_timeout = 'timeout of ssh connection'
-
-        ssh_remote_profile_args = [
-            Argument("hostname", str, optional=False, doc=doc_hostname),
-            Argument("username", str, optional=False, doc=doc_username),
-            Argument("password", str, optional=True, doc=doc_password),
-            Argument("port", int, optional=True, default=22, doc=doc_port),
-            Argument("key_filename", [str, None], optional=True, default=None, doc=doc_key_filename),
-            Argument("passphrase", [str, None], optional=True, default=None, doc=doc_passphrase),
-            Argument("timeout", int, optional=True, default=10, doc=doc_timeout)
-        ]
-        ssh_remote_profile_format = Argument("SSHContext.remote_profile", dict, ssh_remote_profile_args)
-        return ssh_remote_profile_format
-
     @property
     def ssh(self):
         return self.ssh_session.get_ssh_client()