sort imports (#307)

- add hook
- sort imports

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.pre-commit-config.yaml

@@ -20,6 +20,11 @@ repos:
     rev: 22.12.0
     hooks:
     -   id: black-jupyter
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+    files: \.py$
 # numpydoc
 -   repo: https://github.com/Carreau/velin
     rev: 0.0.12

doc/conf.py

@@ -11,8 +11,8 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-import sys
 import subprocess
+import sys
 from datetime import date
 
 # sys.path.insert(0, os.path.abspath('.'))

dpdispatcher/__init__.py

@@ -1,5 +1,6 @@
 import logging
-import os, sys
+import os
+import sys
 import warnings
 
 ROOT_PATH = tuple(__path__)[0]
@@ -39,24 +40,19 @@
 except ImportError:
     __version__ = "unkown"
 
-from .submission import Submission
-from .submission import Task
-from .submission import Job
-from .submission import Resources
-from .slurm import Slurm
-from .pbs import PBS
-from .pbs import Torque
-from .shell import Shell
-from .lsf import LSF
-from .dp_cloud_server import DpCloudServer, Lebesgue
 from .distributed_shell import DistributedShell
-from .machine import Machine
-
+from .dp_cloud_server import DpCloudServer, Lebesgue
+from .dp_cloud_server_context import DpCloudServerContext, LebesgueContext
+from .hdfs_context import HDFSContext
 from .lazy_local_context import LazyLocalContext
 from .local_context import LocalContext
+from .lsf import LSF
+from .machine import Machine
+from .pbs import PBS, Torque
+from .shell import Shell
+from .slurm import Slurm
 from .ssh_context import SSHContext
-from .dp_cloud_server_context import DpCloudServerContext, LebesgueContext
-from .hdfs_context import HDFSContext
+from .submission import Job, Resources, Submission, Task
 
 
 def info():

dpdispatcher/arginfo.py

@@ -1,5 +1,5 @@
-from dpdispatcher.submission import Resources, Task
 from dpdispatcher.machine import Machine
+from dpdispatcher.submission import Resources, Task
 
 resources_dargs = Resources.arginfo
 machine_dargs = Machine.arginfo

dpdispatcher/base_context.py

@@ -1,7 +1,8 @@
 from abc import ABCMeta, abstractmethod
-from dargs import Argument
 from typing import List, Optional, Tuple
 
+from dargs import Argument
+
 from dpdispatcher import dlog
 
 

dpdispatcher/distributed_shell.py

@@ -1,9 +1,9 @@
-from dpdispatcher.JobStatus import JobStatus
+import subprocess as sp
+
 from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
 from dpdispatcher.utils import run_cmd_with_all_output
-import subprocess as sp
-
 
 shell_script_header_template = """
 #!/bin/bash -l

dpdispatcher/dp_cloud_server.py

@@ -5,10 +5,9 @@
 import warnings
 
 from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
-from dpdispatcher.dpcloudserver import zip_file
-from dpdispatcher.dpcloudserver import Client
+from dpdispatcher.dpcloudserver import Client, zip_file
 from dpdispatcher.dpcloudserver.config import ALI_OSS_BUCKET_URL
+from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
 
 shell_script_header_template = """

dpdispatcher/dp_cloud_server_context.py

@@ -1,21 +1,21 @@
 #!/usr/bin/env python
 # coding: utf-8
 # %%
+import os
+import shutil
 import time
 import uuid
+from typing import List
 
+import tqdm
 from dargs.dargs import Argument
-from dpdispatcher.base_context import BaseContext
-from typing import List
-import os
+
 from dpdispatcher import dlog
+from dpdispatcher.base_context import BaseContext
 
 # from dpdispatcher.submission import Machine
 # from . import dlog
-from .dpcloudserver import Client
-from .dpcloudserver import zip_file
-import shutil
-import tqdm
+from .dpcloudserver import Client, zip_file
 
 # from zip_file import zip_files
 from .dpcloudserver.config import ALI_OSS_BUCKET_URL

dpdispatcher/dpcloudserver/client.py

@@ -3,13 +3,15 @@
 import re
 import time
 import urllib.parse
-import requests
-from .retcode import RETCODE
-from .config import HTTP_TIME_OUT, API_HOST, API_LOGGER_STACK_INFO
 from urllib.parse import urljoin
 
+import requests
+
 from dpdispatcher import dlog
 
+from .config import API_HOST, API_LOGGER_STACK_INFO, HTTP_TIME_OUT
+from .retcode import RETCODE
+
 try:
     import oss2
     from oss2 import SizedFileAdapter, determine_part_size

dpdispatcher/dpcloudserver/temp_test.py

@@ -1,8 +1,8 @@
 #%%
 import os
 import sys
-import uuid
 import unittest
+import uuid
 
 from dpdispatcher.dpcloudserver import api
 from dpdispatcher.dpcloudserver.zip_file import zip_files

dpdispatcher/dpcloudserver/zip_file.py

@@ -1,4 +1,5 @@
-import os, glob
+import glob
+import os
 from zipfile import ZipFile
 
 # def zip_file_list(root_path, zip_filename, file_list=[]):

dpdispatcher/hdfs_cli.py

@@ -3,6 +3,7 @@
 
 import os
 import sys
+
 from dpdispatcher.utils import run_cmd_with_all_output
 
 

dpdispatcher/hdfs_context.py

@@ -1,7 +1,11 @@
-from dpdispatcher.base_context import BaseContext
-import os, shutil, hashlib, tarfile
+import hashlib
+import os
+import shutil
+import tarfile
 from glob import glob
+
 from dpdispatcher import dlog
+from dpdispatcher.base_context import BaseContext
 from dpdispatcher.hdfs_cli import HDFS
 
 

dpdispatcher/lazy_local_context.py

@@ -1,7 +1,9 @@
-from dpdispatcher.base_context import BaseContext
-import os, signal
+import os
+import signal
 import subprocess as sp
 
+from dpdispatcher.base_context import BaseContext
+
 
 class SPRetObj(object):
     def __init__(self, ret):

dpdispatcher/local_context.py

@@ -1,10 +1,14 @@
-from dpdispatcher.base_context import BaseContext
-import os, shutil, hashlib, signal
+import hashlib
+import os
+import shutil
+import signal
 import subprocess as sp
 from glob import glob
-from dpdispatcher import dlog
 from subprocess import TimeoutExpired
 
+from dpdispatcher import dlog
+from dpdispatcher.base_context import BaseContext
+
 
 class SPRetObj(object):
     def __init__(self, ret):

dpdispatcher/lsf.py

@@ -3,11 +3,10 @@
 
 from dargs import Argument
 
-from dpdispatcher.machine import Machine
 from dpdispatcher import dlog
 from dpdispatcher.JobStatus import JobStatus
-from dpdispatcher.utils import retry, RetrySignal
-
+from dpdispatcher.machine import Machine
+from dpdispatcher.utils import RetrySignal, retry
 
 lsf_script_header_template = """\
 #!/bin/bash -l

dpdispatcher/machine.py

@@ -1,10 +1,11 @@
 import json
-import shlex
 import pathlib
+import shlex
 from abc import ABCMeta, abstractmethod
 from typing import List, Optional, Tuple
 
 from dargs import Argument, Variant
+
 from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
 

dpdispatcher/pbs.py

@@ -1,10 +1,9 @@
 import shlex
 
-from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
 
-
 pbs_script_header_template = """
 #!/bin/bash -l
 {select_node_line}

dpdispatcher/shell.py

@@ -1,7 +1,7 @@
 import shlex
 
-from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
 
 shell_script_header_template = """

dpdispatcher/slurm.py

@@ -4,11 +4,10 @@
 
 from dargs import Argument
 
-from dpdispatcher.machine import Machine
-from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher import dlog
-from dpdispatcher.machine import script_command_template
-from dpdispatcher.utils import retry, RetrySignal
+from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.machine import Machine, script_command_template
+from dpdispatcher.utils import RetrySignal, retry
 
 # from dpdispatcher.submission import Resources
 

dpdispatcher/ssh_context.py

@@ -1,30 +1,27 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import os, paramiko, tarfile, time
-import uuid
-import shutil
+import os
 import pathlib
-import socket
 import shlex
+import shutil
+import socket
+import tarfile
+import time
+import uuid
 from functools import lru_cache
 from glob import glob
 from typing import List
 
+import paramiko
 import paramiko.ssh_exception
 from dargs.dargs import Argument
 
-from dpdispatcher.base_context import BaseContext
 from dpdispatcher import dlog
+from dpdispatcher.base_context import BaseContext
 
 # from dpdispatcher.submission import Machine
-from dpdispatcher.utils import (
-    get_sha256,
-    generate_totp,
-    rsync,
-    retry,
-    RetrySignal,
-)
+from dpdispatcher.utils import RetrySignal, generate_totp, get_sha256, retry, rsync
 
 
 class SSHSession(object):

dpdispatcher/submission.py

@@ -1,10 +1,16 @@
 # %%
-import time, random, uuid, json, copy, os
-from dargs.dargs import Argument, Variant
-from dpdispatcher.JobStatus import JobStatus
-from dpdispatcher import dlog
+import copy
+import json
+import os
+import random
+import time
+import uuid
 from hashlib import sha1
 
+from dargs.dargs import Argument, Variant
+
+from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
 
 # from dpdispatcher.slurm import SlurmResources
@@ -318,7 +324,8 @@ def remove_unfinished_jobs(self):
                 dlog.info("Can not kill job %s" % job.job_id)
 
             # remove unfinished tasks
-            import os, shutil
+            import os
+            import shutil
 
             for task in job.job_task_list:
                 shutil.rmtree(

dpdispatcher/utils.py

@@ -1,10 +1,10 @@
+import base64
 import hashlib
-import time
-import struct
 import hmac
-import base64
+import struct
 import subprocess
-from typing import Iterable, Optional, Callable, Type, Union
+import time
+from typing import Callable, Iterable, Optional, Type, Union
 
 from dpdispatcher import dlog
 

pyproject.toml

@@ -61,3 +61,6 @@ write_to = "dpdispatcher/_version.py"
 [tool.pyright]
 include = ['dpdispatcher']
 exclude = ['dpdispatcher/dpcloudserver/temp_test.py']
+
+[tool.isort]
+profile = "black"

scripts/script_gen_dargs_docs.py

@@ -2,9 +2,8 @@
 # import sys, os
 # sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..' )))
 # import dpdispatcher
-from dpdispatcher.submission import Resources, Task
 from dpdispatcher.machine import Machine
-
+from dpdispatcher.submission import Resources, Task
 
 # %%
 resources_dargs_doc = Resources.arginfo().gen_doc()