Add isort to pre-commit hooks, package resorting (#4647)

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed`
+- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`

.pre-commit-config.yaml

@@ -1,4 +1,9 @@
 repos:
+  -   repo: https://github.com/pycqa/isort
+      rev: 5.8.0
+      hooks:
+      - id: isort
+        language_version: python3
   -   repo: https://github.com/psf/black
       rev: 20.8b1
       hooks:

conftest.py

@@ -1,7 +1,6 @@
 # https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
 import pytest
 
-
 # Uncomment to enable more logging and checks
 # (https://docs.python.org/3/library/asyncio-dev.html)
 # Note this makes things slower and might consume much memory.

distributed/__init__.py

@@ -1,42 +1,42 @@
-from . import config
+from . import config  # isort:skip; load distributed configuration first
 import dask
 from dask.config import config
+
+from ._version import get_versions
 from .actor import Actor, ActorFuture
-from .core import connect, rpc, Status
-from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
-from .diagnostics.progressbar import progress
-from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin, PipInstall
 from .client import (
     Client,
-    Executor,
     CompatibleExecutor,
-    wait,
+    Executor,
+    Future,
     as_completed,
     default_client,
     fire_and_forget,
-    Future,
     futures_of,
+    get_task_metadata,
     get_task_stream,
     performance_report,
-    get_task_metadata,
+    wait,
 )
+from .core import Status, connect, rpc
+from .deploy import Adaptive, LocalCluster, SpecCluster, SSHCluster
+from .diagnostics.plugin import PipInstall, SchedulerPlugin, WorkerPlugin
+from .diagnostics.progressbar import progress
+from .event import Event
 from .lock import Lock
 from .multi_lock import MultiLock
 from .nanny import Nanny
 from .pubsub import Pub, Sub
 from .queues import Queue
+from .scheduler import Scheduler
 from .security import Security
 from .semaphore import Semaphore
-from .event import Event
-from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
-from .utils import sync, TimeoutError, CancelledError
+from .utils import CancelledError, TimeoutError, sync
 from .variable import Variable
-from .worker import Worker, get_worker, get_client, secede, Reschedule
+from .worker import Reschedule, Worker, get_client, get_worker, secede
 from .worker_client import local_client, worker_client
 
-from ._version import get_versions
-
 versions = get_versions()
 __version__ = versions["version"]
 __git_revision__ = versions["full-revisionid"]

distributed/_concurrent_futures_thread.py

@@ -8,16 +8,17 @@
 __author__ = "Brian Quinlan (brian@sweetapp.com)"
 
 import atexit
-from concurrent.futures import _base
 import itertools
+from concurrent.futures import _base
 
 try:
     import queue
 except ImportError:
     import Queue as queue
+
+import os
 import threading
 import weakref
-import os
 
 # Workers are created as daemon threads. This is done to allow the interpreter
 # to exit when there are still idle threads in a ThreadPoolExecutor's thread

distributed/_ipython_utils.py

@@ -12,19 +12,17 @@
 except ImportError:
     # Python 2
     import Queue as queue
-from subprocess import Popen
+
 import sys
-from threading import Thread
+from subprocess import Popen
+from threading import Event, Thread
 from uuid import uuid4
 
-from tornado.gen import TimeoutError
-from tornado.ioloop import IOLoop
-from threading import Event
-
 from IPython import get_ipython
 from jupyter_client import BlockingKernelClient, write_connection_file
 from jupyter_core.paths import jupyter_runtime_dir
-
+from tornado.gen import TimeoutError
+from tornado.ioloop import IOLoop
 
 OUTPUT_TIMEOUT = 10
 

distributed/actor.py

@@ -5,7 +5,7 @@
 
 from .client import Future, default_client
 from .protocol import to_serialize
-from .utils import iscoroutinefunction, thread_state, sync
+from .utils import iscoroutinefunction, sync, thread_state
 from .utils_comm import WrappedKey
 from .worker import get_worker
 

distributed/batched.py

@@ -1,14 +1,14 @@
-from collections import deque
 import logging
+from collections import deque
 
-import dask
 from tornado import gen, locks
 from tornado.ioloop import IOLoop
 
+import dask
+
 from .core import CommClosedError
 from .utils import parse_timedelta
 
-
 logger = logging.getLogger(__name__)
 
 

distributed/cfexecutor.py

@@ -2,11 +2,10 @@
 import weakref
 
 from tlz import merge
-
 from tornado import gen
 
 from .metrics import time
-from .utils import sync, TimeoutError, parse_timedelta
+from .utils import TimeoutError, parse_timedelta, sync
 
 
 @gen.coroutine

distributed/cli/dask_scheduler.py

@@ -1,18 +1,17 @@
 import atexit
-import logging
 import gc
+import logging
 import os
 import re
 import sys
 import warnings
 
 import click
-
 from tornado.ioloop import IOLoop
 
 from distributed import Scheduler
-from distributed.preloading import validate_preload_argv
 from distributed.cli.utils import check_python_3, install_signal_handlers
+from distributed.preloading import validate_preload_argv
 from distributed.proctitle import (
     enable_proctitle_on_children,
     enable_proctitle_on_current,

distributed/cli/dask_spec.py

@@ -1,7 +1,8 @@
 import asyncio
-import click
 import json
 import sys
+
+import click
 import yaml
 
 from distributed.deploy.spec import run_spec

distributed/cli/dask_ssh.py

@@ -1,7 +1,7 @@
-from distributed.deploy.old_ssh import SSHCluster
 import click
 
 from distributed.cli.utils import check_python_3
+from distributed.deploy.old_ssh import SSHCluster
 
 
 @click.command(

distributed/cli/dask_worker.py

@@ -1,16 +1,20 @@
 import asyncio
 import atexit
-from contextlib import suppress
-import logging
 import gc
+import logging
 import os
 import signal
 import sys
 import warnings
+from contextlib import suppress
 
 import click
+from tlz import valmap
+from tornado.ioloop import IOLoop, TimeoutError
+
 import dask
 from dask.system import CPU_COUNT
+
 from distributed import Nanny
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
@@ -22,9 +26,6 @@
 )
 from distributed.utils import import_term
 
-from tlz import valmap
-from tornado.ioloop import IOLoop, TimeoutError
-
 logger = logging.getLogger("distributed.dask_worker")
 
 

distributed/cli/tests/test_dask_scheduler.py

@@ -3,26 +3,26 @@
 pytest.importorskip("requests")
 
 import os
-import requests
-import socket
 import shutil
+import socket
 import sys
 import tempfile
 from time import sleep
 
+import requests
 from click.testing import CliRunner
 
 import distributed
-from distributed import Scheduler, Client
+import distributed.cli.dask_scheduler
+from distributed import Client, Scheduler
+from distributed.metrics import time
 from distributed.utils import get_ip, get_ip_interface, tmpfile
+from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
-    popen,
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
+    popen,
 )
-from distributed.utils_test import loop  # noqa: F401
-from distributed.metrics import time
-import distributed.cli.dask_scheduler
 
 
 def test_defaults(loop):

distributed/cli/tests/test_dask_spec.py

@@ -1,10 +1,11 @@
-import pytest
 import sys
+
+import pytest
 import yaml
 
 from distributed import Client
-from distributed.utils_test import popen
 from distributed.utils_test import cleanup  # noqa: F401
+from distributed.utils_test import popen
 
 
 @pytest.mark.asyncio

distributed/cli/tests/test_dask_ssh.py

@@ -1,4 +1,5 @@
 from click.testing import CliRunner
+
 from distributed.cli.dask_ssh import main
 
 

distributed/cli/tests/test_dask_worker.py

@@ -1,22 +1,29 @@
 import asyncio
+
 import pytest
 from click.testing import CliRunner
 
 pytest.importorskip("requests")
 
-import requests
-import sys
 import os
-from time import sleep
+import sys
 from multiprocessing import cpu_count
+from time import sleep
+
+import requests
 
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils import sync, tmpfile, parse_ports
-from distributed.utils_test import popen, terminate_process, wait_for_port
-from distributed.utils_test import loop, cleanup  # noqa: F401
+from distributed.utils import parse_ports, sync, tmpfile
+from distributed.utils_test import (  # noqa: F401
+    cleanup,
+    loop,
+    popen,
+    terminate_process,
+    wait_for_port,
+)
 
 
 def test_nanny_worker_ports(loop):

distributed/cli/tests/test_tls_cli.py

@@ -1,16 +1,15 @@
 from time import sleep
 
 from distributed import Client
+from distributed.metrics import time
+from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
-    popen,
     get_cert,
     new_config_file,
-    tls_security,
+    popen,
     tls_only_config,
+    tls_security,
 )
-from distributed.utils_test import loop  # noqa: F401
-from distributed.metrics import time
-
 
 ca_file = get_cert("tls-ca-cert.pem")
 cert = get_cert("tls-cert.pem")

distributed/cli/utils.py

@@ -1,6 +1,5 @@
 from tornado.ioloop import IOLoop
 
-
 py3_err_msg = """
 Warning: Your terminal does not set locales.