ci: Introduce isort (#495)

* setup: Add isort configuration
* setup: Update isort configs
* docs: Add guide for running auto formatters
* style: Apply isort to all source and test files

changes/495.misc.md

@@ -0,0 +1 @@
+Introduce `isort` as our linter and formatter to ensure consistency of the code style of import statements

docs/dev/daily-workflows.rst

@@ -98,6 +98,16 @@ smaller target of files that you work on and `use an option to select the
 targets only changed
 <https://www.pantsbuild.org/docs/advanced-target-selection#running-over-changed-files-with---changed-since>`_ (``--changed-since``).
 
+Running formatters
+------------------
+
+If you encounter failure from ``isort``, you may run the formatter to automatically fix the import ordering issues.
+
+.. code-block:: console
+
+   $ ./pants fmt ::
+   $ ./pants fmt src/ai/backend/common::
+
 Running unit tests
 ------------------
 

pants.toml

@@ -5,6 +5,7 @@ local_execution_root_dir="%(buildroot)s/.tmp"
 backend_packages = [
     "pants.backend.python",
     "pants.backend.python.lint.flake8",
+    "pants.backend.python.lint.isort",
     "pants.backend.python.typecheck.mypy",
     "pants.backend.experimental.python",
     "setupgen",
@@ -50,6 +51,9 @@ extra_requirements.add = [
 ]
 lockfile = "tools/flake8.lock"
 
+[isort]
+lockfile = "tools/isort.lock"
+
 [pytest]
 version = "pytest>=7.0"
 extra_requirements.add = [

pyproject.toml

@@ -53,3 +53,8 @@ ignore_missing_imports = true
 mypy_path = "stubs:src"
 namespace_packages = true
 explicit_package_bases = true
+
+[tool.isort]
+profile = "black"
+sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
+known_first_party = ["ai.backend"]

src/ai/backend/agent/agent.py

@@ -1,21 +1,23 @@
 from __future__ import annotations
 
-from abc import ABCMeta, abstractmethod
 import asyncio
-from collections import defaultdict
-from decimal import Decimal
-from io import BytesIO, SEEK_END
 import json
 import logging
-from pathlib import Path
 import pickle
-import pkg_resources
 import re
 import signal
 import sys
+import time
 import traceback
+import weakref
+from abc import ABCMeta, abstractmethod
+from collections import defaultdict
+from decimal import Decimal
+from io import SEEK_END, BytesIO
+from pathlib import Path
 from types import TracebackType
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncIterator,
     Awaitable,
@@ -24,74 +26,49 @@
     Dict,
     FrozenSet,
     Generic,
-    Optional,
     List,
     Literal,
     Mapping,
     MutableMapping,
     MutableSequence,
+    Optional,
     Sequence,
     Set,
     Tuple,
     Type,
     TypeVar,
     Union,
-    TYPE_CHECKING,
     cast,
 )
-import weakref
 
 import aioredis
 import aiotools
-from async_timeout import timeout
 import attr
-from cachetools import cached, LRUCache
+import pkg_resources
 import snappy
+import zmq
+import zmq.asyncio
+from async_timeout import timeout
+from cachetools import LRUCache, cached
 from tenacity import (
     AsyncRetrying,
+    retry_if_exception_type,
     stop_after_attempt,
     stop_after_delay,
-    retry_if_exception_type,
     wait_fixed,
 )
-import time
-import zmq, zmq.asyncio
 
 from ai.backend.common import msgpack, redis
-from ai.backend.common.docker import (
-    ImageRef,
-    MIN_KERNELSPEC,
-    MAX_KERNELSPEC,
-)
-from ai.backend.common.logging import BraceStyleAdapter, pretty
-from ai.backend.common.types import (
-    AutoPullBehavior,
-    ContainerId,
-    KernelId,
-    SessionId,
-    DeviceName,
-    SlotName,
-    HardwareMetadata,
-    ImageRegistry,
-    ClusterInfo,
-    KernelCreationConfig,
-    KernelCreationResult,
-    MountTypes,
-    MountPermission,
-    Sentinel,
-    ServicePortProtocols,
-    VFolderMount,
-    aobject,
-)
+from ai.backend.common.docker import MAX_KERNELSPEC, MIN_KERNELSPEC, ImageRef
 from ai.backend.common.events import (
-    EventProducer,
     AbstractEvent,
     AgentErrorEvent,
     AgentHeartbeatEvent,
     AgentStartedEvent,
     AgentTerminatedEvent,
     DoSyncKernelLogsEvent,
     DoSyncKernelStatsEvent,
+    EventProducer,
     ExecutionCancelledEvent,
     ExecutionFinishedEvent,
     ExecutionStartedEvent,
@@ -104,37 +81,44 @@
     SessionFailureEvent,
     SessionSuccessEvent,
 )
-from ai.backend.common.utils import cancel_tasks, current_loop
+from ai.backend.common.logging import BraceStyleAdapter, pretty
 from ai.backend.common.plugin.monitor import ErrorPluginContext, StatsPluginContext
 from ai.backend.common.service_ports import parse_service_ports
-from . import __version__ as VERSION
-from .exception import AgentError, ResourceError
-from .kernel import (
-    AbstractKernel,
-    KernelFeatures,
-    match_distro_data,
+from ai.backend.common.types import (
+    AutoPullBehavior,
+    ClusterInfo,
+    ContainerId,
+    DeviceName,
+    HardwareMetadata,
+    ImageRegistry,
+    KernelCreationConfig,
+    KernelCreationResult,
+    KernelId,
+    MountPermission,
+    MountTypes,
+    Sentinel,
+    ServicePortProtocols,
+    SessionId,
+    SlotName,
+    VFolderMount,
+    aobject,
 )
+from ai.backend.common.utils import cancel_tasks, current_loop
+
+from . import __version__ as VERSION
 from . import resources as resources_mod
+from .exception import AgentError, ResourceError
+from .kernel import AbstractKernel, KernelFeatures, match_distro_data
 from .resources import (
+    AbstractAllocMap,
     AbstractComputeDevice,
     AbstractComputePlugin,
-    AbstractAllocMap,
     KernelResourceSpec,
     Mount,
 )
-from .stats import (
-    StatContext, StatModes,
-)
-from .types import (
-    Container,
-    ContainerStatus,
-    ContainerLifecycleEvent,
-    LifecycleEvent,
-)
-from .utils import (
-    generate_local_instance_id,
-    get_arch_name,
-)
+from .stats import StatContext, StatModes
+from .types import Container, ContainerLifecycleEvent, ContainerStatus, LifecycleEvent
+from .utils import generate_local_instance_id, get_arch_name
 
 if TYPE_CHECKING:
     from ai.backend.common.etcd import AsyncEtcd

src/ai/backend/agent/config.py

@@ -8,7 +8,6 @@
 from .stats import StatModes
 from .types import AgentBackend
 
-
 coredump_defaults = {
     'enabled': False,
     'path': './coredumps',

src/ai/backend/agent/docker/agent.py

@@ -2,25 +2,24 @@
 
 import asyncio
 import base64
-from decimal import Decimal
-from functools import partial
-from io import StringIO
 import json
 import logging
 import os
-from pathlib import Path
-from aiohttp import web
-import pkg_resources
 import secrets
 import shutil
 import signal
 import struct
-from subprocess import CalledProcessError
 import sys
+from decimal import Decimal
+from functools import partial
+from io import StringIO
+from pathlib import Path
+from subprocess import CalledProcessError
 from typing import (
+    TYPE_CHECKING,
     Any,
-    FrozenSet,
     Dict,
+    FrozenSet,
     List,
     Literal,
     Mapping,
@@ -29,79 +28,68 @@
     Sequence,
     Set,
     Tuple,
-    TYPE_CHECKING,
     Union,
 )
 
+import aiotools
+import pkg_resources
+import zmq
 from aiodocker.docker import Docker, DockerContainer
 from aiodocker.exceptions import DockerError
-import aiotools
+from aiohttp import web
 from async_timeout import timeout
-import zmq
 
-from ai.backend.common.docker import (
-    ImageRef,
-    MIN_KERNELSPEC,
-    MAX_KERNELSPEC,
-)
+from ai.backend.common.docker import MAX_KERNELSPEC, MIN_KERNELSPEC, ImageRef
 from ai.backend.common.exception import ImageNotAvailable
 from ai.backend.common.logging import BraceStyleAdapter, pretty
 from ai.backend.common.plugin.monitor import ErrorPluginContext, StatsPluginContext
 from ai.backend.common.types import (
     AutoPullBehavior,
     BinarySize,
     ClusterInfo,
+    ContainerId,
+    DeviceName,
     ImageRegistry,
     KernelCreationConfig,
     KernelId,
-    ContainerId,
-    DeviceName,
-    SlotName,
     MountPermission,
     MountTypes,
     ResourceSlot,
     Sentinel,
+    SlotName,
     current_resource_slots,
 )
 from ai.backend.common.utils import AsyncFileWriter, current_loop
-from .kernel import DockerKernel, prepare_kernel_metadata_uri_handling
-from .metadata.server import create_server as create_metadata_server
-from .resources import detect_resources
-from .utils import PersistentServiceContainer
-from ..exception import UnsupportedResource, InitializationError
-from ..fs import create_scratch_filesystem, destroy_scratch_filesystem
-from ..kernel import AbstractKernel, KernelFeatures
-from ..resources import (
-    Mount,
-    KernelResourceSpec,
-)
+
 from ..agent import (
+    ACTIVE_STATUS_SET,
     AbstractAgent,
     AbstractKernelCreationContext,
-    ACTIVE_STATUS_SET,
     ComputerContext,
 )
-from ..proxy import proxy_connection, DomainSocketProxy
+from ..exception import InitializationError, UnsupportedResource
+from ..fs import create_scratch_filesystem, destroy_scratch_filesystem
+from ..kernel import AbstractKernel, KernelFeatures
+from ..proxy import DomainSocketProxy, proxy_connection
 from ..resources import (
     AbstractComputePlugin,
+    KernelResourceSpec,
+    Mount,
     known_slot_types,
 )
-from ..server import (
-    get_extra_volumes,
-)
-from ..types import (
-    Container,
-    Port,
-    ContainerStatus,
-    LifecycleEvent,
-)
+from ..server import get_extra_volumes
+from ..types import Container, ContainerStatus, LifecycleEvent, Port
 from ..utils import (
     closing_async,
-    update_nested_dict,
+    container_pid_to_host_pid,
     get_kernel_id_from_container,
     host_pid_to_container_pid,
-    container_pid_to_host_pid,
+    update_nested_dict,
 )
+from .kernel import DockerKernel, prepare_kernel_metadata_uri_handling
+from .metadata.server import create_server as create_metadata_server
+from .resources import detect_resources
+from .utils import PersistentServiceContainer
 
 if TYPE_CHECKING:
     from ai.backend.common.etcd import AsyncEtcd