Merge pull request #36 from icgood/wip

Add an admin service for running backends

.coveragerc

@@ -1,2 +1,2 @@
 [report]
-omit = */maildir/*, */redis/*
+omit = */maildir/*, */redis/*, */grpc/*

README.md

@@ -21,6 +21,7 @@ Everything runs in an [asyncio][2] event loop.
   * [dict Plugin](#dict-plugin)
   * [maildir Plugin](#maildir-plugin)
   * [redis Plugin](#redis-plugin)
+* [Admin Tool](#admin-tool)
 * [Supported Extensions](#supported-extensions)
 * [Development and Testing](#development-and-testing)
   * [Type Hinting](#type-hinting)
@@ -158,6 +159,44 @@ $ pymap --insecure-login --debug redis redis://localhost
 Once started, check out the dict plugin example above to connect and see it in
 action.
 
+## Admin Tool
+
+The `pymap-admin` tool can be used to perform various admin functions against a
+running pymap server. This is a separate [grpc][10] service using [grpclib][11]
+listening on a UNIX socket, typically `/tmp/pymap/admin-<pid>.sock`.
+
+The admin tool and service have extra dependencies you must install first:
+
+```
+pip install grpclib protobuf
+```
+
+#### `append` Command
+
+To append a message directly to a mailbox, without using IMAP, use the
+`append` admin command. First, check out the help:
+
+```
+$ pymap-admin append --help
+```
+
+As a basic example, you can append a message to a `dict` plugin backend like
+this:
+
+```
+$ cat <<EOF | pymap-admin append demouser
+> From: user@example.com
+>
+> test message!
+> EOF
+validity: 1784302999
+uid: 101
+```
+
+The output is the UID validity value of the mailbox the message was appended
+to, and the UID of the appended message. In this example, `demouser` is the
+login user and the default mailbox is `INBOX`.
+
 ## Supported Extensions
 
 In addition to [RFC 3501][1], pymap supports a number of IMAP extensions to
@@ -255,3 +294,5 @@ no need to attempt `--strict` mode.
 [7]: http://mypy-lang.org/
 [8]: https://redis.io/
 [9]: https://github.com/aio-libs/aioredis
+[10]: https://grpc.io/
+[11]: https://github.com/vmagamedov/grpclib

doc/source/pymap.admin.rst

@@ -0,0 +1,12 @@
+
+``pymap.admin``
+===============
+
+.. automodule:: pymap.admin
+   :members:
+
+``pymap.admin.handlers``
+----------------------------
+
+.. automodule:: pymap.admin.handlers
+   :members:

pymap/admin/__init__.py

@@ -0,0 +1,70 @@
+
+import os
+import os.path
+import asyncio
+from argparse import ArgumentParser
+from asyncio import CancelledError
+from typing_extensions import Final
+
+from grpclib.server import Server  # type: ignore
+from pymap.interfaces.backend import BackendInterface, ServiceInterface
+
+from .handlers import GrpcHandlers
+
+__all__ = ['AdminService']
+
+
+class AdminService(ServiceInterface):
+    """A pymap service implemented using a `grpc <https://grpc.io/>`_ server
+    to perform admin functions on a running backend.
+
+    """
+
+    def __init__(self, path: str, server: Server) -> None:
+        super().__init__()
+        self.path: Final = path
+        self.server: Final = server
+
+    @classmethod
+    def add_arguments(cls, parser: ArgumentParser) -> None:
+        admin = parser.add_argument_group('admin arguments')
+        admin.add_argument('--admin-path', metavar='PATH',
+                           help='path to POSIX socket file')
+
+    @classmethod
+    async def init(cls, backend: BackendInterface) -> 'AdminService':
+        config = backend.config
+        if config.args.admin_path is not None:
+            path = config.args.admin_path
+        else:
+            pid = os.getpid()
+            path = os.path.join(os.sep, 'tmp', 'pymap', f'admin-{pid}.sock')
+        cls._create_dir(path)
+        handlers = GrpcHandlers(backend)
+        server = Server([handlers], loop=asyncio.get_event_loop())
+        return cls(path, server)
+
+    @classmethod
+    def _create_dir(cls, path: str) -> None:
+        try:
+            dirname = os.path.dirname(path)
+            os.mkdir(dirname, 0o755)
+        except FileExistsError:
+            pass
+
+    def _unlink_path(self) -> None:
+        try:
+            os.unlink(self.path)
+        except OSError:
+            pass
+
+    async def run_forever(self) -> None:
+        server = self.server
+        await server.start(path=self.path)
+        try:
+            await server.wait_closed()
+        except CancelledError:
+            server.close()
+            await server.wait_closed()
+        finally:
+            self._unlink_path()

pymap/admin/client/__init__.py

@@ -0,0 +1,58 @@
+"""Admin functions for a running pymap server."""
+
+import os
+import os.path
+import re
+import asyncio
+from argparse import ArgumentParser, Namespace
+
+from grpclib.client import Channel  # type: ignore
+from pymap.core import __version__
+
+from .append import AppendCommand
+from .command import ClientCommand
+from ..grpc.admin_grpc import AdminStub
+
+
+def _find_path(parser: ArgumentParser) -> str:
+    dirname = os.path.join(os.sep, 'tmp', 'pymap')
+    try:
+        paths = [os.path.join(dirname, fn)
+                 for fn in os.listdir(dirname)
+                 if re.match(r'^admin-\d+\.sock$', fn)]
+    except FileNotFoundError:
+        paths = []
+    if len(paths) == 1:
+        return paths[0]
+    parser.error('Cannot determine admin socket path')
+
+
+def main() -> None:
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s' + __version__)
+    parser.add_argument('--path', help='path to admin socket file')
+
+    subparsers = parser.add_subparsers(dest='command',
+                                       help='which admin command to run')
+    commands = dict([AppendCommand.init(parser, subparsers)])
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.error('Expected command name.')
+    command = commands[args.command]
+
+    asyncio.run(run(parser, args, command), debug=False)
+
+
+async def run(parser: ArgumentParser, args: Namespace,
+              command: ClientCommand) -> None:
+    loop = asyncio.get_event_loop()
+    path = args.path or _find_path(parser)
+    channel = Channel(path=path, loop=loop)
+    stub = AdminStub(channel)
+    try:
+        ret = await command.run(stub, args)
+        print(ret)
+    finally:
+        channel.close()

pymap/admin/client/append.py

@@ -0,0 +1,49 @@
+"""Append a message directly to a user's mailbox."""
+
+import sys
+import time
+from argparse import ArgumentParser, FileType, Namespace
+from typing import Tuple
+
+from .command import ClientCommand
+from ..grpc.admin_grpc import AdminStub
+from ..grpc.admin_pb2 import AppendRequest, AppendResponse
+
+
+class AppendCommand(ClientCommand):
+
+    @classmethod
+    def init(cls, parser: ArgumentParser, subparsers) \
+            -> Tuple[str, 'AppendCommand']:
+        subparser = subparsers.add_parser(
+            'append', description=__doc__,
+            help='append a message to a mailbox')
+        subparser.add_argument('--mailbox', default='INBOX',
+                               help='the mailbox name (default: INBOX)')
+        subparser.add_argument('--timestamp', type=float, metavar='SECONDS',
+                               help='the message timestamp')
+        subparser.add_argument('--data', type=FileType('rb'), metavar='FILE',
+                               default=sys.stdin.buffer,
+                               help='the message data (default: stdin)')
+        subparser.add_argument('user', help='the user name')
+        flags = subparser.add_argument_group('message flags')
+        flags.add_argument('--flag', dest='flags', action='append',
+                           metavar='VAL', help='a message flag or keyword')
+        flags.add_argument('--flagged', dest='flags', action='append_const',
+                           const='\\Flagged', help='the message is flagged')
+        flags.add_argument('--seen', dest='flags', action='append_const',
+                           const='\\Seen', help='the message is seen')
+        flags.add_argument('--draft', dest='flags', action='append_const',
+                           const='\\Draft', help='the message is a draft')
+        flags.add_argument('--deleted', dest='flags', action='append_const',
+                           const='\\Deleted', help='the message is deleted')
+        flags.add_argument('--answered', dest='flags', action='append_const',
+                           const='\\Answered', help='the message is answered')
+        return 'append', cls()
+
+    async def run(self, stub: AdminStub, args: Namespace) -> AppendResponse:
+        data = args.data.read()
+        when: float = args.timestamp or time.time()
+        req = AppendRequest(user=args.user, mailbox=args.mailbox,
+                            data=data, flags=args.flags, when=when)
+        return await stub.Append(req)

pymap/admin/client/command.py

@@ -0,0 +1,38 @@
+
+from abc import abstractmethod
+from argparse import ArgumentParser, Namespace
+from typing import Any, Tuple
+from typing_extensions import Protocol
+
+from ..grpc.admin_grpc import AdminStub
+
+__all__ = ['ClientCommand']
+
+
+class ClientCommand(Protocol):
+
+    @classmethod
+    @abstractmethod
+    def init(cls, parser: ArgumentParser, subparsers) \
+            -> Tuple[str, 'ClientCommand']:
+        """Initialize the client command, adding its subparser and returning
+        the command name and object.
+
+        Args:
+            parser: The argument parser object.
+            subparsers: The special action object as returned by
+                :meth:`~argparse.ArgumentParser.add_subparsers`.
+
+        """
+        ...
+
+    @abstractmethod
+    async def run(self, stub: AdminStub, args: Namespace) -> Any:
+        """Run the command and return its result.
+
+        Args:
+            stub: The GRPC stub for executing commands.
+            args: The command line arguments.
+
+        """
+        ...

pymap/admin/grpc/admin.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package admin;
+
+enum Result {
+  SUCCESS = 0;
+  USER_NOT_FOUND = 1;
+  MAILBOX_NOT_FOUND = 2;
+}
+
+message AppendRequest {
+  string user = 1;
+  string mailbox = 2;
+  bytes data = 3;
+  repeated string flags = 4;
+  uint64 when = 5;
+}
+
+message AppendResponse {
+  Result result = 1;
+  uint32 validity = 2;
+  uint32 uid = 3;
+}
+
+service Admin {
+  rpc Append (AppendRequest) returns (AppendResponse) {}
+}

pymap/admin/grpc/admin_grpc.py

@@ -0,0 +1,37 @@
+# Generated by the Protocol Buffers compiler. DO NOT EDIT!
+# source: pymap/admin/grpc/admin.proto
+# plugin: grpclib.plugin.main
+import abc
+
+import grpclib.const
+import grpclib.client
+
+import pymap.admin.grpc.admin_pb2
+
+
+class AdminBase(abc.ABC):
+
+    @abc.abstractmethod
+    async def Append(self, stream):
+        pass
+
+    def __mapping__(self):
+        return {
+            '/admin.Admin/Append': grpclib.const.Handler(
+                self.Append,
+                grpclib.const.Cardinality.UNARY_UNARY,
+                pymap.admin.grpc.admin_pb2.AppendRequest,
+                pymap.admin.grpc.admin_pb2.AppendResponse,
+            ),
+        }
+
+
+class AdminStub:
+
+    def __init__(self, channel: grpclib.client.Channel) -> None:
+        self.Append = grpclib.client.UnaryUnaryMethod(
+            channel,
+            '/admin.Admin/Append',
+            pymap.admin.grpc.admin_pb2.AppendRequest,
+            pymap.admin.grpc.admin_pb2.AppendResponse,
+        )

pymap/admin/grpc/admin_grpc.pyi

@@ -0,0 +1,14 @@
+
+import abc
+import grpclib.client  # type: ignore
+from typing import Any
+
+from .admin_pb2 import AppendRequest, AppendResponse
+
+class AdminBase(abc.ABC):
+    async def Append(self, stream: Any) -> None: ...
+    def __mapping__(self): ...
+
+class AdminStub:
+    def __init__(self, channel: grpclib.client.Channel) -> None: ...
+    async def Append(self, request: AppendRequest) -> AppendResponse: ...