Skip to content

Commit

Permalink
Merge 606eb1b into 596ed91
Browse files Browse the repository at this point in the history
  • Loading branch information
@icgood
icgood committed Sep 24, 2019
2 parents 596ed91 + 606eb1b commit c8d9181
Show file tree
Hide file tree
Showing 18 changed files with 309 additions and 567 deletions.
49 changes: 0 additions & 49 deletions pymap/backend/redis/_util.py
View file

This file was deleted.

19 changes: 3 additions & 16 deletions pymap/backend/redis/cleanup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from aioredis import Redis, ConnectionClosedError # type: ignore

from .keys import GlobalKeys, CleanupKeys, NamespaceKeys, ContentKeys, \
MailboxKeys, MessageKeys
MailboxKeys
from .scripts.cleanup import CleanupScripts

__all__ = ['CleanupTask', 'CleanupThread']
Expand Down Expand Up @@ -60,16 +60,14 @@ class CleanupThread:

namespace_ttl: ClassVar[int] = 0
mailbox_ttl: ClassVar[int] = 600
message_ttl: ClassVar[int] = 600
content_ttl: ClassVar[int] = 3600

def __init__(self, redis: Redis, global_keys: GlobalKeys) -> None:
super().__init__()
self._redis = redis
self._global_keys = global_keys
self._keys = keys = CleanupKeys(global_keys)
self._order = (keys.messages, keys.mailboxes, keys.namespaces,
keys.contents)
self._order = (keys.mailboxes, keys.namespaces, keys.contents)

async def run(self) -> NoReturn:
"""Run the cleanup loop indefinitely.
Expand Down Expand Up @@ -99,9 +97,6 @@ async def _run_one(self, cleanup_key: bytes, cleanup_val: bytes) -> None:
elif cleanup_key == keys.mailboxes:
namespace, mailbox_id = cleanup_val.split(b'\x00', 1)
await self._run_mailbox(namespace, mailbox_id)
elif cleanup_key == keys.messages:
namespace, mailbox_id, msg_uid = cleanup_val.split(b'\x00', 2)
await self._run_message(namespace, mailbox_id, msg_uid)
elif cleanup_key == keys.contents:
namespace, email_id = cleanup_val.split(b'\x00', 1)
await self._run_content(namespace, email_id)
Expand All @@ -117,16 +112,8 @@ async def _run_mailbox(self, namespace: bytes, mailbox_id: bytes) -> None:
await _scripts.mailbox(self._redis, self._keys, mbx_keys,
ttl=self.mailbox_ttl)

async def _run_message(self, namespace: bytes, mailbox_id: bytes,
msg_uid: bytes) -> None:
ns_keys = NamespaceKeys(self._global_keys, namespace)
mbx_keys = MailboxKeys(ns_keys, mailbox_id)
msg_keys = MessageKeys(mbx_keys, msg_uid)
await _scripts.message(self._redis, self._keys, mbx_keys, msg_keys,
ttl=self.message_ttl)

async def _run_content(self, namespace: bytes, email_id: bytes) -> None:
ns_keys = NamespaceKeys(self._global_keys, namespace)
ct_keys = ContentKeys(ns_keys, email_id)
await _scripts.content(self._redis, ct_keys,
await _scripts.content(self._redis, ns_keys, ct_keys,
ttl=self.content_ttl)
62 changes: 17 additions & 45 deletions pymap/backend/redis/keys.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@
from pymap.bytes import MaybeBytes, BytesFormat

__all__ = ['RedisKey', 'KeysGroup', 'GlobalKeys', 'CleanupKeys',
'NamespaceKeys', 'ContentKeys', 'FilterKeys', 'MailboxKeys',
'MessageKeys']
'NamespaceKeys', 'ContentKeys', 'FilterKeys', 'MailboxKeys']

#: The version number of this key layout, to track backwards incompatible
#: changes.
DATA_VERSION: Final = 2
DATA_VERSION: Final = 3

_Value = Union[int, MaybeBytes]

Expand Down Expand Up @@ -121,19 +120,18 @@ class CleanupKeys(KeysGroup):
"""

__slots__ = ['namespaces', 'mailboxes', 'messages', 'contents']
__slots__ = ['namespaces', 'mailboxes', 'contents']

def __init__(self, parent: GlobalKeys) -> None:
root = parent.cleanup_root.fork(DATA_VERSION, name='version')
super().__init__(root)
self.namespaces: Final = root.end(b'ns')
self.mailboxes: Final = root.end(b'mbx')
self.messages: Final = root.end(b'msg')
self.contents: Final = root.end(b'content')

@property
def keys(self) -> Sequence[bytes]:
return [self.namespaces, self.mailboxes, self.messages, self.contents]
return [self.namespaces, self.mailboxes, self.contents]


class NamespaceKeys(KeysGroup):
Expand All @@ -148,7 +146,7 @@ class NamespaceKeys(KeysGroup):

__slots__ = ['mailbox_root', 'content_root', 'filter_root', 'mailboxes',
'max_order', 'order', 'uid_validity', 'subscribed',
'email_ids', 'thread_ids']
'content_refs', 'email_ids', 'thread_ids']

def __init__(self, parent: GlobalKeys, namespace: _Value) -> None:
root = parent.namespace_root.fork(namespace, name='namespace')
Expand All @@ -161,13 +159,15 @@ def __init__(self, parent: GlobalKeys, namespace: _Value) -> None:
self.order: Final = root.end(b'order')
self.uid_validity: Final = root.end(b'uidv')
self.subscribed: Final = root.end(b'subscribed')
self.content_refs: Final = root.end(b'contentrefs')
self.email_ids: Final = root.end(b'emailids')
self.thread_ids: Final = root.end(b'threadids')

@property
def keys(self) -> Sequence[bytes]:
return [self.mailboxes, self.max_order, self.order, self.uid_validity,
self.subscribed, self.email_ids, self.thread_ids]
self.subscribed, self.content_refs, self.email_ids,
self.thread_ids]


class ContentKeys(KeysGroup):
Expand Down Expand Up @@ -202,10 +202,10 @@ class FilterKeys(KeysGroup):
__slots__ = ['data', 'names']

def __init__(self, parent: NamespaceKeys) -> None:
root = parent.root
root = parent.root.fork(b'sieve')
super().__init__(root)
self.data: Final = root.end(b'sieve-data')
self.names: Final = root.end(b'sieve')
self.data: Final = root.end(b'data')
self.names: Final = root.end(b'names')

@property
def keys(self) -> Sequence[bytes]:
Expand All @@ -221,51 +221,23 @@ class MailboxKeys(KeysGroup):
"""

__slots__ = ['message_root', 'abort', 'max_mod', 'max_uid', 'uids',
'mod_seq', 'seq', 'expunged', 'recent', 'deleted', 'unseen',
'dates', 'email_ids', 'thread_ids']
__slots__ = ['message_root', 'max_uid', 'uids', 'seq', 'content',
'changes', 'recent', 'deleted', 'unseen']

def __init__(self, parent: NamespaceKeys, mailbox_id: _Value) -> None:
root = parent.mailbox_root.fork(mailbox_id, name='mailbox_id')
super().__init__(root)
self.message_root: Final = root.fork(b'msg')
self.abort: Final = root.end(b'abort')
self.max_mod: Final = root.end(b'max-mod')
self.max_uid: Final = root.end(b'max-uid')
self.uids: Final = root.end(b'uids')
self.mod_seq: Final = root.end(b'mod-seq')
self.seq: Final = root.end(b'seq')
self.expunged: Final = root.end(b'expunged')
self.content: Final = root.end(b'content')
self.changes: Final = root.end(b'changes')
self.recent: Final = root.end(b'recent')
self.deleted: Final = root.end(b'deleted')
self.unseen: Final = root.end(b'unseen')
self.dates: Final = root.end(b'dates')
self.email_ids: Final = root.end(b'emailids')
self.thread_ids: Final = root.end(b'threadids')

@property
def keys(self) -> Sequence[bytes]:
return [self.abort, self.max_mod, self.max_uid, self.uids,
self.mod_seq, self.seq, self.expunged, self.recent,
self.deleted, self.unseen]


class MessageKeys(KeysGroup):
"""The key group for managing a single message, in a single mailbox.
Args:
root: The root redis key.
uid: The message UID.
"""

__slots__ = ['flags']

def __init__(self, parent: MailboxKeys, uid: _Value) -> None:
root = parent.message_root.fork(uid, name='uid')
super().__init__(root)
self.flags: Final = root.end(b'flags')

@property
def keys(self) -> Sequence[bytes]:
return [self.flags]
return [self.max_uid, self.uids, self.seq, self.content, self.changes,
self.recent, self.deleted, self.unseen]
Loading

0 comments on commit c8d9181

Please sign in to comment.